mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-07-22 01:08:13 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'fuse-update-7.2' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse

Browse Source 
Pull fuse updates from Miklos Szeredi:

 - Fix lots of bugs, most from the late 6.x era, but some going back
   to 2.6.x

 - Add subsystems (io-uring, passthrough) and respective maintainers
   (Bernd, Joanne and Amir)

 - Separate transport and fs layers (Miklos)

 - Don't block on cat /dev/fuse (Joanne)

 - Perform some refactoring in fuse-uring (Joanne)

 - Don't use bounce-buffer for READDIR reply in virtio-fs (Matthew Ochs)

 - Clean up documentation (Randy)

 - Improve tracing (Amir)

 - Extend page cache invalidation after DIO (Cheng Ding)

 - Invalidate readdir cache on epoch change (Jun Wu)

 - Misc cleanups

* tag 'fuse-update-7.2' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse: (81 commits)
  fuse-uring: clear ent->fuse_req in commit_fetch error path
  fuse-uring: use named constants for io-uring iovec indices
  fuse-uring: refactor setting up copy state for payload copying
  fuse-uring: use enum types for header copying
  fuse-uring: refactor io-uring header copying from ring
  fuse-uring: refactor io-uring header copying to ring
  fuse-uring: separate next request fetching from sending logic
  fuse: invalidate readdir cache on epoch bump
  virtio-fs: avoid double-free on failed queue setup
  fuse: invalidate page cache after DIO and async DIO writes
  fuse: set ff->flock only on success
  fuse: clean up interrupt reading
  fuse: remove stray newline in fuse_dev_do_read()
  fuse: use READ_ONCE in fuse_chan_num_background()
  fuse: dax: Move long delayed work on system_dfl_long_wq
  fuse: add fuse_request_sent tracepoint
  fuse: Add SPDX ID lines to some files
  fuse: use QSTR() instead of QSTR_INIT() in fuse_get_dentry
  fuse: convert page array allocation to kcalloc()
  fuse: use current creds for backing files
  ...
This commit is contained in:
Linus Torvalds
2026-06-18 08:50:52 -07:00
parent 9e7e663345 7d87a5a284
commit 6edc20078a
29 changed files with 2788 additions and 2148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
MAINTAINERS
View File

@@ -10616,10 +10616,10 @@ L: netdev@vger.kernel.org
S: Maintained
F: drivers/net/ethernet/fungible/
FUSE: FILESYSTEM IN USERSPACE
FUSE FILESYSTEM [CORE]
M: Miklos Szeredi <miklos@szeredi.hu>
L: linux-fsdevel@vger.kernel.org
S: Maintained
L: fuse-devel@lists.linux.dev
S: Supported
W: https://github.com/libfuse/
T: git git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git
F: Documentation/filesystems/fuse/*
@@ -10627,6 +10627,26 @@ F: fs/fuse/
F: include/uapi/linux/fuse.h
F: tools/testing/selftests/filesystems/fuse/
FUSE FILESYSTEM [IO-URING]
M: Bernd Schubert <bernd@bsbernd.com>
M: Joanne Koong <joannelkoong@gmail.com>
L: fuse-devel@lists.linux.dev
S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git
F: Documentation/filesystems/fuse/fuse-io-uring.rst
F: fs/fuse/dev_uring.c
F: fs/fuse/dev_uring_i.h
FUSE FILESYSTEM [PASSTHROUGH]
M: Amir Goldstein <amir73il@gmail.com>
L: fuse-devel@lists.linux.dev
S: Maintained
T: git git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git
F: Documentation/filesystems/fuse/fuse-passthrough.rst
F: fs/fuse/backing.c
F: fs/fuse/iomode.c
F: fs/fuse/passthrough.c
FUTEX SUBSYSTEM
M: Thomas Gleixner <tglx@kernel.org>
M: Ingo Molnar <mingo@redhat.com>

3
fs/fuse/Makefile
View File

@@ -11,7 +11,8 @@ obj-$(CONFIG_CUSE) += cuse.o
obj-$(CONFIG_VIRTIO_FS) += virtiofs.o
fuse-y := trace.o # put trace.o first so we see ftrace errors sooner
fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o
fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o req_timeout.o req.o
fuse-y += poll.o notify.o
fuse-y += iomode.o
fuse-$(CONFIG_FUSE_DAX) += dax.o
fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o backing.o

4
fs/fuse/acl.c
View File

@@ -1,9 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/*
* FUSE: Filesystem in Userspace
* Copyright (C) 2016 Canonical Ltd. <seth.forshee@canonical.com>
*
* This program can be distributed under the terms of the GNU GPL.
* See the file COPYING.
*/
#include "fuse_i.h"

65
fs/fuse/args.h Normal file
View File

@@ -0,0 +1,65 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _FS_FUSE_ARGS_H
#define _FS_FUSE_ARGS_H
#include <linux/types.h>
struct fuse_mount;
/** One input argument of a request */
struct fuse_in_arg {
unsigned size;
const void *value;
};
/** One output argument of a request */
struct fuse_arg {
unsigned size;
void *value;
};
struct fuse_args {
u64 nodeid;
u32 opcode;
u32 uid;
u32 gid;
u32 pid;
u8 in_numargs;
u8 out_numargs;
u8 ext_idx;
bool force:1;
bool noreply:1;
bool nocreds:1;
bool in_pages:1;
bool out_pages:1;
bool user_pages:1;
bool out_argvar:1;
bool page_zeroing:1;
bool page_replace:1;
bool may_block:1;
bool is_ext:1;
bool is_pinned:1;
bool invalidate_vmap:1;
bool abort_on_kill:1;
struct fuse_in_arg in_args[4];
struct fuse_arg out_args[2];
void (*end)(struct fuse_args *args, int error);
/* Used for kvec iter backed by vmalloc address */
void *vmap_base;
};
/** FUSE folio descriptor */
struct fuse_folio_desc {
unsigned int length;
unsigned int offset;
};
struct fuse_args_pages {
struct fuse_args args;
struct folio **folios;
struct fuse_folio_desc *descs;
unsigned int num_folios;
};
#endif /* _FS_FUSE_ARGS_H */

3
fs/fuse/backing.c
View File

@@ -5,6 +5,7 @@
* Copyright (c) 2023 CTERA Networks.
*/
#include "dev.h"
#include "fuse_i.h"
#include <linux/file.h>
@@ -118,7 +119,7 @@ int fuse_backing_open(struct fuse_conn *fc, struct fuse_backing_map *map)
goto out_fput;
fb->file = file;
fb->cred = prepare_creds();
fb->cred = get_current_cred();
refcount_set(&fb->count, 1);
res = fuse_backing_id_alloc(fc, fb);

20
fs/fuse/control.c
View File

@@ -1,12 +1,11 @@
// SPDX-License-Identifier: GPL-2.0
/*
FUSE: Filesystem in Userspace
Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu>
This program can be distributed under the terms of the GNU GPL.
See the file COPYING.
*/
#include "fuse_i.h"
#include "dev.h"
#include <linux/init.h>
#include <linux/module.h>
@@ -37,9 +36,7 @@ static ssize_t fuse_conn_abort_write(struct file *file, const char __user *buf,
{
struct fuse_conn *fc = fuse_ctl_file_conn_get(file);
if (fc) {
if (fc->abort_err)
fc->aborted = true;
fuse_abort_conn(fc);
fuse_chan_abort(fc->chan, fc->abort_err);
fuse_conn_put(fc);
}
return count;
@@ -57,7 +54,7 @@ static ssize_t fuse_conn_waiting_read(struct file *file, char __user *buf,
if (!fc)
return 0;
value = atomic_read(&fc->num_waiting);
value = fuse_chan_num_waiting(fc->chan);
file->private_data = (void *)value;
fuse_conn_put(fc);
}
@@ -111,7 +108,7 @@ static ssize_t fuse_conn_max_background_read(struct file *file,
if (!fc)
return 0;
val = READ_ONCE(fc->max_background);
val = fuse_chan_max_background(fc->chan);
fuse_conn_put(fc);
return fuse_conn_limit_read(file, buf, len, ppos, val);
@@ -129,12 +126,7 @@ static ssize_t fuse_conn_max_background_write(struct file *file,
if (ret > 0) {
struct fuse_conn *fc = fuse_ctl_file_conn_get(file);
if (fc) {
spin_lock(&fc->bg_lock);
fc->max_background = val;
fc->blocked = fc->num_background >= fc->max_background;
if (!fc->blocked)
wake_up(&fc->blocked_waitq);
spin_unlock(&fc->bg_lock);
fuse_chan_max_background_set(fc->chan, val);
fuse_conn_put(fc);
}
}

32
fs/fuse/cuse.c
View File

@@ -51,6 +51,7 @@
#include <linux/uio.h>
#include <linux/user_namespace.h>
#include "dev.h"
#include "fuse_i.h"
#include "fuse_dev_i.h"
@@ -306,6 +307,7 @@ struct cuse_init_args {
struct cuse_init_out out;
struct folio *folio;
struct fuse_folio_desc desc;
struct fuse_conn *fc;
};
/**
@@ -319,11 +321,10 @@ struct cuse_init_args {
* required data structures for it. Please read the comment at the
* top of this file for high level overview.
*/
static void cuse_process_init_reply(struct fuse_mount *fm,
struct fuse_args *args, int error)
static void cuse_process_init_reply(struct fuse_args *args, int error)
{
struct fuse_conn *fc = fm->fc;
struct cuse_init_args *ia = container_of(args, typeof(*ia), ap.args);
struct fuse_conn *fc = ia->fc;
struct fuse_args_pages *ap = &ia->ap;
struct cuse_conn *cc = fc_to_cc(fc), *pos;
struct cuse_init_out *arg = &ia->out;
@@ -391,7 +392,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm,
rc = -ENOMEM;
cdev = cdev_alloc();
if (!cdev)
goto err_unlock;
goto err_dev;
cdev->owner = THIS_MODULE;
cdev->ops = &cuse_frontend_fops;
@@ -417,13 +418,15 @@ static void cuse_process_init_reply(struct fuse_mount *fm,
err_cdev:
cdev_del(cdev);
err_dev:
device_del(dev);
err_unlock:
mutex_unlock(&cuse_lock);
put_device(dev);
err_region:
unregister_chrdev_region(devt, 1);
err:
fuse_abort_conn(fc);
fuse_chan_abort(fc->chan, false);
goto out;
}
@@ -466,6 +469,7 @@ static int cuse_send_init(struct cuse_conn *cc)
ap->descs = &ia->desc;
ia->folio = folio;
ia->desc.length = ap->args.out_args[1].size;
ia->fc = &cc->fc;
ap->args.end = cuse_process_init_reply;
rc = fuse_simple_background(fm, &ap->args, GFP_KERNEL);
@@ -502,8 +506,12 @@ static int cuse_channel_open(struct inode *inode, struct file *file)
{
struct fuse_dev *fud;
struct cuse_conn *cc;
struct fuse_chan *fch __free(fuse_chan_free) = fuse_dev_chan_new();
int rc;
if (!fch)
return -ENOMEM;
/* set up cuse_conn */
cc = kzalloc_obj(*cc);
if (!cc)
@@ -513,18 +521,16 @@ static int cuse_channel_open(struct inode *inode, struct file *file)
* Limit the cuse channel to requests that can
* be represented in file->f_cred->user_ns.
*/
fuse_conn_init(&cc->fc, &cc->fm, file->f_cred->user_ns,
&fuse_dev_fiq_ops, NULL);
fuse_conn_init(&cc->fc, &cc->fm, file->f_cred->user_ns, no_free_ptr(fch));
cc->fc.release = cuse_fc_release;
fud = fuse_dev_alloc_install(&cc->fc);
fud = fuse_dev_alloc_install(cc->fc.chan);
fuse_conn_put(&cc->fc);
if (!fud)
return -ENOMEM;
INIT_LIST_HEAD(&cc->list);
cc->fc.initialized = 1;
cc->fc.chan->initialized = 1;
rc = cuse_send_init(cc);
if (rc) {
fuse_dev_put(fud);
@@ -549,7 +555,7 @@ static int cuse_channel_open(struct inode *inode, struct file *file)
static int cuse_channel_release(struct inode *inode, struct file *file)
{
struct fuse_dev *fud = __fuse_get_dev(file);
struct cuse_conn *cc = fc_to_cc(fud->fc);
struct cuse_conn *cc = fc_to_cc(fud->chan->conn);
/* remove from the conntbl, no more access from this point on */
mutex_lock(&cuse_lock);
@@ -581,7 +587,7 @@ static ssize_t cuse_class_waiting_show(struct device *dev,
{
struct cuse_conn *cc = dev_get_drvdata(dev);
return sprintf(buf, "%d\n", atomic_read(&cc->fc.num_waiting));
return sprintf(buf, "%d\n", atomic_read(&cc->fc.chan->num_waiting));
}
static DEVICE_ATTR(waiting, 0400, cuse_class_waiting_show, NULL);
@@ -591,7 +597,7 @@ static ssize_t cuse_class_abort_store(struct device *dev,
{
struct cuse_conn *cc = dev_get_drvdata(dev);
fuse_abort_conn(&cc->fc);
fuse_chan_abort(cc->fc.chan, false);
return count;
}
static DEVICE_ATTR(abort, 0200, NULL, cuse_class_abort_store);

2
fs/fuse/dax.c
View File

@@ -113,7 +113,7 @@ __kick_dmap_free_worker(struct fuse_conn_dax *fcd, unsigned long delay_ms)
free_threshold = max_t(unsigned long, fcd->nr_ranges * FUSE_DAX_RECLAIM_THRESHOLD / 100,
1);
if (fcd->nr_free_ranges < free_threshold)
queue_delayed_work(system_long_wq, &fcd->free_work,
queue_delayed_work(system_dfl_long_wq, &fcd->free_work,
msecs_to_jiffies(delay_ms));
}

1374
fs/fuse/dev.c
View File

File diff suppressed because it is too large Load Diff

110
fs/fuse/dev.h Normal file
View File

@@ -0,0 +1,110 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _FS_FUSE_DEV_H
#define _FS_FUSE_DEV_H
#include <linux/cleanup.h>
/** Maximum number of outstanding background requests */
#define FUSE_DEFAULT_MAX_BACKGROUND 12
struct fuse_conn;
struct fuse_chan;
struct fuse_dev;
struct fuse_args;
struct fuse_copy_state;
struct fuse_backing_map;
struct file;
struct folio;
enum fuse_notify_code;
struct fuse_chan_param {
unsigned int minor;
unsigned int max_write;
unsigned int max_pages;
};
struct fuse_chan *fuse_chan_new(void);
struct fuse_chan *fuse_dev_chan_new(void);
void fuse_chan_release(struct fuse_chan *fch);
void fuse_chan_free(struct fuse_chan *fch);
unsigned int fuse_chan_num_background(struct fuse_chan *fch);
unsigned int fuse_chan_max_background(struct fuse_chan *fch);
void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val);
unsigned int fuse_chan_num_waiting(struct fuse_chan *fch);
void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc);
void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *param);
void fuse_chan_io_uring_enable(struct fuse_chan *fch);
ssize_t fuse_chan_send(struct fuse_chan *fch, struct fuse_args *args);
int fuse_chan_send_bg(struct fuse_chan *fch, struct fuse_args *args, gfp_t gfp_flags);
int fuse_chan_send_notify_reply(struct fuse_chan *fch, struct fuse_args *args, u64 unique);
void fuse_chan_resend(struct fuse_chan *fch);
struct fuse_forget_link *fuse_alloc_forget(void);
void fuse_chan_queue_forget(struct fuse_chan *fch, struct fuse_forget_link *forget,
u64 nodeid, u64 nlookup);
DEFINE_FREE(fuse_chan_free, struct fuse_chan *, if (_T) fuse_chan_free(_T))
/**
* Initialize the client device
*/
int fuse_dev_init(void);
/**
* Cleanup the client device
*/
void fuse_dev_cleanup(void);
void fuse_dev_install(struct fuse_dev *fud, struct fuse_chan *fch);
bool fuse_dev_verify(struct fuse_dev *fud, struct fuse_chan *fch);
void fuse_dev_put(struct fuse_dev *fud);
bool fuse_dev_is_installed(struct fuse_dev *fud);
bool fuse_dev_is_sync_init(struct fuse_dev *fud);
struct fuse_dev *fuse_dev_grab(struct file *file);
void fuse_init_server_timeout(struct fuse_chan *fch, unsigned int timeout);
/* Abort all requests */
void fuse_chan_abort(struct fuse_chan *fch, bool abort_with_err);
void fuse_chan_wait_aborted(struct fuse_chan *fch);
/**
* Acquire reference to fuse_conn
*/
struct fuse_conn *fuse_conn_get(struct fuse_conn *fc);
/**
* Release reference to fuse_conn
*/
void fuse_conn_put(struct fuse_conn *fc);
dev_t fuse_conn_get_id(struct fuse_conn *fc);
void fuse_end_polls(struct fuse_conn *fc);
int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
unsigned int size, struct fuse_copy_state *cs);
int fuse_backing_open(struct fuse_conn *fc, struct fuse_backing_map *map);
int fuse_backing_close(struct fuse_conn *fc, int backing_id);
int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size);
int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop,
unsigned offset, unsigned count, int zeroing);
void fuse_copy_finish(struct fuse_copy_state *cs);
#ifdef CONFIG_FUSE_IO_URING
bool fuse_uring_enabled(void);
void fuse_uring_destruct(struct fuse_chan *fch);
#else /* CONFIG_FUSE_IO_URING */
static inline bool fuse_uring_enabled(void)
{
return false;
}
static inline void fuse_uring_destruct(struct fuse_chan *fch)
{
}
#endif /* CONFIG_FUSE_IO_URING */
#endif /* _FS_FUSE_DEV_H */

493
fs/fuse/dev_uring.c
View File

@@ -4,9 +4,9 @@
* Copyright (c) 2023-2024 DataDirect Networks.
*/
#include "fuse_i.h"
#include "dev.h"
#include "args.h"
#include "dev_uring_i.h"
#include "fuse_dev_i.h"
#include "fuse_trace.h"
#include <linux/fs.h>
@@ -18,7 +18,8 @@ MODULE_PARM_DESC(enable_uring,
"Enable userspace communication through io-uring");
#define FUSE_URING_IOV_SEGS 2 /* header and payload */
#define FUSE_URING_IOV_HEADERS 0
#define FUSE_URING_IOV_PAYLOAD 1
bool fuse_uring_enabled(void)
{
@@ -31,6 +32,15 @@ struct fuse_uring_pdu {
static const struct fuse_iqueue_ops fuse_io_uring_ops;
enum fuse_uring_header_type {
/* struct fuse_in_header / struct fuse_out_header */
FUSE_URING_HEADER_IN_OUT,
/* per op code header */
FUSE_URING_HEADER_OP,
/* struct fuse_uring_ent_in_out header */
FUSE_URING_HEADER_RING_ENT,
};
static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd,
struct fuse_ring_ent *ring_ent)
{
@@ -51,10 +61,10 @@ static struct fuse_ring_ent *uring_cmd_to_ring_ent(struct io_uring_cmd *cmd)
static void fuse_uring_flush_bg(struct fuse_ring_queue *queue)
{
struct fuse_ring *ring = queue->ring;
struct fuse_conn *fc = ring->fc;
struct fuse_chan *fch = ring->chan;
lockdep_assert_held(&queue->lock);
lockdep_assert_held(&fc->bg_lock);
lockdep_assert_held(&fch->bg_lock);
/*
* Allow one bg request per queue, ignoring global fc limits.
@@ -62,14 +72,14 @@ static void fuse_uring_flush_bg(struct fuse_ring_queue *queue)
* eliminates the need for remote queue wake-ups when global
* limits are met but this queue has no more waiting requests.
*/
while ((fc->active_background < fc->max_background ||
while ((fch->active_background < fch->max_background ||
!queue->active_background) &&
(!list_empty(&queue->fuse_req_bg_queue))) {
struct fuse_req *req;
req = list_first_entry(&queue->fuse_req_bg_queue,
struct fuse_req, list);
fc->active_background++;
fch->active_background++;
queue->active_background++;
list_move_tail(&req->list, &queue->fuse_req_queue);
@@ -81,7 +91,7 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req,
{
struct fuse_ring_queue *queue = ent->queue;
struct fuse_ring *ring = queue->ring;
struct fuse_conn *fc = ring->fc;
struct fuse_chan *fch = ring->chan;
lockdep_assert_not_held(&queue->lock);
spin_lock(&queue->lock);
@@ -89,9 +99,10 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req,
list_del_init(&req->list);
if (test_bit(FR_BACKGROUND, &req->flags)) {
queue->active_background--;
spin_lock(&fc->bg_lock);
spin_lock(&fch->bg_lock);
fuse_request_bg_finish(fch, req);
fuse_uring_flush_bg(queue);
spin_unlock(&fc->bg_lock);
spin_unlock(&fch->bg_lock);
}
spin_unlock(&queue->lock);
@@ -123,26 +134,25 @@ void fuse_uring_abort_end_requests(struct fuse_ring *ring)
{
int qid;
struct fuse_ring_queue *queue;
struct fuse_conn *fc = ring->fc;
struct fuse_chan *fch = ring->chan;
for (qid = 0; qid < ring->nr_queues; qid++) {
queue = READ_ONCE(ring->queues[qid]);
if (!queue)
continue;
queue->stopped = true;
WARN_ON_ONCE(ring->fc->max_background != UINT_MAX);
WARN_ON_ONCE(fch->max_background != UINT_MAX);
spin_lock(&queue->lock);
spin_lock(&fc->bg_lock);
queue->stopped = true;
spin_lock(&fch->bg_lock);
fuse_uring_flush_bg(queue);
spin_unlock(&fc->bg_lock);
spin_unlock(&fch->bg_lock);
spin_unlock(&queue->lock);
fuse_uring_abort_end_queue_requests(queue);
}
}
static bool ent_list_request_expired(struct fuse_conn *fc, struct list_head *list)
static bool ent_list_request_expired(struct fuse_chan *fch, struct list_head *list)
{
struct fuse_ring_ent *ent;
struct fuse_req *req;
@@ -154,12 +164,12 @@ static bool ent_list_request_expired(struct fuse_conn *fc, struct list_head *lis
req = ent->fuse_req;
return time_is_before_jiffies(req->create_time +
fc->timeout.req_timeout);
fch->timeout.req_timeout);
}
bool fuse_uring_request_expired(struct fuse_conn *fc)
bool fuse_uring_request_expired(struct fuse_chan *fch)
{
struct fuse_ring *ring = fc->ring;
struct fuse_ring *ring = fch->ring;
struct fuse_ring_queue *queue;
int qid;
@@ -172,10 +182,10 @@ bool fuse_uring_request_expired(struct fuse_conn *fc)
continue;
spin_lock(&queue->lock);
if (fuse_request_expired(fc, &queue->fuse_req_queue) ||
fuse_request_expired(fc, &queue->fuse_req_bg_queue) ||
ent_list_request_expired(fc, &queue->ent_w_req_queue) ||
ent_list_request_expired(fc, &queue->ent_in_userspace)) {
if (fuse_request_expired(fch, &queue->fuse_req_queue) ||
fuse_request_expired(fch, &queue->fuse_req_bg_queue) ||
ent_list_request_expired(fch, &queue->ent_w_req_queue) ||
ent_list_request_expired(fch, &queue->ent_in_userspace)) {
spin_unlock(&queue->lock);
return true;
}
@@ -185,9 +195,9 @@ bool fuse_uring_request_expired(struct fuse_conn *fc)
return false;
}
void fuse_uring_destruct(struct fuse_conn *fc)
void fuse_uring_destruct(struct fuse_chan *fch)
{
struct fuse_ring *ring = fc->ring;
struct fuse_ring *ring = fch->ring;
int qid;
if (!ring)
@@ -218,20 +228,20 @@ void fuse_uring_destruct(struct fuse_conn *fc)
kfree(ring->queues);
kfree(ring);
fc->ring = NULL;
fch->ring = NULL;
}
/*
* Basic ring setup for this connection based on the provided configuration
*/
static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc)
static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch)
{
struct fuse_ring *ring;
size_t nr_queues = num_possible_cpus();
struct fuse_ring *res = NULL;
size_t max_payload_size;
ring = kzalloc_obj(*fc->ring, GFP_KERNEL_ACCOUNT);
ring = kzalloc_obj(*ring, GFP_KERNEL_ACCOUNT);
if (!ring)
return NULL;
@@ -240,25 +250,29 @@ static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc)
if (!ring->queues)
goto out_err;
max_payload_size = max(FUSE_MIN_READ_BUFFER, fc->max_write);
max_payload_size = max(max_payload_size, fc->max_pages * PAGE_SIZE);
max_payload_size = max(FUSE_MIN_READ_BUFFER, fch->max_write);
max_payload_size = max(max_payload_size, fch->max_pages * PAGE_SIZE);
spin_lock(&fc->lock);
if (fc->ring) {
spin_lock(&fch->lock);
if (!fch->connected) {
spin_unlock(&fch->lock);
goto out_err;
}
if (fch->ring) {
/* race, another thread created the ring in the meantime */
spin_unlock(&fc->lock);
res = fc->ring;
spin_unlock(&fch->lock);
res = fch->ring;
goto out_err;
}
init_waitqueue_head(&ring->stop_waitq);
ring->nr_queues = nr_queues;
ring->fc = fc;
ring->chan = fch;
ring->max_payload_sz = max_payload_size;
smp_store_release(&fc->ring, ring);
smp_store_release(&fch->ring, ring);
spin_unlock(&fc->lock);
spin_unlock(&fch->lock);
return ring;
out_err:
@@ -270,14 +284,14 @@ static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc)
static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring,
int qid)
{
struct fuse_conn *fc = ring->fc;
struct fuse_chan *fch = ring->chan;
struct fuse_ring_queue *queue;
struct list_head *pq;
queue = kzalloc_obj(*queue, GFP_KERNEL_ACCOUNT);
if (!queue)
return NULL;
pq = kzalloc_objs(struct list_head, FUSE_PQ_HASH_SIZE);
pq = fuse_pqueue_alloc();
if (!pq) {
kfree(queue);
return NULL;
@@ -295,12 +309,12 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring,
INIT_LIST_HEAD(&queue->fuse_req_bg_queue);
INIT_LIST_HEAD(&queue->ent_released);
queue->fpq.processing = pq;
fuse_pqueue_init(&queue->fpq);
queue->fpq.processing = pq;
spin_lock(&fc->lock);
spin_lock(&fch->lock);
if (ring->queues[qid]) {
spin_unlock(&fc->lock);
spin_unlock(&fch->lock);
kfree(queue->fpq.processing);
kfree(queue);
return ring->queues[qid];
@@ -310,7 +324,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring,
* write_once and lock as the caller mostly doesn't take the lock at all
*/
WRITE_ONCE(ring->queues[qid], queue);
spin_unlock(&fc->lock);
spin_unlock(&fch->lock);
return queue;
}
@@ -466,6 +480,7 @@ static void fuse_uring_async_stop_queues(struct work_struct *work)
FUSE_URING_TEARDOWN_INTERVAL);
} else {
wake_up_all(&ring->stop_waitq);
fuse_conn_put(ring->chan->conn);
}
}
@@ -477,6 +492,7 @@ void fuse_uring_stop_queues(struct fuse_ring *ring)
fuse_uring_teardown_all_queues(ring);
if (atomic_read(&ring->queue_refs) > 0) {
fuse_conn_get(ring->chan->conn);
ring->teardown_time = jiffies;
INIT_DELAYED_WORK(&ring->async_teardown_work,
fuse_uring_async_stop_queues);
@@ -507,8 +523,7 @@ static void fuse_uring_cancel(struct io_uring_cmd *cmd,
queue = ent->queue;
spin_lock(&queue->lock);
if (ent->state == FRRS_AVAILABLE) {
ent->state = FRRS_USERSPACE;
list_move_tail(&ent->list, &queue->ent_in_userspace);
list_del_init(&ent->list);
need_cmd_done = true;
ent->cmd = NULL;
}
@@ -517,6 +532,9 @@ static void fuse_uring_cancel(struct io_uring_cmd *cmd,
if (need_cmd_done) {
/* no queue lock to avoid lock order issues */
io_uring_cmd_done(cmd, -ENOTCONN, issue_flags);
kfree(ent);
if (atomic_dec_and_test(&queue->ring->queue_refs))
wake_up_all(&queue->ring->stop_waitq);
}
}
@@ -531,8 +549,7 @@ static void fuse_uring_prepare_cancel(struct io_uring_cmd *cmd, int issue_flags,
* Checks for errors and stores it into the request
*/
static int fuse_uring_out_header_has_err(struct fuse_out_header *oh,
struct fuse_req *req,
struct fuse_conn *fc)
struct fuse_req *req)
{
int err;
@@ -571,6 +588,82 @@ static int fuse_uring_out_header_has_err(struct fuse_out_header *oh,
return err;
}
static int ring_header_type_offset(enum fuse_uring_header_type type)
{
switch (type) {
case FUSE_URING_HEADER_IN_OUT:
return 0;
case FUSE_URING_HEADER_OP:
return offsetof(struct fuse_uring_req_header, op_in);
case FUSE_URING_HEADER_RING_ENT:
return offsetof(struct fuse_uring_req_header, ring_ent_in_out);
default:
WARN_ONCE(1, "Invalid header type: %d\n", type);
return -EINVAL;
}
}
static int copy_header_to_ring(struct fuse_ring_ent *ent,
enum fuse_uring_header_type type,
const void *header, size_t header_size)
{
int offset = ring_header_type_offset(type);
void __user *ring;
if (offset < 0)
return offset;
ring = (void __user *)ent->headers + offset;
if (copy_to_user(ring, header, header_size)) {
pr_info_ratelimited("Copying header to ring failed.\n");
return -EFAULT;
}
return 0;
}
static int copy_header_from_ring(struct fuse_ring_ent *ent,
enum fuse_uring_header_type type, void *header,
size_t header_size)
{
int offset = ring_header_type_offset(type);
const void __user *ring;
if (offset < 0)
return offset;
ring = (void __user *)ent->headers + offset;
if (copy_from_user(header, ring, header_size)) {
pr_info_ratelimited("Copying header from ring failed.\n");
return -EFAULT;
}
return 0;
}
static int setup_fuse_copy_state(struct fuse_copy_state *cs,
struct fuse_ring *ring, struct fuse_req *req,
struct fuse_ring_ent *ent, int dir,
struct iov_iter *iter)
{
int err;
err = import_ubuf(dir, ent->payload, ring->max_payload_sz, iter);
if (err) {
pr_info_ratelimited("fuse: Import of user buffer failed\n");
return err;
}
fuse_copy_init(cs, dir == ITER_DEST, iter);
cs->is_uring = true;
cs->req = req;
return 0;
}
static int fuse_uring_copy_from_ring(struct fuse_ring *ring,
struct fuse_req *req,
struct fuse_ring_ent *ent)
@@ -581,19 +674,14 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring,
int err;
struct fuse_uring_ent_in_out ring_in_out;
err = copy_from_user(&ring_in_out, &ent->headers->ring_ent_in_out,
sizeof(ring_in_out));
if (err)
return -EFAULT;
err = import_ubuf(ITER_SOURCE, ent->payload, ring->max_payload_sz,
&iter);
err = copy_header_from_ring(ent, FUSE_URING_HEADER_RING_ENT,
&ring_in_out, sizeof(ring_in_out));
if (err)
return err;
fuse_copy_init(&cs, false, &iter);
cs.is_uring = true;
cs.req = req;
err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_SOURCE, &iter);
if (err)
return err;
err = fuse_copy_out_args(&cs, args, ring_in_out.payload_sz);
fuse_copy_finish(&cs);
@@ -617,15 +705,9 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req,
.commit_id = req->in.h.unique,
};
err = import_ubuf(ITER_DEST, ent->payload, ring->max_payload_sz, &iter);
if (err) {
pr_info_ratelimited("fuse: Import of user buffer failed\n");
err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_DEST, &iter);
if (err)
return err;
}
fuse_copy_init(&cs, true, &iter);
cs.is_uring = true;
cs.req = req;
if (num_args > 0) {
/*
@@ -633,13 +715,11 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req,
* Some op code have that as zero size.
*/
if (args->in_args[0].size > 0) {
err = copy_to_user(&ent->headers->op_in, in_args->value,
in_args->size);
if (err) {
pr_info_ratelimited(
"Copying the header failed.\n");
return -EFAULT;
}
err = copy_header_to_ring(ent, FUSE_URING_HEADER_OP,
in_args->value,
in_args->size);
if (err)
return err;
}
in_args++;
num_args--;
@@ -655,9 +735,8 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req,
}
ent_in_out.payload_sz = cs.ring.copied_sz;
err = copy_to_user(&ent->headers->ring_ent_in_out, &ent_in_out,
sizeof(ent_in_out));
return err ? -EFAULT : 0;
return copy_header_to_ring(ent, FUSE_URING_HEADER_RING_ENT,
&ent_in_out, sizeof(ent_in_out));
}
static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent,
@@ -686,14 +765,8 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent,
}
/* copy fuse_in_header */
err = copy_to_user(&ent->headers->in_out, &req->in.h,
sizeof(req->in.h));
if (err) {
err = -EFAULT;
return err;
}
return 0;
return copy_header_to_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->in.h,
sizeof(req->in.h));
}
static int fuse_uring_prepare_send(struct fuse_ring_ent *ent,
@@ -702,40 +775,36 @@ static int fuse_uring_prepare_send(struct fuse_ring_ent *ent,
int err;
err = fuse_uring_copy_to_ring(ent, req);
if (!err)
if (!err) {
set_bit(FR_SENT, &req->flags);
else
trace_fuse_request_sent(req);
} else {
/*
* Copying the request failed. Remove the entry from the
* ent_w_req_queue list and terminate the request
*/
spin_lock(&ent->queue->lock);
list_del_init(&ent->list);
ent->state = FRRS_INVALID;
spin_unlock(&ent->queue->lock);
fuse_uring_req_end(ent, req, err);
}
return err;
}
/*
* Write data to the ring buffer and send the request to userspace,
* userspace will read it
* This is comparable with classical read(/dev/fuse)
*/
static int fuse_uring_send_next_to_ring(struct fuse_ring_ent *ent,
struct fuse_req *req,
unsigned int issue_flags)
/* Used to find the request on SQE commit */
static void fuse_uring_add_to_pq(struct fuse_ring_ent *ent)
{
struct fuse_ring_queue *queue = ent->queue;
int err;
struct io_uring_cmd *cmd;
struct fuse_pqueue *fpq = &queue->fpq;
unsigned int hash;
struct fuse_req *req = ent->fuse_req;
err = fuse_uring_prepare_send(ent, req);
if (err)
return err;
spin_lock(&queue->lock);
cmd = ent->cmd;
ent->cmd = NULL;
ent->state = FRRS_USERSPACE;
list_move_tail(&ent->list, &queue->ent_in_userspace);
spin_unlock(&queue->lock);
io_uring_cmd_done(cmd, 0, issue_flags);
return 0;
req->ring_entry = ent;
hash = fuse_req_hash(req->in.h.unique);
list_move_tail(&req->list, &fpq->processing[hash]);
}
/*
@@ -749,19 +818,6 @@ static void fuse_uring_ent_avail(struct fuse_ring_ent *ent,
ent->state = FRRS_AVAILABLE;
}
/* Used to find the request on SQE commit */
static void fuse_uring_add_to_pq(struct fuse_ring_ent *ent,
struct fuse_req *req)
{
struct fuse_ring_queue *queue = ent->queue;
struct fuse_pqueue *fpq = &queue->fpq;
unsigned int hash;
req->ring_entry = ent;
hash = fuse_req_hash(req->in.h.unique);
list_move_tail(&req->list, &fpq->processing[hash]);
}
/*
* Assign a fuse queue entry to the given entry
*/
@@ -779,10 +835,13 @@ static void fuse_uring_add_req_to_ring_ent(struct fuse_ring_ent *ent,
}
clear_bit(FR_PENDING, &req->flags);
/* Until fuse_uring_add_to_pq() the req is not attached to any list */
list_del_init(&req->list);
ent->fuse_req = req;
ent->state = FRRS_FUSE_REQ;
list_move_tail(&ent->list, &queue->ent_w_req_queue);
fuse_uring_add_to_pq(ent, req);
}
/* Fetch the next fuse request if available */
@@ -812,17 +871,13 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req,
unsigned int issue_flags)
{
struct fuse_ring *ring = ent->queue->ring;
struct fuse_conn *fc = ring->fc;
ssize_t err = 0;
ssize_t err = -EFAULT;
err = copy_from_user(&req->out.h, &ent->headers->in_out,
sizeof(req->out.h));
if (err) {
req->out.h.error = -EFAULT;
if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->out.h,
sizeof(req->out.h)))
goto out;
}
err = fuse_uring_out_header_has_err(&req->out.h, req, fc);
err = fuse_uring_out_header_has_err(&req->out.h, req);
if (err) {
/* req->out.h.error already set */
goto out;
@@ -834,11 +889,13 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req,
}
/*
* Get the next fuse req and send it
* Get the next fuse req.
*
* Returns true if the next fuse request has been assigned to the ent.
* Else, there is no next fuse request and this returns false.
*/
static void fuse_uring_next_fuse_req(struct fuse_ring_ent *ent,
struct fuse_ring_queue *queue,
unsigned int issue_flags)
static bool fuse_uring_get_next_fuse_req(struct fuse_ring_ent *ent,
struct fuse_ring_queue *queue)
{
int err;
struct fuse_req *req;
@@ -850,10 +907,12 @@ static void fuse_uring_next_fuse_req(struct fuse_ring_ent *ent,
spin_unlock(&queue->lock);
if (req) {
err = fuse_uring_send_next_to_ring(ent, req, issue_flags);
err = fuse_uring_prepare_send(ent, req);
if (err)
goto retry;
}
return req != NULL;
}
static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent)
@@ -871,15 +930,30 @@ static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent)
return 0;
}
static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd,
ssize_t ret, unsigned int issue_flags)
{
struct fuse_ring_queue *queue = ent->queue;
spin_lock(&queue->lock);
ent->state = FRRS_USERSPACE;
list_move_tail(&ent->list, &queue->ent_in_userspace);
ent->cmd = NULL;
fuse_uring_add_to_pq(ent);
spin_unlock(&queue->lock);
io_uring_cmd_done(cmd, ret, issue_flags);
}
/* FUSE_URING_CMD_COMMIT_AND_FETCH handler */
static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags,
struct fuse_conn *fc)
struct fuse_chan *fch)
{
const struct fuse_uring_cmd_req *cmd_req = io_uring_sqe128_cmd(cmd->sqe,
struct fuse_uring_cmd_req);
struct fuse_ring_ent *ent;
int err;
struct fuse_ring *ring = fc->ring;
struct fuse_ring *ring = fch->ring;
struct fuse_ring_queue *queue;
uint64_t commit_id = READ_ONCE(cmd_req->commit_id);
unsigned int qid = READ_ONCE(cmd_req->qid);
@@ -898,10 +972,15 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags,
return err;
fpq = &queue->fpq;
if (!READ_ONCE(fc->connected) || READ_ONCE(queue->stopped))
if (!READ_ONCE(fch->connected))
return err;
spin_lock(&queue->lock);
if (unlikely(queue->stopped)) {
spin_unlock(&queue->lock);
return err;
}
/* Find a request based on the unique ID of the fuse request
* This should get revised, as it needs a hash calculation and list
* search. And full struct fuse_pqueue is needed (memory overhead).
@@ -924,9 +1003,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags,
pr_info_ratelimited("qid=%d commit_id %llu state %d",
queue->qid, commit_id, ent->state);
spin_unlock(&queue->lock);
req->out.h.error = err;
clear_bit(FR_SENT, &req->flags);
fuse_request_end(req);
fuse_uring_req_end(ent, req, err);
return err;
}
@@ -943,7 +1020,8 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags,
* and fetching is done in one step vs legacy fuse, which has separated
* read (fetch request) and write (commit result).
*/
fuse_uring_next_fuse_req(ent, queue, issue_flags);
if (fuse_uring_get_next_fuse_req(ent, queue))
fuse_uring_send(ent, cmd, 0, issue_flags);
return 0;
}
@@ -975,14 +1053,25 @@ static bool is_ring_ready(struct fuse_ring *ring, int current_qid)
/*
* fuse_uring_req_fetch command handling
*/
static void fuse_uring_do_register(struct fuse_ring_ent *ent,
struct io_uring_cmd *cmd,
unsigned int issue_flags)
static int fuse_uring_do_register(struct fuse_ring_ent *ent,
struct io_uring_cmd *cmd,
unsigned int issue_flags)
{
struct fuse_ring_queue *queue = ent->queue;
struct fuse_ring *ring = queue->ring;
struct fuse_conn *fc = ring->fc;
struct fuse_iqueue *fiq = &fc->iq;
struct fuse_chan *fch = ring->chan;
struct fuse_iqueue *fiq = &fch->iq;
spin_lock(&fch->lock);
/* abort teardown path is running or has run */
if (!fch->connected) {
spin_unlock(&fch->lock);
if (atomic_dec_and_test(&ring->queue_refs))
wake_up_all(&ring->stop_waitq);
kfree(ent);
return -ECONNABORTED;
}
spin_unlock(&fch->lock);
fuse_uring_prepare_cancel(cmd, issue_flags, ent);
@@ -991,20 +1080,21 @@ static void fuse_uring_do_register(struct fuse_ring_ent *ent,
fuse_uring_ent_avail(ent, queue);
spin_unlock(&queue->lock);
if (!ring->ready) {
if (!READ_ONCE(ring->ready)) {
bool ready = is_ring_ready(ring, queue->qid);
if (ready) {
WRITE_ONCE(fiq->ops, &fuse_io_uring_ops);
WRITE_ONCE(ring->ready, true);
wake_up_all(&fc->blocked_waitq);
smp_store_release(&ring->ready, true);
wake_up_all(&fch->blocked_waitq);
}
}
return 0;
}
/*
* sqe->addr is a ptr to an iovec array, iov[0] has the headers, iov[1]
* the payload
* sqe->addr is a ptr to an iovec array, iov[FUSE_URING_IOV_HEADERS] has the
* headers, iov[FUSE_URING_IOV_PAYLOAD] the payload
*/
static int fuse_uring_get_iovec_from_sqe(const struct io_uring_sqe *sqe,
struct iovec iov[FUSE_URING_IOV_SEGS])
@@ -1034,8 +1124,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd,
{
struct fuse_ring *ring = queue->ring;
struct fuse_ring_ent *ent;
size_t payload_size;
struct iovec iov[FUSE_URING_IOV_SEGS];
struct iovec *headers, *payload;
int err;
err = fuse_uring_get_iovec_from_sqe(cmd->sqe, iov);
@@ -1046,15 +1136,16 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd,
}
err = -EINVAL;
if (iov[0].iov_len < sizeof(struct fuse_uring_req_header)) {
pr_info_ratelimited("Invalid header len %zu\n", iov[0].iov_len);
headers = &iov[FUSE_URING_IOV_HEADERS];
if (headers->iov_len < sizeof(struct fuse_uring_req_header)) {
pr_info_ratelimited("Invalid header len %zu\n", headers->iov_len);
return ERR_PTR(err);
}
payload_size = iov[1].iov_len;
if (payload_size < ring->max_payload_sz) {
payload = &iov[FUSE_URING_IOV_PAYLOAD];
if (payload->iov_len < ring->max_payload_sz) {
pr_info_ratelimited("Invalid req payload len %zu\n",
payload_size);
payload->iov_len);
return ERR_PTR(err);
}
@@ -1066,8 +1157,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd,
INIT_LIST_HEAD(&ent->list);
ent->queue = queue;
ent->headers = iov[0].iov_base;
ent->payload = iov[1].iov_base;
ent->headers = headers->iov_base;
ent->payload = payload->iov_base;
atomic_inc(&ring->queue_refs);
return ent;
@@ -1078,11 +1169,11 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd,
* entry as "ready to get fuse requests" on the queue
*/
static int fuse_uring_register(struct io_uring_cmd *cmd,
unsigned int issue_flags, struct fuse_conn *fc)
unsigned int issue_flags, struct fuse_chan *fch)
{
const struct fuse_uring_cmd_req *cmd_req = io_uring_sqe128_cmd(cmd->sqe,
struct fuse_uring_cmd_req);
struct fuse_ring *ring = smp_load_acquire(&fc->ring);
struct fuse_ring *ring = smp_load_acquire(&fch->ring);
struct fuse_ring_queue *queue;
struct fuse_ring_ent *ent;
int err;
@@ -1090,7 +1181,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd,
err = -ENOMEM;
if (!ring) {
ring = fuse_uring_create(fc);
ring = fuse_uring_create(fch);
if (!ring)
return err;
}
@@ -1116,9 +1207,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd,
if (IS_ERR(ent))
return PTR_ERR(ent);
fuse_uring_do_register(ent, cmd, issue_flags);
return 0;
return fuse_uring_do_register(ent, cmd, issue_flags);
}
/*
@@ -1128,7 +1217,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd,
int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
{
struct fuse_dev *fud;
struct fuse_conn *fc;
struct fuse_chan *fch;
u32 cmd_op = cmd->cmd_op;
int err;
@@ -1146,39 +1235,39 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
pr_info_ratelimited("No fuse device found\n");
return PTR_ERR(fud);
}
fc = fud->fc;
fch = fud->chan;
/* Once a connection has io-uring enabled on it, it can't be disabled */
if (!enable_uring && !fc->io_uring) {
if (!enable_uring && !fch->io_uring) {
pr_info_ratelimited("fuse-io-uring is disabled\n");
return -EOPNOTSUPP;
}
if (fc->aborted)
if (fch->abort_with_err)
return -ECONNABORTED;
if (!fc->connected)
if (!fch->connected)
return -ENOTCONN;
/*
* fuse_uring_register() needs the ring to be initialized,
* we need to know the max payload size
*/
if (!fc->initialized)
if (!fch->initialized)
return -EAGAIN;
switch (cmd_op) {
case FUSE_IO_URING_CMD_REGISTER:
err = fuse_uring_register(cmd, issue_flags, fc);
err = fuse_uring_register(cmd, issue_flags, fch);
if (err) {
pr_info_once("FUSE_IO_URING_CMD_REGISTER failed err=%d\n",
err);
fc->io_uring = 0;
wake_up_all(&fc->blocked_waitq);
fch->io_uring = 0;
wake_up_all(&fch->blocked_waitq);
return err;
}
break;
case FUSE_IO_URING_CMD_COMMIT_AND_FETCH:
err = fuse_uring_commit_fetch(cmd, issue_flags, fc);
err = fuse_uring_commit_fetch(cmd, issue_flags, fch);
if (err) {
pr_info_once("FUSE_IO_URING_COMMIT_AND_FETCH failed err=%d\n",
err);
@@ -1192,20 +1281,6 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
return -EIOCBQUEUED;
}
static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd,
ssize_t ret, unsigned int issue_flags)
{
struct fuse_ring_queue *queue = ent->queue;
spin_lock(&queue->lock);
ent->state = FRRS_USERSPACE;
list_move_tail(&ent->list, &queue->ent_in_userspace);
ent->cmd = NULL;
spin_unlock(&queue->lock);
io_uring_cmd_done(cmd, ret, issue_flags);
}
/*
* This prepares and sends the ring request in fuse-uring task context.
* User buffers are not mapped yet - the application does not have permission
@@ -1222,14 +1297,25 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw)
if (!tw.cancel) {
err = fuse_uring_prepare_send(ent, ent->fuse_req);
if (err) {
fuse_uring_next_fuse_req(ent, queue, issue_flags);
return;
if (!fuse_uring_get_next_fuse_req(ent, queue))
return;
err = 0;
}
fuse_uring_send(ent, cmd, err, issue_flags);
} else {
err = -ECANCELED;
}
fuse_uring_send(ent, cmd, err, issue_flags);
spin_lock(&queue->lock);
list_del_init(&ent->list);
spin_unlock(&queue->lock);
io_uring_cmd_done(cmd, err, issue_flags);
fuse_uring_req_end(ent, ent->fuse_req, err);
kfree(ent);
if (atomic_dec_and_test(&queue->ring->queue_refs))
wake_up_all(&queue->ring->stop_waitq);
}
}
static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring)
@@ -1261,8 +1347,7 @@ static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent)
/* queue a fuse request and send it if a ring entry is available */
void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req)
{
struct fuse_conn *fc = req->fm->fc;
struct fuse_ring *ring = fc->ring;
struct fuse_ring *ring = req->chan->ring;
struct fuse_ring_queue *queue;
struct fuse_ring_ent *ent = NULL;
int err;
@@ -1304,8 +1389,8 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req)
bool fuse_uring_queue_bq_req(struct fuse_req *req)
{
struct fuse_conn *fc = req->fm->fc;
struct fuse_ring *ring = fc->ring;
struct fuse_chan *fch = req->chan;
struct fuse_ring *ring = fch->ring;
struct fuse_ring_queue *queue;
struct fuse_ring_ent *ent = NULL;
@@ -1325,12 +1410,12 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req)
ent = list_first_entry_or_null(&queue->ent_avail_queue,
struct fuse_ring_ent, list);
spin_lock(&fc->bg_lock);
fc->num_background++;
if (fc->num_background == fc->max_background)
fc->blocked = 1;
spin_lock(&fch->bg_lock);
fch->num_background++;
if (fch->num_background == fch->max_background)
fch->blocked = 1;
fuse_uring_flush_bg(queue);
spin_unlock(&fc->bg_lock);
spin_unlock(&fch->bg_lock);
/*
* Due to bg_queue flush limits there might be other bg requests

47
fs/fuse/dev_uring_i.h
View File

@@ -7,7 +7,7 @@
#ifndef _FS_FUSE_DEV_URING_I_H
#define _FS_FUSE_DEV_URING_I_H
#include "fuse_i.h"
#include "fuse_dev_i.h"
#ifdef CONFIG_FUSE_IO_URING
@@ -101,13 +101,13 @@ struct fuse_ring_queue {
bool stopped;
};
/**
/*
* Describes if uring is for communication and holds alls the data needed
* for uring communication
*/
struct fuse_ring {
/* back pointer */
struct fuse_conn *fc;
struct fuse_chan *chan;
/* number of ring queues */
size_t nr_queues;
@@ -135,63 +135,54 @@ struct fuse_ring {
bool ready;
};
bool fuse_uring_enabled(void);
void fuse_uring_destruct(struct fuse_conn *fc);
void fuse_uring_stop_queues(struct fuse_ring *ring);
void fuse_uring_abort_end_requests(struct fuse_ring *ring);
int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags);
void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req);
bool fuse_uring_queue_bq_req(struct fuse_req *req);
bool fuse_uring_remove_pending_req(struct fuse_req *req);
bool fuse_uring_request_expired(struct fuse_conn *fc);
bool fuse_uring_request_expired(struct fuse_chan *fch);
static inline void fuse_uring_abort(struct fuse_conn *fc)
static inline void fuse_uring_abort(struct fuse_chan *fch)
{
struct fuse_ring *ring = fc->ring;
struct fuse_ring *ring = fch->ring;
if (ring == NULL)
return;
if (atomic_read(&ring->queue_refs) > 0) {
fuse_uring_abort_end_requests(ring);
fuse_uring_abort_end_requests(ring);
if (atomic_read(&ring->queue_refs) > 0)
fuse_uring_stop_queues(ring);
}
}
static inline void fuse_uring_wait_stopped_queues(struct fuse_conn *fc)
static inline void fuse_uring_wait_stopped_queues(struct fuse_chan *fch)
{
struct fuse_ring *ring = fc->ring;
struct fuse_ring *ring = fch->ring;
if (ring)
wait_event(ring->stop_waitq,
atomic_read(&ring->queue_refs) == 0);
}
static inline bool fuse_uring_ready(struct fuse_conn *fc)
static inline bool fuse_uring_ready(struct fuse_chan *fch)
{
return fc->ring && fc->ring->ready;
struct fuse_ring *ring = READ_ONCE(fch->ring);
return ring && smp_load_acquire(&ring->ready);
}
#else /* CONFIG_FUSE_IO_URING */
static inline void fuse_uring_destruct(struct fuse_conn *fc)
static inline void fuse_uring_abort(struct fuse_chan *fch)
{
}
static inline bool fuse_uring_enabled(void)
{
return false;
}
static inline void fuse_uring_abort(struct fuse_conn *fc)
static inline void fuse_uring_wait_stopped_queues(struct fuse_chan *fch)
{
}
static inline void fuse_uring_wait_stopped_queues(struct fuse_conn *fc)
{
}
static inline bool fuse_uring_ready(struct fuse_conn *fc)
static inline bool fuse_uring_ready(struct fuse_chan *fch)
{
return false;
}
@@ -201,7 +192,7 @@ static inline bool fuse_uring_remove_pending_req(struct fuse_req *req)
return false;
}
static inline bool fuse_uring_request_expired(struct fuse_conn *fc)
static inline bool fuse_uring_request_expired(struct fuse_chan *fch)
{
return false;
}

52
fs/fuse/dir.c
View File

@@ -1,11 +1,10 @@
// SPDX-License-Identifier: GPL-2.0
/*
FUSE: Filesystem in Userspace
Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu>
This program can be distributed under the terms of the GNU GPL.
See the file COPYING.
*/
#include "dev.h"
#include "fuse_i.h"
#include <linux/pagemap.h>
@@ -317,7 +316,7 @@ void fuse_invalidate_attr(struct inode *inode)
static void fuse_dir_changed(struct inode *dir)
{
fuse_invalidate_attr(dir);
fuse_invalidate_attr_mask(dir, FUSE_STATX_MODDIR);
inode_maybe_inc_iversion(dir, false);
}
@@ -430,7 +429,7 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name,
fi = get_fuse_inode(inode);
if (outarg.nodeid != get_node_id(inode) ||
(bool) IS_AUTOMOUNT(inode) != (bool) (outarg.attr.flags & FUSE_ATTR_SUBMOUNT)) {
fuse_queue_forget(fm->fc, forget,
fuse_chan_queue_forget(fm->fc->chan, forget,
outarg.nodeid, 1);
goto invalid;
}
@@ -593,7 +592,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name
attr_version, evict_ctr);
err = -ENOMEM;
if (!*inode) {
fuse_queue_forget(fm->fc, forget, outarg->nodeid, 1);
fuse_chan_queue_forget(fm->fc->chan, forget, outarg->nodeid, 1);
goto out;
}
err = 0;
@@ -837,7 +836,6 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir,
if (!forget)
goto out_err;
err = -ENOMEM;
ff = fuse_file_alloc(fm, true);
if (!ff)
goto out_put_forget_req;
@@ -894,7 +892,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir,
if (!inode) {
flags &= ~(O_CREAT | O_EXCL | O_TRUNC);
fuse_sync_release(NULL, ff, flags);
fuse_queue_forget(fm->fc, forget, outentry.nodeid, 1);
fuse_chan_queue_forget(fm->fc->chan, forget, outentry.nodeid, 1);
err = -ENOMEM;
goto out_err;
}
@@ -1019,7 +1017,7 @@ static struct dentry *create_new_entry(struct mnt_idmap *idmap, struct fuse_moun
inode = fuse_iget(dir->i_sb, outarg.nodeid, outarg.generation,
&outarg.attr, ATTR_TIMEOUT(&outarg), 0, 0);
if (!inode) {
fuse_queue_forget(fm->fc, forget, outarg.nodeid, 1);
fuse_chan_queue_forget(fm->fc->chan, forget, outarg.nodeid, 1);
return ERR_PTR(-ENOMEM);
}
kfree(forget);
@@ -1587,40 +1585,34 @@ int fuse_reverse_inval_entry(struct fuse_conn *fc, u64 parent_nodeid,
{
int err = -ENOTDIR;
struct inode *parent;
struct dentry *dir = NULL;
struct dentry *entry = NULL;
struct dentry *dir;
struct dentry *entry;
parent = fuse_ilookup(fc, parent_nodeid, NULL);
if (!parent)
return -ENOENT;
inode_lock_nested(parent, I_MUTEX_PARENT);
if (!S_ISDIR(parent->i_mode))
goto put_parent;
goto unlock;
err = -ENOENT;
dir = d_find_alias(parent);
if (!dir)
goto put_parent;
while (!entry) {
struct dentry *child = try_lookup_noperm(name, dir);
if (!child || IS_ERR(child))
goto put_parent;
entry = start_removing_dentry(dir, child);
dput(child);
if (IS_ERR(entry))
goto put_parent;
if (!d_same_name(entry, dir, name)) {
end_removing(entry);
entry = NULL;
}
}
goto unlock;
name->hash = full_name_hash(dir, name->name, name->len);
entry = d_lookup(dir, name);
dput(dir);
if (!entry)
goto unlock;
fuse_dir_changed(parent);
if (!(flags & FUSE_EXPIRE_ONLY))
d_invalidate(entry);
fuse_invalidate_entry_cache(entry);
if (child_nodeid != 0) {
if (child_nodeid != 0 && d_really_is_positive(entry)) {
inode_lock(d_inode(entry));
if (get_node_id(d_inode(entry)) != child_nodeid) {
err = -ENOENT;
@@ -1648,10 +1640,10 @@ int fuse_reverse_inval_entry(struct fuse_conn *fc, u64 parent_nodeid,
} else {
err = 0;
}
dput(entry);
end_removing(entry);
put_parent:
dput(dir);
unlock:
inode_unlock(parent);
iput(parent);
return err;
}

220
fs/fuse/file.c
View File

@@ -1,12 +1,11 @@
// SPDX-License-Identifier: GPL-2.0
/*
FUSE: Filesystem in Userspace
Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu>
This program can be distributed under the terms of the GNU GPL.
See the file COPYING.
*/
#include "fuse_i.h"
#include "dev.h"
#include <linux/pagemap.h>
#include <linux/slab.h>
@@ -91,8 +90,7 @@ static struct fuse_file *fuse_file_get(struct fuse_file *ff)
return ff;
}
static void fuse_release_end(struct fuse_mount *fm, struct fuse_args *args,
int error)
static void fuse_release_end(struct fuse_args *args, int error)
{
struct fuse_release_args *ra = container_of(args, typeof(*ra), args);
@@ -112,10 +110,10 @@ static void fuse_file_put(struct fuse_file *ff, bool sync)
if (!args) {
/* Do nothing when server does not implement 'opendir' */
} else if (args->opcode == FUSE_RELEASE && ff->fm->fc->no_open) {
fuse_release_end(ff->fm, args, 0);
fuse_release_end(args, 0);
} else if (sync) {
fuse_simple_request(ff->fm, args);
fuse_release_end(ff->fm, args, 0);
fuse_release_end(args, 0);
} else {
/*
* DAX inodes may need to issue a number of synchronous
@@ -126,7 +124,7 @@ static void fuse_file_put(struct fuse_file *ff, bool sync)
args->end = fuse_release_end;
if (fuse_simple_background(ff->fm, args,
GFP_KERNEL | __GFP_NOFAIL))
fuse_release_end(ff->fm, args, -ENOTCONN);
fuse_release_end(args, -ENOTCONN);
}
kfree(ff);
}
@@ -380,8 +378,14 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff,
* aio and closes the fd before the aio completes. Since aio takes its
* own ref to the file, the IO completion has to drop the ref, which is
* how the fuse server can end up closing its clients' files.
*
* Exception is virtio-fs, which is not affected by the above (server is
* on host, cannot close open files in guest). Virtio-fs needs sync
* release, because the num_waiting mechanism to wait for all requests
* before commencing with fs shutdown doesn't work if submounts are
* used.
*/
fuse_file_put(ff, false);
fuse_file_put(ff, ff->fm->fc->auto_submounts);
}
void fuse_release_common(struct file *file, bool isdir)
@@ -635,6 +639,19 @@ static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io)
return io->bytes < 0 ? io->size : io->bytes;
}
static void fuse_aio_invalidate_worker(struct work_struct *work)
{
struct fuse_io_priv *io = container_of(work, struct fuse_io_priv, work);
struct address_space *mapping = io->iocb->ki_filp->f_mapping;
ssize_t res = fuse_get_res_by_io(io);
pgoff_t start = io->offset >> PAGE_SHIFT;
pgoff_t end = (io->offset + res - 1) >> PAGE_SHIFT;
invalidate_inode_pages2_range(mapping, start, end);
io->iocb->ki_complete(io->iocb, res);
kref_put(&io->refcnt, fuse_io_release);
}
/*
* In case of short read, the caller sets 'pos' to the position of
* actual end of fuse request in IO request. Otherwise, if bytes_requested
@@ -667,10 +684,11 @@ static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
spin_unlock(&io->lock);
if (!left && !io->blocking) {
struct inode *inode = file_inode(io->iocb->ki_filp);
struct address_space *mapping = io->iocb->ki_filp->f_mapping;
ssize_t res = fuse_get_res_by_io(io);
if (res >= 0) {
struct inode *inode = file_inode(io->iocb->ki_filp);
struct fuse_conn *fc = get_fuse_conn(inode);
struct fuse_inode *fi = get_fuse_inode(inode);
@@ -679,6 +697,17 @@ static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
spin_unlock(&fi->lock);
}
if (io->write && res > 0 && mapping->nrpages) {
/*
* As in generic_file_direct_write(), invalidate after the
* write, to invalidate read-ahead cache that may have competed
* with the write.
*/
INIT_WORK(&io->work, fuse_aio_invalidate_worker);
queue_work(inode->i_sb->s_dio_done_wq, &io->work);
return;
}
io->iocb->ki_complete(io->iocb, res);
}
@@ -709,8 +738,7 @@ static void fuse_io_free(struct fuse_io_args *ia)
kfree(ia);
}
static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args,
int err)
static void fuse_aio_complete_req(struct fuse_args *args, int err)
{
struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
struct fuse_io_priv *io = ia->io;
@@ -758,7 +786,7 @@ static ssize_t fuse_async_req_send(struct fuse_mount *fm,
ia->ap.args.may_block = io->should_dirty;
err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL);
if (err)
fuse_aio_complete_req(fm, &ia->ap.args, err);
fuse_aio_complete_req(&ia->ap.args, err);
return num_bytes;
}
@@ -902,7 +930,7 @@ static int fuse_handle_readahead(struct folio *folio,
ia = NULL;
}
if (!ia) {
if (fc->num_background >= fc->congestion_threshold &&
if (fuse_chan_num_background(fc->chan) >= fc->congestion_threshold &&
rac->ra->async_size >= readahead_count(rac))
/*
* Congested and only async pages left, so skip the
@@ -1001,8 +1029,7 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter,
return fuse_do_readfolio(file, folio, off, len);
}
static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args,
int err)
static void fuse_readpages_end(struct fuse_args *args, int err)
{
int i;
struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
@@ -1068,7 +1095,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file,
res = fuse_simple_request(fm, &ap->args);
err = res < 0 ? res : 0;
}
fuse_readpages_end(fm, &ap->args, err);
fuse_readpages_end(&ap->args, err);
}
static void fuse_readahead(struct readahead_control *rac)
@@ -1586,7 +1613,7 @@ static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
* manually extract pages using iov_iter_extract_pages() and then
* copy that to a folios array.
*/
struct page **pages = kzalloc(max_pages * sizeof(struct page *),
struct page **pages = kcalloc(max_pages, sizeof(struct page *),
GFP_KERNEL);
if (!pages) {
ret = -ENOMEM;
@@ -1743,15 +1770,6 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
if (res > 0)
*ppos = pos;
if (res > 0 && write && fopen_direct_io) {
/*
* As in generic_file_direct_write(), invalidate after the
* write, to invalidate read-ahead cache that may have competed
* with the write.
*/
invalidate_inode_pages2_range(mapping, idx_from, idx_to);
}
return res > 0 ? res : err;
}
EXPORT_SYMBOL_GPL(fuse_direct_io);
@@ -1790,6 +1808,8 @@ static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to)
static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
{
struct inode *inode = file_inode(iocb->ki_filp);
struct address_space *mapping = inode->i_mapping;
loff_t pos = iocb->ki_pos;
ssize_t res;
bool exclusive;
@@ -1806,6 +1826,16 @@ static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
FUSE_DIO_WRITE);
fuse_write_update_attr(inode, iocb->ki_pos, res);
}
if (res > 0 && mapping->nrpages) {
/*
* As in generic_file_direct_write(), invalidate after
* write, to invalidate read-ahead cache that may have
* with the write.
*/
invalidate_inode_pages2_range(mapping,
pos >> PAGE_SHIFT,
(pos + res - 1) >> PAGE_SHIFT);
}
}
fuse_dio_unlock(iocb, exclusive);
@@ -1985,8 +2015,7 @@ __acquires(fi->lock)
}
}
static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args,
int error)
static void fuse_writepage_end(struct fuse_args *args, int error)
{
struct fuse_writepage_args *wpa =
container_of(args, typeof(*wpa), ia.ap.args);
@@ -2297,7 +2326,7 @@ static int fuse_writepages(struct address_space *mapping,
return -EIO;
if (wbc->sync_mode == WB_SYNC_NONE &&
fc->num_background >= fc->congestion_threshold)
fuse_chan_num_background(fc->chan) >= fc->congestion_threshold)
return 0;
return iomap_writepages(&wpc);
@@ -2571,8 +2600,9 @@ static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl)
struct fuse_file *ff = file->private_data;
/* emulate flock with POSIX locks */
ff->flock = true;
err = fuse_setlk(file, fl, 1);
if (!err)
ff->flock = true;
}
return err;
@@ -2683,125 +2713,6 @@ static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence)
return retval;
}
/*
* All files which have been polled are linked to RB tree
* fuse_conn->polled_files which is indexed by kh. Walk the tree and
* find the matching one.
*/
static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
struct rb_node **parent_out)
{
struct rb_node **link = &fc->polled_files.rb_node;
struct rb_node *last = NULL;
while (*link) {
struct fuse_file *ff;
last = *link;
ff = rb_entry(last, struct fuse_file, polled_node);
if (kh < ff->kh)
link = &last->rb_left;
else if (kh > ff->kh)
link = &last->rb_right;
else
return link;
}
if (parent_out)
*parent_out = last;
return link;
}
/*
* The file is about to be polled. Make sure it's on the polled_files
* RB tree. Note that files once added to the polled_files tree are
* not removed before the file is released. This is because a file
* polled once is likely to be polled again.
*/
static void fuse_register_polled_file(struct fuse_conn *fc,
struct fuse_file *ff)
{
spin_lock(&fc->lock);
if (RB_EMPTY_NODE(&ff->polled_node)) {
struct rb_node **link, *parent;
link = fuse_find_polled_node(fc, ff->kh, &parent);
BUG_ON(*link);
rb_link_node(&ff->polled_node, parent, link);
rb_insert_color(&ff->polled_node, &fc->polled_files);
}
spin_unlock(&fc->lock);
}
__poll_t fuse_file_poll(struct file *file, poll_table *wait)
{
struct fuse_file *ff = file->private_data;
struct fuse_mount *fm = ff->fm;
struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
struct fuse_poll_out outarg;
FUSE_ARGS(args);
int err;
if (fm->fc->no_poll)
return DEFAULT_POLLMASK;
poll_wait(file, &ff->poll_wait, wait);
inarg.events = mangle_poll(poll_requested_events(wait));
/*
* Ask for notification iff there's someone waiting for it.
* The client may ignore the flag and always notify.
*/
if (waitqueue_active(&ff->poll_wait)) {
inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
fuse_register_polled_file(fm->fc, ff);
}
args.opcode = FUSE_POLL;
args.nodeid = ff->nodeid;
args.in_numargs = 1;
args.in_args[0].size = sizeof(inarg);
args.in_args[0].value = &inarg;
args.out_numargs = 1;
args.out_args[0].size = sizeof(outarg);
args.out_args[0].value = &outarg;
err = fuse_simple_request(fm, &args);
if (!err)
return demangle_poll(outarg.revents);
if (err == -ENOSYS) {
fm->fc->no_poll = 1;
return DEFAULT_POLLMASK;
}
return EPOLLERR;
}
EXPORT_SYMBOL_GPL(fuse_file_poll);
/*
* This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and
* wakes up the poll waiters.
*/
int fuse_notify_poll_wakeup(struct fuse_conn *fc,
struct fuse_notify_poll_wakeup_out *outarg)
{
u64 kh = outarg->kh;
struct rb_node **link;
spin_lock(&fc->lock);
link = fuse_find_polled_node(fc, kh, NULL);
if (*link) {
struct fuse_file *ff;
ff = rb_entry(*link, struct fuse_file, polled_node);
wake_up_interruptible_sync(&ff->poll_wait);
}
spin_unlock(&fc->lock);
return 0;
}
static void fuse_do_truncate(struct file *file)
{
struct inode *inode = file->f_mapping->host;
@@ -2834,6 +2745,7 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
size_t count = iov_iter_count(iter), shortened = 0;
loff_t offset = iocb->ki_pos;
struct fuse_io_priv *io;
bool async = ff->fm->fc->async_dio;
pos = offset;
inode = file->f_mapping->host;
@@ -2842,6 +2754,12 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
if ((iov_iter_rw(iter) == READ) && (offset >= i_size))
return 0;
if ((iov_iter_rw(iter) == WRITE) && async && !inode->i_sb->s_dio_done_wq) {
ret = sb_init_dio_done_wq(inode->i_sb);
if (ret < 0)
return ret;
}
io = kmalloc_obj(struct fuse_io_priv);
if (!io)
return -ENOMEM;
@@ -2857,7 +2775,7 @@ fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
* By default, we want to optimize all I/Os with async request
* submission to the client filesystem if supported.
*/
io->async = ff->fm->fc->async_dio;
io->async = async;
io->iocb = iocb;
io->blocking = is_sync_kiocb(iocb);

351
fs/fuse/fuse_dev_i.h
View File

@@ -6,20 +6,311 @@
#ifndef _FS_FUSE_DEV_I_H
#define _FS_FUSE_DEV_I_H
#include <linux/fuse.h>
#include <linux/types.h>
#include <linux/refcount.h>
#include <linux/wait.h>
#include <linux/workqueue.h>
#include <linux/fs.h>
/* Ordinary requests have even IDs, while interrupts IDs are odd */
#define FUSE_INT_REQ_BIT (1ULL << 0)
#define FUSE_REQ_ID_STEP (1ULL << 1)
extern struct wait_queue_head fuse_dev_waitq;
struct fuse_arg;
struct fuse_args;
struct fuse_pqueue;
struct fuse_req;
struct fuse_iqueue;
struct fuse_forget_link;
/**
* enum fuse_req_flag - Request flags
*
* @FR_ISREPLY: set if the request has reply
* @FR_FORCE: force sending of the request even if interrupted
* @FR_BACKGROUND: request is sent in the background
* @FR_WAITING: request is counted as "waiting"
* @FR_ABORTED: the request was aborted
* @FR_INTERRUPTED: the request has been interrupted
* @FR_LOCKED: data is being copied to/from the request
* @FR_PENDING: request is not yet in userspace
* @FR_SENT: request is in userspace, waiting for an answer
* @FR_FINISHED: request is finished
* @FR_PRIVATE: request is on private list
* @FR_ASYNC: request is asynchronous
* @FR_URING: request is handled through fuse-io-uring
*/
enum fuse_req_flag {
FR_ISREPLY,
FR_FORCE,
FR_BACKGROUND,
FR_WAITING,
FR_ABORTED,
FR_INTERRUPTED,
FR_LOCKED,
FR_PENDING,
FR_SENT,
FR_FINISHED,
FR_PRIVATE,
FR_ASYNC,
FR_URING,
};
/**
* struct fuse_req - A request to the client
*
* .waitq.lock protects the following fields:
* - FR_ABORTED
* - FR_LOCKED (may also be modified under fpq->lock, tested under both)
*/
struct fuse_req {
/**
* @list: This can be on either pending processing or io lists in
* fuse_conn
*/
struct list_head list;
/** @intr_entry: Entry on the interrupts list */
struct list_head intr_entry;
/** @args: Input/output arguments */
struct fuse_args *args;
/** @count: refcount */
refcount_t count;
/** @flags: Request flags, updated with test/set/clear_bit() */
unsigned long flags;
/** @in: The request input header */
struct {
/** @in.h: The request input header */
struct fuse_in_header h;
} in;
/** @out: The request output header */
struct {
/** @out.h: The request output header */
struct fuse_out_header h;
} out;
/** @waitq: Used to wake up the task waiting for completion of request */
wait_queue_head_t waitq;
#if IS_ENABLED(CONFIG_VIRTIO_FS)
/**
* @argbuf: virtio-fs's physically contiguous buffer for in and out
* args
*/
void *argbuf;
#endif
/** @chan: fuse_chan this request belongs to */
struct fuse_chan *chan;
#ifdef CONFIG_FUSE_IO_URING
void *ring_entry;
void *ring_queue;
#endif
/** @create_time: When (in jiffies) the request was created */
unsigned long create_time;
};
/* One forget request */
struct fuse_forget_link {
struct fuse_forget_one forget_one;
struct fuse_forget_link *next;
};
/**
* struct fuse_iqueue_ops - Input queue callbacks
*
* Input queue signalling is device-specific. For example, the /dev/fuse file
* uses fiq->waitq and fasync to wake processes that are waiting on queue
* readiness. These callbacks allow other device types to respond to input
* queue activity.
*/
struct fuse_iqueue_ops {
/**
* @send_forget: Send one forget
*/
void (*send_forget)(struct fuse_iqueue *fiq, struct fuse_forget_link *link);
/**
* @send_interrupt: Send interrupt for request
*/
void (*send_interrupt)(struct fuse_iqueue *fiq, struct fuse_req *req);
/**
* @send_req: Send one request
*/
void (*send_req)(struct fuse_iqueue *fiq, struct fuse_req *req);
/**
* @release: Clean up when fuse_iqueue is destroyed
*/
void (*release)(struct fuse_iqueue *fiq);
};
struct fuse_iqueue {
/** Connection established */
unsigned connected;
/** Lock protecting accesses to members of this structure */
spinlock_t lock;
/** Readers of the connection are waiting on this */
wait_queue_head_t waitq;
/** The next unique request id */
u64 reqctr;
/** The list of pending requests */
struct list_head pending;
/** Pending interrupts */
struct list_head interrupts;
/** Queue of pending forgets */
struct fuse_forget_link forget_list_head;
struct fuse_forget_link *forget_list_tail;
/** Batching of FORGET requests (positive indicates FORGET batch) */
int forget_batch;
/** O_ASYNC requests */
struct fasync_struct *fasync;
/** Device-specific callbacks */
const struct fuse_iqueue_ops *ops;
/** Device-specific state */
void *priv;
};
struct fuse_chan {
/** Lock protecting:
- devices
- connected
- ring
- ring->queues[qid]
*/
spinlock_t lock;
/* back pointer: fc->chan->conn == fc */
struct fuse_conn *conn;
/** Input queue */
struct fuse_iqueue iq;
/** List of device instances belonging to this connection */
struct list_head devices;
/** Maximum number of outstanding background requests */
unsigned max_background;
/** Number of requests currently in the background */
unsigned num_background;
/** Number of background requests currently queued for userspace */
unsigned active_background;
/** The list of background requests set aside for later queuing */
struct list_head bg_queue;
/** Protects: max_background, num_background, active_background, bg_queue, blocked */
spinlock_t bg_lock;
/** Flag indicating that INIT reply has been received. Allocating
* any fuse request will be suspended until the flag is set */
int initialized;
/** Flag indicating if connection is blocked. This will be
the case before the INIT reply is received, and if there
are too many outstading backgrounds requests */
int blocked;
/** waitq for blocked connection */
wait_queue_head_t blocked_waitq;
/** Connection established, cleared on umount, connection
abort and device release */
unsigned connected;
/** The number of requests waiting for completion */
atomic_t num_waiting;
/** Is interrupt not implemented by fs? */
bool no_interrupt;
/* Use io_uring for communication */
unsigned int io_uring;
/* Negotiated minor version */
unsigned int minor;
/* Maximum write size */
unsigned int max_write;
/* Maximum number of pages that can be used in a single request */
unsigned int max_pages;
/* Before being installed into fud, contains the preallocated pq array*/
struct list_head *pq_prealloc;
/** Connection aborted via sysfs, respond with ECONNABORTED on device I/O */
bool abort_with_err;
#ifdef CONFIG_FUSE_IO_URING
/** uring connection information*/
struct fuse_ring *ring;
#endif
/** Only used if the connection opts into request timeouts */
struct {
/* Worker for checking if any requests have timed out */
struct delayed_work work;
/* Request timeout (in jiffies). 0 = no timeout */
unsigned int req_timeout;
} timeout;
};
#define FUSE_PQ_HASH_BITS 8
#define FUSE_PQ_HASH_SIZE (1 << FUSE_PQ_HASH_BITS)
struct fuse_pqueue {
/** Connection established */
unsigned connected;
/** Lock protecting accessess to members of this structure */
spinlock_t lock;
/** Hash table of requests being processed */
struct list_head *processing;
/** The list of requests under I/O */
struct list_head io;
};
/**
* struct fuse_dev - Fuse device instance
*/
struct fuse_dev {
/** @ref: Reference count of this object */
refcount_t ref;
/** @sync_init: Issue FUSE_INIT synchronously */
bool sync_init;
/** @chan: Fuse channel for this device */
struct fuse_chan *chan;
/** @pq: Processing queue */
struct fuse_pqueue pq;
/** @entry: list entry on fch->devices */
struct list_head entry;
};
struct fuse_copy_state {
struct fuse_req *req;
@@ -39,21 +330,21 @@ struct fuse_copy_state {
} ring;
};
/* fud->fc gets assigned to this value when /dev/fuse is closed */
#define FUSE_DEV_FC_DISCONNECTED ((struct fuse_conn *) 1)
/* fud->chan gets assigned to this value when /dev/fuse is closed */
#define FUSE_DEV_CHAN_DISCONNECTED ((struct fuse_chan *) 1)
/*
* Lockless access is OK, because fud->fc is set once during mount and is valid
* Lockless access is OK, because fud->chan is set once during mount and is valid
* until the file is released.
*
* fud->fc is set to FUSE_DEV_FC_DISCONNECTED only after the containing file is
* fud->chan is set to FUSE_DEV_CHAN_DISCONNECTED only after the containing file is
* released, so result is safe to dereference in most cases. Exceptions are:
* fuse_dev_put() and fuse_fill_super_common().
*/
static inline struct fuse_conn *fuse_dev_fc_get(struct fuse_dev *fud)
static inline struct fuse_chan *fuse_dev_chan_get(struct fuse_dev *fud)
{
/* Pairs with xchg() in fuse_dev_install() */
return smp_load_acquire(&fud->fc);
return smp_load_acquire(&fud->chan);
}
static inline struct fuse_dev *fuse_file_to_fud(struct file *file)
@@ -65,22 +356,29 @@ static inline struct fuse_dev *__fuse_get_dev(struct file *file)
{
struct fuse_dev *fud = fuse_file_to_fud(file);
if (!fuse_dev_fc_get(fud))
if (!fuse_dev_chan_get(fud))
return NULL;
return fud;
}
void fuse_iqueue_init(struct fuse_iqueue *fiq, const struct fuse_iqueue_ops *ops, void *priv);
struct fuse_dev *fuse_get_dev(struct file *file);
unsigned int fuse_req_hash(u64 unique);
struct fuse_req *fuse_request_find(struct fuse_pqueue *fpq, u64 unique);
void fuse_dev_end_requests(struct list_head *head);
void fuse_request_bg_finish(struct fuse_chan *fch, struct fuse_req *req);
void fuse_copy_init(struct fuse_copy_state *cs, bool write,
struct iov_iter *iter);
void fuse_copy_finish(struct fuse_copy_state *cs);
/*
* Return the number of bytes in an arguments list
*/
unsigned int fuse_len_args(unsigned int numargs, struct fuse_arg *args);
int fuse_copy_args(struct fuse_copy_state *cs, unsigned int numargs,
unsigned int argpages, struct fuse_arg *args,
int zeroing);
@@ -91,7 +389,34 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq,
void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req);
bool fuse_remove_pending_req(struct fuse_req *req, spinlock_t *lock);
bool fuse_request_expired(struct fuse_conn *fc, struct list_head *list);
bool fuse_request_expired(struct fuse_chan *fch, struct list_head *list);
/*
* Assign a unique id to a fuse request
*/
void fuse_request_assign_unique(struct fuse_iqueue *fiq, struct fuse_req *req);
/*
* Get the next unique ID for a request
*/
u64 fuse_get_unique(struct fuse_iqueue *fiq);
struct fuse_dev *fuse_dev_alloc_install(struct fuse_chan *fch);
struct fuse_dev *fuse_dev_alloc(void);
int fuse_dev_release(struct inode *inode, struct file *file);
struct list_head *fuse_pqueue_alloc(void);
/*
* Initialize the fuse processing queue
*/
void fuse_pqueue_init(struct fuse_pqueue *fpq);
/*
* End a finished request
*/
void fuse_request_end(struct fuse_req *req);
#endif

872
fs/fuse/fuse_i.h
View File

File diff suppressed because it is too large Load Diff

26
fs/fuse/fuse_trace.h
View File

@@ -90,7 +90,7 @@ TRACE_EVENT(fuse_request_send,
),
TP_fast_assign(
__entry->connection = req->fm->fc->dev;
__entry->connection = req->chan->conn->dev;
__entry->unique = req->in.h.unique;
__entry->opcode = req->in.h.opcode;
__entry->len = req->in.h.len;
@@ -101,6 +101,28 @@ TRACE_EVENT(fuse_request_send,
__print_symbolic(__entry->opcode, OPCODES), __entry->len)
);
TRACE_EVENT(fuse_request_sent,
TP_PROTO(const struct fuse_req *req),
TP_ARGS(req),
TP_STRUCT__entry(
__field(dev_t, connection)
__field(uint64_t, unique)
__field(enum fuse_opcode, opcode)
),
TP_fast_assign(
__entry->connection = req->chan->conn->dev;
__entry->unique = req->in.h.unique;
__entry->opcode = req->in.h.opcode;
),
TP_printk("connection %u req %llu opcode %u (%s)",
__entry->connection, __entry->unique, __entry->opcode,
__print_symbolic(__entry->opcode, OPCODES))
);
TRACE_EVENT(fuse_request_end,
TP_PROTO(const struct fuse_req *req),
@@ -114,7 +136,7 @@ TRACE_EVENT(fuse_request_end,
),
TP_fast_assign(
__entry->connection = req->fm->fc->dev;
__entry->connection = req->chan->conn->dev;
__entry->unique = req->in.h.unique;
__entry->len = req->out.h.len;
__entry->error = req->out.h.error;

270
fs/fuse/inode.c
View File

@@ -1,14 +1,11 @@
// SPDX-License-Identifier: GPL-2.0
/*
FUSE: Filesystem in Userspace
Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu>
This program can be distributed under the terms of the GNU GPL.
See the file COPYING.
*/
#include "dev.h"
#include "fuse_i.h"
#include "fuse_dev_i.h"
#include "dev_uring_i.h"
#include <linux/dax.h>
#include <linux/pagemap.h>
@@ -35,14 +32,11 @@ MODULE_LICENSE("GPL");
static struct kmem_cache *fuse_inode_cachep;
struct list_head fuse_conn_list;
DEFINE_MUTEX(fuse_mutex);
DECLARE_WAIT_QUEUE_HEAD(fuse_dev_waitq);
static int set_global_limit(const char *val, const struct kernel_param *kp);
unsigned int fuse_max_pages_limit = 256;
/* default is no timeout */
unsigned int fuse_default_req_timeout;
unsigned int fuse_max_req_timeout;
unsigned int max_user_bgreq;
module_param_call(max_user_bgreq, set_global_limit, param_get_uint,
@@ -62,9 +56,6 @@ MODULE_PARM_DESC(max_user_congthresh,
#define FUSE_DEFAULT_BLKSIZE 512
/** Maximum number of outstanding background requests */
#define FUSE_DEFAULT_MAX_BACKGROUND 12
/** Congestion starts at 75% of maximum */
#define FUSE_DEFAULT_CONGESTION_THRESHOLD (FUSE_DEFAULT_MAX_BACKGROUND * 3 / 4)
@@ -72,11 +63,6 @@ MODULE_PARM_DESC(max_user_congthresh,
static struct file_system_type fuseblk_fs_type;
#endif
struct fuse_forget_link *fuse_alloc_forget(void)
{
return kzalloc_obj(struct fuse_forget_link, GFP_KERNEL_ACCOUNT);
}
static struct fuse_submount_lookup *fuse_alloc_submount_lookup(void)
{
struct fuse_submount_lookup *sl;
@@ -150,7 +136,7 @@ static void fuse_cleanup_submount_lookup(struct fuse_conn *fc,
if (!refcount_dec_and_test(&sl->count))
return;
fuse_queue_forget(fc, sl->forget, sl->nodeid, 1);
fuse_chan_queue_forget(fc->chan, sl->forget, sl->nodeid, 1);
sl->forget = NULL;
kfree(sl);
}
@@ -173,8 +159,8 @@ static void fuse_evict_inode(struct inode *inode)
if (FUSE_IS_DAX(inode))
fuse_dax_inode_cleanup(inode);
if (fi->nlookup) {
fuse_queue_forget(fc, fi->forget, fi->nodeid,
fi->nlookup);
fuse_chan_queue_forget(fc->chan, fi->forget, fi->nodeid,
fi->nlookup);
fi->forget = NULL;
}
@@ -624,7 +610,7 @@ static void fuse_umount_begin(struct super_block *sb)
if (fc->no_force_umount)
return;
fuse_abort_conn(fc);
fuse_chan_abort(fc->chan, false);
// Only retire block-device-based superblocks.
if (sb->s_bdev != NULL)
@@ -688,11 +674,9 @@ static struct fuse_sync_bucket *fuse_sync_bucket_alloc(void)
struct fuse_sync_bucket *bucket;
bucket = kzalloc_obj(*bucket, GFP_KERNEL | __GFP_NOFAIL);
if (bucket) {
init_waitqueue_head(&bucket->waitq);
/* Initial active count */
atomic_set(&bucket->count, 1);
}
init_waitqueue_head(&bucket->waitq);
/* Initial active count */
atomic_set(&bucket->count, 1);
return bucket;
}
@@ -816,8 +800,7 @@ static int fuse_opt_fd(struct fs_context *fsc, struct file *file)
if (file->f_cred->user_ns != fsc->user_ns)
return invalfc(fsc, "wrong user namespace for fuse device");
ctx->fud = file->private_data;
refcount_inc(&ctx->fud->ref);
ctx->fud = fuse_dev_grab(file);
return 0;
}
@@ -970,56 +953,19 @@ static int fuse_show_options(struct seq_file *m, struct dentry *root)
return 0;
}
static void fuse_iqueue_init(struct fuse_iqueue *fiq,
const struct fuse_iqueue_ops *ops,
void *priv)
{
memset(fiq, 0, sizeof(struct fuse_iqueue));
spin_lock_init(&fiq->lock);
init_waitqueue_head(&fiq->waitq);
INIT_LIST_HEAD(&fiq->pending);
INIT_LIST_HEAD(&fiq->interrupts);
fiq->forget_list_tail = &fiq->forget_list_head;
fiq->connected = 1;
fiq->ops = ops;
fiq->priv = priv;
}
void fuse_pqueue_init(struct fuse_pqueue *fpq)
{
unsigned int i;
spin_lock_init(&fpq->lock);
for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
INIT_LIST_HEAD(&fpq->processing[i]);
INIT_LIST_HEAD(&fpq->io);
fpq->connected = 1;
}
void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm,
struct user_namespace *user_ns,
const struct fuse_iqueue_ops *fiq_ops, void *fiq_priv)
struct user_namespace *user_ns, struct fuse_chan *fch)
{
memset(fc, 0, sizeof(*fc));
spin_lock_init(&fc->lock);
spin_lock_init(&fc->bg_lock);
init_rwsem(&fc->killsb);
refcount_set(&fc->count, 1);
atomic_set(&fc->epoch, 1);
INIT_WORK(&fc->epoch_work, fuse_epoch_work);
init_waitqueue_head(&fc->blocked_waitq);
fuse_iqueue_init(&fc->iq, fiq_ops, fiq_priv);
INIT_LIST_HEAD(&fc->bg_queue);
INIT_LIST_HEAD(&fc->entry);
INIT_LIST_HEAD(&fc->devices);
atomic_set(&fc->num_waiting, 0);
fc->max_background = FUSE_DEFAULT_MAX_BACKGROUND;
fc->congestion_threshold = FUSE_DEFAULT_CONGESTION_THRESHOLD;
atomic64_set(&fc->khctr, 0);
fc->polled_files = RB_ROOT;
fc->blocked = 0;
fc->initialized = 0;
fc->connected = 1;
atomic64_set(&fc->attr_version, 1);
atomic64_set(&fc->evict_ctr, 1);
get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key));
@@ -1028,7 +974,6 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm,
fc->max_pages = FUSE_DEFAULT_MAX_PAGES_PER_REQ;
fc->max_pages_limit = fuse_max_pages_limit;
fc->name_max = FUSE_NAME_LOW_MAX;
fc->timeout.req_timeout = 0;
if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH))
fuse_backing_files_init(fc);
@@ -1036,6 +981,8 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm,
INIT_LIST_HEAD(&fc->mounts);
list_add(&fm->fc_entry, &fc->mounts);
fm->fc = fc;
fuse_chan_set_fc(fch, fc);
fc->chan = fch;
}
EXPORT_SYMBOL_GPL(fuse_conn_init);
@@ -1043,7 +990,8 @@ static void delayed_release(struct rcu_head *p)
{
struct fuse_conn *fc = container_of(p, struct fuse_conn, rcu);
fuse_uring_destruct(fc);
fuse_uring_destruct(fc->chan);
fuse_chan_free(fc->chan);
put_user_ns(fc->user_ns);
fc->release(fc);
@@ -1051,7 +999,6 @@ static void delayed_release(struct rcu_head *p)
void fuse_conn_put(struct fuse_conn *fc)
{
struct fuse_iqueue *fiq = &fc->iq;
struct fuse_sync_bucket *bucket;
if (!refcount_dec_and_test(&fc->count))
@@ -1059,11 +1006,8 @@ void fuse_conn_put(struct fuse_conn *fc)
if (IS_ENABLED(CONFIG_FUSE_DAX))
fuse_dax_conn_free(fc);
if (fc->timeout.req_timeout)
cancel_delayed_work_sync(&fc->timeout.work);
cancel_work_sync(&fc->epoch_work);
if (fiq->ops->release)
fiq->ops->release(fiq);
fuse_chan_release(fc->chan);
put_pid_ns(fc->pid_ns);
bucket = rcu_dereference_protected(fc->curr_bucket, 1);
if (bucket) {
@@ -1083,6 +1027,11 @@ struct fuse_conn *fuse_conn_get(struct fuse_conn *fc)
}
EXPORT_SYMBOL_GPL(fuse_conn_get);
dev_t fuse_conn_get_id(struct fuse_conn *fc)
{
return fc->dev;
}
static struct inode *fuse_get_root_inode(struct super_block *sb, unsigned int mode)
{
struct fuse_attr attr;
@@ -1113,12 +1062,11 @@ static struct dentry *fuse_get_dentry(struct super_block *sb,
inode = ilookup5(sb, handle->nodeid, fuse_inode_eq, &handle->nodeid);
if (!inode) {
struct fuse_entry_out outarg;
const struct qstr name = QSTR_INIT(".", 1);
if (!fc->export_support)
goto out_err;
err = fuse_lookup_name(sb, handle->nodeid, &name, &outarg,
err = fuse_lookup_name(sb, handle->nodeid, &QSTR("."), &outarg,
&inode);
if (err && err != -ENOENT)
goto out_err;
@@ -1294,12 +1242,13 @@ static void process_init_limits(struct fuse_conn *fc, struct fuse_init_out *arg)
sanitize_global_limit(&max_user_bgreq);
sanitize_global_limit(&max_user_congthresh);
spin_lock(&fc->bg_lock);
if (arg->max_background) {
fc->max_background = arg->max_background;
unsigned int max_background = arg->max_background;
if (!cap_sys_admin && fc->max_background > max_user_bgreq)
fc->max_background = max_user_bgreq;
if (!cap_sys_admin && max_background > max_user_bgreq)
max_background = max_user_bgreq;
fuse_chan_max_background_set(fc->chan, max_background);
}
if (arg->congestion_threshold) {
fc->congestion_threshold = arg->congestion_threshold;
@@ -1308,48 +1257,20 @@ static void process_init_limits(struct fuse_conn *fc, struct fuse_init_out *arg)
fc->congestion_threshold > max_user_congthresh)
fc->congestion_threshold = max_user_congthresh;
}
spin_unlock(&fc->bg_lock);
}
static void set_request_timeout(struct fuse_conn *fc, unsigned int timeout)
{
fc->timeout.req_timeout = secs_to_jiffies(timeout);
INIT_DELAYED_WORK(&fc->timeout.work, fuse_check_timeout);
queue_delayed_work(system_percpu_wq, &fc->timeout.work,
fuse_timeout_timer_freq);
}
static void init_server_timeout(struct fuse_conn *fc, unsigned int timeout)
{
if (!timeout && !fuse_max_req_timeout && !fuse_default_req_timeout)
return;
if (!timeout)
timeout = fuse_default_req_timeout;
if (fuse_max_req_timeout) {
if (timeout)
timeout = min(fuse_max_req_timeout, timeout);
else
timeout = fuse_max_req_timeout;
}
timeout = max(FUSE_TIMEOUT_TIMER_FREQ, timeout);
set_request_timeout(fc, timeout);
}
struct fuse_init_args {
struct fuse_args args;
struct fuse_init_in in;
struct fuse_init_out out;
struct fuse_mount *fm;
};
static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args,
int error)
static void process_init_reply(struct fuse_args *args, int error)
{
struct fuse_conn *fc = fm->fc;
struct fuse_init_args *ia = container_of(args, typeof(*ia), args);
struct fuse_mount *fm = ia->fm;
struct fuse_conn *fc = fm->fc;
struct fuse_init_out *arg = &ia->out;
bool ok = true;
@@ -1481,7 +1402,7 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args,
ok = false;
}
if (flags & FUSE_OVER_IO_URING && fuse_uring_enabled())
fc->io_uring = 1;
fuse_chan_io_uring_enable(fc->chan);
if (flags & FUSE_REQUEST_TIMEOUT)
timeout = arg->request_timeout;
@@ -1491,7 +1412,7 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args,
fc->no_flock = 1;
}
init_server_timeout(fc, timeout);
fuse_init_server_timeout(fc->chan, timeout);
fm->sb->s_bdi->ra_pages =
min(fm->sb->s_bdi->ra_pages, ra_pages);
@@ -1505,10 +1426,15 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args,
if (!ok) {
fc->conn_init = 0;
fc->conn_error = 1;
fuse_chan_set_initialized(fc->chan, NULL);
} else {
struct fuse_chan_param cp = {
.minor = fc->minor,
.max_write = fc->max_write,
.max_pages = fc->max_pages,
};
fuse_chan_set_initialized(fc->chan, &cp);
}
fuse_set_initialized(fc);
wake_up_all(&fc->blocked_waitq);
}
static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm)
@@ -1518,6 +1444,7 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm)
ia = kzalloc_obj(*ia, GFP_KERNEL | __GFP_NOFAIL);
ia->fm = fm;
ia->in.major = FUSE_KERNEL_VERSION;
ia->in.minor = FUSE_KERNEL_MINOR_VERSION;
ia->in.max_readahead = fm->sb->s_bdi->ra_pages * PAGE_SIZE;
@@ -1591,7 +1518,7 @@ int fuse_send_init(struct fuse_mount *fm)
if (!err)
return 0;
}
process_init_reply(fm, &ia->args, err);
process_init_reply(&ia->args, err);
if (fm->fc->conn_error)
return -ENOTCONN;
return 0;
@@ -1600,7 +1527,6 @@ EXPORT_SYMBOL_GPL(fuse_send_init);
void fuse_free_conn(struct fuse_conn *fc)
{
WARN_ON(!list_empty(&fc->devices));
kfree(fc);
}
EXPORT_SYMBOL_GPL(fuse_free_conn);
@@ -1643,89 +1569,6 @@ static int fuse_bdi_init(struct fuse_conn *fc, struct super_block *sb)
return 0;
}
struct fuse_dev *fuse_dev_alloc(void)
{
struct fuse_dev *fud;
struct list_head *pq;
fud = kzalloc_obj(struct fuse_dev);
if (!fud)
return NULL;
refcount_set(&fud->ref, 1);
pq = kzalloc_objs(struct list_head, FUSE_PQ_HASH_SIZE);
if (!pq) {
kfree(fud);
return NULL;
}
fud->pq.processing = pq;
fuse_pqueue_init(&fud->pq);
return fud;
}
EXPORT_SYMBOL_GPL(fuse_dev_alloc);
void fuse_dev_install(struct fuse_dev *fud, struct fuse_conn *fc)
{
struct fuse_conn *old_fc;
spin_lock(&fc->lock);
/*
* Pairs with:
* - xchg() in fuse_dev_release()
* - smp_load_acquire() in fuse_dev_fc_get()
*/
old_fc = cmpxchg(&fud->fc, NULL, fc);
if (old_fc) {
/*
* failed to set fud->fc because
* - it was already set to a different fc
* - it was set to disconneted
*/
fc->connected = 0;
} else {
list_add_tail(&fud->entry, &fc->devices);
fuse_conn_get(fc);
}
spin_unlock(&fc->lock);
}
EXPORT_SYMBOL_GPL(fuse_dev_install);
struct fuse_dev *fuse_dev_alloc_install(struct fuse_conn *fc)
{
struct fuse_dev *fud;
fud = fuse_dev_alloc();
if (!fud)
return NULL;
fuse_dev_install(fud, fc);
return fud;
}
EXPORT_SYMBOL_GPL(fuse_dev_alloc_install);
void fuse_dev_put(struct fuse_dev *fud)
{
struct fuse_conn *fc;
if (!refcount_dec_and_test(&fud->ref))
return;
fc = fuse_dev_fc_get(fud);
if (fc && fc != FUSE_DEV_FC_DISCONNECTED) {
/* This is the virtiofs case (fuse_dev_release() not called) */
spin_lock(&fc->lock);
list_del(&fud->entry);
spin_unlock(&fc->lock);
fuse_conn_put(fc);
}
kfree(fud->pq.processing);
kfree(fud);
}
EXPORT_SYMBOL_GPL(fuse_dev_put);
static void fuse_fill_attr_from_inode(struct fuse_attr *attr,
const struct fuse_inode *fi)
{
@@ -1941,9 +1784,9 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx)
mutex_lock(&fuse_mutex);
err = -EINVAL;
if (fud) {
if (fuse_dev_fc_get(fud))
if (fuse_dev_is_installed(fud))
goto err_unlock;
if (fud->sync_init)
if (fuse_dev_is_sync_init(fud))
fc->sync_init = 1;
}
@@ -1953,10 +1796,9 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx)
list_add_tail(&fc->entry, &fuse_conn_list);
sb->s_root = root_dentry;
if (fud) {
fuse_dev_install(fud, fc);
wake_up_all(&fuse_dev_waitq);
}
if (fud)
fuse_dev_install(fud, fc->chan);
mutex_unlock(&fuse_mutex);
return 0;
@@ -2001,9 +1843,7 @@ static int fuse_set_no_super(struct super_block *sb, struct fs_context *fsc)
static int fuse_test_super(struct super_block *sb, struct fs_context *fsc)
{
struct fuse_dev *fud = fsc->sget_key;
return fuse_dev_fc_get(fud) == get_fuse_conn_super(sb);
return fuse_dev_verify(fsc->sget_key, get_fuse_conn_super(sb)->chan);
}
static int fuse_get_tree(struct fs_context *fsc)
@@ -2012,8 +1852,12 @@ static int fuse_get_tree(struct fs_context *fsc)
struct fuse_conn *fc;
struct fuse_mount *fm;
struct super_block *sb;
struct fuse_chan *fch __free(fuse_chan_free) = fuse_dev_chan_new();
int err;
if (!fch)
return -ENOMEM;
fc = kmalloc_obj(*fc);
if (!fc)
return -ENOMEM;
@@ -2024,7 +1868,7 @@ static int fuse_get_tree(struct fs_context *fsc)
return -ENOMEM;
}
fuse_conn_init(fc, fm, fsc->user_ns, &fuse_dev_fiq_ops, NULL);
fuse_conn_init(fc, fm, fsc->user_ns, no_free_ptr(fch));
fc->release = fuse_free_conn;
fsc->s_fs_info = fm;
@@ -2045,7 +1889,7 @@ static int fuse_get_tree(struct fs_context *fsc)
* Allow creating a fuse mount with an already initialized fuse
* connection
*/
if (fuse_dev_fc_get(ctx->fud)) {
if (fuse_dev_is_installed(ctx->fud)) {
fsc->sget_key = ctx->fud;
sb = sget_fc(fsc, fuse_test_super, fuse_set_no_super);
err = PTR_ERR_OR_ZERO(sb);
@@ -2116,8 +1960,8 @@ void fuse_conn_destroy(struct fuse_mount *fm)
if (fc->destroy)
fuse_send_destroy(fm);
fuse_abort_conn(fc);
fuse_wait_aborted(fc);
fuse_chan_abort(fc->chan, false);
fuse_chan_wait_aborted(fc->chan);
if (!list_empty(&fc->entry)) {
mutex_lock(&fuse_mutex);

444
fs/fuse/notify.c Normal file
View File

@@ -0,0 +1,444 @@
// SPDX-License-Identifier: GPL-2.0-only
#include "dev.h"
#include "fuse_i.h"
#include <linux/pagemap.h>
static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
struct fuse_copy_state *cs)
{
struct fuse_notify_poll_wakeup_out outarg;
int err;
if (size != sizeof(outarg))
return -EINVAL;
err = fuse_copy_one(cs, &outarg, sizeof(outarg));
if (err)
return err;
fuse_copy_finish(cs);
return fuse_notify_poll_wakeup(fc, &outarg);
}
static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
struct fuse_copy_state *cs)
{
struct fuse_notify_inval_inode_out outarg;
int err;
if (size != sizeof(outarg))
return -EINVAL;
err = fuse_copy_one(cs, &outarg, sizeof(outarg));
if (err)
return err;
fuse_copy_finish(cs);
down_read(&fc->killsb);
err = fuse_reverse_inval_inode(fc, outarg.ino,
outarg.off, outarg.len);
up_read(&fc->killsb);
return err;
}
static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
struct fuse_copy_state *cs)
{
struct fuse_notify_inval_entry_out outarg;
int err;
char *buf;
struct qstr name;
if (size < sizeof(outarg))
return -EINVAL;
err = fuse_copy_one(cs, &outarg, sizeof(outarg));
if (err)
return err;
if (outarg.namelen > fc->name_max)
return -ENAMETOOLONG;
err = -EINVAL;
if (size != sizeof(outarg) + outarg.namelen + 1)
return -EINVAL;
buf = kzalloc(outarg.namelen + 1, GFP_KERNEL);
if (!buf)
return -ENOMEM;
name.name = buf;
name.len = outarg.namelen;
err = fuse_copy_one(cs, buf, outarg.namelen + 1);
if (err)
goto err;
fuse_copy_finish(cs);
buf[outarg.namelen] = 0;
down_read(&fc->killsb);
err = fuse_reverse_inval_entry(fc, outarg.parent, 0, &name, outarg.flags);
up_read(&fc->killsb);
err:
kfree(buf);
return err;
}
static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
struct fuse_copy_state *cs)
{
struct fuse_notify_delete_out outarg;
int err;
char *buf;
struct qstr name;
if (size < sizeof(outarg))
return -EINVAL;
err = fuse_copy_one(cs, &outarg, sizeof(outarg));
if (err)
return err;
if (outarg.namelen > fc->name_max)
return -ENAMETOOLONG;
if (size != sizeof(outarg) + outarg.namelen + 1)
return -EINVAL;
buf = kzalloc(outarg.namelen + 1, GFP_KERNEL);
if (!buf)
return -ENOMEM;
name.name = buf;
name.len = outarg.namelen;
err = fuse_copy_one(cs, buf, outarg.namelen + 1);
if (err)
goto err;
fuse_copy_finish(cs);
buf[outarg.namelen] = 0;
down_read(&fc->killsb);
err = fuse_reverse_inval_entry(fc, outarg.parent, outarg.child, &name, 0);
up_read(&fc->killsb);
err:
kfree(buf);
return err;
}
static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
struct fuse_copy_state *cs)
{
struct fuse_notify_store_out outarg;
struct inode *inode;
struct address_space *mapping;
u64 nodeid;
int err;
unsigned int num;
loff_t file_size;
loff_t pos;
loff_t end;
if (size < sizeof(outarg))
return -EINVAL;
err = fuse_copy_one(cs, &outarg, sizeof(outarg));
if (err)
return err;
if (size - sizeof(outarg) != outarg.size)
return -EINVAL;
if (outarg.offset >= MAX_LFS_FILESIZE)
return -EINVAL;
nodeid = outarg.nodeid;
pos = outarg.offset;
num = min(outarg.size, MAX_LFS_FILESIZE - pos);
down_read(&fc->killsb);
err = -ENOENT;
inode = fuse_ilookup(fc, nodeid, NULL);
if (!inode)
goto out_up_killsb;
if (!S_ISREG(inode->i_mode)) {
err = -EINVAL;
goto out_iput;
}
mapping = inode->i_mapping;
file_size = i_size_read(inode);
end = pos + num;
if (end > file_size) {
file_size = end;
fuse_write_update_attr(inode, file_size, num);
}
while (num) {
struct folio *folio;
unsigned int folio_offset;
unsigned int nr_bytes;
pgoff_t index = pos >> PAGE_SHIFT;
folio = filemap_grab_folio(mapping, index);
err = PTR_ERR(folio);
if (IS_ERR(folio))
goto out_iput;
folio_offset = offset_in_folio(folio, pos);
nr_bytes = min(num, folio_size(folio) - folio_offset);
err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes, 0);
if (!folio_test_uptodate(folio) && !err && folio_offset == 0 &&
(nr_bytes == folio_size(folio) || file_size == end)) {
folio_zero_segment(folio, nr_bytes, folio_size(folio));
folio_mark_uptodate(folio);
}
folio_unlock(folio);
folio_put(folio);
if (err)
goto out_iput;
pos += nr_bytes;
num -= nr_bytes;
}
err = 0;
out_iput:
iput(inode);
out_up_killsb:
up_read(&fc->killsb);
return err;
}
struct fuse_retrieve_args {
struct fuse_args_pages ap;
struct fuse_notify_retrieve_in inarg;
};
static void fuse_retrieve_end(struct fuse_args *args, int error)
{
struct fuse_retrieve_args *ra =
container_of(args, typeof(*ra), ap.args);
release_pages(ra->ap.folios, ra->ap.num_folios);
kfree(ra);
}
static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode,
struct fuse_notify_retrieve_out *outarg)
{
int err;
struct address_space *mapping = inode->i_mapping;
loff_t file_size;
unsigned int num;
unsigned int offset;
size_t total_len = 0;
unsigned int num_pages;
struct fuse_conn *fc = fm->fc;
struct fuse_retrieve_args *ra;
size_t args_size = sizeof(*ra);
struct fuse_args_pages *ap;
struct fuse_args *args;
loff_t pos = outarg->offset;
offset = offset_in_page(pos);
file_size = i_size_read(inode);
num = min(outarg->size, fc->max_write);
if (pos > file_size)
num = 0;
else if (num > file_size - pos)
num = file_size - pos;
num_pages = DIV_ROUND_UP(num + offset, PAGE_SIZE);
num_pages = min(num_pages, fc->max_pages);
num = min(num, num_pages << PAGE_SHIFT);
args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0]));
ra = kzalloc(args_size, GFP_KERNEL);
if (!ra)
return -ENOMEM;
ap = &ra->ap;
ap->folios = (void *) (ra + 1);
ap->descs = (void *) (ap->folios + num_pages);
args = &ap->args;
args->nodeid = outarg->nodeid;
args->opcode = FUSE_NOTIFY_REPLY;
args->in_numargs = 3;
args->in_pages = true;
args->end = fuse_retrieve_end;
while (num && ap->num_folios < num_pages) {
struct folio *folio;
unsigned int folio_offset;
unsigned int nr_bytes;
pgoff_t index = pos >> PAGE_SHIFT;
folio = filemap_get_folio(mapping, index);
if (IS_ERR(folio))
break;
if (!folio_test_uptodate(folio)) {
folio_put(folio);
break;
}
folio_offset = offset_in_folio(folio, pos);
nr_bytes = min(folio_size(folio) - folio_offset, num);
ap->folios[ap->num_folios] = folio;
ap->descs[ap->num_folios].offset = folio_offset;
ap->descs[ap->num_folios].length = nr_bytes;
ap->num_folios++;
pos += nr_bytes;
num -= nr_bytes;
total_len += nr_bytes;
}
ra->inarg.offset = outarg->offset;
ra->inarg.size = total_len;
fuse_set_zero_arg0(args);
args->in_args[1].size = sizeof(ra->inarg);
args->in_args[1].value = &ra->inarg;
args->in_args[2].size = total_len;
err = fuse_simple_notify_reply(fm, args, outarg->notify_unique);
if (err)
fuse_retrieve_end(args, err);
return err;
}
static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
struct fuse_copy_state *cs)
{
struct fuse_notify_retrieve_out outarg;
struct fuse_mount *fm;
struct inode *inode;
u64 nodeid;
int err;
if (size != sizeof(outarg))
return -EINVAL;
err = fuse_copy_one(cs, &outarg, sizeof(outarg));
if (err)
return err;
fuse_copy_finish(cs);
if (outarg.offset >= MAX_LFS_FILESIZE)
return -EINVAL;
down_read(&fc->killsb);
err = -ENOENT;
nodeid = outarg.nodeid;
inode = fuse_ilookup(fc, nodeid, &fm);
if (inode) {
err = -EINVAL;
if (S_ISREG(inode->i_mode))
err = fuse_retrieve(fm, inode, &outarg);
iput(inode);
}
up_read(&fc->killsb);
return err;
}
static int fuse_notify_resend(struct fuse_conn *fc)
{
fuse_chan_resend(fc->chan);
return 0;
}
/*
* Increments the fuse connection epoch. This will cause dentries and
* readdir caches from previous epochs to be invalidated. Additionally,
* if inval_wq is set, a work queue is scheduled to trigger the invalidation.
*/
static int fuse_notify_inc_epoch(struct fuse_conn *fc)
{
atomic_inc(&fc->epoch);
if (inval_wq)
schedule_work(&fc->epoch_work);
return 0;
}
static int fuse_notify_prune(struct fuse_conn *fc, unsigned int size,
struct fuse_copy_state *cs)
{
struct fuse_notify_prune_out outarg;
const unsigned int batch = 512;
u64 *nodeids __free(kfree) = kmalloc(sizeof(u64) * batch, GFP_KERNEL);
unsigned int num, i;
int err;
if (!nodeids)
return -ENOMEM;
if (size < sizeof(outarg))
return -EINVAL;
err = fuse_copy_one(cs, &outarg, sizeof(outarg));
if (err)
return err;
if (size - sizeof(outarg) != array_size(outarg.count, sizeof(u64)))
return -EINVAL;
for (; outarg.count; outarg.count -= num) {
num = min(batch, outarg.count);
err = fuse_copy_one(cs, nodeids, num * sizeof(u64));
if (err)
return err;
scoped_guard(rwsem_read, &fc->killsb) {
for (i = 0; i < num; i++)
fuse_try_prune_one_inode(fc, nodeids[i]);
}
}
return 0;
}
int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
unsigned int size, struct fuse_copy_state *cs)
{
switch (code) {
case FUSE_NOTIFY_POLL:
return fuse_notify_poll(fc, size, cs);
case FUSE_NOTIFY_INVAL_INODE:
return fuse_notify_inval_inode(fc, size, cs);
case FUSE_NOTIFY_INVAL_ENTRY:
return fuse_notify_inval_entry(fc, size, cs);
case FUSE_NOTIFY_STORE:
return fuse_notify_store(fc, size, cs);
case FUSE_NOTIFY_RETRIEVE:
return fuse_notify_retrieve(fc, size, cs);
case FUSE_NOTIFY_DELETE:
return fuse_notify_delete(fc, size, cs);
case FUSE_NOTIFY_RESEND:
return fuse_notify_resend(fc);
case FUSE_NOTIFY_INC_EPOCH:
return fuse_notify_inc_epoch(fc);
case FUSE_NOTIFY_PRUNE:
return fuse_notify_prune(fc, size, cs);
default:
return -EINVAL;
}
}

141
fs/fuse/poll.c Normal file
View File

@@ -0,0 +1,141 @@
// SPDX-License-Identifier: GPL-2.0-only
#include "dev.h"
#include "fuse_i.h"
void fuse_end_polls(struct fuse_conn *fc)
{
struct rb_node *p;
spin_lock(&fc->lock);
p = rb_first(&fc->polled_files);
while (p) {
struct fuse_file *ff;
ff = rb_entry(p, struct fuse_file, polled_node);
wake_up_interruptible_all(&ff->poll_wait);
p = rb_next(p);
}
spin_unlock(&fc->lock);
}
/*
* All files which have been polled are linked to RB tree
* fuse_conn->polled_files which is indexed by kh. Walk the tree and
* find the matching one.
*/
static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
struct rb_node **parent_out)
{
struct rb_node **link = &fc->polled_files.rb_node;
struct rb_node *last = NULL;
while (*link) {
struct fuse_file *ff;
last = *link;
ff = rb_entry(last, struct fuse_file, polled_node);
if (kh < ff->kh)
link = &last->rb_left;
else if (kh > ff->kh)
link = &last->rb_right;
else
return link;
}
if (parent_out)
*parent_out = last;
return link;
}
/*
* The file is about to be polled. Make sure it's on the polled_files
* RB tree. Note that files once added to the polled_files tree are
* not removed before the file is released. This is because a file
* polled once is likely to be polled again.
*/
static void fuse_register_polled_file(struct fuse_conn *fc,
struct fuse_file *ff)
{
spin_lock(&fc->lock);
if (RB_EMPTY_NODE(&ff->polled_node)) {
struct rb_node **link, *parent;
link = fuse_find_polled_node(fc, ff->kh, &parent);
BUG_ON(*link);
rb_link_node(&ff->polled_node, parent, link);
rb_insert_color(&ff->polled_node, &fc->polled_files);
}
spin_unlock(&fc->lock);
}
__poll_t fuse_file_poll(struct file *file, poll_table *wait)
{
struct fuse_file *ff = file->private_data;
struct fuse_mount *fm = ff->fm;
struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
struct fuse_poll_out outarg;
FUSE_ARGS(args);
int err;
if (fm->fc->no_poll)
return DEFAULT_POLLMASK;
poll_wait(file, &ff->poll_wait, wait);
inarg.events = mangle_poll(poll_requested_events(wait));
/*
* Ask for notification iff there's someone waiting for it.
* The client may ignore the flag and always notify.
*/
if (waitqueue_active(&ff->poll_wait)) {
inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
fuse_register_polled_file(fm->fc, ff);
}
args.opcode = FUSE_POLL;
args.nodeid = ff->nodeid;
args.in_numargs = 1;
args.in_args[0].size = sizeof(inarg);
args.in_args[0].value = &inarg;
args.out_numargs = 1;
args.out_args[0].size = sizeof(outarg);
args.out_args[0].value = &outarg;
err = fuse_simple_request(fm, &args);
if (!err)
return demangle_poll(outarg.revents);
if (err == -ENOSYS) {
fm->fc->no_poll = 1;
return DEFAULT_POLLMASK;
}
return EPOLLERR;
}
EXPORT_SYMBOL_GPL(fuse_file_poll);
/*
* This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and
* wakes up the poll waiters.
*/
int fuse_notify_poll_wakeup(struct fuse_conn *fc,
struct fuse_notify_poll_wakeup_out *outarg)
{
u64 kh = outarg->kh;
struct rb_node **link;
spin_lock(&fc->lock);
link = fuse_find_polled_node(fc, kh, NULL);
if (*link) {
struct fuse_file *ff;
ff = rb_entry(*link, struct fuse_file, polled_node);
wake_up_interruptible_sync(&ff->poll_wait);
}
spin_unlock(&fc->lock);
return 0;
}

94
fs/fuse/readdir.c
View File

@@ -1,9 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/*
FUSE: Filesystem in Userspace
Copyright (C) 2001-2018 Miklos Szeredi <miklos@szeredi.hu>
This program can be distributed under the terms of the GNU GPL.
See the file COPYING.
*/
@@ -12,6 +10,7 @@
#include <linux/posix_acl.h>
#include <linux/pagemap.h>
#include <linux/highmem.h>
#include <linux/vmalloc.h>
static bool fuse_use_readdirplus(struct inode *dir, struct dir_context *ctx)
{
@@ -334,6 +333,43 @@ static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file,
return 0;
}
static struct page **fuse_readdir_alloc_buf(struct fuse_args_pages *ap, size_t *bufsize)
{
unsigned int i, nr_alloc, nr_pages = DIV_ROUND_UP(*bufsize, PAGE_SIZE);
struct page **pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL);
if (!pages)
return NULL;
nr_alloc = alloc_pages_bulk(GFP_KERNEL, nr_pages, pages);
if (!nr_alloc)
goto free_array;
if (nr_alloc < nr_pages) {
nr_pages = nr_alloc;
*bufsize = (size_t) nr_pages << PAGE_SHIFT;
}
ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs);
if (!ap->folios)
goto release_pages;
for (i = 0; i < nr_pages; i++) {
ap->folios[i] = page_folio(pages[i]);
ap->descs[i].length = min_t(size_t, *bufsize - (size_t)i * PAGE_SIZE, PAGE_SIZE);
}
ap->num_folios = nr_pages;
ap->args.out_pages = true;
return pages;
release_pages:
release_pages(pages, nr_pages);
free_array:
kfree(pages);
return NULL;
}
static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx)
{
int plus;
@@ -342,18 +378,16 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx)
struct fuse_mount *fm = get_fuse_mount(inode);
struct fuse_conn *fc = fm->fc;
struct fuse_io_args ia = {};
struct fuse_args *args = &ia.ap.args;
struct fuse_args_pages *ap = &ia.ap;
void *buf;
size_t bufsize = clamp((unsigned int) ctx->count, PAGE_SIZE, fc->max_pages << PAGE_SHIFT);
u64 attr_version = 0, evict_ctr = 0;
bool locked;
struct page **pages = fuse_readdir_alloc_buf(ap, &bufsize);
buf = kvmalloc(bufsize, GFP_KERNEL);
if (!buf)
if (!pages)
return -ENOMEM;
args->out_args[0].value = buf;
plus = fuse_use_readdirplus(inode, ctx);
if (plus) {
attr_version = fuse_get_attr_version(fm->fc);
@@ -363,24 +397,37 @@ static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx)
fuse_read_args_fill(&ia, file, ctx->pos, bufsize, FUSE_READDIR);
}
locked = fuse_lock_inode(inode);
res = fuse_simple_request(fm, args);
res = fuse_simple_request(fm, &ap->args);
fuse_unlock_inode(inode, locked);
if (res >= 0) {
if (!res) {
struct fuse_file *ff = file->private_data;
if (res < 0)
goto out;
if (ff->open_flags & FOPEN_CACHE_DIR)
fuse_readdir_cache_end(file, ctx->pos);
} else if (plus) {
res = parse_dirplusfile(buf, res, file, ctx, attr_version,
evict_ctr);
} else {
res = parse_dirfile(buf, res, file, ctx);
}
if (!res) {
struct fuse_file *ff = file->private_data;
if (ff->open_flags & FOPEN_CACHE_DIR)
fuse_readdir_cache_end(file, ctx->pos);
goto out;
}
kvfree(buf);
buf = vm_map_ram(pages, ap->num_folios, -1);
if (!buf) {
res = -ENOMEM;
} else {
if (plus)
res = parse_dirplusfile(buf, res, file, ctx, attr_version, evict_ctr);
else
res = parse_dirfile(buf, res, file, ctx);
vm_unmap_ram(buf, ap->num_folios);
}
out:
kfree(ap->folios);
release_pages(pages, ap->num_folios);
kfree(pages);
fuse_invalidate_atime(inode);
return res;
}
@@ -441,6 +488,7 @@ static void fuse_rdc_reset(struct inode *inode)
fi->rdc.version++;
fi->rdc.size = 0;
fi->rdc.pos = 0;
fi->rdc.epoch = 0;
}
#define UNCACHED 1
@@ -482,6 +530,7 @@ static int fuse_readdir_cached(struct file *file, struct dir_context *ctx)
if (!ctx->pos && !fi->rdc.size) {
fi->rdc.mtime = inode_get_mtime(inode);
fi->rdc.iversion = inode_query_iversion(inode);
fi->rdc.epoch = atomic_read(&fc->epoch);
}
spin_unlock(&fi->rdc.lock);
return UNCACHED;
@@ -495,7 +544,8 @@ static int fuse_readdir_cached(struct file *file, struct dir_context *ctx)
struct timespec64 mtime = inode_get_mtime(inode);
if (inode_peek_iversion(inode) != fi->rdc.iversion ||
!timespec64_equal(&fi->rdc.mtime, &mtime)) {
!timespec64_equal(&fi->rdc.mtime, &mtime) ||
fi->rdc.epoch != atomic_read(&fc->epoch)) {
fuse_rdc_reset(inode);
goto retry_locked;
}

99
fs/fuse/req.c Normal file
View File

@@ -0,0 +1,99 @@
// SPDX-License-Identifier: GPL-2.0-only
#include "dev.h"
#include "fuse_i.h"
static int fuse_fill_creds(struct fuse_mount *fm, struct fuse_args *args, struct mnt_idmap *idmap)
{
struct fuse_conn *fc = fm->fc;
bool no_idmap = !fm->sb || (fm->sb->s_iflags & SB_I_NOIDMAP);
kuid_t fsuid = mapped_fsuid(idmap, fc->user_ns);
kgid_t fsgid = mapped_fsgid(idmap, fc->user_ns);
args->pid = pid_nr_ns(task_pid(current), fc->pid_ns);
if (args->force) {
if (args->nocreds)
return 0;
if (no_idmap) {
args->uid = from_kuid_munged(fc->user_ns, current_fsuid());
args->gid = from_kgid_munged(fc->user_ns, current_fsgid());
} else {
args->uid = FUSE_INVALID_UIDGID;
args->gid = FUSE_INVALID_UIDGID;
}
return 0;
}
WARN_ON(args->nocreds);
/*
* Keep the old behavior when idmappings support was not
* declared by a FUSE server.
*
* For those FUSE servers who support idmapped mounts, we send UID/GID
* only along with "inode creation" fuse requests, otherwise idmap ==
* &invalid_mnt_idmap and req->in.h.{u,g}id will be equal to
* FUSE_INVALID_UIDGID.
*/
if (no_idmap) {
fsuid = current_fsuid();
fsgid = current_fsgid();
}
args->uid = from_kuid(fc->user_ns, fsuid);
args->gid = from_kgid(fc->user_ns, fsgid);
if (no_idmap && unlikely(args->uid == ((uid_t)-1) || args->gid == ((gid_t)-1)))
return -EOVERFLOW;
return 0;
}
static int fuse_req_prep(struct fuse_mount *fm, struct fuse_args *args, struct mnt_idmap *idmap)
{
if (!args->force && fm->fc->conn_error)
return -ECONNREFUSED;
return fuse_fill_creds(fm, args, idmap);
}
ssize_t __fuse_simple_request(struct mnt_idmap *idmap, struct fuse_mount *fm,
struct fuse_args *args)
{
struct fuse_conn *fc = fm->fc;
int err = fuse_req_prep(fm, args, idmap);
if (err)
return err;
return fuse_chan_send(fc->chan, args);
}
int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, gfp_t gfp_flags)
{
struct fuse_conn *fc = fm->fc;
int err;
WARN_ON(args->force && !args->nocreds);
err = fuse_req_prep(fm, args, &invalid_mnt_idmap);
if (err)
return err;
return fuse_chan_send_bg(fc->chan, args, gfp_flags);
}
EXPORT_SYMBOL_GPL(fuse_simple_background);
int fuse_simple_notify_reply(struct fuse_mount *fm, struct fuse_args *args, u64 unique)
{
struct fuse_conn *fc = fm->fc;
int err;
WARN_ON(args->force && !args->nocreds);
err = fuse_req_prep(fm, args, &invalid_mnt_idmap);
if (err)
return err;
return fuse_chan_send_notify_reply(fc->chan, args, unique);
}

148
fs/fuse/req_timeout.c Normal file
View File

@@ -0,0 +1,148 @@
// SPDX-License-Identifier: GPL-2.0-only
#include "dev.h"
#include "sysctl.h"
#include "fuse_dev_i.h"
#include "dev_uring_i.h"
/* Frequency (in seconds) of request timeout checks, if opted into */
#define FUSE_TIMEOUT_TIMER_FREQ 15
/* Frequency (in jiffies) of request timeout checks, if opted into */
static const unsigned long fuse_timeout_timer_freq =
secs_to_jiffies(FUSE_TIMEOUT_TIMER_FREQ);
/*
* Default timeout (in seconds) for the server to reply to a request
* before the connection is aborted, if no timeout was specified on mount.
*
* Exported via sysctl
*/
unsigned int fuse_default_req_timeout;
/*
* Max timeout (in seconds) for the server to reply to a request before
* the connection is aborted.
*
* Exported via sysctl
*/
unsigned int fuse_max_req_timeout;
bool fuse_request_expired(struct fuse_chan *fch, struct list_head *list)
{
struct fuse_req *req;
req = list_first_entry_or_null(list, struct fuse_req, list);
if (!req)
return false;
return time_is_before_jiffies(req->create_time + fch->timeout.req_timeout);
}
static bool fuse_fpq_processing_expired(struct fuse_chan *fch, struct list_head *processing)
{
int i;
for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
if (fuse_request_expired(fch, &processing[i]))
return true;
return false;
}
/*
* Check if any requests aren't being completed by the time the request timeout
* elapses. To do so, we:
* - check the fiq pending list
* - check the bg queue
* - check the fpq io and processing lists
*
* To make this fast, we only check against the head request on each list since
* these are generally queued in order of creation time (eg newer requests get
* queued to the tail). We might miss a few edge cases (eg requests transitioning
* between lists, re-sent requests at the head of the pending list having a
* later creation time than other requests on that list, etc.) but that is fine
* since if the request never gets fulfilled, it will eventually be caught.
*/
static void fuse_check_timeout(struct work_struct *work)
{
struct delayed_work *dwork = to_delayed_work(work);
struct fuse_chan *fch = container_of(dwork, struct fuse_chan, timeout.work);
struct fuse_iqueue *fiq = &fch->iq;
struct fuse_dev *fud;
struct fuse_pqueue *fpq;
bool expired = false;
if (!atomic_read(&fch->num_waiting))
goto out;
spin_lock(&fiq->lock);
expired = fuse_request_expired(fch, &fiq->pending);
spin_unlock(&fiq->lock);
if (expired)
goto chan_abort;
spin_lock(&fch->bg_lock);
expired = fuse_request_expired(fch, &fch->bg_queue);
spin_unlock(&fch->bg_lock);
if (expired)
goto chan_abort;
spin_lock(&fch->lock);
if (!fch->connected) {
spin_unlock(&fch->lock);
return;
}
list_for_each_entry(fud, &fch->devices, entry) {
fpq = &fud->pq;
spin_lock(&fpq->lock);
if (fuse_request_expired(fch, &fpq->io) ||
fuse_fpq_processing_expired(fch, fpq->processing)) {
spin_unlock(&fpq->lock);
spin_unlock(&fch->lock);
goto chan_abort;
}
spin_unlock(&fpq->lock);
}
spin_unlock(&fch->lock);
if (fuse_uring_request_expired(fch))
goto chan_abort;
out:
queue_delayed_work(system_percpu_wq, &fch->timeout.work,
fuse_timeout_timer_freq);
return;
chan_abort:
fuse_chan_abort(fch, false);
}
static void set_request_timeout(struct fuse_chan *fch, unsigned int timeout)
{
fch->timeout.req_timeout = secs_to_jiffies(timeout);
INIT_DELAYED_WORK(&fch->timeout.work, fuse_check_timeout);
queue_delayed_work(system_percpu_wq, &fch->timeout.work,
fuse_timeout_timer_freq);
}
void fuse_init_server_timeout(struct fuse_chan *fch, unsigned int timeout)
{
if (!timeout && !fuse_max_req_timeout && !fuse_default_req_timeout)
return;
if (!timeout)
timeout = fuse_default_req_timeout;
if (fuse_max_req_timeout) {
if (timeout)
timeout = min(fuse_max_req_timeout, timeout);
else
timeout = fuse_max_req_timeout;
}
timeout = max(FUSE_TIMEOUT_TIMER_FREQ, timeout);
set_request_timeout(fch, timeout);
}

1
fs/fuse/sysctl.c
View File

@@ -6,6 +6,7 @@
*/
#include <linux/sysctl.h>
#include "sysctl.h"
#include "fuse_i.h"
static struct ctl_table_header *fuse_table_header;

9
fs/fuse/sysctl.h Normal file
View File

@@ -0,0 +1,9 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _FS_FUSE_SYSCTL_H
#define _FS_FUSE_SYSCTL_H
extern unsigned int fuse_default_req_timeout;
extern unsigned int fuse_max_req_timeout;
#endif /* _FS_FUSE_SYSCTL_H */

23
fs/fuse/virtio_fs.c
View File

@@ -19,6 +19,7 @@
#include <linux/highmem.h>
#include <linux/cleanup.h>
#include <linux/uio.h>
#include "dev.h"
#include "fuse_i.h"
#include "fuse_dev_i.h"
@@ -1009,7 +1010,9 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev,
kfree(vqs);
if (ret) {
kfree(fs->vqs);
fs->vqs = NULL;
kfree(fs->mq_map);
fs->mq_map = NULL;
}
return ret;
}
@@ -1519,7 +1522,7 @@ static void virtio_fs_send_req(struct fuse_iqueue *fiq, struct fuse_req *req)
if (ret == -ENOSPC) {
/*
* Virtqueue full. Retry submission from worker
* context as we might be holding fc->bg_lock.
* context as we might be holding fc->chan->bg_lock.
*/
spin_lock(&fsvq->lock);
list_add_tail(&req->list, &fsvq->queued_reqs);
@@ -1562,7 +1565,7 @@ static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc)
{
struct fuse_mount *fm = get_fuse_mount_super(sb);
struct fuse_conn *fc = fm->fc;
struct virtio_fs *fs = fc->iq.priv;
struct virtio_fs *fs = fc->chan->iq.priv;
struct fuse_fs_context *ctx = fsc->fs_private;
unsigned int i;
int err;
@@ -1606,7 +1609,7 @@ static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc)
for (i = 0; i < fs->nvqs; i++) {
struct virtio_fs_vq *fsvq = &fs->vqs[i];
fuse_dev_install(fsvq->fud, fc);
fuse_dev_install(fsvq->fud, fc->chan);
}
/* Previous unmount will stop all queues. Start these again */
@@ -1625,7 +1628,7 @@ static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc)
static void virtio_fs_conn_destroy(struct fuse_mount *fm)
{
struct fuse_conn *fc = fm->fc;
struct virtio_fs *vfs = fc->iq.priv;
struct virtio_fs *vfs = fc->chan->iq.priv;
struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO];
/* Stop dax worker. Soon evict_inodes() will be called which
@@ -1673,7 +1676,7 @@ static int virtio_fs_test_super(struct super_block *sb,
struct fuse_mount *fsc_fm = fsc->s_fs_info;
struct fuse_mount *sb_fm = get_fuse_mount_super(sb);
return fsc_fm->fc->iq.priv == sb_fm->fc->iq.priv;
return fsc_fm->fc->chan->iq.priv == sb_fm->fc->chan->iq.priv;
}
static int virtio_fs_get_tree(struct fs_context *fsc)
@@ -1683,13 +1686,17 @@ static int virtio_fs_get_tree(struct fs_context *fsc)
struct fuse_conn *fc = NULL;
struct fuse_mount *fm;
unsigned int virtqueue_size;
struct fuse_chan *fch __free(fuse_chan_free) = fuse_chan_new();
int err = -EIO;
if (!fch)
return -ENOMEM;
if (!fsc->source)
return invalf(fsc, "No source specified");
/* This gets a reference on virtio_fs object. This ptr gets installed
* in fc->iq->priv. Once fuse_conn is going away, it calls ->put()
* in chan->iq->priv. Once fuse_conn is going away, it calls ->put()
* to drop the reference to this object.
*/
fs = virtio_fs_find_instance(fsc->source);
@@ -1711,7 +1718,9 @@ static int virtio_fs_get_tree(struct fs_context *fsc)
if (!fm)
goto out_err;
fuse_conn_init(fc, fm, fsc->user_ns, &virtio_fs_fiq_ops, fs);
fuse_iqueue_init(&fch->iq, &virtio_fs_fiq_ops, fs);
fuse_conn_init(fc, fm, fsc->user_ns, no_free_ptr(fch));
fc->release = fuse_free_conn;
fc->delete_stale = true;
fc->auto_submounts = true;

4
fs/fuse/xattr.c
View File

@@ -1,9 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/*
* FUSE: Filesystem in Userspace
* Copyright (C) 2001-2016 Miklos Szeredi <miklos@szeredi.hu>
*
* This program can be distributed under the terms of the GNU GPL.
* See the file COPYING.
*/
#include "fuse_i.h"

1
fs/internal.h
View File

@@ -138,7 +138,6 @@ extern bool super_trylock_shared(struct super_block *sb);
struct super_block *user_get_super(dev_t, bool excl);
void put_super(struct super_block *sb);
extern bool mount_capable(struct fs_context *);
int sb_init_dio_done_wq(struct super_block *sb);
/*
* Prepare superblock for changing its read-only state (i.e., either remount

2
include/linux/fs/super.h
View File

@@ -235,4 +235,6 @@ int freeze_super(struct super_block *super, enum freeze_holder who,
int thaw_super(struct super_block *super, enum freeze_holder who,
const void *freeze_owner);
int sb_init_dio_done_wq(struct super_block *sb);
#endif /* _LINUX_FS_SUPER_H */