mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-28 06:23:42 -04:00
Code Issues Packages Projects Releases Wiki Activity

pipe: only enable the extra wake_up(rd_wait) for EPOLLET consumers

Browse Source 
pipe_poll() unconditionally sets ->poll_usage on the first call, forcing
anon_pipe_write() to wake up readers on every write even if the pipe was
not empty.

The reason is that some legacy epoll(EPOLLET) users depend on historical
per-write wakeups, see commit 3a34b13a88 ("pipe: make pipe writes always
wake up readers").

Test-case:

	#include <unistd.h>
	#include <sys/epoll.h>
	#include <assert.h>

	int main(void)
	{
		int pfd[2], efd;
		struct epoll_event evt = { .events = EPOLLIN | EPOLLET };

		pipe(pfd);
		efd = epoll_create1(0);
		epoll_ctl(efd, EPOLL_CTL_ADD, pfd[0], &evt);

		for (int i = 0; i < 2; ++i) {
			write(pfd[1], "", 1);
			assert(epoll_wait(efd, &evt, 1, 0) == 1);
		}

		return 0;
	}

it fails if WRITE_ONCE(poll_usage, true) is removed from pipe_poll().
However, without EPOLLET in .events, it does not need the extra wakeup
and succeeds even if write() is called only once before the main loop.

Currently io_uring without (unsupported) IORING_POLL_ADD_LEVEL always
sets EPOLLET, and in IORING_POLL_ADD_MULTI mode it depends on per-write
wakeups the same way:

	#include <unistd.h>
	#include <sys/mman.h>
	#include <sys/epoll.h>
	#include <sys/syscall.h>
	#include <linux/io_uring.h>
	#include <assert.h>

	int main(void)
	{
		struct io_uring_params p = {};
		int fd, pfd[2];

		pipe(pfd);

		fd = syscall(SYS_io_uring_setup, 2, &p);
		assert(fd >= 0);

		void *ring = mmap(0, p.cq_off.cqes + p.cq_entries * sizeof(struct io_uring_cqe),
				  PROT_READ | PROT_WRITE, MAP_SHARED, fd, IORING_OFF_SQ_RING);
		assert(ring != MAP_FAILED);
		*(unsigned *)(ring + p.sq_off.tail) = 1;

		struct io_uring_sqe *sqes = mmap(0, p.sq_entries * sizeof(*sqes),
				  PROT_READ | PROT_WRITE, MAP_SHARED, fd, IORING_OFF_SQES);
		assert(sqes != MAP_FAILED);
		sqes[0].opcode = IORING_OP_POLL_ADD;
		sqes[0].fd = pfd[0];
		sqes[0].len = IORING_POLL_ADD_MULTI;
		sqes[0].poll32_events = EPOLLIN;

		syscall(SYS_io_uring_enter, fd, 1, 0, 0, 0, 0);

		unsigned *cq_head = ring + p.cq_off.head;
		unsigned *cq_tail = ring + p.cq_off.tail;
		for (int i = 0; i < 2; ++i) {
			write(pfd[1], "", 1);
			syscall(SYS_io_uring_enter, fd, 0, 0, IORING_ENTER_GETEVENTS, 0, 0);
			assert(*cq_tail == ++*cq_head);
		}

		return 0;
	}

the 2nd assert() in the main loop fails without ->poll_usage == true.

Rename ->poll_usage to ->pseudo_edgetrigger to make the purpose clearer,
update the comments, and change pipe_poll() to set ->pseudo_edgetrigger
only if wait->_key & EPOLLET is true. This check should catch both users,
and this way poll/select and epoll without EPOLLET users will not pay for
the extra wakeup.

Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Link: https://patch.msgid.link/anCNoW-x0bcB2ggg@redhat.com
Signed-off-by: Christian Brauner (Amutable) <brauner@kernel.org>
This commit is contained in:
Oleg Nesterov
2026-08-03 14:46:25 +02:00
committed by Christian Brauner
parent 9688a46802
commit f1cebdec97
2 changed files with 15 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
fs/pipe.c
View File

@@ -665,10 +665,9 @@ anon_pipe_write(struct kiocb *iocb, struct iov_iter *from)
* how (for example) the GNU make jobserver uses small writes to
* wake up pending jobs
*
* Epoll nonsensically wants a wakeup whether the pipe
* was already empty or not.
* ->pseudo_edgetrigger enables per-write wakeups, see pipe_poll()
*/
if (was_empty || pipe->poll_usage)
if (was_empty || READ_ONCE(pipe->pseudo_edgetrigger))
wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
if (wake_next_writer)
@@ -731,7 +730,6 @@ static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
}
}
/* No kernel lock held - fine */
static __poll_t
pipe_poll(struct file *filp, poll_table *wait)
{
@@ -739,9 +737,17 @@ pipe_poll(struct file *filp, poll_table *wait)
struct pipe_inode_info *pipe = filp->private_data;
union pipe_index idx;
/* Epoll has some historical nasty semantics, this enables them */
if (unlikely(!READ_ONCE(pipe->poll_usage)))
WRITE_ONCE(pipe->poll_usage, true);
/*
* Legacy epoll(EPOLLET) users depend on historical per-write wakeups,
* see 3a34b13a88ca ("pipe: make pipe writes always wake up readers")
* and the ->pseudo_edgetrigger check in anon_pipe_write().
* Currently io_uring sets EPOLLET for multishot polls, so it gets the
* same behaviour.
*/
if ((filp->f_mode & FMODE_READ) &&
wait && (wait->_key & EPOLLET) &&
unlikely(!READ_ONCE(pipe->pseudo_edgetrigger)))
WRITE_ONCE(pipe->pseudo_edgetrigger, true);
/*
* Reading pipe state only -- no need for acquiring the semaphore.

4
include/linux/pipe_fs_i.h
View File

@@ -92,7 +92,7 @@ struct anon_pipe_prealloc {
* @files: number of struct file referring this pipe (protected by ->i_lock)
* @r_counter: reader counter
* @w_counter: writer counter
* @poll_usage: is this pipe used for epoll, which has crazy wakeups?
* @pseudo_edgetrigger: has an EPOLLET consumer, enable per-write wakeups
* @fasync_readers: reader side fasync
* @fasync_writers: writer side fasync
* @bufs: the circular array of pipe buffers
@@ -113,7 +113,7 @@ struct pipe_inode_info {
unsigned int files;
unsigned int r_counter;
unsigned int w_counter;
bool poll_usage;
bool pseudo_edgetrigger;
#ifdef CONFIG_WATCH_QUEUE
bool note_loss;
#endif