diff --git a/fs/eventpoll.c b/fs/eventpoll.c
index 0e65c7431dfc..128d7fd3d0ea 100644
--- a/fs/eventpoll.c
+++ b/fs/eventpoll.c
@@ -2248,7 +2248,6 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
 	lockdep_assert_irqs_enabled();
 
 	if (timeout && (timeout->tv_sec | timeout->tv_nsec)) {
-		slack = select_estimate_accuracy(timeout);
 		to = &expires;
 		*to = timespec64_to_ktime(*timeout);
 	} else if (timeout) {
@@ -2327,10 +2326,13 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
 
 		spin_unlock_irq(&ep->lock);
 
-		if (!eavail)
+		if (!eavail) {
+			if (to)
+				slack = select_estimate_accuracy(timeout);
 			timed_out = !ep_schedule_timeout(to) ||
 				!schedule_hrtimeout_range(to, slack,
 							  HRTIMER_MODE_ABS);
+		}
 		__set_current_state(TASK_RUNNING);
 
 		/*