[Devel] [PATCH hci-8.1 1/4] sched: fix cfs_rq::nr_iowait leak on the remote wakeup path

[Vasileios Almpanis]

try_to_wake_up() decrements cfs_rq::nr_iowait only after the
p->on_cpu check. If the task's CPU is still in schedule(), the wakeup
is queued with ttwu_queue_wakelist() and try_to_wake_up() returns
early, so the counter is never decremented. It keeps growing and shows
up as a bogus procs_blocked in the CT's /proc/stat:

  # grep procs_blocked /proc/stat
  procs_blocked 70

With NO_TTWU_QUEUE the leak goes away.

Fix it by doing the decrement where rq::nr_iowait is handled: in
try_to_wake_up() before set_task_cpu() for migrating wakeups, and in
ttwu_do_activate() for all others, which also covers the wakelist.
This way non migrating wakeups don't take the rq lock for it anymore.

A task on the wakelist is TASK_WAKING but still counted, so add
task_in_nr_iowait() and use it in sched_move_task() instead of
skipping every TASK_WAKING task.

https://virtuozzo.atlassian.net/browse/VSTOR-147823
Fixes: 415fe063c53e ("sched: Account cfs_rq::nr_iowait")
Signed-off-by: Vasileios Almpanis <vasileios.almpanis at virtuozzo.com>

Feature: sched: enhance task states granularity and accounting
---
 kernel/sched/core.c  | 26 +++++++++++++-------------
 kernel/sched/sched.h | 12 ++++++++++++
 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9275501bc3ad..dc016d264d19 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3775,6 +3775,8 @@ ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
 	if (p->in_iowait) {
 		delayacct_blkio_end(p);
 		atomic_dec(&task_rq(p)->nr_iowait);
+		if (p->sched_class->nr_iowait_dec)
+			p->sched_class->nr_iowait_dec(p);
 	}
 
 	if (p->sched_interruptible_sleep)
@@ -4355,20 +4357,20 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
 		 */
 		smp_cond_load_acquire(&p->on_cpu, !VAL);
 
-		if (p->in_iowait && p->sched_class->nr_iowait_dec) {
-			struct rq_flags rf;
-			struct rq *rq;
-
-			rq = __task_rq_lock(p, &rf);
-			p->sched_class->nr_iowait_dec(p);
-			__task_rq_unlock(rq, &rf);
-		}
-
 		cpu = select_task_rq(p, p->wake_cpu, &wake_flags);
 		if (task_cpu(p) != cpu) {
 			if (p->in_iowait) {
 				delayacct_blkio_end(p);
 				atomic_dec(&task_rq(p)->nr_iowait);
+				/* On the old CPU, see also ttwu_do_activate() */
+				if (p->sched_class->nr_iowait_dec) {
+					struct rq_flags rf;
+					struct rq *rq;
+
+					rq = __task_rq_lock(p, &rf);
+					p->sched_class->nr_iowait_dec(p);
+					__task_rq_unlock(rq, &rf);
+				}
 			}
 
 			wake_flags |= WF_MIGRATED;
@@ -9139,8 +9141,7 @@ void sched_move_task(struct task_struct *tsk, bool for_autogroup)
 	if (queued)
 		dequeue_task(rq, tsk, queue_flags);
 	else {
-		if (!(READ_ONCE(tsk->__state) ==  TASK_WAKING) && tsk->in_iowait &&
-		    tsk->sched_class->nr_iowait_dec)
+		if (task_in_nr_iowait(tsk) && tsk->sched_class->nr_iowait_dec)
 			tsk->sched_class->nr_iowait_dec(tsk);
 
 		if (tsk->sched_contributes_to_load)
@@ -9159,8 +9160,7 @@ void sched_move_task(struct task_struct *tsk, bool for_autogroup)
 	if (queued)
 		enqueue_task(rq, tsk, queue_flags);
 	else {
-		if (!(READ_ONCE(tsk->__state) ==  TASK_WAKING) && tsk->in_iowait &&
-		    tsk->sched_class->nr_iowait_inc)
+		if (task_in_nr_iowait(tsk) && tsk->sched_class->nr_iowait_inc)
 			tsk->sched_class->nr_iowait_inc(tsk);
 
 		if (tsk->sched_contributes_to_load)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 2a3f73933456..176077ba4201 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2337,6 +2337,18 @@ static inline int task_on_rq_queued(struct task_struct *p)
 	return READ_ONCE(p->on_rq) == TASK_ON_RQ_QUEUED;
 }
 
+/*
+ * A sleeping iowait task stays counted in nr_iowait of its sched class until
+ * it is woken up: a migrating wakeup drops it in try_to_wake_up(), any other
+ * in ttwu_do_activate(), which for a wakelist wakeup runs later while the
+ * task is TASK_WAKING. Called with p->pi_lock and the task's rq lock held.
+ */
+static inline bool task_in_nr_iowait(struct task_struct *p)
+{
+	return p->in_iowait && !task_on_rq_queued(p) &&
+	       (READ_ONCE(p->__state) != TASK_WAKING || !p->sched_remote_wakeup);
+}
+
 static inline int task_on_rq_migrating(struct task_struct *p)
 {
 	return READ_ONCE(p->on_rq) == TASK_ON_RQ_MIGRATING;

-- 
2.43.0