5.15-stable review patch. If anyone has any objections, please let me know.
------------------
From: Chengming Zhou zhouchengming@bytedance.com
[ Upstream commit 52b33d87b9197c51e8ffdc61873739d90dd0a16f ]
The commit d583d360a620 ("psi: Fix psi state corruption when schedule() races with cgroup move") fixed a race problem by making cgroup_move_task() use task->psi_flags instead of looking at the scheduler state.
We can extend task->psi_flags usage to CPU migration, which should be a minor optimization for performance and code simplicity.
Signed-off-by: Chengming Zhou zhouchengming@bytedance.com Signed-off-by: Peter Zijlstra (Intel) peterz@infradead.org Acked-by: Johannes Weiner hannes@cmpxchg.org Link: https://lore.kernel.org/r/20220926081931.45420-1-zhouchengming@bytedance.com Stable-dep-of: a430d99e3490 ("sched/fair: Fix value reported by hot tasks pulled in /proc/schedstat") Signed-off-by: Sasha Levin sashal@kernel.org --- include/linux/sched.h | 3 --- kernel/sched/core.c | 2 +- kernel/sched/stats.h | 22 ++++------------------ 3 files changed, 5 insertions(+), 22 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h index 9b3cfe685cb45..875f3d317b9c8 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -874,9 +874,6 @@ struct task_struct { unsigned sched_reset_on_fork:1; unsigned sched_contributes_to_load:1; unsigned sched_migrated:1; -#ifdef CONFIG_PSI - unsigned sched_psi_wake_requeue:1; -#endif
/* Force alignment to the next boundary: */ unsigned :0; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index ed92b75f7e024..fee8e2a7c7530 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1967,7 +1967,7 @@ static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
if (!(flags & ENQUEUE_RESTORE)) { sched_info_enqueue(rq, p); - psi_enqueue(p, flags & ENQUEUE_WAKEUP); + psi_enqueue(p, (flags & ENQUEUE_WAKEUP) && !(flags & ENQUEUE_MIGRATED)); }
uclamp_rq_inc(rq, p); diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h index 975703572bc0d..cee3100c74cd5 100644 --- a/kernel/sched/stats.h +++ b/kernel/sched/stats.h @@ -91,11 +91,9 @@ static inline void psi_enqueue(struct task_struct *p, bool wakeup) if (p->in_memstall) set |= TSK_MEMSTALL_RUNNING;
- if (!wakeup || p->sched_psi_wake_requeue) { + if (!wakeup) { if (p->in_memstall) set |= TSK_MEMSTALL; - if (p->sched_psi_wake_requeue) - p->sched_psi_wake_requeue = 0; } else { if (p->in_iowait) clear |= TSK_IOWAIT; @@ -106,8 +104,6 @@ static inline void psi_enqueue(struct task_struct *p, bool wakeup)
static inline void psi_dequeue(struct task_struct *p, bool sleep) { - int clear = TSK_RUNNING; - if (static_branch_likely(&psi_disabled)) return;
@@ -120,10 +116,7 @@ static inline void psi_dequeue(struct task_struct *p, bool sleep) if (sleep) return;
- if (p->in_memstall) - clear |= (TSK_MEMSTALL | TSK_MEMSTALL_RUNNING); - - psi_task_change(p, clear, 0); + psi_task_change(p, p->psi_flags, 0); }
static inline void psi_ttwu_dequeue(struct task_struct *p) @@ -135,19 +128,12 @@ static inline void psi_ttwu_dequeue(struct task_struct *p) * deregister its sleep-persistent psi states from the old * queue, and let psi_enqueue() know it has to requeue. */ - if (unlikely(p->in_iowait || p->in_memstall)) { + if (unlikely(p->psi_flags)) { struct rq_flags rf; struct rq *rq; - int clear = 0; - - if (p->in_iowait) - clear |= TSK_IOWAIT; - if (p->in_memstall) - clear |= TSK_MEMSTALL;
rq = __task_rq_lock(p, &rf); - psi_task_change(p, clear, 0); - p->sched_psi_wake_requeue = 1; + psi_task_change(p, p->psi_flags, 0); __task_rq_unlock(rq, &rf); } }