5.10-stable review patch. If anyone has any objections, please let me know.
------------------
From: Vincent Guittot vincent.guittot@linaro.org
[ Upstream commit c6f886546cb8a38617cdbe755fe50d3acd2463e4 ]
Instead of waking up a random and already idle CPU, we can take advantage of this_cpu being about to enter idle to run the ILB and update the blocked load.
Signed-off-by: Vincent Guittot vincent.guittot@linaro.org Signed-off-by: Peter Zijlstra (Intel) peterz@infradead.org Signed-off-by: Ingo Molnar mingo@kernel.org Reviewed-by: Valentin Schneider valentin.schneider@arm.com Link: https://lkml.kernel.org/r/20210224133007.28644-7-vincent.guittot@linaro.org Stable-dep-of: ff47a0acfcce ("sched/fair: Check idle_cpu() before need_resched() to detect ilb CPU turning busy") Signed-off-by: Sasha Levin sashal@kernel.org --- kernel/sched/core.c | 2 +- kernel/sched/fair.c | 24 +++++++++++++++++++++--- kernel/sched/idle.c | 6 ++++++ kernel/sched/sched.h | 7 +++++++ 4 files changed, 35 insertions(+), 4 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 8e30041cecf94..1f4bf91c27d22 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -746,7 +746,7 @@ static void nohz_csd_func(void *info) /* * Release the rq::nohz_csd. */ - flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(cpu)); + flags = atomic_fetch_andnot(NOHZ_KICK_MASK | NOHZ_NEWILB_KICK, nohz_flags(cpu)); WARN_ON(!(flags & NOHZ_KICK_MASK));
rq->idle_balance = idle_cpu(cpu); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ab29666eb50ed..8121cfd60b8fb 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10747,6 +10747,24 @@ static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) return true; }
+/* + * Check if we need to run the ILB for updating blocked load before entering + * idle state. + */ +void nohz_run_idle_balance(int cpu) +{ + unsigned int flags; + + flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu)); + + /* + * Update the blocked load only if no SCHED_SOFTIRQ is about to happen + * (ie NOHZ_STATS_KICK set) and will do the same. + */ + if ((flags == NOHZ_NEWILB_KICK) && !need_resched()) + _nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK, CPU_IDLE); +} + static void nohz_newidle_balance(struct rq *this_rq) { int this_cpu = this_rq->cpu; @@ -10768,10 +10786,10 @@ static void nohz_newidle_balance(struct rq *this_rq) return;
/* - * Blocked load of idle CPUs need to be updated. - * Kick an ILB to update statistics. + * Set the need to trigger ILB in order to update blocked load + * before entering idle state. */ - kick_ilb(NOHZ_STATS_KICK); + atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu)); }
#else /* !CONFIG_NO_HZ_COMMON */ diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 2593a733c0849..cdc3e690de714 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -261,6 +261,12 @@ static void cpuidle_idle_call(void) static void do_idle(void) { int cpu = smp_processor_id(); + + /* + * Check if we need to update blocked load + */ + nohz_run_idle_balance(cpu); + /* * If the arch has a polling bit, we maintain an invariant: * diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index df6cf8aa59f89..66e3ecb7c10e4 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2292,9 +2292,11 @@ extern void cfs_bandwidth_usage_dec(void); #ifdef CONFIG_NO_HZ_COMMON #define NOHZ_BALANCE_KICK_BIT 0 #define NOHZ_STATS_KICK_BIT 1 +#define NOHZ_NEWILB_KICK_BIT 2
#define NOHZ_BALANCE_KICK BIT(NOHZ_BALANCE_KICK_BIT) #define NOHZ_STATS_KICK BIT(NOHZ_STATS_KICK_BIT) +#define NOHZ_NEWILB_KICK BIT(NOHZ_NEWILB_KICK_BIT)
#define NOHZ_KICK_MASK (NOHZ_BALANCE_KICK | NOHZ_STATS_KICK)
@@ -2305,6 +2307,11 @@ extern void nohz_balance_exit_idle(struct rq *rq); static inline void nohz_balance_exit_idle(struct rq *rq) { } #endif
+#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON) +extern void nohz_run_idle_balance(int cpu); +#else +static inline void nohz_run_idle_balance(int cpu) { } +#endif
#ifdef CONFIG_SMP static inline