6.14-stable review patch. If anyone has any objections, please let me know.
------------------
From: Juri Lelli juri.lelli@redhat.com
[ Upstream commit 45007c6fb5860cf63556a9cadc87c8984927e23d ]
Bandwidth checks and updates that work on root domains currently employ a cookie mechanism for efficiency. This mechanism is very much tied to when root domains are first created and initialized.
Generalize the cookie mechanism so that it can be used also later at runtime while updating root domains. Also, additionally guard it with sched_domains_mutex, since domains need to be stable while updating them (and it will be required for further dynamic changes).
Fixes: 53916d5fd3c0 ("sched/deadline: Check bandwidth overflow earlier for hotplug") Reported-by: Jon Hunter jonathanh@nvidia.com Signed-off-by: Juri Lelli juri.lelli@redhat.com Signed-off-by: Peter Zijlstra (Intel) peterz@infradead.org Reviewed-by: Valentin Schneider vschneid@redhat.com Reviewed-by: Dietmar Eggemann dietmar.eggemann@arm.com Tested-by: Waiman Long longman@redhat.com Tested-by: Jon Hunter jonathanh@nvidia.com Tested-by: Dietmar Eggemann dietmar.eggemann@arm.com Link: https://lore.kernel.org/r/Z9MQaiXPvEeW_v7x@jlelli-thinkpadt14gen4.remote.csb Signed-off-by: Sasha Levin sashal@kernel.org --- include/linux/sched/deadline.h | 3 +++ kernel/sched/deadline.c | 23 +++++++++++++---------- kernel/sched/rt.c | 2 ++ kernel/sched/sched.h | 2 +- kernel/sched/topology.c | 2 +- 5 files changed, 20 insertions(+), 12 deletions(-)
diff --git a/include/linux/sched/deadline.h b/include/linux/sched/deadline.h index 3a912ab42bb55..6ec578600b24c 100644 --- a/include/linux/sched/deadline.h +++ b/include/linux/sched/deadline.h @@ -37,4 +37,7 @@ extern void dl_clear_root_domain(struct root_domain *rd);
#endif /* CONFIG_SMP */
+extern u64 dl_cookie; +extern bool dl_bw_visited(int cpu, u64 cookie); + #endif /* _LINUX_SCHED_DEADLINE_H */ diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 1a041c1fc0d1e..3e05032e9e0eb 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -166,14 +166,14 @@ static inline unsigned long dl_bw_capacity(int i) } }
-static inline bool dl_bw_visited(int cpu, u64 gen) +static inline bool dl_bw_visited(int cpu, u64 cookie) { struct root_domain *rd = cpu_rq(cpu)->rd;
- if (rd->visit_gen == gen) + if (rd->visit_cookie == cookie) return true;
- rd->visit_gen = gen; + rd->visit_cookie = cookie; return false; }
@@ -207,7 +207,7 @@ static inline unsigned long dl_bw_capacity(int i) return SCHED_CAPACITY_SCALE; }
-static inline bool dl_bw_visited(int cpu, u64 gen) +static inline bool dl_bw_visited(int cpu, u64 cookie) { return false; } @@ -3171,15 +3171,18 @@ DEFINE_SCHED_CLASS(dl) = { #endif };
-/* Used for dl_bw check and update, used under sched_rt_handler()::mutex */ -static u64 dl_generation; +/* + * Used for dl_bw check and update, used under sched_rt_handler()::mutex and + * sched_domains_mutex. + */ +u64 dl_cookie;
int sched_dl_global_validate(void) { u64 runtime = global_rt_runtime(); u64 period = global_rt_period(); u64 new_bw = to_ratio(period, runtime); - u64 gen = ++dl_generation; + u64 cookie = ++dl_cookie; struct dl_bw *dl_b; int cpu, cpus, ret = 0; unsigned long flags; @@ -3192,7 +3195,7 @@ int sched_dl_global_validate(void) for_each_online_cpu(cpu) { rcu_read_lock_sched();
- if (dl_bw_visited(cpu, gen)) + if (dl_bw_visited(cpu, cookie)) goto next;
dl_b = dl_bw_of(cpu); @@ -3229,7 +3232,7 @@ static void init_dl_rq_bw_ratio(struct dl_rq *dl_rq) void sched_dl_do_global(void) { u64 new_bw = -1; - u64 gen = ++dl_generation; + u64 cookie = ++dl_cookie; struct dl_bw *dl_b; int cpu; unsigned long flags; @@ -3240,7 +3243,7 @@ void sched_dl_do_global(void) for_each_possible_cpu(cpu) { rcu_read_lock_sched();
- if (dl_bw_visited(cpu, gen)) { + if (dl_bw_visited(cpu, cookie)) { rcu_read_unlock_sched(); continue; } diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 4b8e33c615b12..8cebe71d2bb16 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -2910,6 +2910,7 @@ static int sched_rt_handler(const struct ctl_table *table, int write, void *buff int ret;
mutex_lock(&mutex); + sched_domains_mutex_lock(); old_period = sysctl_sched_rt_period; old_runtime = sysctl_sched_rt_runtime;
@@ -2936,6 +2937,7 @@ static int sched_rt_handler(const struct ctl_table *table, int write, void *buff sysctl_sched_rt_period = old_period; sysctl_sched_rt_runtime = old_runtime; } + sched_domains_mutex_unlock(); mutex_unlock(&mutex);
return ret; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 023b844159c94..1aa65a0ac5864 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -998,7 +998,7 @@ struct root_domain { * Also, some corner cases, like 'wrap around' is dangerous, but given * that u64 is 'big enough'. So that shouldn't be a concern. */ - u64 visit_gen; + u64 visit_cookie;
#ifdef HAVE_RT_PUSH_IPI /* diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 296ff2acfd321..44093339761c9 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -568,7 +568,7 @@ static int init_rootdomain(struct root_domain *rd) rd->rto_push_work = IRQ_WORK_INIT_HARD(rto_push_irq_work_func); #endif
- rd->visit_gen = 0; + rd->visit_cookie = 0; init_dl_bw(&rd->dl_bw); if (cpudl_init(&rd->cpudl) != 0) goto free_rto_mask;