Full dynticks can only be enabled if "nohz_full" boot option has been been specified with or without parameter. Any change in the list of nohz_full CPUs have to be reflected in tick_nohz_full_mask. So the newly introduced tick_nohz_full_update_cpus() will be called to update the mask.
We also need to enable CPU context tracking for those CPUs that are in tick_nohz_full_mask. So remove __init from tick_nohz_init() and ct_cpu_track_user() so that they be called later when an isolated cpuset partition is being created. The __ro_after_init attribute is taken away from context_tracking_key as well.
Also add a new ct_cpu_untrack_user() function to reverse the action of ct_cpu_track_user() in case we need to disable the nohz_full mode of a CPU.
With nohz_full enabled, the boot CPU (typically CPU 0) will be the tick CPU which cannot be shut down easily. So the boot CPU should not be used in an isolated cpuset partition.
With runtime modification of nohz_full CPUs, tick_do_timer_cpu can become TICK_DO_TIMER_NONE. So remove the two TICK_DO_TIMER_NONE WARN_ON_ONCE() calls in tick-sched.c to avoid unnecessary warnings.
Signed-off-by: Waiman Long longman@redhat.com --- include/linux/context_tracking.h | 1 + kernel/cgroup/cpuset.c | 23 ++++++++++++++++++++++- kernel/context_tracking.c | 17 ++++++++++++++--- kernel/time/tick-sched.c | 7 ------- 4 files changed, 37 insertions(+), 11 deletions(-)
diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index a3fea7f9fef6..1a6b816f1ad6 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -17,6 +17,7 @@ #define CONTEXT_TRACKING_FORCE_ENABLE (-1)
extern void ct_cpu_track_user(int cpu); +extern void ct_cpu_untrack_user(int cpu);
/* Called with interrupts disabled. */ extern void __ct_user_enter(enum ctx_state state); diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 6308bb14e018..45c82c18bec4 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -23,6 +23,7 @@ */ #include "cpuset-internal.h"
+#include <linux/context_tracking.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/kernel.h> @@ -1361,7 +1362,7 @@ static void partition_xcpus_del(int old_prs, struct cpuset *parent, */ static int do_housekeeping_exclude_cpumask(void *arg __maybe_unused) { - int ret; + int cpu, ret; struct cpumask *icpus = isolated_cpus; unsigned long flags = BIT(HK_TYPE_DOMAIN) | BIT(HK_TYPE_KERNEL_NOISE);
@@ -1395,6 +1396,26 @@ static int do_housekeeping_exclude_cpumask(void *arg __maybe_unused) ret = housekeeping_exclude_cpumask(icpus, flags); WARN_ON_ONCE((ret < 0) && (ret != -EOPNOTSUPP));
+#ifdef CONFIG_NO_HZ_FULL + /* + * To properly enable/disable nohz_full dynticks for the affected CPUs, + * the new nohz_full CPUs have to be copied to tick_nohz_full_mask and + * ct_cpu_track_user/ct_cpu_untrack_user() will have to be called + * for those CPUs that have their states changed. + */ + if (tick_nohz_full_enabled()) { + tick_nohz_full_update_cpus(icpus); + for_each_cpu(cpu, isolcpus_update_state.cpus) { + if (cpumask_test_cpu(cpu, icpus)) + ct_cpu_track_user(cpu); + else + ct_cpu_untrack_user(cpu); + } + } else { + pr_warn_once("Full dynticks cannot be enabled without the nohz_full kernel boot parameter!\n"); + } +#endif + if (icpus != isolated_cpus) kfree(icpus); return ret; diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 734354bbfdbb..ed5653a3d6f7 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -431,7 +431,7 @@ static __always_inline void ct_kernel_enter(bool user, int offset) { } #define CREATE_TRACE_POINTS #include <trace/events/context_tracking.h>
-DEFINE_STATIC_KEY_FALSE_RO(context_tracking_key); +DEFINE_STATIC_KEY_FALSE(context_tracking_key); EXPORT_SYMBOL_GPL(context_tracking_key);
static noinstr bool context_tracking_recursion_enter(void) @@ -694,9 +694,9 @@ void user_exit_callable(void) } NOKPROBE_SYMBOL(user_exit_callable);
-void __init ct_cpu_track_user(int cpu) +void ct_cpu_track_user(int cpu) { - static __initdata bool initialized = false; + static bool initialized;
if (cpu == CONTEXT_TRACKING_FORCE_ENABLE) { static_branch_inc(&context_tracking_key); @@ -720,6 +720,17 @@ void __init ct_cpu_track_user(int cpu) initialized = true; }
+void ct_cpu_untrack_user(int cpu) +{ +#ifndef CONFIG_CONTEXT_TRACKING_USER_FORCE + if (!per_cpu(context_tracking.active, cpu)) + return; + + per_cpu(context_tracking.active, cpu) = false; + static_branch_dec(&context_tracking_key); +#endif +} + #ifdef CONFIG_CONTEXT_TRACKING_USER_FORCE void __init context_tracking_init(void) { diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 9204808b7a55..c16250c6a79f 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -220,9 +220,6 @@ static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now) tick_cpu = READ_ONCE(tick_do_timer_cpu);
if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && unlikely(tick_cpu == TICK_DO_TIMER_NONE)) { -#ifdef CONFIG_NO_HZ_FULL - WARN_ON_ONCE(tick_nohz_full_running); -#endif WRITE_ONCE(tick_do_timer_cpu, cpu); tick_cpu = cpu; } @@ -1201,10 +1198,6 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) */ if (tick_cpu == cpu) return false; - - /* Should not happen for nohz-full */ - if (WARN_ON_ONCE(tick_cpu == TICK_DO_TIMER_NONE)) - return false; }
return true;