Enable or disable the appropriate DL servers (ext and fair) depending on whether an scx scheduler is started in full or partial mode:
- in full mode, disable the fair DL server and enable the ext DL server on all online CPUs, - in partial mode (%SCX_OPS_SWITCH_PARTIAL), keep both fair and ext DL servers active to support tasks in both scheduling classes.
Additionally, handle CPU hotplug events by selectively enabling or disabling the relevant DL servers on the CPU that is going offline/online. This ensures correct bandwidth reservation also when CPUs are brought online or offline.
v2: - start the dl_server if there's any scx task running in the rq (Christian Loehle)
Co-developed-by: Joel Fernandes joelagnelf@nvidia.com Signed-off-by: Joel Fernandes joelagnelf@nvidia.com Signed-off-by: Andrea Righi arighi@nvidia.com --- kernel/sched/ext.c | 99 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 89 insertions(+), 10 deletions(-)
diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 2a25749c54ba1..9e23ead618e16 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -2600,6 +2600,59 @@ static void set_cpus_allowed_scx(struct task_struct *p, p, (struct cpumask *)p->cpus_ptr); }
+static void dl_server_on(struct rq *rq, bool switch_all) +{ + struct rq_flags rf; + int err; + + rq_lock_irqsave(rq, &rf); + update_rq_clock(rq); + + if (switch_all) { + /* + * If all fair tasks are moved to the scx scheduler, we + * don't need the fair DL server anymore, so remove it. + * + * When the current scx scheduler is unloaded, the fair DL + * server will be re-initialized. + */ + if (dl_server_active(&rq->fair_server)) + dl_server_stop(&rq->fair_server); + dl_server_remove_params(&rq->fair_server, rq, &rf); + } + + err = dl_server_init_params(&rq->ext_server); + WARN_ON_ONCE(err); + if (rq->scx.nr_running) + dl_server_start(&rq->ext_server); + + rq_unlock_irqrestore(rq, &rf); +} + +static void dl_server_off(struct rq *rq, bool switch_all) +{ + struct rq_flags rf; + int err; + + rq_lock_irqsave(rq, &rf); + update_rq_clock(rq); + + if (dl_server_active(&rq->ext_server)) + dl_server_stop(&rq->ext_server); + dl_server_remove_params(&rq->ext_server, rq, &rf); + + if (switch_all) { + /* + * Re-initialize the fair DL server if it was previously disabled + * because all fair tasks had been moved to the ext class. + */ + err = dl_server_init_params(&rq->fair_server); + WARN_ON_ONCE(err); + } + + rq_unlock_irqrestore(rq, &rf); +} + static void handle_hotplug(struct rq *rq, bool online) { struct scx_sched *sch = scx_root; @@ -2615,9 +2668,20 @@ static void handle_hotplug(struct rq *rq, bool online) if (unlikely(!sch)) return;
- if (scx_enabled()) + if (scx_enabled()) { + bool is_switching_all = READ_ONCE(scx_switching_all); + scx_idle_update_selcpu_topology(&sch->ops);
+ /* + * Update ext and fair DL servers on hotplug events. + */ + if (online) + dl_server_on(rq, is_switching_all); + else + dl_server_off(rq, is_switching_all); + } + if (online && SCX_HAS_OP(sch, cpu_online)) SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cpu_online, NULL, cpu); else if (!online && SCX_HAS_OP(sch, cpu_offline)) @@ -3976,6 +4040,7 @@ static void scx_disable_workfn(struct kthread_work *work) struct scx_exit_info *ei = sch->exit_info; struct scx_task_iter sti; struct task_struct *p; + bool is_switching_all = READ_ONCE(scx_switching_all); int kind, cpu;
kind = atomic_read(&sch->exit_kind); @@ -4031,6 +4096,22 @@ static void scx_disable_workfn(struct kthread_work *work)
scx_init_task_enabled = false;
+ for_each_online_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + /* + * Invalidate all the rq clocks to prevent getting outdated + * rq clocks from a previous scx scheduler. + */ + scx_rq_clock_invalidate(rq); + + /* + * We are unloading the sched_ext scheduler, we do not need its + * DL server bandwidth anymore, remove it for all CPUs. + */ + dl_server_off(rq, is_switching_all); + } + scx_task_iter_start(&sti); while ((p = scx_task_iter_next_locked(&sti))) { unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; @@ -4052,15 +4133,6 @@ static void scx_disable_workfn(struct kthread_work *work) scx_task_iter_stop(&sti); percpu_up_write(&scx_fork_rwsem);
- /* - * Invalidate all the rq clocks to prevent getting outdated - * rq clocks from a previous scx scheduler. - */ - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - scx_rq_clock_invalidate(rq); - } - /* no task is on scx, turn off all the switches and flush in-progress calls */ static_branch_disable(&__scx_enabled); bitmap_zero(sch->has_op, SCX_OPI_END); @@ -4834,6 +4906,13 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) } } scx_task_iter_stop(&sti); + + /* + * Enable the ext DL server on all online CPUs. + */ + for_each_online_cpu(cpu) + dl_server_on(cpu_rq(cpu), !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); + percpu_up_write(&scx_fork_rwsem);
scx_bypass(false);