From: Joel Fernandes joelagnelf@nvidia.com

Currently the DL server interface for applying parameters checks CFS-internals to identify if the server is active. This is error-prone and makes it difficult when adding new servers in the future.

Fix it, by using dl_server_active() which is also used by the DL server code to determine if the DL server was started.

Acked-by: Tejun Heo tj@kernel.org Reviewed-by: Juri Lelli juri.lelli@redhat.com Reviewed-by: Andrea Righi arighi@nvidia.com Signed-off-by: Joel Fernandes joelagnelf@nvidia.com --- kernel/sched/debug.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 6cf9be6eea49a..e71f6618c1a6a 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -354,6 +354,8 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu return err;

scoped_guard (rq_lock_irqsave, rq) { + bool is_active; + runtime = rq->fair_server.dl_runtime; period = rq->fair_server.dl_period;

@@ -376,8 +378,11 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu return -EINVAL; }

- update_rq_clock(rq); - dl_server_stop(&rq->fair_server); + is_active = dl_server_active(&rq->fair_server); + if (is_active) { + update_rq_clock(rq); + dl_server_stop(&rq->fair_server); + }

retval = dl_server_apply_params(&rq->fair_server, runtime, period, 0);

@@ -385,7 +390,7 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu printk_deferred("Fair server disabled in CPU %d, system may crash due to starvation.\n", cpu_of(rq));

- if (rq->cfs.h_nr_queued) + if (is_active) dl_server_start(&rq->fair_server);

if (retval < 0)

During switching from sched_ext to fair tasks and vice-versa, we need support for intializing and removing the bandwidth contribution of either DL server.

Add support for handling these transitions.

Moreover, remove references specific to the fair server, in preparation for adding the ext server.

v2: - wait for inactive_task_timer to fire before removing the bandwidth reservation (Juri Lelli) - add WARN_ON_ONCE(!cpus) sanity check in dl_server_apply_params() (Andrea Righi)

Co-developed-by: Joel Fernandes joelagnelf@nvidia.com Signed-off-by: Joel Fernandes joelagnelf@nvidia.com Signed-off-by: Andrea Righi arighi@nvidia.com --- kernel/sched/deadline.c | 96 ++++++++++++++++++++++++++++++++++------- kernel/sched/sched.h | 3 ++ 2 files changed, 84 insertions(+), 15 deletions(-)

diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 4aefb34a1d38b..8aff1aba7b8a9 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -1441,8 +1441,8 @@ static void update_curr_dl_se(struct rq *rq, struct sched_dl_entity *dl_se, s64 dl_se->runtime -= scaled_delta_exec;

/* - * The fair server can consume its runtime while throttled (not queued/ - * running as regular CFS). + * The dl_server can consume its runtime while throttled (not + * queued / running as regular fair task). * * If the server consumes its entire runtime in this state. The server * is not required for the current period. Thus, reset the server by @@ -1501,10 +1501,10 @@ static void update_curr_dl_se(struct rq *rq, struct sched_dl_entity *dl_se, s64 }

/* - * The fair server (sole dl_server) does not account for real-time - * workload because it is running fair work. + * The dl_server does not account real-time workload because it + * runs non-RT tasks. */ - if (dl_se == &rq->fair_server) + if (dl_se->dl_server) return;

#ifdef CONFIG_RT_GROUP_SCHED @@ -1540,8 +1540,8 @@ static void update_curr_dl_se(struct rq *rq, struct sched_dl_entity *dl_se, s64 * server provides a guarantee. * * If the dl_server is in defer mode, the idle time is also considered - * as time available for the fair server, avoiding a penalty for the - * rt scheduler that did not consumed that time. + * as time available for the dl_server, avoiding a penalty for the rt + * scheduler that did not consumed that time. */ void dl_server_update_idle_time(struct rq *rq, struct task_struct *p) { @@ -1570,11 +1570,37 @@ void dl_server_update_idle_time(struct rq *rq, struct task_struct *p)

void dl_server_update(struct sched_dl_entity *dl_se, s64 delta_exec) { - /* 0 runtime = fair server disabled */ + /* 0 runtime = dl_server disabled */ if (dl_se->dl_runtime) update_curr_dl_se(dl_se->rq, dl_se, delta_exec); }

+/** + * dl_server_init_params - Initialize bandwidth reservation for a DL server + * @dl_se: The DL server entity to remove bandwidth for + * + * This function initializes the bandwidth reservation for a DL server + * entity, its bandwidth accounting and server state. + * + * Returns: 0 on success, negative error code on failure + */ +int dl_server_init_params(struct sched_dl_entity *dl_se) +{ + u64 runtime = 50 * NSEC_PER_MSEC; + u64 period = 1000 * NSEC_PER_MSEC; + int err; + + err = dl_server_apply_params(dl_se, runtime, period, 1); + if (err) + return err; + + dl_se->dl_server = 1; + dl_se->dl_defer = 1; + setup_new_dl_entity(dl_se); + + return err; +} + void dl_server_start(struct sched_dl_entity *dl_se) { struct rq *rq = dl_se->rq; @@ -1614,8 +1640,7 @@ void sched_init_dl_servers(void) struct sched_dl_entity *dl_se;

for_each_online_cpu(cpu) { - u64 runtime = 50 * NSEC_PER_MSEC; - u64 period = 1000 * NSEC_PER_MSEC; + int err;

rq = cpu_rq(cpu);

@@ -1625,11 +1650,8 @@ void sched_init_dl_servers(void)

WARN_ON(dl_server(dl_se));

- dl_server_apply_params(dl_se, runtime, period, 1); - - dl_se->dl_server = 1; - dl_se->dl_defer = 1; - setup_new_dl_entity(dl_se); + err = dl_server_init_params(dl_se); + WARN_ON_ONCE(err); } }

@@ -1663,6 +1685,9 @@ int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 perio guard(raw_spinlock)(&dl_b->lock);

cpus = dl_bw_cpus(cpu); + if (WARN_ON_ONCE(!cpus)) + return -ENODEV; + cap = dl_bw_capacity(cpu);

if (__dl_overflow(dl_b, cap, old_bw, new_bw)) @@ -1678,6 +1703,12 @@ int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 perio dl_rq_change_utilization(rq, dl_se, new_bw); }

+ /* Clear these so that the dl_server is reinitialized */ + if (new_bw == 0) { + dl_se->dl_defer = 0; + dl_se->dl_server = 0; + } + dl_se->dl_runtime = runtime; dl_se->dl_deadline = period; dl_se->dl_period = period; @@ -1691,6 +1722,41 @@ int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 perio return retval; }

+/** + * dl_server_remove_params - Remove bandwidth reservation for a DL server + * @dl_se: The DL server entity to remove bandwidth for + * + * This function removes the bandwidth reservation for a DL server entity, + * cleaning up all bandwidth accounting and server state. + * + * Returns: 0 on success, negative error code on failure + */ +int dl_server_remove_params(struct sched_dl_entity *dl_se, + struct rq *rq, struct rq_flags *rf) +{ + if (!dl_se->dl_server) + return 0; /* Already disabled */ + + /* + * First dequeue if still queued. It should not be queued since + * we call this only after the last dl_server_stop(). + */ + if (WARN_ON_ONCE(on_dl_rq(dl_se))) + dequeue_dl_entity(dl_se, DEQUEUE_SLEEP); + + if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == -1) { + rq_unlock_irqrestore(rq, rf); + + hrtimer_cancel(&dl_se->inactive_timer); + + rq_lock_irqsave(rq, rf); + update_rq_clock(rq); + } + + /* Remove bandwidth reservation */ + return dl_server_apply_params(dl_se, 0, dl_se->dl_period, false); +} + /* * Update the current task's runtime statistics (provided it is still * a -deadline task and has not been removed from the dl_rq). diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 27aae2a298f8b..4a0bf38dc71e9 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -417,6 +417,9 @@ extern void fair_server_init(struct rq *rq); extern void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq); extern int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 period, bool init); +extern int dl_server_init_params(struct sched_dl_entity *dl_se); +extern int dl_server_remove_params(struct sched_dl_entity *dl_se, + struct rq *rq, struct rq_flags *rf);

static inline bool dl_server_active(struct sched_dl_entity *dl_se) {

From: Joel Fernandes joelagnelf@nvidia.com

Since we are adding more servers, make dl_server_update_idle_time() accept a server argument rather than a specific server.

v2: - rename rq_dl_server to dl_server (Peter Zijlstra)

Acked-by: Juri Lelli juri.lelli@redhat.com Reviewed-by: Andrea Righi arighi@nvidia.com Signed-off-by: Joel Fernandes joelagnelf@nvidia.com --- kernel/sched/deadline.c | 16 ++++++++-------- kernel/sched/fair.c | 2 +- kernel/sched/idle.c | 2 +- kernel/sched/sched.h | 3 ++- 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 8aff1aba7b8a9..6ecfaaa1f912d 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -1543,26 +1543,26 @@ static void update_curr_dl_se(struct rq *rq, struct sched_dl_entity *dl_se, s64 * as time available for the dl_server, avoiding a penalty for the rt * scheduler that did not consumed that time. */ -void dl_server_update_idle_time(struct rq *rq, struct task_struct *p) +void dl_server_update_idle_time(struct rq *rq, struct task_struct *p, + struct sched_dl_entity *dl_server) { s64 delta_exec;

- if (!rq->fair_server.dl_defer) + if (!dl_server->dl_defer) return;

/* no need to discount more */ - if (rq->fair_server.runtime < 0) + if (dl_server->runtime < 0) return;

delta_exec = rq_clock_task(rq) - p->se.exec_start; if (delta_exec < 0) return;

- rq->fair_server.runtime -= delta_exec; - - if (rq->fair_server.runtime < 0) { - rq->fair_server.dl_defer_running = 0; - rq->fair_server.runtime = 0; + dl_server->runtime -= delta_exec; + if (dl_server->runtime < 0) { + dl_server->dl_defer_running = 0; + dl_server->runtime = 0; }

p->se.exec_start = rq_clock_task(rq); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 2554055c1ba13..562cdd253678a 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6999,7 +6999,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (!rq_h_nr_queued && rq->cfs.h_nr_queued) { /* Account for idle runtime */ if (!rq->nr_running) - dl_server_update_idle_time(rq, rq->curr); + dl_server_update_idle_time(rq, rq->curr, &rq->fair_server); dl_server_start(&rq->fair_server); }

diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 7fa0b593bcff7..60a19ea9bdbb7 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -454,7 +454,7 @@ static void wakeup_preempt_idle(struct rq *rq, struct task_struct *p, int flags)

static void put_prev_task_idle(struct rq *rq, struct task_struct *prev, struct task_struct *next) { - dl_server_update_idle_time(rq, prev); + dl_server_update_idle_time(rq, prev, &rq->fair_server); scx_update_idle(rq, false, true); }

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 4a0bf38dc71e9..eaae470841dea 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -412,7 +412,8 @@ extern void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq, extern void sched_init_dl_servers(void);

extern void dl_server_update_idle_time(struct rq *rq, - struct task_struct *p); + struct task_struct *p, + struct sched_dl_entity *rq_dl_server); extern void fair_server_init(struct rq *rq); extern void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq); extern int dl_server_apply_params(struct sched_dl_entity *dl_se,

From: Joel Fernandes joelagnelf@nvidia.com

When a sched_ext server is loaded, tasks in the fair class are automatically moved to the sched_ext class. Add support to modify the ext server parameters similar to how the fair server parameters are modified.

Re-use common code between ext and fair servers as needed.

v2: - use dl_se->dl_server to determine if dl_se is a DL server (Peter Zijlstra) - depend on CONFIG_SCHED_CLASS_EXT (Andrea Righi)

Reviewed-by: Juri Lelli juri.lelli@redhat.com Co-developed-by: Andrea Righi arighi@nvidia.com Signed-off-by: Andrea Righi arighi@nvidia.com Signed-off-by: Joel Fernandes joelagnelf@nvidia.com --- kernel/sched/debug.c | 157 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 133 insertions(+), 24 deletions(-)

diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index e71f6618c1a6a..9c2084d203df5 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -336,14 +336,16 @@ enum dl_param { DL_PERIOD, };

-static unsigned long fair_server_period_max = (1UL << 22) * NSEC_PER_USEC; /* ~4 seconds */ -static unsigned long fair_server_period_min = (100) * NSEC_PER_USEC; /* 100 us */ +static unsigned long dl_server_period_max = (1UL << 22) * NSEC_PER_USEC; /* ~4 seconds */ +static unsigned long dl_server_period_min = (100) * NSEC_PER_USEC; /* 100 us */

-static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubuf, - size_t cnt, loff_t *ppos, enum dl_param param) +static ssize_t sched_server_write_common(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos, enum dl_param param, + void *server) { long cpu = (long) ((struct seq_file *) filp->private_data)->private; struct rq *rq = cpu_rq(cpu); + struct sched_dl_entity *dl_se = (struct sched_dl_entity *)server; u64 runtime, period; int retval = 0; size_t err; @@ -356,8 +358,8 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu scoped_guard (rq_lock_irqsave, rq) { bool is_active;

- runtime = rq->fair_server.dl_runtime; - period = rq->fair_server.dl_period; + runtime = dl_se->dl_runtime; + period = dl_se->dl_period;

switch (param) { case DL_RUNTIME: @@ -373,25 +375,25 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu }

if (runtime > period || - period > fair_server_period_max || - period < fair_server_period_min) { + period > dl_server_period_max || + period < dl_server_period_min) { return -EINVAL; }

- is_active = dl_server_active(&rq->fair_server); + is_active = dl_server_active(dl_se); if (is_active) { update_rq_clock(rq); - dl_server_stop(&rq->fair_server); + dl_server_stop(dl_se); }

- retval = dl_server_apply_params(&rq->fair_server, runtime, period, 0); + retval = dl_server_apply_params(dl_se, runtime, period, 0);

if (!runtime) - printk_deferred("Fair server disabled in CPU %d, system may crash due to starvation.\n", - cpu_of(rq)); + printk_deferred("%s server disabled on CPU %d, system may crash due to starvation.\n", + server == &rq->fair_server ? "Fair" : "Ext", cpu_of(rq));

if (is_active) - dl_server_start(&rq->fair_server); + dl_server_start(dl_se);

if (retval < 0) return retval; @@ -401,36 +403,42 @@ static ssize_t sched_fair_server_write(struct file *filp, const char __user *ubu return cnt; }

-static size_t sched_fair_server_show(struct seq_file *m, void *v, enum dl_param param) +static size_t sched_server_show_common(struct seq_file *m, void *v, enum dl_param param, + void *server) { - unsigned long cpu = (unsigned long) m->private; - struct rq *rq = cpu_rq(cpu); + struct sched_dl_entity *dl_se = (struct sched_dl_entity *)server; u64 value;

switch (param) { case DL_RUNTIME: - value = rq->fair_server.dl_runtime; + value = dl_se->dl_runtime; break; case DL_PERIOD: - value = rq->fair_server.dl_period; + value = dl_se->dl_period; break; }

seq_printf(m, "%llu\n", value); return 0; - }

static ssize_t sched_fair_server_runtime_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { - return sched_fair_server_write(filp, ubuf, cnt, ppos, DL_RUNTIME); + long cpu = (long) ((struct seq_file *) filp->private_data)->private; + struct rq *rq = cpu_rq(cpu); + + return sched_server_write_common(filp, ubuf, cnt, ppos, DL_RUNTIME, + &rq->fair_server); }

static int sched_fair_server_runtime_show(struct seq_file *m, void *v) { - return sched_fair_server_show(m, v, DL_RUNTIME); + unsigned long cpu = (unsigned long) m->private; + struct rq *rq = cpu_rq(cpu); + + return sched_server_show_common(m, v, DL_RUNTIME, &rq->fair_server); }

static int sched_fair_server_runtime_open(struct inode *inode, struct file *filp) @@ -446,16 +454,57 @@ static const struct file_operations fair_server_runtime_fops = { .release = single_release, };

+#ifdef CONFIG_SCHED_CLASS_EXT +static ssize_t +sched_ext_server_runtime_write(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + long cpu = (long) ((struct seq_file *) filp->private_data)->private; + struct rq *rq = cpu_rq(cpu); + + return sched_server_write_common(filp, ubuf, cnt, ppos, DL_RUNTIME, + &rq->ext_server); +} + +static int sched_ext_server_runtime_show(struct seq_file *m, void *v) +{ + unsigned long cpu = (unsigned long) m->private; + struct rq *rq = cpu_rq(cpu); + + return sched_server_show_common(m, v, DL_RUNTIME, &rq->ext_server); +} + +static int sched_ext_server_runtime_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, sched_ext_server_runtime_show, inode->i_private); +} + +static const struct file_operations ext_server_runtime_fops = { + .open = sched_ext_server_runtime_open, + .write = sched_ext_server_runtime_write, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif /* CONFIG_SCHED_CLASS_EXT */ + static ssize_t sched_fair_server_period_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) { - return sched_fair_server_write(filp, ubuf, cnt, ppos, DL_PERIOD); + long cpu = (long) ((struct seq_file *) filp->private_data)->private; + struct rq *rq = cpu_rq(cpu); + + return sched_server_write_common(filp, ubuf, cnt, ppos, DL_PERIOD, + &rq->fair_server); }

static int sched_fair_server_period_show(struct seq_file *m, void *v) { - return sched_fair_server_show(m, v, DL_PERIOD); + unsigned long cpu = (unsigned long) m->private; + struct rq *rq = cpu_rq(cpu); + + return sched_server_show_common(m, v, DL_PERIOD, &rq->fair_server); }

static int sched_fair_server_period_open(struct inode *inode, struct file *filp) @@ -471,6 +520,40 @@ static const struct file_operations fair_server_period_fops = { .release = single_release, };

+#ifdef CONFIG_SCHED_CLASS_EXT +static ssize_t +sched_ext_server_period_write(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + long cpu = (long) ((struct seq_file *) filp->private_data)->private; + struct rq *rq = cpu_rq(cpu); + + return sched_server_write_common(filp, ubuf, cnt, ppos, DL_PERIOD, + &rq->ext_server); +} + +static int sched_ext_server_period_show(struct seq_file *m, void *v) +{ + unsigned long cpu = (unsigned long) m->private; + struct rq *rq = cpu_rq(cpu); + + return sched_server_show_common(m, v, DL_PERIOD, &rq->ext_server); +} + +static int sched_ext_server_period_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, sched_ext_server_period_show, inode->i_private); +} + +static const struct file_operations ext_server_period_fops = { + .open = sched_ext_server_period_open, + .write = sched_ext_server_period_write, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif /* CONFIG_SCHED_CLASS_EXT */ + static struct dentry *debugfs_sched;

static void debugfs_fair_server_init(void) @@ -494,6 +577,29 @@ static void debugfs_fair_server_init(void) } }

+#ifdef CONFIG_SCHED_CLASS_EXT +static void debugfs_ext_server_init(void) +{ + struct dentry *d_ext; + unsigned long cpu; + + d_ext = debugfs_create_dir("ext_server", debugfs_sched); + if (!d_ext) + return; + + for_each_possible_cpu(cpu) { + struct dentry *d_cpu; + char buf[32]; + + snprintf(buf, sizeof(buf), "cpu%lu", cpu); + d_cpu = debugfs_create_dir(buf, d_ext); + + debugfs_create_file("runtime", 0644, d_cpu, (void *) cpu, &ext_server_runtime_fops); + debugfs_create_file("period", 0644, d_cpu, (void *) cpu, &ext_server_period_fops); + } +} +#endif /* CONFIG_SCHED_CLASS_EXT */ + static __init int sched_init_debug(void) { struct dentry __maybe_unused *numa; @@ -532,6 +638,9 @@ static __init int sched_init_debug(void) debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);

debugfs_fair_server_init(); +#ifdef CONFIG_SCHED_CLASS_EXT + debugfs_ext_server_init(); +#endif

return 0; }

Enable or disable the appropriate DL servers (ext and fair) depending on whether an scx scheduler is started in full or partial mode:

- in full mode, disable the fair DL server and enable the ext DL server on all online CPUs, - in partial mode (%SCX_OPS_SWITCH_PARTIAL), keep both fair and ext DL servers active to support tasks in both scheduling classes.

Additionally, handle CPU hotplug events by selectively enabling or disabling the relevant DL servers on the CPU that is going offline/online. This ensures correct bandwidth reservation also when CPUs are brought online or offline.

v2: - start the dl_server if there's any scx task running in the rq (Christian Loehle)

Co-developed-by: Joel Fernandes joelagnelf@nvidia.com Signed-off-by: Joel Fernandes joelagnelf@nvidia.com Signed-off-by: Andrea Righi arighi@nvidia.com --- kernel/sched/ext.c | 99 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 89 insertions(+), 10 deletions(-)

diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 2a25749c54ba1..9e23ead618e16 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -2600,6 +2600,59 @@ static void set_cpus_allowed_scx(struct task_struct *p, p, (struct cpumask *)p->cpus_ptr); }

+static void dl_server_on(struct rq *rq, bool switch_all) +{ + struct rq_flags rf; + int err; + + rq_lock_irqsave(rq, &rf); + update_rq_clock(rq); + + if (switch_all) { + /* + * If all fair tasks are moved to the scx scheduler, we + * don't need the fair DL server anymore, so remove it. + * + * When the current scx scheduler is unloaded, the fair DL + * server will be re-initialized. + */ + if (dl_server_active(&rq->fair_server)) + dl_server_stop(&rq->fair_server); + dl_server_remove_params(&rq->fair_server, rq, &rf); + } + + err = dl_server_init_params(&rq->ext_server); + WARN_ON_ONCE(err); + if (rq->scx.nr_running) + dl_server_start(&rq->ext_server); + + rq_unlock_irqrestore(rq, &rf); +} + +static void dl_server_off(struct rq *rq, bool switch_all) +{ + struct rq_flags rf; + int err; + + rq_lock_irqsave(rq, &rf); + update_rq_clock(rq); + + if (dl_server_active(&rq->ext_server)) + dl_server_stop(&rq->ext_server); + dl_server_remove_params(&rq->ext_server, rq, &rf); + + if (switch_all) { + /* + * Re-initialize the fair DL server if it was previously disabled + * because all fair tasks had been moved to the ext class. + */ + err = dl_server_init_params(&rq->fair_server); + WARN_ON_ONCE(err); + } + + rq_unlock_irqrestore(rq, &rf); +} + static void handle_hotplug(struct rq *rq, bool online) { struct scx_sched *sch = scx_root; @@ -2615,9 +2668,20 @@ static void handle_hotplug(struct rq *rq, bool online) if (unlikely(!sch)) return;

- if (scx_enabled()) + if (scx_enabled()) { + bool is_switching_all = READ_ONCE(scx_switching_all); + scx_idle_update_selcpu_topology(&sch->ops);

+ /* + * Update ext and fair DL servers on hotplug events. + */ + if (online) + dl_server_on(rq, is_switching_all); + else + dl_server_off(rq, is_switching_all); + } + if (online && SCX_HAS_OP(sch, cpu_online)) SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cpu_online, NULL, cpu); else if (!online && SCX_HAS_OP(sch, cpu_offline)) @@ -3976,6 +4040,7 @@ static void scx_disable_workfn(struct kthread_work *work) struct scx_exit_info *ei = sch->exit_info; struct scx_task_iter sti; struct task_struct *p; + bool is_switching_all = READ_ONCE(scx_switching_all); int kind, cpu;

kind = atomic_read(&sch->exit_kind); @@ -4031,6 +4096,22 @@ static void scx_disable_workfn(struct kthread_work *work)

scx_init_task_enabled = false;

+ for_each_online_cpu(cpu) { + struct rq *rq = cpu_rq(cpu); + + /* + * Invalidate all the rq clocks to prevent getting outdated + * rq clocks from a previous scx scheduler. + */ + scx_rq_clock_invalidate(rq); + + /* + * We are unloading the sched_ext scheduler, we do not need its + * DL server bandwidth anymore, remove it for all CPUs. + */ + dl_server_off(rq, is_switching_all); + } + scx_task_iter_start(&sti); while ((p = scx_task_iter_next_locked(&sti))) { unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK; @@ -4052,15 +4133,6 @@ static void scx_disable_workfn(struct kthread_work *work) scx_task_iter_stop(&sti); percpu_up_write(&scx_fork_rwsem);

- /* - * Invalidate all the rq clocks to prevent getting outdated - * rq clocks from a previous scx scheduler. - */ - for_each_possible_cpu(cpu) { - struct rq *rq = cpu_rq(cpu); - scx_rq_clock_invalidate(rq); - } - /* no task is on scx, turn off all the switches and flush in-progress calls */ static_branch_disable(&__scx_enabled); bitmap_zero(sch->has_op, SCX_OPI_END); @@ -4834,6 +4906,13 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) } } scx_task_iter_stop(&sti); + + /* + * Enable the ext DL server on all online CPUs. + */ + for_each_online_cpu(cpu) + dl_server_on(cpu_rq(cpu), !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); + percpu_up_write(&scx_fork_rwsem);

scx_bypass(false);

From: Joel Fernandes joelagnelf@nvidia.com

Add a new kselftest to verify that the total_bw value in /sys/kernel/debug/sched/debug remains consistent across all CPUs under different sched_ext BPF program states:

1. Before a BPF scheduler is loaded 2. While a BPF scheduler is loaded and active 3. After a BPF scheduler is unloaded

The test runs CPU stress threads to ensure DL server bandwidth values stabilize before checking consistency. This helps catch potential issues with DL server bandwidth accounting during sched_ext transitions.

v2: - small coding style fixes (Andrea Righi)

Signed-off-by: Joel Fernandes joelagnelf@nvidia.com --- tools/testing/selftests/sched_ext/Makefile | 1 + tools/testing/selftests/sched_ext/total_bw.c | 281 +++++++++++++++++++ 2 files changed, 282 insertions(+) create mode 100644 tools/testing/selftests/sched_ext/total_bw.c

diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile index c9255d1499b6e..2c601a7eaff5f 100644 --- a/tools/testing/selftests/sched_ext/Makefile +++ b/tools/testing/selftests/sched_ext/Makefile @@ -185,6 +185,7 @@ auto-test-targets := \ select_cpu_vtime \ rt_stall \ test_example \ + total_bw \

testcase-targets := $(addsuffix .o,$(addprefix $(SCXOBJ_DIR)/,$(auto-test-targets)))

diff --git a/tools/testing/selftests/sched_ext/total_bw.c b/tools/testing/selftests/sched_ext/total_bw.c new file mode 100644 index 0000000000000..5b0a619bab86e --- /dev/null +++ b/tools/testing/selftests/sched_ext/total_bw.c @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test to verify that total_bw value remains consistent across all CPUs + * in different BPF program states. + * + * Copyright (C) 2025 NVIDIA Corporation. + */ +#include <bpf/bpf.h> +#include <errno.h> +#include <pthread.h> +#include <scx/common.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/wait.h> +#include <unistd.h> +#include "minimal.bpf.skel.h" +#include "scx_test.h" + +#define MAX_CPUS 512 +#define STRESS_DURATION_SEC 5 + +struct total_bw_ctx { + struct minimal *skel; + long baseline_bw[MAX_CPUS]; + int nr_cpus; +}; + +static void *cpu_stress_thread(void *arg) +{ + volatile int i; + time_t end_time = time(NULL) + STRESS_DURATION_SEC; + + while (time(NULL) < end_time) + for (i = 0; i < 1000000; i++) + ; + + return NULL; +} + +/* + * The first enqueue on a CPU causes the DL server to start, for that + * reason run stressor threads in the hopes it schedules on all CPUs. + */ +static int run_cpu_stress(int nr_cpus) +{ + pthread_t *threads; + int i, ret = 0; + + threads = calloc(nr_cpus, sizeof(pthread_t)); + if (!threads) + return -ENOMEM; + + /* Create threads to run on each CPU */ + for (i = 0; i < nr_cpus; i++) { + if (pthread_create(&threads[i], NULL, cpu_stress_thread, NULL)) { + ret = -errno; + fprintf(stderr, "Failed to create thread %d: %s\n", i, strerror(-ret)); + break; + } + } + + /* Wait for all threads to complete */ + for (i = 0; i < nr_cpus; i++) { + if (threads[i]) + pthread_join(threads[i], NULL); + } + + free(threads); + return ret; +} + +static int read_total_bw_values(long *bw_values, int max_cpus) +{ + FILE *fp; + char line[256]; + int cpu_count = 0; + + fp = fopen("/sys/kernel/debug/sched/debug", "r"); + if (!fp) { + SCX_ERR("Failed to open debug file"); + return -1; + } + + while (fgets(line, sizeof(line), fp)) { + char *bw_str = strstr(line, "total_bw"); + + if (bw_str) { + bw_str = strchr(bw_str, ':'); + if (bw_str) { + /* Only store up to max_cpus values */ + if (cpu_count < max_cpus) + bw_values[cpu_count] = atol(bw_str + 1); + cpu_count++; + } + } + } + + fclose(fp); + return cpu_count; +} + +static bool verify_total_bw_consistency(long *bw_values, int count) +{ + int i; + long first_value; + + if (count <= 0) + return false; + + first_value = bw_values[0]; + + for (i = 1; i < count; i++) { + if (bw_values[i] != first_value) { + SCX_ERR("Inconsistent total_bw: CPU0=%ld, CPU%d=%ld", + first_value, i, bw_values[i]); + return false; + } + } + + return true; +} + +static int fetch_verify_total_bw(long *bw_values, int nr_cpus) +{ + int attempts = 0; + int max_attempts = 10; + int count; + + /* + * The first enqueue on a CPU causes the DL server to start, for that + * reason run stressor threads in the hopes it schedules on all CPUs. + */ + if (run_cpu_stress(nr_cpus) < 0) { + SCX_ERR("Failed to run CPU stress"); + return -1; + } + + /* Try multiple times to get stable values */ + while (attempts < max_attempts) { + count = read_total_bw_values(bw_values, nr_cpus); + fprintf(stderr, "Read %d total_bw values (testing %d CPUs)\n", count, nr_cpus); + /* If system has more CPUs than we're testing, that's OK */ + if (count < nr_cpus) { + SCX_ERR("Expected at least %d CPUs, got %d", nr_cpus, count); + attempts++; + sleep(1); + continue; + } + + /* Only verify the CPUs we're testing */ + if (verify_total_bw_consistency(bw_values, nr_cpus)) { + fprintf(stderr, "Values are consistent: %ld\n", bw_values[0]); + return 0; + } + + attempts++; + sleep(1); + } + + return -1; +} + +static enum scx_test_status setup(void **ctx) +{ + struct total_bw_ctx *test_ctx; + + if (access("/sys/kernel/debug/sched/debug", R_OK) != 0) { + fprintf(stderr, "Skipping test: debugfs sched/debug not accessible\n"); + return SCX_TEST_SKIP; + } + + test_ctx = calloc(1, sizeof(*test_ctx)); + if (!test_ctx) + return SCX_TEST_FAIL; + + test_ctx->nr_cpus = sysconf(_SC_NPROCESSORS_ONLN); + if (test_ctx->nr_cpus <= 0) { + free(test_ctx); + return SCX_TEST_FAIL; + } + + /* If system has more CPUs than MAX_CPUS, just test the first MAX_CPUS */ + if (test_ctx->nr_cpus > MAX_CPUS) + test_ctx->nr_cpus = MAX_CPUS; + + /* Test scenario 1: BPF program not loaded */ + /* Read and verify baseline total_bw before loading BPF program */ + fprintf(stderr, "BPF prog initially not loaded, reading total_bw values\n"); + if (fetch_verify_total_bw(test_ctx->baseline_bw, test_ctx->nr_cpus) < 0) { + SCX_ERR("Failed to get stable baseline values"); + free(test_ctx); + return SCX_TEST_FAIL; + } + + /* Load the BPF skeleton */ + test_ctx->skel = minimal__open(); + if (!test_ctx->skel) { + free(test_ctx); + return SCX_TEST_FAIL; + } + + SCX_ENUM_INIT(test_ctx->skel); + if (minimal__load(test_ctx->skel)) { + minimal__destroy(test_ctx->skel); + free(test_ctx); + return SCX_TEST_FAIL; + } + + *ctx = test_ctx; + return SCX_TEST_PASS; +} + +static enum scx_test_status run(void *ctx) +{ + struct total_bw_ctx *test_ctx = ctx; + struct bpf_link *link; + long loaded_bw[MAX_CPUS]; + long unloaded_bw[MAX_CPUS]; + int i; + + /* Test scenario 2: BPF program loaded */ + link = bpf_map__attach_struct_ops(test_ctx->skel->maps.minimal_ops); + if (!link) { + SCX_ERR("Failed to attach scheduler"); + return SCX_TEST_FAIL; + } + + fprintf(stderr, "BPF program loaded, reading total_bw values\n"); + if (fetch_verify_total_bw(loaded_bw, test_ctx->nr_cpus) < 0) { + SCX_ERR("Failed to get stable values with BPF loaded"); + bpf_link__destroy(link); + return SCX_TEST_FAIL; + } + bpf_link__destroy(link); + + /* Test scenario 3: BPF program unloaded */ + fprintf(stderr, "BPF program unloaded, reading total_bw values\n"); + if (fetch_verify_total_bw(unloaded_bw, test_ctx->nr_cpus) < 0) { + SCX_ERR("Failed to get stable values after BPF unload"); + return SCX_TEST_FAIL; + } + + /* Verify all three scenarios have the same total_bw values */ + for (i = 0; i < test_ctx->nr_cpus; i++) { + if (test_ctx->baseline_bw[i] != loaded_bw[i]) { + SCX_ERR("CPU%d: baseline_bw=%ld != loaded_bw=%ld", + i, test_ctx->baseline_bw[i], loaded_bw[i]); + return SCX_TEST_FAIL; + } + + if (test_ctx->baseline_bw[i] != unloaded_bw[i]) { + SCX_ERR("CPU%d: baseline_bw=%ld != unloaded_bw=%ld", + i, test_ctx->baseline_bw[i], unloaded_bw[i]); + return SCX_TEST_FAIL; + } + } + + fprintf(stderr, "All total_bw values are consistent across all scenarios\n"); + return SCX_TEST_PASS; +} + +static void cleanup(void *ctx) +{ + struct total_bw_ctx *test_ctx = ctx; + + if (test_ctx) { + if (test_ctx->skel) + minimal__destroy(test_ctx->skel); + free(test_ctx); + } +} + +struct scx_test total_bw = { + .name = "total_bw", + .description = "Verify total_bw consistency across BPF program states", + .setup = setup, + .run = run, + .cleanup = cleanup, +}; +REGISTER_SCX_TEST(&total_bw)