Re: [PATCH 2/3] sched: hmp: unify active migration code

-- Tixy > kernel/sched/fair.c | 188 +++++++++++---------------------------------------- > 1 file changed, 40 insertions(+), 148 deletions(-) > > diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c > index 1b9be59..a5cae4a 100644 > --- a/kernel/sched/fair.c > +++ b/kernel/sched/fair.c > @@ -6214,6 +6214,7 @@ out: > #ifdef CONFIG_SCHED_HMP > static unsigned int hmp_idle_pull(int this_cpu); > #endif > +static int move_specific_task(struct lb_env *env, struct task_struct *pm); > /* > * idle_balance is called by schedule() if this_cpu is about to become > * idle. Attempts to pull tasks from other CPUs. > @@ -6273,21 +6274,17 @@ void idle_balance(int this_cpu, struct rq *this_rq) > } > } > > -/* > - * active_load_balance_cpu_stop is run by cpu stopper. It pushes > - * running tasks off the busiest CPU onto idle CPUs. It requires at > - * least 1 task to be running on each physical CPU where possible, and > - * avoids physical / logical imbalances. > - */ > -static int active_load_balance_cpu_stop(void *data) > +static int __do_active_load_balance_cpu_stop(void *data, bool check_sd_lb_flag) > { > struct rq *busiest_rq = data; > int busiest_cpu = cpu_of(busiest_rq); > int target_cpu = busiest_rq->push_cpu; > struct rq *target_rq = cpu_rq(target_cpu); > struct sched_domain *sd; > + struct task_struct *p; > > raw_spin_lock_irq(&busiest_rq->lock); > + p = busiest_rq->migrate_task; > > /* make sure the requested cpu hasn't gone down in the meantime */ > if (unlikely(busiest_cpu != smp_processor_id() || > @@ -6298,6 +6295,11 @@ static int active_load_balance_cpu_stop(void *data) > if (busiest_rq->nr_running <= 1) > goto out_unlock; > > + if (!check_sd_lb_flag) { > + /* Task has migrated meanwhile, abort forced migration */ > + if (task_rq(p) != busiest_rq) > + goto out_unlock; > + } > /* > * This condition is "impossible", if it occurs > * we need to fix it. Originally reported by > @@ -6311,12 +6313,14 @@ static int active_load_balance_cpu_stop(void *data) > /* Search for an sd spanning us and the target CPU. */ > rcu_read_lock(); > for_each_domain(target_cpu, sd) { > - if ((sd->flags & SD_LOAD_BALANCE) && > - cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) > + if (((check_sd_lb_flag && sd->flags & SD_LOAD_BALANCE) || > + !check_sd_lb_flag) && > + cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) > break; > } > > if (likely(sd)) { > + bool success = false; > struct lb_env env = { > .sd = sd, > .dst_cpu = target_cpu, > @@ -6328,7 +6332,14 @@ static int active_load_balance_cpu_stop(void *data) > changes > schedstat_inc(sd, alb_count); > > - if (move_one_task(&env)) > + if (check_sd_lb_flag) { > + if (move_one_task(&env)) > + success = true; > + } else { > + if (move_specific_task(&env, p)) > + success = true; > + } > + if (success) > schedstat_inc(sd, alb_pushed); > else > schedstat_inc(sd, alb_failed); > @@ -6336,11 +6347,24 @@ static int active_load_balance_cpu_stop(void *data) > rcu_read_unlock(); > double_unlock_balance(busiest_rq, target_rq); > out_unlock: > + if (!check_sd_lb_flag) > + put_task_struct(p); > busiest_rq->active_balance = 0; > raw_spin_unlock_irq(&busiest_rq->lock); > return 0; > } > > +/* > + * active_load_balance_cpu_stop is run by cpu stopper. It pushes > + * running tasks off the busiest CPU onto idle CPUs. It requires at > + * least 1 task to be running on each physical CPU where possible, and > + * avoids physical / logical imbalances. > + */ > +static int active_load_balance_cpu_stop(void *data) > +{ > + return __do_active_load_balance_cpu_stop(data, true); > +} > + > #ifdef CONFIG_NO_HZ_COMMON > /* > * idle load balancing details > @@ -6900,151 +6924,19 @@ static int move_specific_task(struct lb_env *env, struct task_struct *pm) > * hmp_active_task_migration_cpu_stop is run by cpu stopper and used to > * migrate a specific task from one runqueue to another. > * hmp_force_up_migration uses this to push a currently running task > - * off a runqueue. > - * Based on active_load_balance_stop_cpu and can potentially be merged. > + * off a runqueue. hmp_idle_pull uses this to pull a currently > + * running task to an idle runqueue. > + * Reuses __do_active_load_balance_cpu_stop to actually do the work. > */ > static int hmp_active_task_migration_cpu_stop(void *data) > { > - struct rq *busiest_rq = data; > - struct task_struct *p = busiest_rq->migrate_task; > - int busiest_cpu = cpu_of(busiest_rq); > - int target_cpu = busiest_rq->push_cpu; > - struct rq *target_rq = cpu_rq(target_cpu); > - struct sched_domain *sd; > - > - raw_spin_lock_irq(&busiest_rq->lock); > - /* make sure the requested cpu hasn't gone down in the meantime */ > - if (unlikely(busiest_cpu != smp_processor_id() || > - !busiest_rq->active_balance)) { > - goto out_unlock; > - } > - /* Is there any task to move? */ > - if (busiest_rq->nr_running <= 1) > - goto out_unlock; > - /* Task has migrated meanwhile, abort forced migration */ > - if (task_rq(p) != busiest_rq) > - goto out_unlock; > - /* > - * This condition is "impossible", if it occurs > - * we need to fix it. Originally reported by > - * Bjorn Helgaas on a 128-cpu setup. > - */ > - BUG_ON(busiest_rq == target_rq); > - > - /* move a task from busiest_rq to target_rq */ > - double_lock_balance(busiest_rq, target_rq); > - > - /* Search for an sd spanning us and the target CPU. */ > - rcu_read_lock(); > - for_each_domain(target_cpu, sd) { > - if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) > - break; > - } > - > - if (likely(sd)) { > - struct lb_env env = { > - .sd = sd, > - .dst_cpu = target_cpu, > - .dst_rq = target_rq, > - .src_cpu = busiest_rq->cpu, > - .src_rq = busiest_rq, > - .idle = CPU_IDLE, > - }; > - > - schedstat_inc(sd, alb_count); > - > - if (move_specific_task(&env, p)) > - schedstat_inc(sd, alb_pushed); > - else > - schedstat_inc(sd, alb_failed); > - } > - rcu_read_unlock(); > - double_unlock_balance(busiest_rq, target_rq); > -out_unlock: > - put_task_struct(p); > - busiest_rq->active_balance = 0; > - raw_spin_unlock_irq(&busiest_rq->lock); > - return 0; > -} > - > -/* > - * hmp_idle_pull_cpu_stop is run by cpu stopper and used to > - * migrate a specific task from one runqueue to another. > - * hmp_idle_pull uses this to push a currently running task > - * off a runqueue to a faster CPU. > - * Locking is slightly different than usual. > - * Based on active_load_balance_stop_cpu and can potentially be merged. > - */ > -static int hmp_idle_pull_cpu_stop(void *data) > -{ > - struct rq *busiest_rq = data; > - struct task_struct *p = busiest_rq->migrate_task; > - int busiest_cpu = cpu_of(busiest_rq); > - int target_cpu = busiest_rq->push_cpu; > - struct rq *target_rq = cpu_rq(target_cpu); > - struct sched_domain *sd; > - > - raw_spin_lock_irq(&busiest_rq->lock); > - > - /* make sure the requested cpu hasn't gone down in the meantime */ > - if (unlikely(busiest_cpu != smp_processor_id() || > - !busiest_rq->active_balance)) > - goto out_unlock; > - > - /* Is there any task to move? */ > - if (busiest_rq->nr_running <= 1) > - goto out_unlock; > - > - /* Task has migrated meanwhile, abort forced migration */ > - if (task_rq(p) != busiest_rq) > - goto out_unlock; > - > - /* > - * This condition is "impossible", if it occurs > - * we need to fix it. Originally reported by > - * Bjorn Helgaas on a 128-cpu setup. > - */ > - BUG_ON(busiest_rq == target_rq); > - > - /* move a task from busiest_rq to target_rq */ > - double_lock_balance(busiest_rq, target_rq); > - > - /* Search for an sd spanning us and the target CPU. */ > - rcu_read_lock(); > - for_each_domain(target_cpu, sd) { > - if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) > - break; > - } > - if (likely(sd)) { > - struct lb_env env = { > - .sd = sd, > - .dst_cpu = target_cpu, > - .dst_rq = target_rq, > - .src_cpu = busiest_rq->cpu, > - .src_rq = busiest_rq, > - .idle = CPU_IDLE, > - }; > - > - schedstat_inc(sd, alb_count); > - > - if (move_specific_task(&env, p)) > - schedstat_inc(sd, alb_pushed); > - else > - schedstat_inc(sd, alb_failed); > - } > - rcu_read_unlock(); > - double_unlock_balance(busiest_rq, target_rq); > -out_unlock: > - put_task_struct(p); > - busiest_rq->active_balance = 0; > - raw_spin_unlock_irq(&busiest_rq->lock); > - return 0; > + return __do_active_load_balance_cpu_stop(data, false); > } > > /* > * Move task in a runnable state to another CPU. > * > - * Tailored on 'active_load_balance_stop_cpu' with slight > + * Tailored on 'active_load_balance_cpu_stop' with slight > * modification to locking and pre-transfer checks. Note > * rq->lock must be held before calling. > */ > @@ -7284,7 +7176,7 @@ static unsigned int hmp_idle_pull(int this_cpu) > > if (force) { > stop_one_cpu_nowait(cpu_of(target), > - hmp_idle_pull_cpu_stop, > + hmp_active_task_migration_cpu_stop, > target, &target->active_balance_work); > } > done: