From: Vikram Mulukutla markivx@codeaurora.org
Actual frequency on X86 may be a function of the number of non-idle CPUs, the highest frequency amongst the cpufreq policies corresponding to the CPUs, and whether a turbo-boost frequency has been selected. For example, if cpu0 has voted for 2GHz and cpu1 for 1GHz, cpu1 can also run at 2GHz for as long as cpu0 is not idle.
This can result in somewhat inaccurate load tracking when a load statistic needs to be frequency invariant. To address this, calculate the current frequency on supported Intel X86 machines using the PMC counters. As far as I can tell, these counters can't be read by a remote CPU, and IPIs would be too expensive, so try to keep the frequency updated as often as possible on a CPU.
This is a temporary/hack patch included for completeness and not intended for merging. Your machine may crash if the rdpmc instruction is unsupported!
Signed-off-by: Vikram Mulukutla markivx@codeaurora.org --- kernel/sched/core.c | 1 + kernel/sched/sched.h | 3 +++ kernel/sched/walt.c | 42 +++++++++++++++++++++++++++++++++++++++++- kernel/sched/walt.h | 2 ++ 4 files changed, 47 insertions(+), 1 deletion(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 068bde8..250014c 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -111,6 +111,7 @@ void update_rq_clock(struct rq *rq) return; rq->clock += delta; update_rq_clock_task(rq, delta); + walt_update_cpu_freq(rq); }
/* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 52a0ac5..e6acd66 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -666,6 +666,9 @@ struct rq { #ifdef CONFIG_SCHED_WALT unsigned int cur_freq; struct cpumask freq_domain_cpumask; + u64 cycle_update_time; + unsigned long cycles; + unsigned int cur_freq_hw;
int efficiency; /* Differentiate cpus with different IPC capability */ u64 window_start; diff --git a/kernel/sched/walt.c b/kernel/sched/walt.c index 203e02d..ccea3f4 100644 --- a/kernel/sched/walt.c +++ b/kernel/sched/walt.c @@ -125,9 +125,48 @@ update_window_start(struct rq *rq, u64 wallclock) rq->window_start += (u64)nr_windows * (u64)walt_ravg_window; }
+unsigned long rdpmc_actual_cycles(void) +{ + unsigned a, d, c; + c = (1<<30)+1; + __asm__ volatile("rdpmc" : "=a" (a), "=d" (d) : "c" (c)); + return ((unsigned long)a) | (((unsigned long)d) << 32);; +} + +static int start_hw_freq_update; + +inline void walt_update_cpu_freq(struct rq *rq) +{ + u64 cur_time; + unsigned long cur_cycles; + + if (unlikely(!start_hw_freq_update)) + return; + + if (cpu_of(rq) != smp_processor_id()) + return; + + cur_time = sched_clock_cpu(cpu_of(rq)); + cur_cycles = rdpmc_actual_cycles(); + + rq->cur_freq_hw = ((cur_cycles - rq->cycles) * 1000 * 1000)/ + (cur_time - rq->cycle_update_time); + + rq->cycle_update_time = cur_time; + rq->cycles = cur_cycles; +} + +static int __init walt_init_late(void) +{ + start_hw_freq_update = 1; + return 0; +} +late_initcall(walt_init_late); + static u64 scale_exec_time(u64 delta, struct rq *rq) { - unsigned int cur_freq = rq->cur_freq; + unsigned int cur_freq = rq->cur_freq_hw > rq->cur_freq ? rq->cur_freq_hw + : rq->cur_freq; int sf;
/* round up div64 */ @@ -285,6 +324,7 @@ void walt_update_task_ravg(struct task_struct *p, struct rq *rq, if (!p->ravg.mark_start) goto done;
+ walt_update_cpu_freq(rq); update_cpu_busy_time(p, rq, event, wallclock, irqtime);
done: diff --git a/kernel/sched/walt.h b/kernel/sched/walt.h index 5b03995..555192d 100644 --- a/kernel/sched/walt.h +++ b/kernel/sched/walt.h @@ -16,6 +16,7 @@
#ifdef CONFIG_SCHED_WALT
+void walt_update_cpu_freq(struct rq *rq); void walt_update_task_ravg(struct task_struct *p, struct rq *rq, enum task_event event, u64 wallclock, u64 irqtime); void walt_fixup_busy_time(struct task_struct *p, int new_cpu); @@ -47,6 +48,7 @@ static inline unsigned long cpu_walt_util(struct rq *rq)
#else /* CONFIG_SCHED_WALT */
+static inline void walt_update_cpu_freq(struct rq *rq) { }; static inline void walt_update_task_ravg(struct task_struct *p, struct rq *rq, int event, u64 wallclock, u64 irqtime) { } static inline void walt_fixup_busy_time(struct task_struct *p, int new_cpu) { } -- The Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum, a Linux Foundation Collaborative Project