From: Nathan Zimmer nzimmer@sgi.com

This eliminates the contention I am seeing in __cpufreq_cpu_get. It also nicely stages the lock to be replaced by the rcu.

Signed-off-by: Nathan Zimmer nzimmer@sgi.com Acked-by: Viresh Kumar viresh.kumar@linaro.org Signed-off-by: Rafael J. Wysocki rafael.j.wysocki@intel.com --- drivers/cpufreq/cpufreq.c | 52 +++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index b02824d..c5996fe 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -45,7 +45,7 @@ static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data); /* This one keeps track of the previously set governor of a removed CPU */ static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor); #endif -static DEFINE_SPINLOCK(cpufreq_driver_lock); +static DEFINE_RWLOCK(cpufreq_driver_lock);

/* * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure @@ -137,7 +137,7 @@ static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs) goto err_out;

/* get the cpufreq driver */ - spin_lock_irqsave(&cpufreq_driver_lock, flags); + read_lock_irqsave(&cpufreq_driver_lock, flags);

if (!cpufreq_driver) goto err_out_unlock; @@ -155,13 +155,13 @@ static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs) if (!sysfs && !kobject_get(&data->kobj)) goto err_out_put_module;

- spin_unlock_irqrestore(&cpufreq_driver_lock, flags); + read_unlock_irqrestore(&cpufreq_driver_lock, flags); return data;

err_out_put_module: module_put(cpufreq_driver->owner); err_out_unlock: - spin_unlock_irqrestore(&cpufreq_driver_lock, flags); + read_unlock_irqrestore(&cpufreq_driver_lock, flags); err_out: return NULL; } @@ -266,9 +266,9 @@ void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state) pr_debug("notification %u of frequency transition to %u kHz\n", state, freqs->new);

- spin_lock_irqsave(&cpufreq_driver_lock, flags); + read_lock_irqsave(&cpufreq_driver_lock, flags); policy = per_cpu(cpufreq_cpu_data, freqs->cpu); - spin_unlock_irqrestore(&cpufreq_driver_lock, flags); + read_unlock_irqrestore(&cpufreq_driver_lock, flags);

switch (state) {

@@ -765,12 +765,12 @@ static int cpufreq_add_dev_interface(unsigned int cpu, goto err_out_kobj_put; }

- spin_lock_irqsave(&cpufreq_driver_lock, flags); + write_lock_irqsave(&cpufreq_driver_lock, flags); for_each_cpu(j, policy->cpus) { per_cpu(cpufreq_cpu_data, j) = policy; per_cpu(cpufreq_policy_cpu, j) = policy->cpu; } - spin_unlock_irqrestore(&cpufreq_driver_lock, flags); + write_unlock_irqrestore(&cpufreq_driver_lock, flags);

ret = cpufreq_add_dev_symlink(cpu, policy); if (ret) @@ -813,12 +813,12 @@ static int cpufreq_add_policy_cpu(unsigned int cpu, unsigned int sibling,

lock_policy_rwsem_write(sibling);

- spin_lock_irqsave(&cpufreq_driver_lock, flags); + write_lock_irqsave(&cpufreq_driver_lock, flags);

cpumask_set_cpu(cpu, policy->cpus); per_cpu(cpufreq_policy_cpu, cpu) = policy->cpu; per_cpu(cpufreq_cpu_data, cpu) = policy; - spin_unlock_irqrestore(&cpufreq_driver_lock, flags); + write_unlock_irqrestore(&cpufreq_driver_lock, flags);

unlock_policy_rwsem_write(sibling);

@@ -871,15 +871,15 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)

#ifdef CONFIG_HOTPLUG_CPU /* Check if this cpu was hot-unplugged earlier and has siblings */ - spin_lock_irqsave(&cpufreq_driver_lock, flags); + read_lock_irqsave(&cpufreq_driver_lock, flags); for_each_online_cpu(sibling) { struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling); if (cp && cpumask_test_cpu(cpu, cp->related_cpus)) { - spin_unlock_irqrestore(&cpufreq_driver_lock, flags); + read_unlock_irqrestore(&cpufreq_driver_lock, flags); return cpufreq_add_policy_cpu(cpu, sibling, dev); } } - spin_unlock_irqrestore(&cpufreq_driver_lock, flags); + read_unlock_irqrestore(&cpufreq_driver_lock, flags); #endif #endif

@@ -952,10 +952,10 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) return 0;

err_out_unregister: - spin_lock_irqsave(&cpufreq_driver_lock, flags); + write_lock_irqsave(&cpufreq_driver_lock, flags); for_each_cpu(j, policy->cpus) per_cpu(cpufreq_cpu_data, j) = NULL; - spin_unlock_irqrestore(&cpufreq_driver_lock, flags); + write_unlock_irqrestore(&cpufreq_driver_lock, flags);

kobject_put(&policy->kobj); wait_for_completion(&policy->kobj_unregister); @@ -1008,12 +1008,12 @@ static int __cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif

pr_debug("%s: unregistering CPU %u\n", __func__, cpu);

- spin_lock_irqsave(&cpufreq_driver_lock, flags); + write_lock_irqsave(&cpufreq_driver_lock, flags);

data = per_cpu(cpufreq_cpu_data, cpu); per_cpu(cpufreq_cpu_data, cpu) = NULL;

- spin_unlock_irqrestore(&cpufreq_driver_lock, flags); + write_unlock_irqrestore(&cpufreq_driver_lock, flags);

if (!data) { pr_debug("%s: No cpu_data found\n", __func__); @@ -1047,9 +1047,9 @@ static int __cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif WARN_ON(lock_policy_rwsem_write(cpu)); cpumask_set_cpu(cpu, data->cpus);

- spin_lock_irqsave(&cpufreq_driver_lock, flags); + write_lock_irqsave(&cpufreq_driver_lock, flags); per_cpu(cpufreq_cpu_data, cpu) = data; - spin_unlock_irqrestore(&cpufreq_driver_lock, flags); + write_unlock_irqrestore(&cpufreq_driver_lock, flags);

unlock_policy_rwsem_write(cpu);

@@ -1848,13 +1848,13 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) if (driver_data->setpolicy) driver_data->flags |= CPUFREQ_CONST_LOOPS;

- spin_lock_irqsave(&cpufreq_driver_lock, flags); + write_lock_irqsave(&cpufreq_driver_lock, flags); if (cpufreq_driver) { - spin_unlock_irqrestore(&cpufreq_driver_lock, flags); + write_unlock_irqrestore(&cpufreq_driver_lock, flags); return -EBUSY; } cpufreq_driver = driver_data; - spin_unlock_irqrestore(&cpufreq_driver_lock, flags); + write_unlock_irqrestore(&cpufreq_driver_lock, flags);

ret = subsys_interface_register(&cpufreq_interface); if (ret) @@ -1886,9 +1886,9 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) err_if_unreg: subsys_interface_unregister(&cpufreq_interface); err_null_driver: - spin_lock_irqsave(&cpufreq_driver_lock, flags); + write_lock_irqsave(&cpufreq_driver_lock, flags); cpufreq_driver = NULL; - spin_unlock_irqrestore(&cpufreq_driver_lock, flags); + write_unlock_irqrestore(&cpufreq_driver_lock, flags); return ret; } EXPORT_SYMBOL_GPL(cpufreq_register_driver); @@ -1914,9 +1914,9 @@ int cpufreq_unregister_driver(struct cpufreq_driver *driver) subsys_interface_unregister(&cpufreq_interface); unregister_hotcpu_notifier(&cpufreq_cpu_notifier);

- spin_lock_irqsave(&cpufreq_driver_lock, flags); + write_lock_irqsave(&cpufreq_driver_lock, flags); cpufreq_driver = NULL; - spin_unlock_irqrestore(&cpufreq_driver_lock, flags); + write_unlock_irqrestore(&cpufreq_driver_lock, flags);

return 0; }

Currently, there can't be multiple instances of single governor_type. If we have a multi-package system, where we have multiple instances of struct policy (per package), we can't have multiple instances of same governor. i.e. We can't have multiple instances of ondemand governor for multiple packages.

Governors directory in sysfs is created at /sys/devices/system/cpu/cpufreq/ governor-name/. Which again reflects that there can be only one instance of a governor_type in the system.

This is a bottleneck for multicluster system, where we want different packages to use same governor type, but with different tunables.

This patch uses the infrastructure provided by earlier patch and implements init/exit routines for ondemand and conservative governors.

Signed-off-by: Viresh Kumar viresh.kumar@linaro.org Signed-off-by: Rafael J. Wysocki rafael.j.wysocki@intel.com --- drivers/cpufreq/cpufreq.c | 15 +- drivers/cpufreq/cpufreq_conservative.c | 193 ++++++++++++++---------- drivers/cpufreq/cpufreq_governor.c | 212 +++++++++++++++++--------- drivers/cpufreq/cpufreq_governor.h | 117 +++++++++++++-- drivers/cpufreq/cpufreq_ondemand.c | 263 ++++++++++++++++++++------------- include/linux/cpufreq.h | 8 + 6 files changed, 538 insertions(+), 270 deletions(-)

diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 08df7a1..85963fc 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -128,6 +128,11 @@ void disable_cpufreq(void) static LIST_HEAD(cpufreq_governor_list); static DEFINE_MUTEX(cpufreq_governor_mutex);

+bool have_governor_per_policy(void) +{ + return cpufreq_driver->have_governor_per_policy; +} + static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs) { struct cpufreq_policy *data; @@ -1546,10 +1551,12 @@ static int __cpufreq_governor(struct cpufreq_policy *policy, policy->cpu, event); ret = policy->governor->governor(policy, event);

- if (event == CPUFREQ_GOV_START) - policy->governor->initialized++; - else if (event == CPUFREQ_GOV_STOP) - policy->governor->initialized--; + if (!ret) { + if (event == CPUFREQ_GOV_POLICY_INIT) + policy->governor->initialized++; + else if (event == CPUFREQ_GOV_POLICY_EXIT) + policy->governor->initialized--; + }

/* we keep one module reference alive for each CPU governed by this CPU */ diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c index 4fd0006..98b4946 100644 --- a/drivers/cpufreq/cpufreq_conservative.c +++ b/drivers/cpufreq/cpufreq_conservative.c @@ -20,6 +20,7 @@ #include <linux/mutex.h> #include <linux/notifier.h> #include <linux/percpu-defs.h> +#include <linux/slab.h> #include <linux/sysfs.h> #include <linux/types.h>

@@ -31,17 +32,8 @@ #define DEF_SAMPLING_DOWN_FACTOR (1) #define MAX_SAMPLING_DOWN_FACTOR (10)

-static struct dbs_data cs_dbs_data; static DEFINE_PER_CPU(struct cs_cpu_dbs_info_s, cs_cpu_dbs_info);

-static struct cs_dbs_tuners cs_tuners = { - .up_threshold = DEF_FREQUENCY_UP_THRESHOLD, - .down_threshold = DEF_FREQUENCY_DOWN_THRESHOLD, - .sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR, - .ignore_nice = 0, - .freq_step = 5, -}; - /* * Every sampling_rate, we check, if current idle time is less than 20% * (default), then we try to increase frequency Every sampling_rate * @@ -55,24 +47,26 @@ static void cs_check_cpu(int cpu, unsigned int load) { struct cs_cpu_dbs_info_s *dbs_info = &per_cpu(cs_cpu_dbs_info, cpu); struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy; + struct dbs_data *dbs_data = policy->governor_data; + struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; unsigned int freq_target;

/* * break out if we 'cannot' reduce the speed as the user might * want freq_step to be zero */ - if (cs_tuners.freq_step == 0) + if (cs_tuners->freq_step == 0) return;

/* Check for frequency increase */ - if (load > cs_tuners.up_threshold) { + if (load > cs_tuners->up_threshold) { dbs_info->down_skip = 0;

/* if we are already at full speed then break out early */ if (dbs_info->requested_freq == policy->max) return;

- freq_target = (cs_tuners.freq_step * policy->max) / 100; + freq_target = (cs_tuners->freq_step * policy->max) / 100;

/* max freq cannot be less than 100. But who knows.... */ if (unlikely(freq_target == 0)) @@ -92,8 +86,8 @@ static void cs_check_cpu(int cpu, unsigned int load) * support the current CPU usage without triggering the up policy. To be * safe, we focus 10 points under the threshold. */ - if (load < (cs_tuners.down_threshold - 10)) { - freq_target = (cs_tuners.freq_step * policy->max) / 100; + if (load < (cs_tuners->down_threshold - 10)) { + freq_target = (cs_tuners->freq_step * policy->max) / 100;

dbs_info->requested_freq -= freq_target; if (dbs_info->requested_freq < policy->min) @@ -119,11 +113,13 @@ static void cs_dbs_timer(struct work_struct *work) unsigned int cpu = dbs_info->cdbs.cur_policy->cpu; struct cs_cpu_dbs_info_s *core_dbs_info = &per_cpu(cs_cpu_dbs_info, cpu); - int delay = delay_for_sampling_rate(cs_tuners.sampling_rate); + struct dbs_data *dbs_data = dbs_info->cdbs.cur_policy->governor_data; + struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; + int delay = delay_for_sampling_rate(cs_tuners->sampling_rate);

mutex_lock(&core_dbs_info->cdbs.timer_mutex); - if (need_load_eval(&core_dbs_info->cdbs, cs_tuners.sampling_rate)) - dbs_check_cpu(&cs_dbs_data, cpu); + if (need_load_eval(&core_dbs_info->cdbs, cs_tuners->sampling_rate)) + dbs_check_cpu(dbs_data, cpu);

schedule_delayed_work_on(smp_processor_id(), dw, delay); mutex_unlock(&core_dbs_info->cdbs.timer_mutex); @@ -154,16 +150,12 @@ static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val, }

/************************** sysfs interface ************************/ -static ssize_t show_sampling_rate_min(struct kobject *kobj, - struct attribute *attr, char *buf) -{ - return sprintf(buf, "%u\n", cs_dbs_data.min_sampling_rate); -} +static struct common_dbs_data cs_dbs_cdata;

-static ssize_t store_sampling_down_factor(struct kobject *a, - struct attribute *b, - const char *buf, size_t count) +static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data, + const char *buf, size_t count) { + struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; unsigned int input; int ret; ret = sscanf(buf, "%u", &input); @@ -171,13 +163,14 @@ static ssize_t store_sampling_down_factor(struct kobject *a, if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1) return -EINVAL;

- cs_tuners.sampling_down_factor = input; + cs_tuners->sampling_down_factor = input; return count; }

-static ssize_t store_sampling_rate(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_sampling_rate(struct dbs_data *dbs_data, const char *buf, + size_t count) { + struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; unsigned int input; int ret; ret = sscanf(buf, "%u", &input); @@ -185,43 +178,46 @@ static ssize_t store_sampling_rate(struct kobject *a, struct attribute *b, if (ret != 1) return -EINVAL;

- cs_tuners.sampling_rate = max(input, cs_dbs_data.min_sampling_rate); + cs_tuners->sampling_rate = max(input, dbs_data->min_sampling_rate); return count; }

-static ssize_t store_up_threshold(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_up_threshold(struct dbs_data *dbs_data, const char *buf, + size_t count) { + struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; unsigned int input; int ret; ret = sscanf(buf, "%u", &input);

- if (ret != 1 || input > 100 || input <= cs_tuners.down_threshold) + if (ret != 1 || input > 100 || input <= cs_tuners->down_threshold) return -EINVAL;

- cs_tuners.up_threshold = input; + cs_tuners->up_threshold = input; return count; }

-static ssize_t store_down_threshold(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_down_threshold(struct dbs_data *dbs_data, const char *buf, + size_t count) { + struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; unsigned int input; int ret; ret = sscanf(buf, "%u", &input);

/* cannot be lower than 11 otherwise freq will not fall */ if (ret != 1 || input < 11 || input > 100 || - input >= cs_tuners.up_threshold) + input >= cs_tuners->up_threshold) return -EINVAL;

- cs_tuners.down_threshold = input; + cs_tuners->down_threshold = input; return count; }

-static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_ignore_nice(struct dbs_data *dbs_data, const char *buf, + size_t count) { + struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; unsigned int input, j; int ret;

@@ -232,10 +228,10 @@ static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b, if (input > 1) input = 1;

- if (input == cs_tuners.ignore_nice) /* nothing to do */ + if (input == cs_tuners->ignore_nice) /* nothing to do */ return count;

- cs_tuners.ignore_nice = input; + cs_tuners->ignore_nice = input;

/* we need to re-evaluate prev_cpu_idle */ for_each_online_cpu(j) { @@ -243,16 +239,17 @@ static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b, dbs_info = &per_cpu(cs_cpu_dbs_info, j); dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j, &dbs_info->cdbs.prev_cpu_wall); - if (cs_tuners.ignore_nice) + if (cs_tuners->ignore_nice) dbs_info->cdbs.prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE]; } return count; }

-static ssize_t store_freq_step(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_freq_step(struct dbs_data *dbs_data, const char *buf, + size_t count) { + struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; unsigned int input; int ret; ret = sscanf(buf, "%u", &input); @@ -267,43 +264,88 @@ static ssize_t store_freq_step(struct kobject *a, struct attribute *b, * no need to test here if freq_step is zero as the user might actually * want this, they would be crazy though :) */ - cs_tuners.freq_step = input; + cs_tuners->freq_step = input; return count; }

-show_one(cs, sampling_rate, sampling_rate); -show_one(cs, sampling_down_factor, sampling_down_factor); -show_one(cs, up_threshold, up_threshold); -show_one(cs, down_threshold, down_threshold); -show_one(cs, ignore_nice_load, ignore_nice); -show_one(cs, freq_step, freq_step); - -define_one_global_rw(sampling_rate); -define_one_global_rw(sampling_down_factor); -define_one_global_rw(up_threshold); -define_one_global_rw(down_threshold); -define_one_global_rw(ignore_nice_load); -define_one_global_rw(freq_step); -define_one_global_ro(sampling_rate_min); - -static struct attribute *dbs_attributes[] = { - &sampling_rate_min.attr, - &sampling_rate.attr, - &sampling_down_factor.attr, - &up_threshold.attr, - &down_threshold.attr, - &ignore_nice_load.attr, - &freq_step.attr, +show_store_one(cs, sampling_rate); +show_store_one(cs, sampling_down_factor); +show_store_one(cs, up_threshold); +show_store_one(cs, down_threshold); +show_store_one(cs, ignore_nice); +show_store_one(cs, freq_step); +declare_show_sampling_rate_min(cs); + +gov_sys_pol_attr_rw(sampling_rate); +gov_sys_pol_attr_rw(sampling_down_factor); +gov_sys_pol_attr_rw(up_threshold); +gov_sys_pol_attr_rw(down_threshold); +gov_sys_pol_attr_rw(ignore_nice); +gov_sys_pol_attr_rw(freq_step); +gov_sys_pol_attr_ro(sampling_rate_min); + +static struct attribute *dbs_attributes_gov_sys[] = { + &sampling_rate_min_gov_sys.attr, + &sampling_rate_gov_sys.attr, + &sampling_down_factor_gov_sys.attr, + &up_threshold_gov_sys.attr, + &down_threshold_gov_sys.attr, + &ignore_nice_gov_sys.attr, + &freq_step_gov_sys.attr, NULL };

-static struct attribute_group cs_attr_group = { - .attrs = dbs_attributes, +static struct attribute_group cs_attr_group_gov_sys = { + .attrs = dbs_attributes_gov_sys, + .name = "conservative", +}; + +static struct attribute *dbs_attributes_gov_pol[] = { + &sampling_rate_min_gov_pol.attr, + &sampling_rate_gov_pol.attr, + &sampling_down_factor_gov_pol.attr, + &up_threshold_gov_pol.attr, + &down_threshold_gov_pol.attr, + &ignore_nice_gov_pol.attr, + &freq_step_gov_pol.attr, + NULL +}; + +static struct attribute_group cs_attr_group_gov_pol = { + .attrs = dbs_attributes_gov_pol, .name = "conservative", };

/************************** sysfs end ************************/

+static int cs_init(struct dbs_data *dbs_data) +{ + struct cs_dbs_tuners *tuners; + + tuners = kzalloc(sizeof(struct cs_dbs_tuners), GFP_KERNEL); + if (!tuners) { + pr_err("%s: kzalloc failed\n", __func__); + return -ENOMEM; + } + + tuners->up_threshold = DEF_FREQUENCY_UP_THRESHOLD; + tuners->down_threshold = DEF_FREQUENCY_DOWN_THRESHOLD; + tuners->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR; + tuners->ignore_nice = 0; + tuners->freq_step = 5; + + dbs_data->tuners = tuners; + dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO * + jiffies_to_usecs(10); + mutex_init(&dbs_data->mutex); + return 0; +} + +static void cs_exit(struct dbs_data *dbs_data) +{ + kfree(dbs_data->tuners); +} + define_get_cpu_dbs_routines(cs_cpu_dbs_info);

static struct notifier_block cs_cpufreq_notifier_block = { @@ -314,21 +356,23 @@ static struct cs_ops cs_ops = { .notifier_block = &cs_cpufreq_notifier_block, };

-static struct dbs_data cs_dbs_data = { +static struct common_dbs_data cs_dbs_cdata = { .governor = GOV_CONSERVATIVE, - .attr_group = &cs_attr_group, - .tuners = &cs_tuners, + .attr_group_gov_sys = &cs_attr_group_gov_sys, + .attr_group_gov_pol = &cs_attr_group_gov_pol, .get_cpu_cdbs = get_cpu_cdbs, .get_cpu_dbs_info_s = get_cpu_dbs_info_s, .gov_dbs_timer = cs_dbs_timer, .gov_check_cpu = cs_check_cpu, .gov_ops = &cs_ops, + .init = cs_init, + .exit = cs_exit, };

static int cs_cpufreq_governor_dbs(struct cpufreq_policy *policy, unsigned int event) { - return cpufreq_governor_dbs(&cs_dbs_data, policy, event); + return cpufreq_governor_dbs(policy, &cs_dbs_cdata, event); }

#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE @@ -343,7 +387,6 @@ struct cpufreq_governor cpufreq_gov_conservative = {

static int __init cpufreq_gov_dbs_init(void) { - mutex_init(&cs_dbs_data.mutex); return cpufreq_register_governor(&cpufreq_gov_conservative); }

diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c index 5a76086..26fbb72 100644 --- a/drivers/cpufreq/cpufreq_governor.c +++ b/drivers/cpufreq/cpufreq_governor.c @@ -22,12 +22,29 @@ #include <linux/export.h> #include <linux/kernel_stat.h> #include <linux/mutex.h> +#include <linux/slab.h> #include <linux/tick.h> #include <linux/types.h> #include <linux/workqueue.h>

#include "cpufreq_governor.h"

+static struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy) +{ + if (have_governor_per_policy()) + return &policy->kobj; + else + return cpufreq_global_kobject; +} + +static struct attribute_group *get_sysfs_attr(struct dbs_data *dbs_data) +{ + if (have_governor_per_policy()) + return dbs_data->cdata->attr_group_gov_pol; + else + return dbs_data->cdata->attr_group_gov_sys; +} + static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall) { u64 idle_time; @@ -65,7 +82,7 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time);

void dbs_check_cpu(struct dbs_data *dbs_data, int cpu) { - struct cpu_dbs_common_info *cdbs = dbs_data->get_cpu_cdbs(cpu); + struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu); struct od_dbs_tuners *od_tuners = dbs_data->tuners; struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; struct cpufreq_policy *policy; @@ -73,7 +90,7 @@ void dbs_check_cpu(struct dbs_data *dbs_data, int cpu) unsigned int ignore_nice; unsigned int j;

- if (dbs_data->governor == GOV_ONDEMAND) + if (dbs_data->cdata->governor == GOV_ONDEMAND) ignore_nice = od_tuners->ignore_nice; else ignore_nice = cs_tuners->ignore_nice; @@ -87,7 +104,7 @@ void dbs_check_cpu(struct dbs_data *dbs_data, int cpu) unsigned int idle_time, wall_time, iowait_time; unsigned int load;

- j_cdbs = dbs_data->get_cpu_cdbs(j); + j_cdbs = dbs_data->cdata->get_cpu_cdbs(j);

cur_idle_time = get_cpu_idle_time(j, &cur_wall_time);

@@ -117,9 +134,9 @@ void dbs_check_cpu(struct dbs_data *dbs_data, int cpu) idle_time += jiffies_to_usecs(cur_nice_jiffies); }

- if (dbs_data->governor == GOV_ONDEMAND) { + if (dbs_data->cdata->governor == GOV_ONDEMAND) { struct od_cpu_dbs_info_s *od_j_dbs_info = - dbs_data->get_cpu_dbs_info_s(cpu); + dbs_data->cdata->get_cpu_dbs_info_s(cpu);

cur_iowait_time = get_cpu_iowait_time_us(j, &cur_wall_time); @@ -145,7 +162,7 @@ void dbs_check_cpu(struct dbs_data *dbs_data, int cpu)

load = 100 * (wall_time - idle_time) / wall_time;

- if (dbs_data->governor == GOV_ONDEMAND) { + if (dbs_data->cdata->governor == GOV_ONDEMAND) { int freq_avg = __cpufreq_driver_getavg(policy, j); if (freq_avg <= 0) freq_avg = policy->cur; @@ -157,7 +174,7 @@ void dbs_check_cpu(struct dbs_data *dbs_data, int cpu) max_load = load; }

- dbs_data->gov_check_cpu(cpu, max_load); + dbs_data->cdata->gov_check_cpu(cpu, max_load); } EXPORT_SYMBOL_GPL(dbs_check_cpu);

@@ -165,14 +182,14 @@ static inline void dbs_timer_init(struct dbs_data *dbs_data, int cpu, unsigned int sampling_rate) { int delay = delay_for_sampling_rate(sampling_rate); - struct cpu_dbs_common_info *cdbs = dbs_data->get_cpu_cdbs(cpu); + struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu);

schedule_delayed_work_on(cpu, &cdbs->work, delay); }

static inline void dbs_timer_exit(struct dbs_data *dbs_data, int cpu) { - struct cpu_dbs_common_info *cdbs = dbs_data->get_cpu_cdbs(cpu); + struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu);

cancel_delayed_work_sync(&cdbs->work); } @@ -196,31 +213,128 @@ bool need_load_eval(struct cpu_dbs_common_info *cdbs, } EXPORT_SYMBOL_GPL(need_load_eval);

-int cpufreq_governor_dbs(struct dbs_data *dbs_data, - struct cpufreq_policy *policy, unsigned int event) +static void set_sampling_rate(struct dbs_data *dbs_data, + unsigned int sampling_rate) +{ + if (dbs_data->cdata->governor == GOV_CONSERVATIVE) { + struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; + cs_tuners->sampling_rate = sampling_rate; + } else { + struct od_dbs_tuners *od_tuners = dbs_data->tuners; + od_tuners->sampling_rate = sampling_rate; + } +} + +int cpufreq_governor_dbs(struct cpufreq_policy *policy, + struct common_dbs_data *cdata, unsigned int event) { + struct dbs_data *dbs_data; struct od_cpu_dbs_info_s *od_dbs_info = NULL; struct cs_cpu_dbs_info_s *cs_dbs_info = NULL; - struct cs_ops *cs_ops = NULL; struct od_ops *od_ops = NULL; - struct od_dbs_tuners *od_tuners = dbs_data->tuners; - struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; + struct od_dbs_tuners *od_tuners = NULL; + struct cs_dbs_tuners *cs_tuners = NULL; struct cpu_dbs_common_info *cpu_cdbs; - unsigned int *sampling_rate, latency, ignore_nice, j, cpu = policy->cpu; + unsigned int sampling_rate, latency, ignore_nice, j, cpu = policy->cpu; int rc;

- cpu_cdbs = dbs_data->get_cpu_cdbs(cpu); + if (have_governor_per_policy()) + dbs_data = policy->governor_data; + else + dbs_data = cdata->gdbs_data; + + WARN_ON(!dbs_data && (event != CPUFREQ_GOV_POLICY_INIT)); + + switch (event) { + case CPUFREQ_GOV_POLICY_INIT: + if (have_governor_per_policy()) { + WARN_ON(dbs_data); + } else if (dbs_data) { + policy->governor_data = dbs_data; + return 0; + } + + dbs_data = kzalloc(sizeof(*dbs_data), GFP_KERNEL); + if (!dbs_data) { + pr_err("%s: POLICY_INIT: kzalloc failed\n", __func__); + return -ENOMEM; + } + + dbs_data->cdata = cdata; + rc = cdata->init(dbs_data); + if (rc) { + pr_err("%s: POLICY_INIT: init() failed\n", __func__); + kfree(dbs_data); + return rc; + } + + rc = sysfs_create_group(get_governor_parent_kobj(policy), + get_sysfs_attr(dbs_data)); + if (rc) { + cdata->exit(dbs_data); + kfree(dbs_data); + return rc; + } + + policy->governor_data = dbs_data; + + /* policy latency is in nS. Convert it to uS first */ + latency = policy->cpuinfo.transition_latency / 1000; + if (latency == 0) + latency = 1; + + /* Bring kernel and HW constraints together */ + dbs_data->min_sampling_rate = max(dbs_data->min_sampling_rate, + MIN_LATENCY_MULTIPLIER * latency); + set_sampling_rate(dbs_data, max(dbs_data->min_sampling_rate, + latency * LATENCY_MULTIPLIER)); + + if (dbs_data->cdata->governor == GOV_CONSERVATIVE) { + struct cs_ops *cs_ops = dbs_data->cdata->gov_ops; + + cpufreq_register_notifier(cs_ops->notifier_block, + CPUFREQ_TRANSITION_NOTIFIER); + } + + if (!have_governor_per_policy()) + cdata->gdbs_data = dbs_data; + + return 0; + case CPUFREQ_GOV_POLICY_EXIT: + if ((policy->governor->initialized == 1) || + have_governor_per_policy()) { + sysfs_remove_group(get_governor_parent_kobj(policy), + get_sysfs_attr(dbs_data)); + + if (dbs_data->cdata->governor == GOV_CONSERVATIVE) { + struct cs_ops *cs_ops = dbs_data->cdata->gov_ops; + + cpufreq_unregister_notifier(cs_ops->notifier_block, + CPUFREQ_TRANSITION_NOTIFIER); + } + + cdata->exit(dbs_data); + kfree(dbs_data); + cdata->gdbs_data = NULL; + }

- if (dbs_data->governor == GOV_CONSERVATIVE) { - cs_dbs_info = dbs_data->get_cpu_dbs_info_s(cpu); - sampling_rate = &cs_tuners->sampling_rate; + policy->governor_data = NULL; + return 0; + } + + cpu_cdbs = dbs_data->cdata->get_cpu_cdbs(cpu); + + if (dbs_data->cdata->governor == GOV_CONSERVATIVE) { + cs_tuners = dbs_data->tuners; + cs_dbs_info = dbs_data->cdata->get_cpu_dbs_info_s(cpu); + sampling_rate = cs_tuners->sampling_rate; ignore_nice = cs_tuners->ignore_nice; - cs_ops = dbs_data->gov_ops; } else { - od_dbs_info = dbs_data->get_cpu_dbs_info_s(cpu); - sampling_rate = &od_tuners->sampling_rate; + od_tuners = dbs_data->tuners; + od_dbs_info = dbs_data->cdata->get_cpu_dbs_info_s(cpu); + sampling_rate = od_tuners->sampling_rate; ignore_nice = od_tuners->ignore_nice; - od_ops = dbs_data->gov_ops; + od_ops = dbs_data->cdata->gov_ops; }

switch (event) { @@ -232,7 +346,7 @@ int cpufreq_governor_dbs(struct dbs_data *dbs_data,

for_each_cpu(j, policy->cpus) { struct cpu_dbs_common_info *j_cdbs = - dbs_data->get_cpu_cdbs(j); + dbs_data->cdata->get_cpu_cdbs(j);

j_cdbs->cpu = j; j_cdbs->cur_policy = policy; @@ -244,69 +358,34 @@ int cpufreq_governor_dbs(struct dbs_data *dbs_data,

mutex_init(&j_cdbs->timer_mutex); INIT_DEFERRABLE_WORK(&j_cdbs->work, - dbs_data->gov_dbs_timer); - } - - if (!policy->governor->initialized) { - rc = sysfs_create_group(cpufreq_global_kobject, - dbs_data->attr_group); - if (rc) { - mutex_unlock(&dbs_data->mutex); - return rc; - } + dbs_data->cdata->gov_dbs_timer); }

/* * conservative does not implement micro like ondemand * governor, thus we are bound to jiffes/HZ */ - if (dbs_data->governor == GOV_CONSERVATIVE) { + if (dbs_data->cdata->governor == GOV_CONSERVATIVE) { cs_dbs_info->down_skip = 0; cs_dbs_info->enable = 1; cs_dbs_info->requested_freq = policy->cur; - - if (!policy->governor->initialized) { - cpufreq_register_notifier(cs_ops->notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - - dbs_data->min_sampling_rate = - MIN_SAMPLING_RATE_RATIO * - jiffies_to_usecs(10); - } } else { od_dbs_info->rate_mult = 1; od_dbs_info->sample_type = OD_NORMAL_SAMPLE; od_ops->powersave_bias_init_cpu(cpu); - - if (!policy->governor->initialized) - od_tuners->io_is_busy = od_ops->io_busy(); }

- if (policy->governor->initialized) - goto unlock; - - /* policy latency is in nS. Convert it to uS first */ - latency = policy->cpuinfo.transition_latency / 1000; - if (latency == 0) - latency = 1; - - /* Bring kernel and HW constraints together */ - dbs_data->min_sampling_rate = max(dbs_data->min_sampling_rate, - MIN_LATENCY_MULTIPLIER * latency); - *sampling_rate = max(dbs_data->min_sampling_rate, latency * - LATENCY_MULTIPLIER); -unlock: mutex_unlock(&dbs_data->mutex);

/* Initiate timer time stamp */ cpu_cdbs->time_stamp = ktime_get();

for_each_cpu(j, policy->cpus) - dbs_timer_init(dbs_data, j, *sampling_rate); + dbs_timer_init(dbs_data, j, sampling_rate); break;

case CPUFREQ_GOV_STOP: - if (dbs_data->governor == GOV_CONSERVATIVE) + if (dbs_data->cdata->governor == GOV_CONSERVATIVE) cs_dbs_info->enable = 0;

for_each_cpu(j, policy->cpus) @@ -315,13 +394,6 @@ unlock: mutex_lock(&dbs_data->mutex); mutex_destroy(&cpu_cdbs->timer_mutex);

- if (policy->governor->initialized == 1) { - sysfs_remove_group(cpufreq_global_kobject, - dbs_data->attr_group); - if (dbs_data->governor == GOV_CONSERVATIVE) - cpufreq_unregister_notifier(cs_ops->notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - } mutex_unlock(&dbs_data->mutex);

break; diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h index cc4bd2f..3570d09 100644 --- a/drivers/cpufreq/cpufreq_governor.h +++ b/drivers/cpufreq/cpufreq_governor.h @@ -40,14 +40,75 @@ /* Ondemand Sampling types */ enum {OD_NORMAL_SAMPLE, OD_SUB_SAMPLE};

-/* Macro creating sysfs show routines */ -#define show_one(_gov, file_name, object) \ -static ssize_t show_##file_name \ +/* + * Macro for creating governors sysfs routines + * + * - gov_sys: One governor instance per whole system + * - gov_pol: One governor instance per policy + */ + +/* Create attributes */ +#define gov_sys_attr_ro(_name) \ +static struct global_attr _name##_gov_sys = \ +__ATTR(_name, 0444, show_##_name##_gov_sys, NULL) + +#define gov_sys_attr_rw(_name) \ +static struct global_attr _name##_gov_sys = \ +__ATTR(_name, 0644, show_##_name##_gov_sys, store_##_name##_gov_sys) + +#define gov_pol_attr_ro(_name) \ +static struct freq_attr _name##_gov_pol = \ +__ATTR(_name, 0444, show_##_name##_gov_pol, NULL) + +#define gov_pol_attr_rw(_name) \ +static struct freq_attr _name##_gov_pol = \ +__ATTR(_name, 0644, show_##_name##_gov_pol, store_##_name##_gov_pol) + +#define gov_sys_pol_attr_rw(_name) \ + gov_sys_attr_rw(_name); \ + gov_pol_attr_rw(_name) + +#define gov_sys_pol_attr_ro(_name) \ + gov_sys_attr_ro(_name); \ + gov_pol_attr_ro(_name) + +/* Create show/store routines */ +#define show_one(_gov, file_name) \ +static ssize_t show_##file_name##_gov_sys \ (struct kobject *kobj, struct attribute *attr, char *buf) \ { \ - return sprintf(buf, "%u\n", _gov##_tuners.object); \ + struct _gov##_dbs_tuners *tuners = _gov##_dbs_cdata.gdbs_data->tuners; \ + return sprintf(buf, "%u\n", tuners->file_name); \ +} \ + \ +static ssize_t show_##file_name##_gov_pol \ +(struct cpufreq_policy *policy, char *buf) \ +{ \ + struct dbs_data *dbs_data = policy->governor_data; \ + struct _gov##_dbs_tuners *tuners = dbs_data->tuners; \ + return sprintf(buf, "%u\n", tuners->file_name); \ +} + +#define store_one(_gov, file_name) \ +static ssize_t store_##file_name##_gov_sys \ +(struct kobject *kobj, struct attribute *attr, const char *buf, size_t count) \ +{ \ + struct dbs_data *dbs_data = _gov##_dbs_cdata.gdbs_data; \ + return store_##file_name(dbs_data, buf, count); \ +} \ + \ +static ssize_t store_##file_name##_gov_pol \ +(struct cpufreq_policy *policy, const char *buf, size_t count) \ +{ \ + struct dbs_data *dbs_data = policy->governor_data; \ + return store_##file_name(dbs_data, buf, count); \ }

+#define show_store_one(_gov, file_name) \ +show_one(_gov, file_name); \ +store_one(_gov, file_name) + +/* create helper routines */ #define define_get_cpu_dbs_routines(_dbs_info) \ static struct cpu_dbs_common_info *get_cpu_cdbs(int cpu) \ { \ @@ -103,7 +164,7 @@ struct cs_cpu_dbs_info_s { unsigned int enable:1; };

-/* Governers sysfs tunables */ +/* Per policy Governers sysfs tunables */ struct od_dbs_tuners { unsigned int ignore_nice; unsigned int sampling_rate; @@ -123,31 +184,42 @@ struct cs_dbs_tuners { unsigned int freq_step; };

-/* Per Governer data */ -struct dbs_data { +/* Common Governer data across policies */ +struct dbs_data; +struct common_dbs_data { /* Common across governors */ #define GOV_ONDEMAND 0 #define GOV_CONSERVATIVE 1 int governor; - unsigned int min_sampling_rate; - struct attribute_group *attr_group; - void *tuners; + struct attribute_group *attr_group_gov_sys; /* one governor - system */ + struct attribute_group *attr_group_gov_pol; /* one governor - policy */

- /* dbs_mutex protects dbs_enable in governor start/stop */ - struct mutex mutex; + /* Common data for platforms that don't set have_governor_per_policy */ + struct dbs_data *gdbs_data;

struct cpu_dbs_common_info *(*get_cpu_cdbs)(int cpu); void *(*get_cpu_dbs_info_s)(int cpu); void (*gov_dbs_timer)(struct work_struct *work); void (*gov_check_cpu)(int cpu, unsigned int load); + int (*init)(struct dbs_data *dbs_data); + void (*exit)(struct dbs_data *dbs_data);

/* Governor specific ops, see below */ void *gov_ops; };

+/* Governer Per policy data */ +struct dbs_data { + struct common_dbs_data *cdata; + unsigned int min_sampling_rate; + void *tuners; + + /* dbs_mutex protects dbs_enable in governor start/stop */ + struct mutex mutex; +}; + /* Governor specific ops, will be passed to dbs_data->gov_ops */ struct od_ops { - int (*io_busy)(void); void (*powersave_bias_init_cpu)(int cpu); unsigned int (*powersave_bias_target)(struct cpufreq_policy *policy, unsigned int freq_next, unsigned int relation); @@ -169,10 +241,25 @@ static inline int delay_for_sampling_rate(unsigned int sampling_rate) return delay; }

+#define declare_show_sampling_rate_min(_gov) \ +static ssize_t show_sampling_rate_min_gov_sys \ +(struct kobject *kobj, struct attribute *attr, char *buf) \ +{ \ + struct dbs_data *dbs_data = _gov##_dbs_cdata.gdbs_data; \ + return sprintf(buf, "%u\n", dbs_data->min_sampling_rate); \ +} \ + \ +static ssize_t show_sampling_rate_min_gov_pol \ +(struct cpufreq_policy *policy, char *buf) \ +{ \ + struct dbs_data *dbs_data = policy->governor_data; \ + return sprintf(buf, "%u\n", dbs_data->min_sampling_rate); \ +} + u64 get_cpu_idle_time(unsigned int cpu, u64 *wall); void dbs_check_cpu(struct dbs_data *dbs_data, int cpu); bool need_load_eval(struct cpu_dbs_common_info *cdbs, unsigned int sampling_rate); -int cpufreq_governor_dbs(struct dbs_data *dbs_data, - struct cpufreq_policy *policy, unsigned int event); +int cpufreq_governor_dbs(struct cpufreq_policy *policy, + struct common_dbs_data *cdata, unsigned int event); #endif /* _CPUFREQ_GOVERNOR_H */ diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c index f3eb26c..15e80ee 100644 --- a/drivers/cpufreq/cpufreq_ondemand.c +++ b/drivers/cpufreq/cpufreq_ondemand.c @@ -20,6 +20,7 @@ #include <linux/module.h> #include <linux/mutex.h> #include <linux/percpu-defs.h> +#include <linux/slab.h> #include <linux/sysfs.h> #include <linux/tick.h> #include <linux/types.h> @@ -37,22 +38,12 @@ #define MIN_FREQUENCY_UP_THRESHOLD (11) #define MAX_FREQUENCY_UP_THRESHOLD (100)

-static struct dbs_data od_dbs_data; static DEFINE_PER_CPU(struct od_cpu_dbs_info_s, od_cpu_dbs_info);

#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND static struct cpufreq_governor cpufreq_gov_ondemand; #endif

-static struct od_dbs_tuners od_tuners = { - .up_threshold = DEF_FREQUENCY_UP_THRESHOLD, - .sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR, - .adj_up_threshold = DEF_FREQUENCY_UP_THRESHOLD - - DEF_FREQUENCY_DOWN_DIFFERENTIAL, - .ignore_nice = 0, - .powersave_bias = 0, -}; - static void ondemand_powersave_bias_init_cpu(int cpu) { struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu); @@ -98,6 +89,8 @@ static unsigned int powersave_bias_target(struct cpufreq_policy *policy, unsigned int jiffies_total, jiffies_hi, jiffies_lo; struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, policy->cpu); + struct dbs_data *dbs_data = policy->governor_data; + struct od_dbs_tuners *od_tuners = dbs_data->tuners;

if (!dbs_info->freq_table) { dbs_info->freq_lo = 0; @@ -108,7 +101,7 @@ static unsigned int powersave_bias_target(struct cpufreq_policy *policy, cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next, relation, &index); freq_req = dbs_info->freq_table[index].frequency; - freq_reduc = freq_req * od_tuners.powersave_bias / 1000; + freq_reduc = freq_req * od_tuners->powersave_bias / 1000; freq_avg = freq_req - freq_reduc;

/* Find freq bounds for freq_avg in freq_table */ @@ -127,7 +120,7 @@ static unsigned int powersave_bias_target(struct cpufreq_policy *policy, dbs_info->freq_lo_jiffies = 0; return freq_lo; } - jiffies_total = usecs_to_jiffies(od_tuners.sampling_rate); + jiffies_total = usecs_to_jiffies(od_tuners->sampling_rate); jiffies_hi = (freq_avg - freq_lo) * jiffies_total; jiffies_hi += ((freq_hi - freq_lo) / 2); jiffies_hi /= (freq_hi - freq_lo); @@ -148,12 +141,15 @@ static void ondemand_powersave_bias_init(void)

static void dbs_freq_increase(struct cpufreq_policy *p, unsigned int freq) { - if (od_tuners.powersave_bias) + struct dbs_data *dbs_data = p->governor_data; + struct od_dbs_tuners *od_tuners = dbs_data->tuners; + + if (od_tuners->powersave_bias) freq = powersave_bias_target(p, freq, CPUFREQ_RELATION_H); else if (p->cur == p->max) return;

- __cpufreq_driver_target(p, freq, od_tuners.powersave_bias ? + __cpufreq_driver_target(p, freq, od_tuners->powersave_bias ? CPUFREQ_RELATION_L : CPUFREQ_RELATION_H); }

@@ -170,15 +166,17 @@ static void od_check_cpu(int cpu, unsigned int load_freq) { struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu); struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy; + struct dbs_data *dbs_data = policy->governor_data; + struct od_dbs_tuners *od_tuners = dbs_data->tuners;

dbs_info->freq_lo = 0;

/* Check for frequency increase */ - if (load_freq > od_tuners.up_threshold * policy->cur) { + if (load_freq > od_tuners->up_threshold * policy->cur) { /* If switching to max speed, apply sampling_down_factor */ if (policy->cur < policy->max) dbs_info->rate_mult = - od_tuners.sampling_down_factor; + od_tuners->sampling_down_factor; dbs_freq_increase(policy, policy->max); return; } @@ -193,9 +191,10 @@ static void od_check_cpu(int cpu, unsigned int load_freq) * support the current CPU usage without triggering the up policy. To be * safe, we focus 10 points under the threshold. */ - if (load_freq < od_tuners.adj_up_threshold * policy->cur) { + if (load_freq < od_tuners->adj_up_threshold + * policy->cur) { unsigned int freq_next; - freq_next = load_freq / od_tuners.adj_up_threshold; + freq_next = load_freq / od_tuners->adj_up_threshold;

/* No longer fully busy, reset rate_mult */ dbs_info->rate_mult = 1; @@ -203,7 +202,7 @@ static void od_check_cpu(int cpu, unsigned int load_freq) if (freq_next < policy->min) freq_next = policy->min;

- if (!od_tuners.powersave_bias) { + if (!od_tuners->powersave_bias) { __cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_L); } else { @@ -223,12 +222,14 @@ static void od_dbs_timer(struct work_struct *work) unsigned int cpu = dbs_info->cdbs.cur_policy->cpu; struct od_cpu_dbs_info_s *core_dbs_info = &per_cpu(od_cpu_dbs_info, cpu); + struct dbs_data *dbs_data = dbs_info->cdbs.cur_policy->governor_data; + struct od_dbs_tuners *od_tuners = dbs_data->tuners; int delay, sample_type = core_dbs_info->sample_type; bool eval_load;

mutex_lock(&core_dbs_info->cdbs.timer_mutex); eval_load = need_load_eval(&core_dbs_info->cdbs, - od_tuners.sampling_rate); + od_tuners->sampling_rate);

/* Common NORMAL_SAMPLE setup */ core_dbs_info->sample_type = OD_NORMAL_SAMPLE; @@ -240,13 +241,13 @@ static void od_dbs_timer(struct work_struct *work) CPUFREQ_RELATION_H); } else { if (eval_load) - dbs_check_cpu(&od_dbs_data, cpu); + dbs_check_cpu(dbs_data, cpu); if (core_dbs_info->freq_lo) { /* Setup timer for SUB_SAMPLE */ core_dbs_info->sample_type = OD_SUB_SAMPLE; delay = core_dbs_info->freq_hi_jiffies; } else { - delay = delay_for_sampling_rate(od_tuners.sampling_rate + delay = delay_for_sampling_rate(od_tuners->sampling_rate * core_dbs_info->rate_mult); } } @@ -256,12 +257,7 @@ static void od_dbs_timer(struct work_struct *work) }

/************************** sysfs interface ************************/ - -static ssize_t show_sampling_rate_min(struct kobject *kobj, - struct attribute *attr, char *buf) -{ - return sprintf(buf, "%u\n", od_dbs_data.min_sampling_rate); -} +static struct common_dbs_data od_dbs_cdata;

/** * update_sampling_rate - update sampling rate effective immediately if needed. @@ -276,12 +272,14 @@ static ssize_t show_sampling_rate_min(struct kobject *kobj, * reducing the sampling rate, we need to make the new value effective * immediately. */ -static void update_sampling_rate(unsigned int new_rate) +static void update_sampling_rate(struct dbs_data *dbs_data, + unsigned int new_rate) { + struct od_dbs_tuners *od_tuners = dbs_data->tuners; int cpu;

- od_tuners.sampling_rate = new_rate = max(new_rate, - od_dbs_data.min_sampling_rate); + od_tuners->sampling_rate = new_rate = max(new_rate, + dbs_data->min_sampling_rate);

for_each_online_cpu(cpu) { struct cpufreq_policy *policy; @@ -322,34 +320,37 @@ static void update_sampling_rate(unsigned int new_rate) } }

-static ssize_t store_sampling_rate(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_sampling_rate(struct dbs_data *dbs_data, const char *buf, + size_t count) { unsigned int input; int ret; ret = sscanf(buf, "%u", &input); if (ret != 1) return -EINVAL; - update_sampling_rate(input); + + update_sampling_rate(dbs_data, input); return count; }

-static ssize_t store_io_is_busy(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_io_is_busy(struct dbs_data *dbs_data, const char *buf, + size_t count) { + struct od_dbs_tuners *od_tuners = dbs_data->tuners; unsigned int input; int ret;

ret = sscanf(buf, "%u", &input); if (ret != 1) return -EINVAL; - od_tuners.io_is_busy = !!input; + od_tuners->io_is_busy = !!input; return count; }

-static ssize_t store_up_threshold(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_up_threshold(struct dbs_data *dbs_data, const char *buf, + size_t count) { + struct od_dbs_tuners *od_tuners = dbs_data->tuners; unsigned int input; int ret; ret = sscanf(buf, "%u", &input); @@ -359,23 +360,24 @@ static ssize_t store_up_threshold(struct kobject *a, struct attribute *b, return -EINVAL; } /* Calculate the new adj_up_threshold */ - od_tuners.adj_up_threshold += input; - od_tuners.adj_up_threshold -= od_tuners.up_threshold; + od_tuners->adj_up_threshold += input; + od_tuners->adj_up_threshold -= od_tuners->up_threshold;

- od_tuners.up_threshold = input; + od_tuners->up_threshold = input; return count; }

-static ssize_t store_sampling_down_factor(struct kobject *a, - struct attribute *b, const char *buf, size_t count) +static ssize_t store_sampling_down_factor(struct dbs_data *dbs_data, + const char *buf, size_t count) { + struct od_dbs_tuners *od_tuners = dbs_data->tuners; unsigned int input, j; int ret; ret = sscanf(buf, "%u", &input);

if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1) return -EINVAL; - od_tuners.sampling_down_factor = input; + od_tuners->sampling_down_factor = input;

/* Reset down sampling multiplier in case it was active */ for_each_online_cpu(j) { @@ -386,9 +388,10 @@ static ssize_t store_sampling_down_factor(struct kobject *a, return count; }

-static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_ignore_nice(struct dbs_data *dbs_data, const char *buf, + size_t count) { + struct od_dbs_tuners *od_tuners = dbs_data->tuners; unsigned int input; int ret;

@@ -401,10 +404,10 @@ static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b, if (input > 1) input = 1;

- if (input == od_tuners.ignore_nice) { /* nothing to do */ + if (input == od_tuners->ignore_nice) { /* nothing to do */ return count; } - od_tuners.ignore_nice = input; + od_tuners->ignore_nice = input;

/* we need to re-evaluate prev_cpu_idle */ for_each_online_cpu(j) { @@ -412,7 +415,7 @@ static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b, dbs_info = &per_cpu(od_cpu_dbs_info, j); dbs_info->cdbs.prev_cpu_idle = get_cpu_idle_time(j, &dbs_info->cdbs.prev_cpu_wall); - if (od_tuners.ignore_nice) + if (od_tuners->ignore_nice) dbs_info->cdbs.prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE];

@@ -420,9 +423,10 @@ static ssize_t store_ignore_nice_load(struct kobject *a, struct attribute *b, return count; }

-static ssize_t store_powersave_bias(struct kobject *a, struct attribute *b, - const char *buf, size_t count) +static ssize_t store_powersave_bias(struct dbs_data *dbs_data, const char *buf, + size_t count) { + struct od_dbs_tuners *od_tuners = dbs_data->tuners; unsigned int input; int ret; ret = sscanf(buf, "%u", &input); @@ -433,68 +437,138 @@ static ssize_t store_powersave_bias(struct kobject *a, struct attribute *b, if (input > 1000) input = 1000;

- od_tuners.powersave_bias = input; + od_tuners->powersave_bias = input; ondemand_powersave_bias_init(); return count; }

-show_one(od, sampling_rate, sampling_rate); -show_one(od, io_is_busy, io_is_busy); -show_one(od, up_threshold, up_threshold); -show_one(od, sampling_down_factor, sampling_down_factor); -show_one(od, ignore_nice_load, ignore_nice); -show_one(od, powersave_bias, powersave_bias); - -define_one_global_rw(sampling_rate); -define_one_global_rw(io_is_busy); -define_one_global_rw(up_threshold); -define_one_global_rw(sampling_down_factor); -define_one_global_rw(ignore_nice_load); -define_one_global_rw(powersave_bias); -define_one_global_ro(sampling_rate_min); - -static struct attribute *dbs_attributes[] = { - &sampling_rate_min.attr, - &sampling_rate.attr, - &up_threshold.attr, - &sampling_down_factor.attr, - &ignore_nice_load.attr, - &powersave_bias.attr, - &io_is_busy.attr, +show_store_one(od, sampling_rate); +show_store_one(od, io_is_busy); +show_store_one(od, up_threshold); +show_store_one(od, sampling_down_factor); +show_store_one(od, ignore_nice); +show_store_one(od, powersave_bias); +declare_show_sampling_rate_min(od); + +gov_sys_pol_attr_rw(sampling_rate); +gov_sys_pol_attr_rw(io_is_busy); +gov_sys_pol_attr_rw(up_threshold); +gov_sys_pol_attr_rw(sampling_down_factor); +gov_sys_pol_attr_rw(ignore_nice); +gov_sys_pol_attr_rw(powersave_bias); +gov_sys_pol_attr_ro(sampling_rate_min); + +static struct attribute *dbs_attributes_gov_sys[] = { + &sampling_rate_min_gov_sys.attr, + &sampling_rate_gov_sys.attr, + &up_threshold_gov_sys.attr, + &sampling_down_factor_gov_sys.attr, + &ignore_nice_gov_sys.attr, + &powersave_bias_gov_sys.attr, + &io_is_busy_gov_sys.attr, NULL };

-static struct attribute_group od_attr_group = { - .attrs = dbs_attributes, +static struct attribute_group od_attr_group_gov_sys = { + .attrs = dbs_attributes_gov_sys, + .name = "ondemand", +}; + +static struct attribute *dbs_attributes_gov_pol[] = { + &sampling_rate_min_gov_pol.attr, + &sampling_rate_gov_pol.attr, + &up_threshold_gov_pol.attr, + &sampling_down_factor_gov_pol.attr, + &ignore_nice_gov_pol.attr, + &powersave_bias_gov_pol.attr, + &io_is_busy_gov_pol.attr, + NULL +}; + +static struct attribute_group od_attr_group_gov_pol = { + .attrs = dbs_attributes_gov_pol, .name = "ondemand", };

/************************** sysfs end ************************/

+static int od_init(struct dbs_data *dbs_data) +{ + struct od_dbs_tuners *tuners; + u64 idle_time; + int cpu; + + tuners = kzalloc(sizeof(struct od_dbs_tuners), GFP_KERNEL); + if (!tuners) { + pr_err("%s: kzalloc failed\n", __func__); + return -ENOMEM; + } + + cpu = get_cpu(); + idle_time = get_cpu_idle_time_us(cpu, NULL); + put_cpu(); + if (idle_time != -1ULL) { + /* Idle micro accounting is supported. Use finer thresholds */ + tuners->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD; + tuners->adj_up_threshold = MICRO_FREQUENCY_UP_THRESHOLD - + MICRO_FREQUENCY_DOWN_DIFFERENTIAL; + /* + * In nohz/micro accounting case we set the minimum frequency + * not depending on HZ, but fixed (very low). The deferred + * timer might skip some samples if idle/sleeping as needed. + */ + dbs_data->min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE; + } else { + tuners->up_threshold = DEF_FREQUENCY_UP_THRESHOLD; + tuners->adj_up_threshold = DEF_FREQUENCY_UP_THRESHOLD - + DEF_FREQUENCY_DOWN_DIFFERENTIAL; + + /* For correct statistics, we need 10 ticks for each measure */ + dbs_data->min_sampling_rate = MIN_SAMPLING_RATE_RATIO * + jiffies_to_usecs(10); + } + + tuners->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR; + tuners->ignore_nice = 0; + tuners->powersave_bias = 0; + tuners->io_is_busy = should_io_be_busy(); + + dbs_data->tuners = tuners; + pr_info("%s: tuners %p\n", __func__, tuners); + mutex_init(&dbs_data->mutex); + return 0; +} + +static void od_exit(struct dbs_data *dbs_data) +{ + kfree(dbs_data->tuners); +} + define_get_cpu_dbs_routines(od_cpu_dbs_info);

static struct od_ops od_ops = { - .io_busy = should_io_be_busy, .powersave_bias_init_cpu = ondemand_powersave_bias_init_cpu, .powersave_bias_target = powersave_bias_target, .freq_increase = dbs_freq_increase, };

-static struct dbs_data od_dbs_data = { +static struct common_dbs_data od_dbs_cdata = { .governor = GOV_ONDEMAND, - .attr_group = &od_attr_group, - .tuners = &od_tuners, + .attr_group_gov_sys = &od_attr_group_gov_sys, + .attr_group_gov_pol = &od_attr_group_gov_pol, .get_cpu_cdbs = get_cpu_cdbs, .get_cpu_dbs_info_s = get_cpu_dbs_info_s, .gov_dbs_timer = od_dbs_timer, .gov_check_cpu = od_check_cpu, .gov_ops = &od_ops, + .init = od_init, + .exit = od_exit, };

static int od_cpufreq_governor_dbs(struct cpufreq_policy *policy, unsigned int event) { - return cpufreq_governor_dbs(&od_dbs_data, policy, event); + return cpufreq_governor_dbs(policy, &od_dbs_cdata, event); }

#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND @@ -509,29 +583,6 @@ struct cpufreq_governor cpufreq_gov_ondemand = {

static int __init cpufreq_gov_dbs_init(void) { - u64 idle_time; - int cpu = get_cpu(); - - mutex_init(&od_dbs_data.mutex); - idle_time = get_cpu_idle_time_us(cpu, NULL); - put_cpu(); - if (idle_time != -1ULL) { - /* Idle micro accounting is supported. Use finer thresholds */ - od_tuners.up_threshold = MICRO_FREQUENCY_UP_THRESHOLD; - od_tuners.adj_up_threshold = MICRO_FREQUENCY_UP_THRESHOLD - - MICRO_FREQUENCY_DOWN_DIFFERENTIAL; - /* - * In nohz/micro accounting case we set the minimum frequency - * not depending on HZ, but fixed (very low). The deferred - * timer might skip some samples if idle/sleeping as needed. - */ - od_dbs_data.min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE; - } else { - /* For correct statistics, we need 10 ticks for each measure */ - od_dbs_data.min_sampling_rate = MIN_SAMPLING_RATE_RATIO * - jiffies_to_usecs(10); - } - return cpufreq_register_governor(&cpufreq_gov_ondemand); }

diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 848b38b..9fd6bce 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -232,6 +232,13 @@ struct cpufreq_driver { struct module *owner; char name[CPUFREQ_NAME_LEN]; u8 flags; + /* + * This should be set by platforms having multiple clock-domains, i.e. + * supporting multiple policies. With this sysfs directories of governor + * would be created in cpu/cpu<num>/cpufreq/ directory and so they can + * use the same governor with different tunables for different clusters. + */ + bool have_governor_per_policy;

/* needed by all drivers */ int (*init) (struct cpufreq_policy *policy); @@ -332,6 +339,7 @@ const char *cpufreq_get_current_driver(void); *********************************************************************/ int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu); int cpufreq_update_policy(unsigned int cpu); +bool have_governor_per_policy(void);

#ifdef CONFIG_CPU_FREQ /* query the current CPU frequency (in kHz). If zero, cpufreq couldn't detect it */

Some cpufreq drivers implement their own governor and so don't need us to call generic governors interface via __cpufreq_governor(). Few recent commits haven't obeyed this law well and we saw some regressions.

This patch is an attempt to fix the above issue.

Signed-off-by: Viresh Kumar viresh.kumar@linaro.org Reported-and-tested-by: Sedat Dilek sedat.dilek@gmail.com Tested-by: Dirk Brandewie dirk.brandewie@gmail.com Signed-off-by: Rafael J. Wysocki rafael.j.wysocki@intel.com --- drivers/cpufreq/cpufreq.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 0072b46..95a6b14 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -808,13 +808,14 @@ static int cpufreq_add_policy_cpu(unsigned int cpu, unsigned int sibling, struct device *dev) { struct cpufreq_policy *policy; - int ret = 0; + int ret = 0, has_target = !!cpufreq_driver->target; unsigned long flags;

policy = cpufreq_cpu_get(sibling); WARN_ON(!policy);

- __cpufreq_governor(policy, CPUFREQ_GOV_STOP); + if (has_target) + __cpufreq_governor(policy, CPUFREQ_GOV_STOP);

lock_policy_rwsem_write(sibling);

@@ -827,8 +828,10 @@ static int cpufreq_add_policy_cpu(unsigned int cpu, unsigned int sibling,

unlock_policy_rwsem_write(sibling);

- __cpufreq_governor(policy, CPUFREQ_GOV_START); - __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); + if (has_target) { + __cpufreq_governor(policy, CPUFREQ_GOV_START); + __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); + }

ret = sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"); if (ret) { @@ -1077,7 +1080,8 @@ static int __cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif

/* If cpu is last user of policy, free policy */ if (cpus == 1) { - __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT); + if (cpufreq_driver->target) + __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT);

lock_policy_rwsem_read(cpu); kobj = &data->kobj;

We must call __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT) before calling cpufreq_cpu_put(data), so that policy kobject have valid fields. Otherwise, removing last online cpu of policy->cpus causes this crash for ondemand/conservative governor.

[<c00fb076>] (sysfs_find_dirent+0xe/0xa8) from [<c00fb1bd>] (sysfs_get_dirent+0x21/0x58) [<c00fb1bd>] (sysfs_get_dirent+0x21/0x58) from [<c00fc259>] (sysfs_remove_group+0x85/0xbc) [<c00fc259>] (sysfs_remove_group+0x85/0xbc) from [<c02faad9>] (cpufreq_governor_dbs+0x369/0x4a0) [<c02faad9>] (cpufreq_governor_dbs+0x369/0x4a0) from [<c02f66d7>] (__cpufreq_governor+0x2b/0x8c) [<c02f66d7>] (__cpufreq_governor+0x2b/0x8c) from [<c02f6893>] (__cpufreq_remove_dev.isra.12+0x15b/0x250) [<c02f6893>] (__cpufreq_remove_dev.isra.12+0x15b/0x250) from [<c03e91c7>] (cpufreq_cpu_callback+0x2f/0x3c) [<c03e91c7>] (cpufreq_cpu_callback+0x2f/0x3c) from [<c0036fe1>] (notifier_call_chain+0x45/0x54) [<c0036fe1>] (notifier_call_chain+0x45/0x54) from [<c001e611>] (__cpu_notify+0x1d/0x34) [<c001e611>] (__cpu_notify+0x1d/0x34) from [<c03e5833>] (_cpu_down+0x63/0x1ac) [<c03e5833>] (_cpu_down+0x63/0x1ac) from [<c03e5997>] (cpu_down+0x1b/0x30) [<c03e5997>] (cpu_down+0x1b/0x30) from [<c03e60eb>] (store_online+0x27/0x54) [<c03e60eb>] (store_online+0x27/0x54) from [<c0295629>] (dev_attr_store+0x11/0x18) [<c0295629>] (dev_attr_store+0x11/0x18) from [<c00f9edd>] (sysfs_write_file+0xed/0x114) [<c00f9edd>] (sysfs_write_file+0xed/0x114) from [<c00b42a9>] (vfs_write+0x65/0xd8) [<c00b42a9>] (vfs_write+0x65/0xd8) from [<c00b4523>] (sys_write+0x2f/0x50) [<c00b4523>] (sys_write+0x2f/0x50) from [<c000cdc1>] (ret_fast_syscall+0x1/0x52)

Of course this only impacted drivers which have have_governor_per_policy set to true. i.e. big LITTLE cpufreq driver.

Signed-off-by: Viresh Kumar viresh.kumar@linaro.org Signed-off-by: Rafael J. Wysocki rafael.j.wysocki@intel.com --- drivers/cpufreq/cpufreq.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 95a6b14..bb1169d 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1075,14 +1075,14 @@ static int __cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif __func__, cpu_dev->id, cpu); }

+ if ((cpus == 1) && (cpufreq_driver->target)) + __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT); + pr_debug("%s: removing link, cpu: %d\n", __func__, cpu); cpufreq_cpu_put(data);

/* If cpu is last user of policy, free policy */ if (cpus == 1) { - if (cpufreq_driver->target) - __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT); - lock_policy_rwsem_read(cpu); kobj = &data->kobj; cmp = &data->kobj_unregister;

This patch adds: EXPORT_SYMBOL_GPL(have_governor_per_policy), so that this routine can be used by modules too.

Signed-off-by: Viresh Kumar viresh.kumar@linaro.org --- drivers/cpufreq/cpufreq.c | 1 + 1 file changed, 1 insertion(+)

diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index bb1169d..d476bb8 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -132,6 +132,7 @@ bool have_governor_per_policy(void) { return cpufreq_driver->have_governor_per_policy; } +EXPORT_SYMBOL_GPL(have_governor_per_policy);

static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs) {

With this lock around __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT), we get circular dependency when we call sysfs_remove_group().

[ 195.319079] ====================================================== [ 195.337653] [ INFO: possible circular locking dependency detected ] [ 195.356497] 3.9.0-rc7+ #15 Not tainted [ 195.367758] ------------------------------------------------------- [ 195.386613] cat/2387 is trying to acquire lock: [ 195.400176] (&per_cpu(cpu_policy_rwsem, cpu)){+++++.}, at: [<c02f6179>] lock_policy_rwsem_read+0x25/0x34 [ 195.428920] [ 195.428920] but task is already holding lock: [ 195.446393] (s_active#41){++++.+}, at: [<c00f9bf7>] sysfs_read_file+0x4f/0xcc [ 195.468305] [ 195.468305] which lock already depends on the new lock. [ 195.468305] [ 195.492830] [ 195.492830] the existing dependency chain (in reverse order) is: [ 195.515250] -> #1 (s_active#41){++++.+}: [ 195.527647] [<c0055a79>] lock_acquire+0x61/0xbc [ 195.543129] [<c00fabf1>] sysfs_addrm_finish+0xc1/0x128 [ 195.560362] [<c00f9819>] sysfs_hash_and_remove+0x35/0x64 [ 195.578119] [<c00fbe6f>] remove_files.isra.0+0x1b/0x24 [ 195.595497] [<c00fbea5>] sysfs_remove_group+0x2d/0xa8 [ 195.612469] [<c02f9a0b>] cpufreq_governor_interactive+0x13b/0x35c [ 195.632668] [<c02f61df>] __cpufreq_governor+0x2b/0x8c [ 195.649644] [<c02f6579>] __cpufreq_set_policy+0xa9/0xf8 [ 195.667132] [<c02f6b75>] store_scaling_governor+0x61/0x100 [ 195.685404] [<c02f6f4d>] store+0x39/0x60 [ 195.698989] [<c00f9b81>] sysfs_write_file+0xed/0x114 [ 195.715694] [<c00b3fd1>] vfs_write+0x65/0xd8 [ 195.730320] [<c00b424b>] sys_write+0x2f/0x50 [ 195.744943] [<c000cdc1>] ret_fast_syscall+0x1/0x52 [ 195.761135] -> #0 (&per_cpu(cpu_policy_rwsem, cpu)){+++++.}: [ 195.778665] [<c0055253>] __lock_acquire+0xef3/0x13dc [ 195.795371] [<c0055a79>] lock_acquire+0x61/0xbc [ 195.810776] [<c03ee1f5>] down_read+0x25/0x30 [ 195.825398] [<c02f6179>] lock_policy_rwsem_read+0x25/0x34 [ 195.843410] [<c02f6edd>] show+0x21/0x58 [ 195.856731] [<c00f9c0f>] sysfs_read_file+0x67/0xcc [ 195.872919] [<c00b40a7>] vfs_read+0x63/0xd8 [ 195.887282] [<c00b41fb>] sys_read+0x2f/0x50 [ 195.901645] [<c000cdc1>] ret_fast_syscall+0x1/0x52 [ 195.917863] [ 195.917863] other info that might help us debug this: [ 195.917863] [ 195.941853] Possible unsafe locking scenario: [ 195.941853] [ 195.959586] CPU0 CPU1 [ 195.973149] ---- ---- [ 195.986712] lock(s_active#41); [ 195.996407] lock(&per_cpu(cpu_policy_rwsem, cpu)); [ 196.018912] lock(s_active#41); [ 196.036161] lock(&per_cpu(cpu_policy_rwsem, cpu)); [ 196.051051] [ 196.051051] *** DEADLOCK *** [ 196.051051] [ 196.068792] 2 locks held by cat/2387: [ 196.079750] #0: (&buffer->mutex){+.+.+.}, at: [<c00f9bcd>] sysfs_read_file+0x25/0xcc [ 196.103546] #1: (s_active#41){++++.+}, at: [<c00f9bf7>] sysfs_read_file+0x4f/0xcc [ 196.126577] [ 196.126577] stack backtrace: [ 196.139644] [<c0011d55>] (unwind_backtrace+0x1/0x9c) from [<c03e9a09>] (print_circular_bug+0x19d/0x1e8) [ 196.167857] [<c03e9a09>] (print_circular_bug+0x19d/0x1e8) from [<c0055253>] (__lock_acquire+0xef3/0x13dc) [ 196.196542] [<c0055253>] (__lock_acquire+0xef3/0x13dc) from [<c0055a79>] (lock_acquire+0x61/0xbc) [ 196.223139] [<c0055a79>] (lock_acquire+0x61/0xbc) from [<c03ee1f5>] (down_read+0x25/0x30) [ 196.247722] [<c03ee1f5>] (down_read+0x25/0x30) from [<c02f6179>] (lock_policy_rwsem_read+0x25/0x34) [ 196.274905] [<c02f6179>] (lock_policy_rwsem_read+0x25/0x34) from [<c02f6edd>] (show+0x21/0x58) [ 196.300724] [<c02f6edd>] (show+0x21/0x58) from [<c00f9c0f>] (sysfs_read_file+0x67/0xcc) [ 196.324719] [<c00f9c0f>] (sysfs_read_file+0x67/0xcc) from [<c00b40a7>] (vfs_read+0x63/0xd8) [ 196.349756] [<c00b40a7>] (vfs_read+0x63/0xd8) from [<c00b41fb>] (sys_read+0x2f/0x50) [ 196.372970] [<c00b41fb>] (sys_read+0x2f/0x50) from [<c000cdc1>] (ret_fast_syscall+0x1/0x52)

This lock isn't required while calling __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT). Remove it.

Signed-off-by: Viresh Kumar viresh.kumar@linaro.org --- drivers/cpufreq/cpufreq.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 763fc8e1..cdde8cb 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1732,18 +1732,23 @@ static int __cpufreq_set_policy(struct cpufreq_policy *data, /* end old governor */ if (data->governor) { __cpufreq_governor(data, CPUFREQ_GOV_STOP); + unlock_policy_rwsem_write(policy->cpu); __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT); + lock_policy_rwsem_write(policy->cpu); }

/* start new governor */ data->governor = policy->governor; if (!__cpufreq_governor(data, CPUFREQ_GOV_POLICY_INIT)) { - if (!__cpufreq_governor(data, CPUFREQ_GOV_START)) + if (!__cpufreq_governor(data, CPUFREQ_GOV_START)) { failed = 0; - else + } else { + unlock_policy_rwsem_write(policy->cpu); __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT); + lock_policy_rwsem_write(policy->cpu); + } }

if (failed) {

Generic version of get_cpu_idle_time() is present in cpufreq.c file. We must not replicate that for Interactive governor, rather use the same one. Lets do it.

Signed-off-by: Viresh Kumar viresh.kumar@linaro.org --- drivers/cpufreq/cpufreq_interactive.c | 42 ++--------------------------------- 1 file changed, 2 insertions(+), 40 deletions(-)

diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c index baf8740..4ed2fed 100644 --- a/drivers/cpufreq/cpufreq_interactive.c +++ b/drivers/cpufreq/cpufreq_interactive.c @@ -30,8 +30,6 @@ #include <linux/workqueue.h> #include <linux/kthread.h> #include <linux/slab.h> -#include <linux/kernel_stat.h> -#include <asm/cputime.h>

#define CREATE_TRACE_POINTS #include <trace/events/cpufreq_interactive.h> @@ -130,42 +128,6 @@ struct cpufreq_governor cpufreq_gov_interactive = { .owner = THIS_MODULE, };

-static inline cputime64_t get_cpu_idle_time_jiffy(unsigned int cpu, - cputime64_t *wall) -{ - u64 idle_time; - u64 cur_wall_time; - u64 busy_time; - - cur_wall_time = jiffies64_to_cputime64(get_jiffies_64()); - - busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL]; - busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE]; - - idle_time = cur_wall_time - busy_time; - if (wall) - *wall = jiffies_to_usecs(cur_wall_time); - - return jiffies_to_usecs(idle_time); -} - -static inline cputime64_t get_cpu_idle_time(unsigned int cpu, - cputime64_t *wall) -{ - u64 idle_time = get_cpu_idle_time_us(cpu, wall); - - if (idle_time == -1ULL) - idle_time = get_cpu_idle_time_jiffy(cpu, wall); - else if (!io_is_busy) - idle_time += get_cpu_iowait_time_us(cpu, wall); - - return idle_time; -} - static void cpufreq_interactive_timer_resched( struct cpufreq_interactive_cpuinfo *pcpu) { @@ -181,7 +143,7 @@ static void cpufreq_interactive_timer_resched( spin_lock_irqsave(&pcpu->load_lock, flags); pcpu->time_in_idle = get_cpu_idle_time(smp_processor_id(), - &pcpu->time_in_idle_timestamp); + &pcpu->time_in_idle_timestamp, io_is_busy); pcpu->cputime_speedadj = 0; pcpu->cputime_speedadj_timestamp = pcpu->time_in_idle_timestamp; spin_unlock_irqrestore(&pcpu->load_lock, flags); @@ -317,7 +279,7 @@ static u64 update_load(int cpu) unsigned int delta_time; u64 active_time;

- now_idle = get_cpu_idle_time(cpu, &now); + now_idle = get_cpu_idle_time(cpu, &now, io_is_busy); delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle); delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp); active_time = delta_time - delta_idle;

Cpufreq no longer calls governor callback for offlined cpus. i.e. All policy->cpus are guaranteed to be online. Hence we don't need explicit check to see if cpu is online or not.

Signed-off-by: Viresh Kumar viresh.kumar@linaro.org --- drivers/cpufreq/cpufreq_interactive.c | 3 --- 1 file changed, 3 deletions(-)

diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c index 4ed2fed..baa5638 100644 --- a/drivers/cpufreq/cpufreq_interactive.c +++ b/drivers/cpufreq/cpufreq_interactive.c @@ -992,9 +992,6 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy,

switch (event) { case CPUFREQ_GOV_START: - if (!cpu_online(policy->cpu)) - return -EINVAL; - mutex_lock(&gov_lock);

freq_table =

This moves definition of cpufreq_gov_interactive towards the bottom of file, so that we don't have to add prototype of cpufreq_governor_interactive() in the beginning of file.

Signed-off-by: Viresh Kumar viresh.kumar@linaro.org --- drivers/cpufreq/cpufreq_interactive.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/drivers/cpufreq/cpufreq_interactive.c b/drivers/cpufreq/cpufreq_interactive.c index baa5638..8821873 100644 --- a/drivers/cpufreq/cpufreq_interactive.c +++ b/drivers/cpufreq/cpufreq_interactive.c @@ -115,19 +115,6 @@ static int timer_slack_val = DEFAULT_TIMER_SLACK;

static bool io_is_busy;

-static int cpufreq_governor_interactive(struct cpufreq_policy *policy, - unsigned int event); - -#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE -static -#endif -struct cpufreq_governor cpufreq_gov_interactive = { - .name = "interactive", - .governor = cpufreq_governor_interactive, - .max_transition_latency = 10000000, - .owner = THIS_MODULE, -}; - static void cpufreq_interactive_timer_resched( struct cpufreq_interactive_cpuinfo *pcpu) { @@ -1083,6 +1070,16 @@ static int cpufreq_governor_interactive(struct cpufreq_policy *policy, return 0; }

+#ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE +static +#endif +struct cpufreq_governor cpufreq_gov_interactive = { + .name = "interactive", + .governor = cpufreq_governor_interactive, + .max_transition_latency = 10000000, + .owner = THIS_MODULE, +}; + static void cpufreq_interactive_nop_timer(unsigned long data) { }