The current implementation only saves and restores the context for ETM sources while ignoring the context of links. However, if funnels or replicators on a linked path resides in a CPU or cluster power domain, the hardware context for the link will be lost after resuming from low power states.

To support context management for links during CPU low power modes, a better way is to implement CPU PM callbacks in the Arm CoreSight core layer. As the core layer has sufficient information for linked paths, from tracers to links, which can be used for power management.

As a first step, this patch registers CPU PM notifier in the core layer. If a source driver provides callbacks for saving and restoring context, these callbacks will be invoked in CPU suspend and resume.

Further changes will extend for controlling path.

Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/coresight.h | 4 ++++ 2 files changed, 66 insertions(+)

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 1503037662f2..7ab7321ca8d5 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -4,6 +4,7 @@ */

#include <linux/build_bug.h> +#include <linux/cpu_pm.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/types.h> @@ -421,6 +422,21 @@ int coresight_resume_source(struct coresight_device *csdev) } EXPORT_SYMBOL_GPL(coresight_resume_source);

+static int coresight_save_source(struct coresight_device *csdev) +{ + if (csdev && source_ops(csdev)->save) + return source_ops(csdev)->save(csdev); + + /* Return success if callback is not supported */ + return 0; +} + +static void coresight_restore_source(struct coresight_device *csdev) +{ + if (csdev && source_ops(csdev)->restore) + source_ops(csdev)->restore(csdev); +} + /* * coresight_disable_path_from : Disable components in the given path beyond * @nd in the list. If @nd is NULL, all the components, except the SOURCE are @@ -1566,6 +1582,45 @@ char *coresight_alloc_device_name(struct coresight_dev_list *dict, } EXPORT_SYMBOL_GPL(coresight_alloc_device_name);

+static int coresight_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, + void *v) +{ + unsigned int cpu = smp_processor_id(); + struct coresight_device *source = per_cpu(csdev_source, cpu); + + if (!source) + return NOTIFY_OK; + + switch (cmd) { + case CPU_PM_ENTER: + if (coresight_save_source(source)) + return NOTIFY_BAD; + break; + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + coresight_restore_source(source); + break; + default: + return NOTIFY_DONE; + } + + return NOTIFY_OK; +} + +static struct notifier_block coresight_cpu_pm_nb = { + .notifier_call = coresight_cpu_pm_notify, +}; + +static int __init coresight_pm_setup(void) +{ + return cpu_pm_register_notifier(&coresight_cpu_pm_nb); +} + +static void coresight_pm_cleanup(void) +{ + cpu_pm_unregister_notifier(&coresight_cpu_pm_nb); +} + const struct bus_type coresight_bustype = { .name = "coresight", }; @@ -1620,9 +1675,15 @@ static int __init coresight_init(void)

/* initialise the coresight syscfg API */ ret = cscfg_init(); + if (ret) + goto exit_notifier; + + ret = coresight_pm_setup(); if (!ret) return 0;

+ cscfg_exit(); +exit_notifier: atomic_notifier_chain_unregister(&panic_notifier_list, &coresight_notifier); exit_perf: @@ -1634,6 +1695,7 @@ static int __init coresight_init(void)

static void __exit coresight_exit(void) { + coresight_pm_cleanup(); cscfg_exit(); atomic_notifier_chain_unregister(&panic_notifier_list, &coresight_notifier); diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 4ac65c68bbf4..64cee4040daa 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -400,6 +400,8 @@ struct coresight_ops_link { * @disable: disables tracing for a source. * @resume_perf: resumes tracing for a source in perf session. * @pause_perf: pauses tracing for a source in perf session. + * @save: save context for a source. + * @restore: restore context for a source. */ struct coresight_ops_source { int (*cpu_id)(struct coresight_device *csdev); @@ -409,6 +411,8 @@ struct coresight_ops_source { struct perf_event *event); int (*resume_perf)(struct coresight_device *csdev); void (*pause_perf)(struct coresight_device *csdev); + int (*save)(struct coresight_device *csdev); + void (*restore)(struct coresight_device *csdev); };

/**

For a source device, save activated path into the coresight_device structure.

This will be used by sequential changes for controlling the path during CPU idle.

Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 10 ++++++++++ include/linux/coresight.h | 3 +++ 2 files changed, 13 insertions(+)

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 7ab7321ca8d5..8f565bb3c535 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -495,6 +495,12 @@ static void coresight_disable_path_from(struct coresight_path *path,

void coresight_disable_path(struct coresight_path *path) { + struct coresight_device *source; + + source = coresight_get_source(path); + if (coresight_is_percpu_source(source)) + source->path = NULL; + coresight_disable_path_from(path, NULL); } EXPORT_SYMBOL_GPL(coresight_disable_path); @@ -577,6 +583,10 @@ int coresight_enable_path(struct coresight_path *path, enum cs_mode mode, } }

+ source = coresight_get_source(path); + if (coresight_is_percpu_source(source)) + source->path = path; + out: return ret; err_disable_helpers: diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 64cee4040daa..cd9709a36b9c 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -265,6 +265,7 @@ struct coresight_trace_id_map { * CS_MODE_SYSFS. Otherwise it must be accessed from inside the * spinlock. * @orphan: true if the component has connections that haven't been linked. + * @path: activated path pointer (only used for source device). * @sysfs_sink_activated: 'true' when a sink has been selected for use via sysfs * by writing a 1 to the 'enable_sink' file. A sink can be * activated but not yet enabled. Enabling for a _sink_ happens @@ -291,6 +292,8 @@ struct coresight_device { local_t mode; int refcnt; bool orphan; + /* activated path (for source only) */ + struct coresight_path *path; /* sink specific fields */ bool sysfs_sink_activated; struct dev_ext_attribute *ea;

This commit extends control over CoreSight components on an activated path during CPU PM.

An activated path is recorded in the 'path' field of coresight_device structure. The patch pointer is cleared when the path is disabled. Traverses the links on the path to disable and re-enable components during CPU low-power modes.

Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+)

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 8f565bb3c535..075d485dcea5 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -596,6 +596,94 @@ int coresight_enable_path(struct coresight_path *path, enum cs_mode mode, goto out; }

+static int coresight_save_path(struct coresight_path *path) +{ + u32 type; + struct coresight_node *nd; + struct coresight_device *source, *sink; + + if (!path) + return 0; + + source = coresight_get_source(path); + + /* + * Do a sanity check as the source device must have been enabled when + * run at here. + */ + if (coresight_get_mode(source) == CS_MODE_DISABLED) + return -EINVAL; + + sink = coresight_get_sink(path); + + /* + * Disable links. Deliberately to skip disabling the sink to avoid + * latency. + */ + nd = list_first_entry(&path->path_list, struct coresight_node, link); + list_for_each_entry_continue(nd, &path->path_list, link) { + struct coresight_device *csdev, *parent, *child; + + csdev = nd->csdev; + type = csdev->type; + + /* Adjust type for LINKSINK */ + if (type == CORESIGHT_DEV_TYPE_LINKSINK) + type = (csdev == sink) ? CORESIGHT_DEV_TYPE_SINK : + CORESIGHT_DEV_TYPE_LINK; + + if (type != CORESIGHT_DEV_TYPE_LINK) + continue; + + parent = list_prev_entry(nd, link)->csdev; + child = list_next_entry(nd, link)->csdev; + coresight_disable_link(csdev, parent, child, source); + } + + return 0; +} + +static void coresight_restore_path(struct coresight_path *path) +{ + int ret; + u32 type; + struct coresight_device *source, *sink; + struct coresight_node *nd; + + if (!path) + return; + + source = coresight_get_source(path); + if (coresight_get_mode(source) == CS_MODE_DISABLED) + return; + + sink = coresight_get_sink(path); + + list_for_each_entry_reverse(nd, &path->path_list, link) { + struct coresight_device *csdev, *parent, *child; + + csdev = nd->csdev; + type = csdev->type; + + if (type == CORESIGHT_DEV_TYPE_LINKSINK) + type = (csdev == sink) ? CORESIGHT_DEV_TYPE_SINK : + CORESIGHT_DEV_TYPE_LINK; + + if (type != CORESIGHT_DEV_TYPE_LINK) + continue; + + parent = list_prev_entry(nd, link)->csdev; + child = list_next_entry(nd, link)->csdev; + ret = coresight_enable_link(csdev, parent, child, + coresight_get_source(path)); + if (ret) { + dev_err(&csdev->dev, "Failed to restore\n"); + coresight_disable_path_from(path, nd); + return; + } + } +} + struct coresight_device *coresight_get_sink(struct coresight_path *path) { struct coresight_device *csdev; @@ -1605,9 +1693,15 @@ static int coresight_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, case CPU_PM_ENTER: if (coresight_save_source(source)) return NOTIFY_BAD; + + if (coresight_save_path(source->path)) { + coresight_restore_source(source); + return NOTIFY_BAD; + } break; case CPU_PM_EXIT: case CPU_PM_ENTER_FAILED: + coresight_restore_path(source->path); coresight_restore_source(source); break; default:

Unlike a system level's sink, the per-CPU sink may lose power during CPU idle states. So far, this refers to the TRBE as a sink. This commit adds save and restore callbacks for the sink, and these callbacks are via the PM notifier.

For not per-CPU sinks (e.g., ETR, ETB, etc), which are system level components and should not be affected by CPU low power states. The PM notifier does not interact with them to avoid additional latency.

Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 23 +++++++++++++++++++++++ include/linux/coresight.h | 4 ++++ 2 files changed, 27 insertions(+)

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 075d485dcea5..02b80db1da3d 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -437,6 +437,21 @@ static void coresight_restore_source(struct coresight_device *csdev) source_ops(csdev)->restore(csdev); }

+static int coresight_save_sink(struct coresight_device *csdev) +{ + if (csdev && sink_ops(csdev)->save) + return sink_ops(csdev)->save(csdev); + + /* Return success if callback is not supported */ + return 0; +} + +static void coresight_restore_sink(struct coresight_device *csdev) +{ + if (csdev && sink_ops(csdev)->restore) + sink_ops(csdev)->restore(csdev); +} + /* * coresight_disable_path_from : Disable components in the given path beyond * @nd in the list. If @nd is NULL, all the components, except the SOURCE are @@ -616,6 +631,10 @@ static int coresight_save_path(struct coresight_path *path)

sink = coresight_get_sink(path);

+ /* Save per CPU sink and directly bail out */ + if (coresight_is_percpu_sink(sink)) + return coresight_save_sink(sink); + /* * Disable links. Deliberately to skip disabling the sink to avoid * latency. @@ -659,6 +678,10 @@ static void coresight_restore_path(struct coresight_path *path)

sink = coresight_get_sink(path);

+ /* Restore per CPU sink and directly bail out */ + if (coresight_is_percpu_sink(sink)) + return coresight_restore_sink(sink); + list_for_each_entry_reverse(nd, &path->path_list, link) { struct coresight_device *csdev, *parent, *child;

diff --git a/include/linux/coresight.h b/include/linux/coresight.h index cd9709a36b9c..bffcb2238102 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -365,6 +365,8 @@ enum cs_mode { * @alloc_buffer: initialises perf's ring buffer for trace collection. * @free_buffer: release memory allocated in @get_config. * @update_buffer: update buffer pointers after a trace session. + * @save: save context for a sink. + * @restore: restore context for a sink. */ struct coresight_ops_sink { int (*enable)(struct coresight_device *csdev, enum cs_mode mode, @@ -377,6 +379,8 @@ struct coresight_ops_sink { unsigned long (*update_buffer)(struct coresight_device *csdev, struct perf_output_handle *handle, void *sink_config); + int (*save)(struct coresight_device *csdev); + void (*restore)(struct coresight_device *csdev); };

/**

The hotplug lock is acquired and released in the etm4_disable_sysfs() function, which is a low-level function located in the ETM4 driver. This prevents us from a new solution for hotplug.

Firstly, hotplug callbacks cannot invoke etm4_disable_sysfs() to disable the source; otherwise, a deadlock issue occurs. The reason is that, in the hotplug flow, the kernel acquires the hotplug lock before calling callbacks. Subsequently, if coresight_disable_source() is invoked and it calls etm4_disable_sysfs(), the hotplug lock will be acquired twice, leading to a double lock issue.

Secondly, when hotplugging a CPU on or off, if we want to manipulate all components on a path attached to the CPU, we need to maintain atomicity for the entire path. Otherwise, a race condition may occur with users setting the same path via the Sysfs knobs, ultimately causing mess states in CoreSight components.

This patch moves the hotplug locking from etm4_disable_sysfs() into enable_source_store(). As a result, when users control the Sysfs knobs, the whole flow is protected by hotplug locking, ensuring it is mutual exclusive with hotplug callbacks.

Note, the paired function etm4_enable_sysfs() does not use hotplug locking, which is why this patch does not modify it.

Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-etm4x-core.c | 8 -------- drivers/hwtracing/coresight/coresight-sysfs.c | 12 +++++++++++- 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index d5fd9e58a962..e5a7c0dd7f8e 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1032,13 +1032,6 @@ static void etm4_disable_sysfs(struct coresight_device *csdev) { struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

- /* - * Taking hotplug lock here protects from clocks getting disabled - * with tracing being left on (crash scenario) if user disable occurs - * after cpu online mask indicates the cpu is offline but before the - * DYING hotplug callback is serviced by the ETM driver. - */ - cpus_read_lock(); raw_spin_lock(&drvdata->spinlock);

/* @@ -1048,7 +1041,6 @@ static void etm4_disable_sysfs(struct coresight_device *csdev) smp_call_function_single(drvdata->cpu, etm4_disable_hw, drvdata, 1);

raw_spin_unlock(&drvdata->spinlock); - cpus_read_unlock();

/* * we only release trace IDs when resetting sysfs. diff --git a/drivers/hwtracing/coresight/coresight-sysfs.c b/drivers/hwtracing/coresight/coresight-sysfs.c index feadaf065b53..ea839a5e601b 100644 --- a/drivers/hwtracing/coresight/coresight-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-sysfs.c @@ -359,14 +359,24 @@ static ssize_t enable_source_store(struct device *dev, if (ret) return ret;

+ /* + * CoreSight hotplug callbacks in core layer control a activated path + * from its source to sink. Taking hotplug lock here protects a race + * condition with hotplug callbacks. + */ + cpus_read_lock(); + if (val) { ret = coresight_enable_sysfs(csdev); - if (ret) + if (ret) { + cpus_read_unlock(); return ret; + } } else { coresight_disable_sysfs(csdev); }

+ cpus_read_unlock(); return size; } static DEVICE_ATTR_RW(enable_source);

This commit moves CPU hotplug callbacks from ETMv4 driver to core layer. The motivation is the core layer can control all components on an activated path rather but not only managing tracer in ETMv4 driver.

The perf event layer will disable CoreSight PMU event 'cs_etm' when hotplug off a CPU. That means a perf mode will be always converted to disabled mode in CPU hotplug. Arm CoreSight CPU hotplug callbacks only need to handle the Sysfs mode and ignore the perf mode.

The core layer invokes a high level API coresight_disable_source() to disable a source when hotplug-off a CPU. It disables a tracer and changes the tracer's mode to CS_MODE_DISABLED.

When hotplug-in a CPU, if a activated path is detected - when the activated path pointer is not NULL - in this case, the tracer will be re-enabled.

Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 50 +++++++++++++++++++++++++++++++++++++++++- drivers/hwtracing/coresight/coresight-etm4x-core.c | 37 ------------------------------- 2 files changed, 49 insertions(+), 38 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 02b80db1da3d..a1fb00dbdb55 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -1703,6 +1703,41 @@ char *coresight_alloc_device_name(struct coresight_dev_list *dict, } EXPORT_SYMBOL_GPL(coresight_alloc_device_name);

+static int coresight_starting_cpu(unsigned int cpu) +{ + struct coresight_device *csdev = per_cpu(csdev_source, cpu); + + /* + * After a CPU is hotpluged-off, source will be disabled and set as + * CS_MODE_DISABLED mode. Here, if a path is activated, we need to + * re-enable components on the path. + */ + if (!csdev || !csdev->path) + return 0; + + /* Only support SYSFS mode */ + source_ops(csdev)->enable(csdev, NULL, CS_MODE_SYSFS, csdev->path); + return 0; +} + +static int coresight_dying_cpu(unsigned int cpu) +{ + struct coresight_device *csdev = per_cpu(csdev_source, cpu); + + /* If a path is activated, we need to disable components on the path. */ + if (!csdev || !csdev->path) + return 0; + + /* + * The perf event layer will disable PMU events in the CPU hotplug. + * CoreSight driver should never handle the CS_MODE_PERF case. + */ + WARN_ON(coresight_get_mode(csdev) != CS_MODE_SYSFS); + + coresight_disable_source(csdev, NULL); + return 0; +} + static int coresight_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, void *v) { @@ -1740,11 +1775,24 @@ static struct notifier_block coresight_cpu_pm_nb = {

static int __init coresight_pm_setup(void) { - return cpu_pm_register_notifier(&coresight_cpu_pm_nb); + int ret; + + ret = cpu_pm_register_notifier(&coresight_cpu_pm_nb); + if (ret) + return ret; + + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING, + "arm/coresight-core:starting", + coresight_starting_cpu, coresight_dying_cpu); + if (ret) + cpu_pm_unregister_notifier(&coresight_cpu_pm_nb); + + return ret; }

static void coresight_pm_cleanup(void) { + cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); cpu_pm_unregister_notifier(&coresight_cpu_pm_nb); }

diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index e5a7c0dd7f8e..cc20b63f2d35 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -1735,33 +1735,6 @@ static int etm4_online_cpu(unsigned int cpu) return 0; }

-static int etm4_starting_cpu(unsigned int cpu) -{ - if (!etmdrvdata[cpu]) - return 0; - - raw_spin_lock(&etmdrvdata[cpu]->spinlock); - if (!etmdrvdata[cpu]->os_unlock) - etm4_os_unlock(etmdrvdata[cpu]); - - if (coresight_get_mode(etmdrvdata[cpu]->csdev)) - etm4_enable_hw(etmdrvdata[cpu]); - raw_spin_unlock(&etmdrvdata[cpu]->spinlock); - return 0; -} - -static int etm4_dying_cpu(unsigned int cpu) -{ - if (!etmdrvdata[cpu]) - return 0; - - raw_spin_lock(&etmdrvdata[cpu]->spinlock); - if (coresight_get_mode(etmdrvdata[cpu]->csdev)) - etm4_disable_hw(etmdrvdata[cpu]); - raw_spin_unlock(&etmdrvdata[cpu]->spinlock); - return 0; -} - static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) { int i, ret = 0; @@ -2061,13 +2034,6 @@ static int __init etm4_pm_setup(void) { int ret;

- ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING, - "arm/coresight4:starting", - etm4_starting_cpu, etm4_dying_cpu); - - if (ret) - return ret; - ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "arm/coresight4:online", etm4_online_cpu, NULL); @@ -2078,14 +2044,11 @@ static int __init etm4_pm_setup(void) return 0; }

- /* failed dyn state - remove others */ - cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); return ret; }

static void etm4_pm_clear(void) { - cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); if (hp_online) { cpuhp_remove_state_nocalls(hp_online); hp_online = 0;