A PMU NMI can throttle a group during an AUX pause or resume, stopping its CoreSight event before the callback finishes. The callback can then restart tracing or reopen an AUX output handle after teardown.
Use per-CPU atomic STOP and AUX bits to serialize these callbacks. Reject AUX operations while a stop is pending and defer stops until an active AUX operation finishes. Complete deferred stops through etm_event_stop() and set PERF_HES_STOPPED only after disabling the source and path.
Assisted-by: Codex:gpt-6 Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-etm-perf.c | 104 ++++++++++++++++++----- 1 file changed, 84 insertions(+), 20 deletions(-)
diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index 8f803914e9abfb18e080ff68371ad8b32042f180..8020a4d8af18bb669441824cfdb687f4fa299eb4 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -4,6 +4,7 @@ * Author: Mathieu Poirier mathieu.poirier@linaro.org */
+#include <linux/atomic.h> #include <linux/bitfield.h> #include <linux/coresight.h> #include <linux/coresight-pmu.h> @@ -26,6 +27,9 @@ #include "coresight-syscfg.h" #include "coresight-trace-id.h"
+#define ETM_PERF_ACT_STOP BIT(0) +#define ETM_PERF_ACT_AUX BIT(1) + static struct pmu etm_pmu; static bool etm_perf_up;
@@ -46,6 +50,7 @@ static bool etm_perf_up; struct etm_ctxt { struct perf_output_handle handle; struct etm_event_data *event_data; + atomic_t action; };
static DEFINE_PER_CPU(struct etm_ctxt, etm_ctxt); @@ -531,33 +536,69 @@ static void *etm_setup_aux(struct perf_event *event, void **pages, goto out; }
-static int etm_event_resume(struct etm_ctxt *ctxt) +static void etm_event_stop(struct perf_event *event, int mode); + +static bool etm_event_aux_begin(struct perf_event *event, struct etm_ctxt *ctxt) +{ + int action; + + /* Claim exclusive access for the AUX operation */ + action = atomic_fetch_or(ETM_PERF_ACT_AUX, &ctxt->action); + if (action & ETM_PERF_ACT_AUX) + return false; + + /* Leave an existing stop to finish its own work */ + if ((action & ETM_PERF_ACT_STOP) || + (READ_ONCE(event->hw.state) & PERF_HES_STOPPED)) { + atomic_fetch_andnot(ETM_PERF_ACT_AUX, &ctxt->action); + return false; + } + + return true; +} + +static void etm_event_aux_end(struct perf_event *event, struct etm_ctxt *ctxt) +{ + int action; + + /* Complete any stop deferred while the AUX operation was active */ + action = atomic_fetch_andnot(ETM_PERF_ACT_AUX, &ctxt->action); + if (action & ETM_PERF_ACT_STOP) + etm_event_stop(event, PERF_EF_UPDATE); +} + +static int etm_event_resume(struct perf_event *event, struct etm_ctxt *ctxt) { struct perf_output_handle *handle = &ctxt->handle; - struct perf_event *event = handle->event; struct coresight_device *source; struct coresight_path *path; - int ret; + int ret = 0;
- if (!perf_get_aux(handle)) + if (!etm_event_aux_begin(event, ctxt)) return 0;
+ if (!perf_get_aux(handle)) + goto out; + path = etm_event_get_ctxt_path(ctxt); if (!path) - return 0; + goto out;
source = coresight_get_source(path); if (!source) - return 0; + goto out;
ret = coresight_resume_source(source); if (ret < 0) { dev_err(&source->dev, "Failed to resume ETM event.\n"); - return ret; + goto out; }
etm_event_clear_hw_state(event, PERF_HES_UPTODATE); - return 0; + +out: + etm_event_aux_end(event, ctxt); + return ret; }
static void etm_event_start(struct perf_event *event, int flags) @@ -571,7 +612,7 @@ static void etm_event_start(struct perf_event *event, int flags) u64 hw_id;
if (flags & PERF_EF_RESUME) { - WARN_ON_ONCE(etm_event_resume(ctxt)); + WARN_ON_ONCE(etm_event_resume(event, ctxt)); return; }
@@ -697,18 +738,22 @@ static void etm_event_update_buffer(struct perf_event *event, static void etm_event_pause(struct perf_event *event, struct etm_ctxt *ctxt) { - struct coresight_path *path = etm_event_get_ctxt_path(ctxt); + struct coresight_path *path; struct perf_output_handle *handle = &ctxt->handle; struct coresight_device *source, *sink; struct etm_event_data *event_data;
- if (!path) + if (!etm_event_aux_begin(event, ctxt)) return;
+ path = etm_event_get_ctxt_path(ctxt); + if (!path) + goto out; + source = coresight_get_source(path); sink = coresight_get_sink(path); if (WARN_ON_ONCE(!source || !sink)) - return; + goto out;
/* Stop tracer */ coresight_pause_source(source); @@ -720,7 +765,7 @@ static void etm_event_pause(struct perf_event *event, * disallows updating buffer for the per CPU sink case. */ if (coresight_is_percpu_sink(sink)) - return; + goto out;
event_data = READ_ONCE(ctxt->event_data); etm_event_update_buffer(event, handle, event_data, sink, @@ -728,6 +773,9 @@ static void etm_event_pause(struct perf_event *event,
/* Prepare the handle for resuming trace */ perf_aux_output_begin(handle, event); + +out: + etm_event_aux_end(event, ctxt); }
static void etm_event_stop(struct perf_event *event, int mode) @@ -738,13 +786,28 @@ static void etm_event_stop(struct perf_event *event, int mode) struct coresight_path *path; struct hw_perf_event *hwc = &event->hw; struct etm_event_data *event_data; + int action; + + if (mode & PERF_EF_PAUSE) { + etm_event_pause(event, ctxt); + return; + }
/* If we're already stopped, then nothing to do */ if (READ_ONCE(hwc->state) & PERF_HES_STOPPED) return;
- if (mode & PERF_EF_PAUSE) - return etm_event_pause(event, ctxt); + /* Leave STOP pending until the AUX operation releases its action bit */ + action = atomic_fetch_or(ETM_PERF_ACT_STOP, &ctxt->action); + if (action & ETM_PERF_ACT_AUX) + return; + + /* + * A repeated throttling stop must not interrupt an active teardown. + * PERF_EF_UPDATE lets the AUX callback complete a deferred stop. + */ + if ((action & ETM_PERF_ACT_STOP) && !(mode & PERF_EF_UPDATE)) + return;
path = etm_event_get_ctxt_path(ctxt);
@@ -754,7 +817,7 @@ static void etm_event_stop(struct perf_event *event, int mode) */ if (!path) { etm_event_set_hw_state(event, PERF_HES_STOPPED | PERF_HES_UPTODATE); - return; + goto out; }
event_data = READ_ONCE(ctxt->event_data); @@ -764,18 +827,19 @@ static void etm_event_stop(struct perf_event *event, int mode) source = coresight_get_source(path); sink = coresight_get_sink(path); if (!source || !sink) - return; + goto out;
/* stop tracer */ coresight_disable_source(source, event);
- /* tell the core */ - etm_event_set_hw_state(event, PERF_HES_STOPPED); - etm_event_update_buffer(event, handle, event_data, sink, mode);
/* Disabling the path make its elements available to other sessions */ coresight_disable_path(path); + etm_event_set_hw_state(event, PERF_HES_STOPPED); + +out: + atomic_fetch_andnot(ETM_PERF_ACT_STOP, &ctxt->action); }
static int etm_event_add(struct perf_event *event, int mode)