The existing validation rejects negative copy errors but allows a zero return. In that case no AUX data was copied, but it is incorrectly passed to the alignment-padding check.

Reject zero together with negative errors before calculating the padding. This reports the invalid snapshot result and avoids treating a missing AUX payload as alignment padding.

Fixes: a4faf00d994c ("perf/aux: Allow using AUX data in perf samples") Assisted-by: Codex:gpt-6 Signed-off-by: Leo Yan leo.yan@arm.com --- kernel/events/core.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/kernel/events/core.c b/kernel/events/core.c index fe33fe15689d077e7f445c0b17d9c2e638c884a1..17355e4b8b7c6c106117153c9eb8509e6f9a41e3 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -8058,12 +8058,11 @@ static void perf_aux_sample_output(struct perf_event *event, size = perf_pmu_snapshot_aux(rb, sampler, handle, data->aux_size);

/* - * An error here means that perf_output_copy() failed (returned a - * non-zero surplus that it didn't copy), which in its current - * enlightened implementation is not possible. If that changes, we'd - * like to know. + * A negative return means that perf_output_copy() failed, while zero + * means that no AUX data was copied despite a non-zero request. Neither + * can be treated as alignment padding below. */ - if (WARN_ON_ONCE(size < 0)) + if (WARN_ON_ONCE(size <= 0)) goto out_put;

/*

CoreSight AUX sampling needs perf_output_copy_aux() to copy trace data into samples and perf_output_copy() to prepend zero bytes when the available trace is shorter than the requested snapshot.

Export both helpers so the CoreSight snapshot_aux() callback can use them when CONFIG_CORESIGHT=m.

Assisted-by: Codex:gpt-6 Signed-off-by: Leo Yan leo.yan@arm.com --- kernel/events/ring_buffer.c | 2 ++ 1 file changed, 2 insertions(+)

diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 1b1ffe0533e58a9d976d0eb0fc23e09d5936da2b..e69ea5b077a5e94b5b6bb1f4c6f0ca5b8a94a6cc 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -298,6 +298,7 @@ unsigned int perf_output_copy(struct perf_output_handle *handle, { return __output_copy(handle, buf, len); } +EXPORT_SYMBOL_GPL(perf_output_copy);

unsigned int perf_output_skip(struct perf_output_handle *handle, unsigned int len) @@ -617,6 +618,7 @@ long perf_output_copy_aux(struct perf_output_handle *aux_handle,

return len; } +EXPORT_SYMBOL_GPL(perf_output_copy_aux);

#define PERF_AUX_GFP (GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY)

The pause and stop callbacks duplicate the sequence that updates a sink buffer and finishes the current AUX output transaction. Move the common sequence into etm_event_update_buffer() for a single implementation.

The helper first verifies that perf handle is active. It updates the buffer when PERF_EF_UPDATE is requested and otherwise ends the transaction without data. The pause callback can then start a new perf handle for resuming trace.

Assisted-by: Codex:gpt-6 Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-etm-perf.c | 89 ++++++++++-------------- 1 file changed, 36 insertions(+), 53 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index a45f69f39f175f3044322b25fe5413fb97154496..13d865ca86f1161b2e21174f096473c4829afc8c 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -637,6 +637,38 @@ static void etm_event_start(struct perf_event *event, int flags) return; }

+static void etm_event_update_buffer(struct perf_output_handle *handle, + struct etm_event_data *event_data, + struct coresight_device *sink, + int mode) +{ + unsigned long size; + + /* + * Make sure the handle is still valid, as the sink may close it from + * the IRQ handler. E.g., the sink driver may fail to restart the + * handle during interrupt handling due to insufficient buffer space. + * The sink driver must serialize update_buffer() with IRQ handling, + * so this should return either a valid handle with a valid size + * (which may be 0), or no handle at all. + */ + if (!perf_get_aux(handle)) + return; + + if (!sink_ops(sink)->update_buffer) + goto out; + + if (mode & PERF_EF_UPDATE) { + size = sink_ops(sink)->update_buffer(sink, handle, + event_data->snk_config); + perf_aux_output_end(handle, size); + return; + } + +out: + perf_aux_output_end(handle, 0); +} + static void etm_event_pause(struct coresight_path *path, struct perf_event *event, struct etm_ctxt *ctxt) @@ -644,7 +676,6 @@ static void etm_event_pause(struct coresight_path *path, struct perf_output_handle *handle = &ctxt->handle; struct coresight_device *source, *sink; struct etm_event_data *event_data; - unsigned long size;

if (!path) return; @@ -669,27 +700,16 @@ static void etm_event_pause(struct coresight_path *path, if (WARN_ON_ONCE(handle->event != event)) return;

- if (!sink_ops(sink)->update_buffer) - return; - event_data = READ_ONCE(ctxt->event_data); - size = sink_ops(sink)->update_buffer(sink, handle, - event_data->snk_config); - if (READ_ONCE(handle->event)) { - if (!size) - return; + etm_event_update_buffer(handle, event_data, sink, PERF_EF_UPDATE);

- perf_aux_output_end(handle, size); - perf_aux_output_begin(handle, event); - } else { - WARN_ON_ONCE(size); - } + /* Prepare the handle for resuming trace */ + perf_aux_output_begin(handle, event); }

static void etm_event_stop(struct perf_event *event, int mode) { int cpu = smp_processor_id(); - unsigned long size; struct coresight_device *source, *sink; struct etm_ctxt *ctxt = this_cpu_ptr(&etm_ctxt); struct perf_output_handle *handle = &ctxt->handle; @@ -742,45 +762,8 @@ static void etm_event_stop(struct perf_event *event, int mode) /* tell the core */ event->hw.state = PERF_HES_STOPPED;

- /* - * If the handle is not bound to an event anymore - * (e.g, the sink driver was unable to restart the - * handle due to lack of buffer space), we don't - * have to do anything here. - */ - if (!handle->event) - goto out; - - if (mode & PERF_EF_UPDATE) { - if (WARN_ON_ONCE(handle->event != event)) - goto out; - - /* update trace information */ - if (!sink_ops(sink)->update_buffer) - goto out; + etm_event_update_buffer(handle, event_data, sink, mode);

- size = sink_ops(sink)->update_buffer(sink, handle, - event_data->snk_config); - /* - * Make sure the handle is still valid as the - * sink could have closed it from an IRQ. - * The sink driver must handle the race with - * update_buffer() and IRQ. Thus either we - * should get a valid handle and valid size - * (which may be 0). - * - * But we should never get a non-zero size with - * an invalid handle. - */ - if (READ_ONCE(handle->event)) - perf_aux_output_end(handle, size); - else - WARN_ON(size); - } else { - perf_aux_output_end(handle, 0); - } - -out: /* Disabling the path make its elements available to other sessions */ coresight_disable_path(path); }

etm_event_get_ctxt_path() returns a path only when the per-CPU context has published event data and that data contains a path for the current CPU. Perf PMU callbacks run on the same CPU in atomic context, and only the normal stop clears the event data. A returned path therefore establishes that the pause and stop callbacks have a live ETM context.

The AUX handle has a separate lifetime. It can be absent after an AUX transaction could not be restarted while the CoreSight path still needs to be paused or disabled. The handle is acquired for the callback event before event data is published, and another ETM event cannot replace it while the published context remains live. Thus any active handle belongs to that context, and etm_event_update_buffer() uses perf_get_aux() to determine whether it remains valid.

Remove the redundant comparisons of the handle with the event and event data, together with the redundant event_data check. Use the path to validate the ETM context lifetime and perf_get_aux() to validate the AUX handle lifetime.

Assisted-by: Codex:gpt-6 Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-etm-perf.c | 15 --------------- 1 file changed, 15 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index afd22fd4a851e600ab69fd5b6883e6f8518b2cab..c7454da5c881c4a8eea5cc77902c9113c297096e 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -698,9 +698,6 @@ static void etm_event_pause(struct coresight_path *path, if (coresight_is_percpu_sink(sink)) return;

- if (WARN_ON_ONCE(handle->event != event)) - return; - event_data = READ_ONCE(ctxt->event_data); etm_event_update_buffer(handle, event_data, sink, PERF_EF_UPDATE);

@@ -728,14 +725,6 @@ static void etm_event_stop(struct perf_event *event, int mode) return; }

- /* - * If we still have access to the event_data via handle, - * confirm that we haven't messed up the tracking. - */ - if (handle->event && - WARN_ON(perf_get_aux(handle) != ctxt->event_data)) - return; - event_data = READ_ONCE(ctxt->event_data); /* Clear the event_data as this ETM is stopping the trace. */ WRITE_ONCE(ctxt->event_data, NULL); @@ -743,10 +732,6 @@ static void etm_event_stop(struct perf_event *event, int mode) if (event->hw.state == PERF_HES_STOPPED) return;

- /* We must have a valid event_data for a running event */ - if (WARN_ON(!event_data)) - return; - source = coresight_get_source(path); sink = coresight_get_sink(path); if (!source || !sink)

Keep hw.state consistent with event activation and AUX buffer updates. Initialize events as stopped and up to date, clear both flags after a successful start, and set UPTODATE after committing an AUX buffer update. On resume, clear UPTODATE only on success. Preserve state on resume failure and ignore duplicate stops.

Add helpers for setting and clearing state bits. Use READ_ONCE() and WRITE_ONCE() to prevent compiler merging or refetching of state accesses.

Assisted-by: Codex:gpt-6 Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-etm-perf.c | 58 ++++++++++++++++-------- 1 file changed, 40 insertions(+), 18 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index e3c9d3cf84c502f357a430bc3bfa854a776618f1..f66c99b26e1862ab04105704276c901b21728207 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -150,6 +150,20 @@ etm_event_cpu_path(struct etm_event_data *data, int cpu) return *etm_event_cpu_path_ptr(data, cpu); }

+static inline void etm_event_set_hw_state(struct perf_event *event, int mask) +{ + struct hw_perf_event *hwc = &event->hw; + + WRITE_ONCE(hwc->state, READ_ONCE(hwc->state) | mask); +} + +static inline void etm_event_clear_hw_state(struct perf_event *event, int mask) +{ + struct hw_perf_event *hwc = &event->hw; + + WRITE_ONCE(hwc->state, READ_ONCE(hwc->state) & ~mask); +} + static void etm_event_read(struct perf_event *event) {}

static int etm_addr_filters_alloc(struct perf_event *event) @@ -520,6 +534,7 @@ static void *etm_setup_aux(struct perf_event *event, void **pages, static int etm_event_resume(struct etm_ctxt *ctxt) { struct perf_output_handle *handle = &ctxt->handle; + struct perf_event *event = handle->event; struct coresight_device *source; struct coresight_path *path; int ret; @@ -536,10 +551,13 @@ static int etm_event_resume(struct etm_ctxt *ctxt) return 0;

ret = coresight_resume_source(source); - if (ret < 0) + if (ret < 0) { dev_err(&source->dev, "Failed to resume ETM event.\n"); + return ret; + }

- return ret; + etm_event_clear_hw_state(event, PERF_HES_UPTODATE); + return 0; }

static void etm_event_start(struct perf_event *event, int flags) @@ -553,8 +571,7 @@ static void etm_event_start(struct perf_event *event, int flags) u64 hw_id;

if (flags & PERF_EF_RESUME) { - if (etm_event_resume(ctxt) < 0) - goto fail; + WARN_ON_ONCE(etm_event_resume(ctxt)); return; }

@@ -583,7 +600,7 @@ static void etm_event_start(struct perf_event *event, int flags) */ if (!cpumask_test_cpu(cpu, &event_data->mask)) { perf_aux_output_end(handle, 0); - event->hw.state = 0; + etm_event_clear_hw_state(event, PERF_HES_STOPPED); return; }

@@ -621,7 +638,7 @@ static void etm_event_start(struct perf_event *event, int flags) }

/* Tell the perf core the event is alive */ - event->hw.state = 0; + etm_event_clear_hw_state(event, PERF_HES_STOPPED | PERF_HES_UPTODATE); /* Save the event_data for this ETM */ WRITE_ONCE(ctxt->event_data, event_data); return; @@ -639,11 +656,12 @@ static void etm_event_start(struct perf_event *event, int flags) perf_aux_output_end(handle, 0); } fail: - event->hw.state = PERF_HES_STOPPED; + etm_event_set_hw_state(event, PERF_HES_STOPPED); return; }

-static void etm_event_update_buffer(struct perf_output_handle *handle, +static void etm_event_update_buffer(struct perf_event *event, + struct perf_output_handle *handle, struct etm_event_data *event_data, struct coresight_device *sink, int mode) @@ -668,6 +686,7 @@ static void etm_event_update_buffer(struct perf_output_handle *handle, size = sink_ops(sink)->update_buffer(sink, handle, event_data->snk_config); perf_aux_output_end(handle, size); + etm_event_set_hw_state(event, PERF_HES_UPTODATE); return; }

@@ -704,7 +723,8 @@ static void etm_event_pause(struct coresight_path *path, return;

event_data = READ_ONCE(ctxt->event_data); - etm_event_update_buffer(handle, event_data, sink, PERF_EF_UPDATE); + etm_event_update_buffer(event, handle, event_data, sink, + PERF_EF_UPDATE);

/* Prepare the handle for resuming trace */ perf_aux_output_begin(handle, event); @@ -716,8 +736,13 @@ static void etm_event_stop(struct perf_event *event, int mode) struct etm_ctxt *ctxt = this_cpu_ptr(&etm_ctxt); struct perf_output_handle *handle = &ctxt->handle; struct coresight_path *path = etm_event_get_ctxt_path(ctxt); + struct hw_perf_event *hwc = &event->hw; struct etm_event_data *event_data;

+ /* If we're already stopped, then nothing to do */ + if (READ_ONCE(hwc->state) & PERF_HES_STOPPED) + return; + if (mode & PERF_EF_PAUSE) return etm_event_pause(path, event, ctxt);

@@ -726,7 +751,7 @@ static void etm_event_stop(struct perf_event *event, int mode) * to synchronize. Mark the event stopped and up to date. */ if (!path) { - event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE; + etm_event_set_hw_state(event, PERF_HES_STOPPED | PERF_HES_UPTODATE); return; }

@@ -734,9 +759,6 @@ static void etm_event_stop(struct perf_event *event, int mode) /* Clear the event_data as this ETM is stopping the trace. */ WRITE_ONCE(ctxt->event_data, NULL);

- if (event->hw.state == PERF_HES_STOPPED) - return; - source = coresight_get_source(path); sink = coresight_get_sink(path); if (!source || !sink) @@ -746,9 +768,9 @@ static void etm_event_stop(struct perf_event *event, int mode) coresight_disable_source(source, event);

/* tell the core */ - event->hw.state = PERF_HES_STOPPED; + etm_event_set_hw_state(event, PERF_HES_STOPPED);

- etm_event_update_buffer(handle, event_data, sink, mode); + etm_event_update_buffer(event, handle, event_data, sink, mode);

/* Disabling the path make its elements available to other sessions */ coresight_disable_path(path); @@ -759,12 +781,12 @@ static int etm_event_add(struct perf_event *event, int mode) int ret = 0; struct hw_perf_event *hwc = &event->hw;

+ etm_event_set_hw_state(event, PERF_HES_STOPPED | PERF_HES_UPTODATE); + if (mode & PERF_EF_START) { etm_event_start(event, 0); - if (hwc->state & PERF_HES_STOPPED) + if (READ_ONCE(hwc->state) & PERF_HES_STOPPED) ret = -EINVAL; - } else { - hwc->state = PERF_HES_STOPPED; }

return ret;

TRBE uses Fill mode for snapshots, stopping trace collection and raising a maintenance interrupt at each wrap. Circular buffer mode retains the most recent trace without stopping collection or generating wrap interrupts.

Use circular buffer mode for snapshots unless the CPU requires the write out-of-range workaround. Keep Fill mode and its guard page on affected CPUs. Select the mode on each enable to handle task migration.

Calculate the minimum known trace advance from the write pointer and the WRAP flag, since additional complete laps cannot be counted. Advance the AUX head by this amount, then clamp the reported size to one buffer of retained trace.

Assisted-by: Codex:gpt-6 Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-trbe.c | 61 +++++++++++++++++++++------- 1 file changed, 47 insertions(+), 14 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index a31f2ebc327cd681344a62de28c0b3840505a9cb..ab3c5c7d27f3d272b07eade1e6452559808b196c 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -69,6 +69,7 @@ struct trbe_buf { int nr_pages; void **pages; bool snapshot; + bool circular; struct trbe_cpudata *cpudata; };

@@ -324,9 +325,11 @@ static void trbe_stop_and_truncate_event(struct perf_output_handle *handle) * When the write pointer reaches the address just before the limit pointer, it gets * wrapped around again to the base pointer. This is called a TRBE wrap event, which * generates a maintenance interrupt when operated in WRAP or FILL mode. This driver - * uses FILL mode, where the TRBE stops the trace collection at wrap event. The IRQ - * handler updates the AUX buffer and re-enables the TRBE with updated WRITE and - * LIMIT pointers. + * uses FILL mode for non-overwrite buffers, where the TRBE stops trace collection + * at a wrap event. The IRQ handler updates the AUX buffer and re-enables the TRBE + * with updated WRITE and LIMIT pointers. Snapshot buffers use Circular Buffer + * mode where possible, overwriting old trace without stopping or interrupting + * the CPU when the write pointer wraps. * * Wrap around with an IRQ * ------ < ------ < ------- < ----- < ----- @@ -628,6 +631,7 @@ static void set_trbe_limit_pointer_enabled(struct trbe_buf *buf) { u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1); unsigned long addr = buf->trbe_limit; + u64 mode = buf->circular ? TRBLIMITR_EL1_FM_CBUF : TRBLIMITR_EL1_FM_FILL;

WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_EL1_LIMIT_SHIFT))); WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE)); @@ -638,15 +642,11 @@ static void set_trbe_limit_pointer_enabled(struct trbe_buf *buf) trblimitr &= ~TRBLIMITR_EL1_LIMIT_MASK;

/* - * Fill trace buffer mode is used here while configuring the - * TRBE for trace capture. In this particular mode, the trace - * collection is stopped and a maintenance interrupt is raised - * when the current write pointer wraps. This pause in trace - * collection gives the software an opportunity to capture the - * trace data in the interrupt handler, before reconfiguring - * the TRBE. + * Circular mode keeps the most recent trace in a snapshot buffer + * without generating wrap interrupts. Use Fill mode otherwise so + * the IRQ handler can collect the trace before it is overwritten. */ - trblimitr |= (TRBLIMITR_EL1_FM_FILL << TRBLIMITR_EL1_FM_SHIFT) & + trblimitr |= (mode << TRBLIMITR_EL1_FM_SHIFT) & TRBLIMITR_EL1_FM_MASK;

/* @@ -757,6 +757,26 @@ static unsigned long trbe_get_trace_size(struct perf_output_handle *handle, return size; }

+static unsigned long trbe_get_circular_size(struct perf_output_handle *handle, + struct trbe_buf *buf, u64 status) +{ + u64 start = PERF_IDX2OFF(handle->head, buf); + u64 write = get_trbe_write_pointer() - buf->trbe_base; + + /* + * WRAP indicates at least one crossing of the limit. Any additional + * full laps cannot be determined, so account only for the trace + * known to have been collected. This also preserves the write offset + * when updating the head. + */ + if (is_trbe_wrap(status)) + write += (u64)buf->nr_pages << PAGE_SHIFT; + else if (WARN_ON_ONCE(write < start)) + return 0; + + return write - start; +} + static void *arm_trbe_alloc_buffer(struct coresight_device *csdev, struct perf_event *event, void **pages, int nr_pages, bool snapshot) @@ -887,14 +907,19 @@ static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev, wrap = true; }

- size = trbe_get_trace_size(handle, buf, wrap); + if (buf->circular) + size = trbe_get_circular_size(handle, buf, status); + else + size = trbe_get_trace_size(handle, buf, wrap);

done: local_irq_restore(flags);

if (buf->snapshot) handle->head += size; - return size; + + /* At most one full buffer of trace is available */ + return min(size, (unsigned long)buf->nr_pages << PAGE_SHIFT); }

@@ -973,7 +998,8 @@ static int trbe_apply_work_around_before_enable(struct trbe_buf *buf) * - At trace collection: * - Pad the 256bytes skipped above again with IGNORE packets. */ - if (trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE)) { + if (!buf->circular && + trbe_has_erratum(buf->cpudata, TRBE_WORKAROUND_OVERWRITE_FILL_MODE)) { if (WARN_ON(!IS_ALIGNED(buf->trbe_write, PAGE_SIZE))) return -EINVAL; buf->trbe_hw_base = buf->trbe_write; @@ -1056,6 +1082,13 @@ static int arm_trbe_enable(struct coresight_device *csdev, enum cs_mode mode, cpudata->mode = mode; buf->cpudata = cpudata;

+ /* + * A CPU affected by the write out-of-range erratum cannot use circular + * mode, as the TRBE continues tracing after wrapping and may write out + * of range. Fall back to FILL mode so the IRQ handler can fix this up. + */ + buf->circular = buf->snapshot && !trbe_may_write_out_of_range(cpudata); + return __arm_trbe_enable(buf, handle); }

arm_trbe_disable() detaches the trace buffer and clears cpudata->buf. A subsequent disable dereferences this NULL pointer in the buffer consistency check before testing whether the sink is still in perf mode.

Check the mode before accessing the buffer and return success when the sink is already disabled. This allows deferred event cleanup to revisit a path that an AUX snapshot has already disabled after a failure.

Assisted-by: Codex:gpt-6 Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-trbe.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 8eb625d3fc99db57a18e35ef59ef814dacf20a09..6da89a890712d32e87a15f7c564ff4dc4d197bb1 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -1098,11 +1098,15 @@ static int arm_trbe_disable(struct coresight_device *csdev) struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev); struct trbe_buf *buf = cpudata->buf;

+ /* A failed AUX snapshot may have already disabled the sink. */ + if (cpudata->mode == CS_MODE_DISABLED) + return 0; + if (cpudata->mode != CS_MODE_PERF) + return -EINVAL; + WARN_ON(buf->cpudata != cpudata); WARN_ON(cpudata->cpu != smp_processor_id()); WARN_ON(cpudata->drvdata != drvdata); - if (cpudata->mode != CS_MODE_PERF) - return -EINVAL;

trbe_drain_and_disable_local(cpudata); buf->cpudata = NULL;

A PMU NMI can throttle a group during an AUX pause or resume, stopping its CoreSight event before the callback finishes. The callback can then restart tracing or reopen an AUX output handle after teardown.

Use per-CPU atomic STOP and AUX bits to serialize these callbacks. Reject AUX operations while a stop is pending and defer stops until an active AUX operation finishes. Complete deferred stops through etm_event_stop() and set PERF_HES_STOPPED only after disabling the source and path.

Assisted-by: Codex:gpt-6 Signed-off-by: Leo Yan leo.yan@arm.com --- drivers/hwtracing/coresight/coresight-etm-perf.c | 104 ++++++++++++++++++----- 1 file changed, 84 insertions(+), 20 deletions(-)

diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index 8f803914e9abfb18e080ff68371ad8b32042f180..8020a4d8af18bb669441824cfdb687f4fa299eb4 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -4,6 +4,7 @@ * Author: Mathieu Poirier mathieu.poirier@linaro.org */

+#include <linux/atomic.h> #include <linux/bitfield.h> #include <linux/coresight.h> #include <linux/coresight-pmu.h> @@ -26,6 +27,9 @@ #include "coresight-syscfg.h" #include "coresight-trace-id.h"

+#define ETM_PERF_ACT_STOP BIT(0) +#define ETM_PERF_ACT_AUX BIT(1) + static struct pmu etm_pmu; static bool etm_perf_up;

@@ -46,6 +50,7 @@ static bool etm_perf_up; struct etm_ctxt { struct perf_output_handle handle; struct etm_event_data *event_data; + atomic_t action; };

static DEFINE_PER_CPU(struct etm_ctxt, etm_ctxt); @@ -531,33 +536,69 @@ static void *etm_setup_aux(struct perf_event *event, void **pages, goto out; }

-static int etm_event_resume(struct etm_ctxt *ctxt) +static void etm_event_stop(struct perf_event *event, int mode); + +static bool etm_event_aux_begin(struct perf_event *event, struct etm_ctxt *ctxt) +{ + int action; + + /* Claim exclusive access for the AUX operation */ + action = atomic_fetch_or(ETM_PERF_ACT_AUX, &ctxt->action); + if (action & ETM_PERF_ACT_AUX) + return false; + + /* Leave an existing stop to finish its own work */ + if ((action & ETM_PERF_ACT_STOP) || + (READ_ONCE(event->hw.state) & PERF_HES_STOPPED)) { + atomic_fetch_andnot(ETM_PERF_ACT_AUX, &ctxt->action); + return false; + } + + return true; +} + +static void etm_event_aux_end(struct perf_event *event, struct etm_ctxt *ctxt) +{ + int action; + + /* Complete any stop deferred while the AUX operation was active */ + action = atomic_fetch_andnot(ETM_PERF_ACT_AUX, &ctxt->action); + if (action & ETM_PERF_ACT_STOP) + etm_event_stop(event, PERF_EF_UPDATE); +} + +static int etm_event_resume(struct perf_event *event, struct etm_ctxt *ctxt) { struct perf_output_handle *handle = &ctxt->handle; - struct perf_event *event = handle->event; struct coresight_device *source; struct coresight_path *path; - int ret; + int ret = 0;

- if (!perf_get_aux(handle)) + if (!etm_event_aux_begin(event, ctxt)) return 0;

+ if (!perf_get_aux(handle)) + goto out; + path = etm_event_get_ctxt_path(ctxt); if (!path) - return 0; + goto out;

source = coresight_get_source(path); if (!source) - return 0; + goto out;

ret = coresight_resume_source(source); if (ret < 0) { dev_err(&source->dev, "Failed to resume ETM event.\n"); - return ret; + goto out; }

etm_event_clear_hw_state(event, PERF_HES_UPTODATE); - return 0; + +out: + etm_event_aux_end(event, ctxt); + return ret; }

static void etm_event_start(struct perf_event *event, int flags) @@ -571,7 +612,7 @@ static void etm_event_start(struct perf_event *event, int flags) u64 hw_id;

if (flags & PERF_EF_RESUME) { - WARN_ON_ONCE(etm_event_resume(ctxt)); + WARN_ON_ONCE(etm_event_resume(event, ctxt)); return; }

@@ -697,18 +738,22 @@ static void etm_event_update_buffer(struct perf_event *event, static void etm_event_pause(struct perf_event *event, struct etm_ctxt *ctxt) { - struct coresight_path *path = etm_event_get_ctxt_path(ctxt); + struct coresight_path *path; struct perf_output_handle *handle = &ctxt->handle; struct coresight_device *source, *sink; struct etm_event_data *event_data;

- if (!path) + if (!etm_event_aux_begin(event, ctxt)) return;

+ path = etm_event_get_ctxt_path(ctxt); + if (!path) + goto out; + source = coresight_get_source(path); sink = coresight_get_sink(path); if (WARN_ON_ONCE(!source || !sink)) - return; + goto out;

/* Stop tracer */ coresight_pause_source(source); @@ -720,7 +765,7 @@ static void etm_event_pause(struct perf_event *event, * disallows updating buffer for the per CPU sink case. */ if (coresight_is_percpu_sink(sink)) - return; + goto out;

event_data = READ_ONCE(ctxt->event_data); etm_event_update_buffer(event, handle, event_data, sink, @@ -728,6 +773,9 @@ static void etm_event_pause(struct perf_event *event,

/* Prepare the handle for resuming trace */ perf_aux_output_begin(handle, event); + +out: + etm_event_aux_end(event, ctxt); }

static void etm_event_stop(struct perf_event *event, int mode) @@ -738,13 +786,28 @@ static void etm_event_stop(struct perf_event *event, int mode) struct coresight_path *path; struct hw_perf_event *hwc = &event->hw; struct etm_event_data *event_data; + int action; + + if (mode & PERF_EF_PAUSE) { + etm_event_pause(event, ctxt); + return; + }

/* If we're already stopped, then nothing to do */ if (READ_ONCE(hwc->state) & PERF_HES_STOPPED) return;

- if (mode & PERF_EF_PAUSE) - return etm_event_pause(event, ctxt); + /* Leave STOP pending until the AUX operation releases its action bit */ + action = atomic_fetch_or(ETM_PERF_ACT_STOP, &ctxt->action); + if (action & ETM_PERF_ACT_AUX) + return; + + /* + * A repeated throttling stop must not interrupt an active teardown. + * PERF_EF_UPDATE lets the AUX callback complete a deferred stop. + */ + if ((action & ETM_PERF_ACT_STOP) && !(mode & PERF_EF_UPDATE)) + return;

path = etm_event_get_ctxt_path(ctxt);

@@ -754,7 +817,7 @@ static void etm_event_stop(struct perf_event *event, int mode) */ if (!path) { etm_event_set_hw_state(event, PERF_HES_STOPPED | PERF_HES_UPTODATE); - return; + goto out; }

event_data = READ_ONCE(ctxt->event_data); @@ -764,18 +827,19 @@ static void etm_event_stop(struct perf_event *event, int mode) source = coresight_get_source(path); sink = coresight_get_sink(path); if (!source || !sink) - return; + goto out;

/* stop tracer */ coresight_disable_source(source, event);

- /* tell the core */ - etm_event_set_hw_state(event, PERF_HES_STOPPED); - etm_event_update_buffer(event, handle, event_data, sink, mode);

/* Disabling the path make its elements available to other sessions */ coresight_disable_path(path); + etm_event_set_hw_state(event, PERF_HES_STOPPED); + +out: + atomic_fetch_andnot(ETM_PERF_ACT_STOP, &ctxt->action); }

static int etm_event_add(struct perf_event *event, int mode)

Perf clears the IP of a user-only PMU sample when its interrupt skids into the kernel. The timestamp and TID remain valid, but the zero-IP check prevents --itrace=L from attaching decoded branch history.

Remove the IP check and keep matching samples to trace by time and thread.

Assisted-by: Codex:gpt-6 Signed-off-by: Leo Yan leo.yan@arm.com --- tools/perf/util/cs-etm.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 00407a80933e1c0b76260aed5894dece50b9aecd..baa962085dba5226d9729a0ee8039374d5fcb1a0 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -3097,6 +3097,8 @@ static int cs_etm__br_stack_init(struct cs_etm_auxtrace *etm, /* * Add decoded branch history to an existing sample. The sample keeps its own * ip, callchain and event identity; only an absent branch stack is filled in. + * A user-only PMU sample can have a zero IP when the interrupt skids into + * the kernel; its thread and trace window still identify valid history. */ static int cs_etm__process_sample(struct cs_etm_auxtrace *etm, struct perf_session *session, @@ -3107,7 +3109,7 @@ static int cs_etm__process_sample(struct cs_etm_auxtrace *etm, int err;

if (!etm->synth_opts.add_last_branch || sample->branch_stack || - !sample->ip || !sample->time || sample->time == (u64)-1) + !sample->time || sample->time == (u64)-1) return 0;

/* Adding branch history to existing samples supports the host only */

cs_etm__flush() can fail while synthesizing or delivering a sample at a trace discontinuity. cs_etm__process_traceid_queue() ignores the result and continues even though the packet swap and thread stack flush may not have completed.

The timestamped decoder also loses errors when it processes packets left at the end of an AUX buffer: cs_etm__clear_all_traceid_queues() discards the result from each trace ID queue and allows decoding to fetch the next buffer.

Check the discontinuity flush result, make cs_etm__clear_all_traceid_queues() return the first queue-processing error, and propagate that error through cs_etm__process_timestamped_queues(). This stops decoding on failure and reports the error to the caller, including sample delivery errors from a perf script dlfilter.

Assisted-by: Codex:gpt-6 Signed-off-by: Leo Yan leo.yan@arm.com --- tools/perf/util/cs-etm.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 1a53431161e995ba6b3cfdc86e338ed26e675774..8340b8a16fed363795fc78ffbc419661a61442a3 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -2644,10 +2644,13 @@ static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq, break; case CS_ETM_DISCONTINUITY: /* - * Discontinuity in trace, flush - * previous branch stack + * Flush the previous branch stack at a discontinuity. + * Propagate sample delivery errors, which can occur before + * the packet swap and thread stack flush are complete. */ - cs_etm__flush(etmq, tidq); + ret = cs_etm__flush(etmq, tidq); + if (ret) + goto out; break; case CS_ETM_EMPTY: /* @@ -2665,9 +2668,9 @@ static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq, return ret; }

-static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq) +static int cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq) { - int idx; + int idx, ret; struct int_node *inode; struct cs_etm_traceid_queue *tidq; struct intlist *traceid_queues_list = etmq->traceid_queues_list; @@ -2676,9 +2679,12 @@ static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq) idx = (int)(intptr_t)inode->priv; tidq = etmq->traceid_queues[idx];

- /* Ignore return value */ - cs_etm__process_traceid_queue(etmq, tidq); + ret = cs_etm__process_traceid_queue(etmq, tidq); + if (ret) + return ret; } + + return 0; }

static int cs_etm__run_timeless_decoder(struct cs_etm_queue *etmq) @@ -2893,7 +2899,9 @@ static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm, * process in this auxtrace_buffer. As such empty and * flush all traceID queues. */ - cs_etm__clear_all_traceid_queues(etmq); + ret = cs_etm__clear_all_traceid_queues(etmq); + if (ret) + goto out;

/* Fetch another auxtrace_buffer for this etmq */ goto refetch;

OpenCSD can retain output at the end of an input block even after returning CONT. WAIT driven flushing alone does not use out that output before the decoder is reset for the next block.

Submit OCSD_OP_EOT from cs_etm_decoder__drain_packets() after any pending WAIT response has been flushed. Track submission with the eot_sent flag and clear it on reset, make sure to submit EOT only once. This applies to timeless decoding, timestamped decoding and raw packet dumping through their existing drain calls.

Return OCSD_RESP_CONT for a clean EOT without queuing a discontinuity. This lets the frontend finish draining while preserving trace history for sample processing. EOT reporting trace loss, NO_SYNC and TRACE_ON still generate discontinuities and flush the thread stack.

Assisted-by: Codex:gpt-6 Signed-off-by: Leo Yan leo.yan@arm.com --- tools/perf/util/cs-etm-decoder/cs-etm-decoder.c | 24 +++++++++++++++++++++++- tools/perf/util/cs-etm-decoder/cs-etm-decoder.h | 2 +- 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c index a7a6410481e5c99be14a951493092c145a8512ba..0fcbf91aac63bcc5c40bf1453359d3be3ece684f 100644 --- a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c +++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c @@ -46,6 +46,7 @@ struct cs_etm_decoder { void *data; void (*packet_printer)(const char *msg, void *data); bool suppress_printing; + bool eot_sent; dcd_tree_handle_t dcd_tree; cs_etm_mem_cb_type mem_access; ocsd_datapath_resp_t prev_return; @@ -86,6 +87,7 @@ int cs_etm_decoder__reset(struct cs_etm_decoder *decoder) ocsd_datapath_resp_t dp_ret;

decoder->prev_return = OCSD_RESP_CONT; + decoder->eot_sent = false; decoder->suppress_printing = true; dp_ret = ocsd_dt_process_data(decoder->dcd_tree, OCSD_OP_RESET, 0, 0, NULL, NULL); @@ -592,6 +594,17 @@ static ocsd_datapath_resp_t cs_etm_decoder__gen_trace_elem_printer(

type = elem->elem_type;

+ /* + * The frontend uses EOT to drain packets buffered in OpenCSD at the + * end of a data block. Return CONT for a clean EOT so the frontend + * can finish draining without generating a discontinuity, preserving + * trace history for sample processing. EOTs reporting trace loss + * still generate a discontinuity below. + */ + if (type == OCSD_GEN_TRC_ELEM_EO_TRACE && + elem->unsync_eot_info == UNSYNC_EOT) + return OCSD_RESP_CONT; + if (type == OCSD_GEN_TRC_ELEM_EO_TRACE || type == OCSD_GEN_TRC_ELEM_NO_SYNC || type == OCSD_GEN_TRC_ELEM_TRACE_ON) @@ -803,10 +816,19 @@ int cs_etm_decoder__drain_packets(struct cs_etm_decoder *decoder) NULL, NULL);

+ if (OCSD_DATA_RESP_IS_CONT(decoder->prev_return) && !decoder->eot_sent) { + decoder->prev_return = ocsd_dt_process_data(decoder->dcd_tree, + OCSD_OP_EOT, + 0, + 0, + NULL, + NULL); + decoder->eot_sent = true; + } + if (OCSD_DATA_RESP_IS_WAIT(decoder->prev_return)) return 1;

- /* No further WAIT driven flushing is needed */ return OCSD_DATA_RESP_IS_CONT(decoder->prev_return) ? 0 : -EINVAL; }

diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.h b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.h index c187f3809ba0a350ca6ee549d2c0c67ed8e16d1e..876b28f6fda099db5694fdfa9552104cc270b49a 100644 --- a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.h +++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.h @@ -92,7 +92,7 @@ int cs_etm_decoder__process_data_block(struct cs_etm_decoder *decoder, size_t len, size_t *consumed);

/* - * Drain pending packets after consuming a data block. + * Drain pending packets and submit EOT once after consuming a data block. * Process queued packets after each call, including the final call. * Returns 1 for WAIT, 0 when done, or a negative error. */

Decode raw AUX trace embedded in PERF_SAMPLE_AUX and attach missing callchains and branch stacks to the owning PMU samples.

Select the decoder by the sampled CPU and initialize its thread context from the sample's PID and TID. Queue each payload temporarily for the shared timeless decoder, clearing packet queues and flushing thread stacks before decoding an independent window. The embedded trace already belongs to the sample, so timestamp correlation is not needed.

Extend cs_etm__save_sample_history() to collect callchains as well as branch stacks after decoding. Support --itrace=G and --itrace=L while preserving recorded callchains and branch stacks.

Record CPU ID on AUX sampling events so each sample selects the correct decoder.

Assisted-by: Codex:gpt-6 Signed-off-by: Leo Yan leo.yan@arm.com --- tools/perf/arch/arm/util/cs-etm.c | 6 + tools/perf/util/cs-etm.c | 308 +++++++++++++++++++++++++++++++++----- 2 files changed, 274 insertions(+), 40 deletions(-)

diff --git a/tools/perf/arch/arm/util/cs-etm.c b/tools/perf/arch/arm/util/cs-etm.c index d2861d66a6612ea213066258672ba41769e578cb..710887daad0e4cbb5f48a45aa396c7bbab4c7ead 100644 --- a/tools/perf/arch/arm/util/cs-etm.c +++ b/tools/perf/arch/arm/util/cs-etm.c @@ -450,6 +450,12 @@ static int cs_etm_recording_options(struct auxtrace_record *itr, */ evsel__set_sample_bit(cs_etm_evsel, CPU);

+ /* Raw AUX samples need the sampling CPU to select the trace decoder. */ + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.aux_sample_size) + evsel__set_sample_bit(evsel, CPU); + } + /* * Also the case of per-cpu mmaps, need the contextID in order to be notified * when a context switch happened. diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 815a22cedc49e9281b7087f01355bb71b5e41c57..8a1b532df0d3f45fbec85b32ade9bc5ce5b58c98 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -66,6 +66,7 @@ struct cs_etm_auxtrace { */ bool per_thread_decoding; bool snapshot_mode; + bool sampling_mode; bool data_queued; bool has_virtual_ts; /* Virtual/Kernel timestamps in the trace. */ bool use_thread_stack; @@ -77,6 +78,7 @@ struct cs_etm_auxtrace { /* Internal reconstruction depth, see cs_etm__br_stack_init() */ unsigned int br_stack_sz_plus; struct branch_stack *br_stack; + struct ip_callchain *chain; u64 latest_kernel_timestamp; u32 auxtrace_type; u32 branches_filter; @@ -804,7 +806,8 @@ static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm, struct cs_etm_packet *tmp;

if (etm->synth_opts.branches || etm->synth_opts.last_branch || - etm->synth_opts.add_last_branch || etm->synth_opts.instructions) { + etm->synth_opts.add_last_branch || etm->synth_opts.add_callchain || + etm->synth_opts.instructions) { /* * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for * the next incoming packet. @@ -961,7 +964,7 @@ static int cs_etm__flush_events(struct perf_session *session, auxtrace); int ret;

- if (dump_trace) + if (dump_trace || etm->sampling_mode) return 0;

if (!tool->ordered_events) @@ -1077,6 +1080,7 @@ static void cs_etm__free(struct perf_session *session)

zfree(&aux->metadata); zfree(&aux->br_stack); + zfree(&aux->chain); zfree(&aux); }

@@ -1102,6 +1106,13 @@ static struct machine *cs_etm__get_machine(struct cs_etm_queue *etmq, if (pid_fmt == CS_ETM_PIDFMT_CTXTID) return &etmq->etm->session->machines.host;

+ /* + * AUX samples without context IDs use the owning host sample's PID/TID. + * Keep EL1 trace in that host context as well. + */ + if (etmq->etm->sampling_mode && pid_fmt == CS_ETM_PIDFMT_NONE) + return &etmq->etm->session->machines.host; + /* * Not perfect, but otherwise assume anything in EL1 is the default * guest, and everything else is the host. Distinguishing between guest @@ -1615,7 +1626,7 @@ static void cs_etm__add_stack_event(struct cs_etm_queue *etmq, int size;

if (!etm->synth_opts.branches && !etm->synth_opts.instructions && - !etm->synth_opts.add_last_branch) + !etm->synth_opts.add_last_branch && !etm->synth_opts.add_callchain) return;

if (!cs_etm__packet_has_taken_branch(tidq->prev_packet)) @@ -2059,10 +2070,17 @@ static int cs_etm__save_sample_history(struct cs_etm_auxtrace *etm, return -ENOMEM;

/* Consume branch history so later samples cannot reuse the same window. */ - thread_stack__br_sample_late(thread, sample->cpu, etm->br_stack, - etm->br_stack_sz, sample->ip, - machine__kernel_start(machine)); - thread_stack__br_stack_consume(thread, sample->cpu); + if (etm->synth_opts.add_last_branch && !sample->branch_stack) { + thread_stack__br_sample_late(thread, sample->cpu, etm->br_stack, + etm->br_stack_sz, sample->ip, + machine__kernel_start(machine)); + thread_stack__br_stack_consume(thread, sample->cpu); + } + + if (etm->synth_opts.add_callchain && !sample->callchain) + thread_stack__sample_late(thread, sample->cpu, etm->chain, + etm->synth_opts.callchain_sz + 1, + sample->ip, machine__kernel_start(machine));

thread__put(thread); return 0; @@ -2187,6 +2205,13 @@ static void cs_etm__flush_all_stack(struct cs_etm_queue *etmq) cs_etm__flush_machine_stack(etmq, HOST_KERNEL_ID); break; case CS_ETM_PIDFMT_NONE: + /* + * AUX samples provide PID/TID via PERF_SAMPLE_TID, so flush host + * stacks even without traced context IDs. + */ + if (etmq->etm->sampling_mode) + cs_etm__flush_machine_stack(etmq, HOST_KERNEL_ID); + break; default: break;

@@ -3064,6 +3089,61 @@ static bool cs_etm__tracing_kernel(struct cs_etm_auxtrace *etm, return false; }

+static bool cs_etm__sampling_mode(struct perf_session *session) +{ + struct evsel *evsel; + + evlist__for_each_entry(session->evlist, evsel) { + if ((evsel->core.attr.sample_type & PERF_SAMPLE_AUX) && + evsel->core.attr.aux_sample_size) + return true; + } + return false; +} + +static int cs_etm__aux_sample_init(struct cs_etm_auxtrace *etm) +{ + struct evsel *evsel; + + if (!etm->sampling_mode) { + if (etm->synth_opts.add_callchain) { + pr_err("CS ETM: --itrace=G requires AUX samples\n"); + return -EINVAL; + } + + return 0; + } + + evlist__for_each_entry(etm->session->evlist, evsel) { + u64 type = evsel->core.attr.sample_type; + u64 required = PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_CPU; + + if (!(type & PERF_SAMPLE_AUX)) + continue; + + if ((type & required) != required) { + pr_err("CS ETM: AUX samples require IP, TID and CPU\n"); + return -EINVAL; + } + + /* + * Synthesize the callchain from AUX trace when it is not + * provided by the PMU sample. + */ + if (etm->synth_opts.add_callchain && !(type & PERF_SAMPLE_CALLCHAIN)) + evsel->synth_sample_type |= PERF_SAMPLE_CALLCHAIN; + } + + if (etm->synth_opts.add_callchain) { + etm->chain = zalloc(struct_size(etm->chain, ips, + etm->synth_opts.callchain_sz + 1)); + if (!etm->chain) + return -ENOMEM; + } + + return 0; +} + static int cs_etm__br_stack_init(struct cs_etm_auxtrace *etm, struct perf_session *session) { @@ -3071,10 +3151,11 @@ static int cs_etm__br_stack_init(struct cs_etm_auxtrace *etm,

evlist__for_each_entry(session->evlist, evsel) { /* - * Only timestamped events can be matched against the decoded - * trace, so do not advertise a branch stack on any other. + * AUX samples own their trace window. Other samples need a + * timestamp to match against the decoded trace. */ - if (!(evsel->core.attr.sample_type & PERF_SAMPLE_TIME)) + if (!(evsel->core.attr.sample_type & + (etm->sampling_mode ? PERF_SAMPLE_AUX : PERF_SAMPLE_TIME))) continue; if (!(evsel->core.attr.sample_type & PERF_SAMPLE_BRANCH_STACK)) evsel->synth_sample_type |= PERF_SAMPLE_BRANCH_STACK; @@ -3101,9 +3182,85 @@ static int cs_etm__br_stack_init(struct cs_etm_auxtrace *etm, return 0; }

+static int cs_etm__set_sample_context(struct cs_etm_queue *etmq, + const struct perf_sample *sample) +{ + struct machine *machine = &etmq->etm->session->machines.host; + struct cs_etm_traceid_queue *tidq; + u64 *metadata = get_cpu_data(etmq->etm, sample->cpu); + struct thread *thread; + u8 trace_id; + int ret; + + if (!metadata) + return -EINVAL; + + ret = cs_etm__metadata_get_trace_id(&trace_id, metadata); + if (ret) + return ret; + + tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_id); + if (!tidq) + return -ENOMEM; + + thread = machine__findnew_thread(machine, sample->pid, sample->tid); + if (!thread) + return -ENOMEM; + + tidq->kernel_start = machine__kernel_start(machine); + tidq->decode_el = ocsd_EL_unknown; + thread__put(tidq->decode_thread); + thread__put(tidq->frontend_thread); + /* Thread already holds a reference from machine__findnew_thread() */ + tidq->decode_thread = thread; + tidq->frontend_thread = thread__get(thread); + + return 0; +} + +/* Consume the queued AUX window while its owning sample is still available. */ +static int cs_etm__process_aux_sample(struct cs_etm_auxtrace *etm, + struct perf_sample *sample) +{ + struct cs_etm_queue *etmq = cs_etm__get_queue(etm, sample->cpu); + struct auxtrace_buffer buffer = { + .data = sample->aux_sample.data, + .size = sample->aux_sample.size, + .pid = sample->pid, + .tid = sample->tid, + .cpu = { sample->cpu }, + }; + struct auxtrace_queue *queue; + int ret; + + if (!etmq || !etmq->decoder) + return -EINVAL; + + queue = &etm->queues.queue_array[etmq->queue_nr]; + if (!list_empty(&queue->head) || etmq->buffer) + return -EINVAL; + + ret = cs_etm__set_sample_context(etmq, sample); + if (ret) + return ret; + + cs_etm__clear_all_packet_queues(etmq); + + buffer.buffer_nr = etm->queues.next_buffer_nr++; + list_add_tail(&buffer.list, &queue->head); + ret = cs_etm__run_timeless_decoder(etmq); + + /* The buffer is borrowed for this decode only. */ + list_del_init(&buffer.list); + etmq->buffer = NULL; + etmq->buf = NULL; + etmq->buf_len = 0; + return ret; +} + /* - * Add decoded branch history to an existing sample. The sample keeps its own - * ip, callchain and event identity; only an absent branch stack is filled in. + * Decode the trace belonging to this sample and fill in missing history. + * The sample keeps its IP, event identity and any recorded stacks. * A user-only PMU sample can have a zero IP when the interrupt skids into * the kernel; its thread and trace window still identify valid history. */ @@ -3112,25 +3269,35 @@ static int cs_etm__process_sample(struct cs_etm_auxtrace *etm, { int err;

- if (!etm->synth_opts.add_last_branch || sample->branch_stack || - !sample->time || sample->time == (u64)-1) + if ((!etm->synth_opts.add_last_branch || sample->branch_stack) && + (!etm->synth_opts.add_callchain || sample->callchain)) return 0;

- /* Adding branch history to existing samples supports the host only */ + /* Adding history to existing samples supports the host only */ if (sample->cpumode == PERF_RECORD_MISC_GUEST_KERNEL || sample->cpumode == PERF_RECORD_MISC_GUEST_USER) return 0;

- err = cs_etm__update_queues(etm); - if (err) - return err; + if (etm->sampling_mode) { + if (!sample->aux_sample.size) + return 0;

- /* - * Decode every queue up to this sample's time. Afterwards the thread - * stack holds the branches that executed before the sample, and - * nothing that executed after it. - */ - err = cs_etm__process_timestamped_queues(etm, sample->time); + err = cs_etm__process_aux_sample(etm, sample); + } else { + if (!sample->time || sample->time == (u64)-1) + return 0; + + err = cs_etm__update_queues(etm); + if (err) + return err; + + /* + * Decode every queue up to this sample's time. Afterwards the + * thread stack holds the branches that executed before the + * sample, and nothing that executed after it. + */ + err = cs_etm__process_timestamped_queues(etm, sample->time); + } if (err) return err;

@@ -3138,9 +3305,16 @@ static int cs_etm__process_sample(struct cs_etm_auxtrace *etm, if (err) return err;

- if (etm->br_stack->nr) + if (etm->synth_opts.add_last_branch && !sample->branch_stack && etm->br_stack->nr) sample->branch_stack = etm->br_stack;

+ if (etm->synth_opts.add_callchain && !sample->callchain) { + /* An empty history produces only a context marker and sample IP */ + if (etm->chain->nr > 2 || + (etm->chain->nr == 2 && etm->chain->ips[1] != sample->ip)) + sample->callchain = etm->chain; + } + return 0; }

@@ -3225,6 +3399,10 @@ static int cs_etm__process_auxtrace_event(struct perf_session *session, struct cs_etm_auxtrace *etm = container_of(session->auxtrace, struct cs_etm_auxtrace, auxtrace); + + if (etm->sampling_mode) + return 0; + if (!etm->data_queued) { struct auxtrace_buffer *buffer; off_t data_offset; @@ -3529,14 +3707,28 @@ static int cs_etm__queue_aux_records_cb(struct perf_session *session, union perf return ret; }

-static int cs_etm__queue_aux_records(struct perf_session *session) +static int cs_etm__prepare_auxtrace_queues(struct cs_etm_auxtrace *etm, + struct perf_session *session) { - struct auxtrace_index *index = list_first_entry_or_null(&session->auxtrace_index, - struct auxtrace_index, list); - if (index && index->nr > 0) - return perf_session__peek_events(session, session->header.data_offset, - session->header.data_size, - cs_etm__queue_aux_records_cb, NULL); + struct auxtrace_index *index; + unsigned int i; + + /* + * AUX samples have no AUX record to describe the formatter framing. + * Since it is only supported by TRBE, the trace is always unformatted. + */ + if (etm->sampling_mode) { + for (i = 0; i < etm->queues.nr_queues; i++) { + struct cs_etm_queue *etmq = etm->queues.queue_array[i].priv; + + etmq->format = UNFORMATTED; + } + + return 0; + } + + index = list_first_entry_or_null(&session->auxtrace_index, + struct auxtrace_index, list);

/* * We would get here if there are no entries in the index (either no auxtrace @@ -3546,7 +3738,12 @@ static int cs_etm__queue_aux_records(struct perf_session *session) * * In that scenario, buffers will not be split by AUX records. */ - return 0; + if (!index || index->nr <= 0) + return 0; + + return perf_session__peek_events(session, session->header.data_offset, + session->header.data_size, + cs_etm__queue_aux_records_cb, NULL); }

#define HAS_PARAM(j, type, param) (metadata[(j)][CS_ETM_NR_TRC_PARAMS] <= \ @@ -3622,6 +3819,11 @@ static int cs_etm__create_queue_decoders(struct cs_etm_queue *etmq) if (decoders == 0) return 0;

+ if (etmq->etm->sampling_mode && decoders != 1) { + pr_err("CS ETM Trace: AUX samples require a per-CPU raw trace source\n"); + return -EINVAL; + } + /* * Each queue can only contain data from one CPU when unformatted, so only one decoder is * needed. @@ -3679,11 +3881,12 @@ static int cs_etm__create_decoders(struct cs_etm_auxtrace *etm) int ret;

/* - * Don't create decoders for empty queues, mainly because - * etmq->format is unknown for empty queues. + * AUX sample buffers are queued when their samples are processed, + * so create their decoders even though the queues are still empty. + * Other empty queues have no known format or data to decode. */ assert(empty || etmq->format != UNSET); - if (empty) + if (empty && !etm->sampling_mode) continue;

ret = cs_etm__create_queue_decoders(etmq); @@ -3818,6 +4021,19 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, etm->synth_opts.thread_stack = session->itrace_synth_opts->thread_stack; }

+ etm->sampling_mode = cs_etm__sampling_mode(session); + if (etm->sampling_mode) { + /* AUX windows augment their owning samples, without synthesizing events. */ + etm->synth_opts.instructions = false; + etm->synth_opts.branches = false; + etm->synth_opts.callchain = false; + etm->synth_opts.last_branch = false; + if (!session->itrace_synth_opts->set) { + etm->synth_opts.add_callchain = true; + etm->synth_opts.add_last_branch = true; + } + } + if (etm->synth_opts.calls) etm->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_TRACE_BEGIN | @@ -3828,11 +4044,13 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, PERF_IP_FLAG_TRACE_BEGIN | PERF_IP_FLAG_TRACE_END;

- if (etm->synth_opts.callchain && !symbol_conf.use_callchain) { + if ((etm->synth_opts.callchain || etm->synth_opts.add_callchain) && + !symbol_conf.use_callchain) { symbol_conf.use_callchain = true; if (callchain_register_param(&callchain_param) < 0) { symbol_conf.use_callchain = false; etm->synth_opts.callchain = false; + etm->synth_opts.add_callchain = false; } }

@@ -3859,7 +4077,7 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, /* Use virtual timestamps if all ETMs report ts_source = 1 */ etm->has_virtual_ts = cs_etm__has_virtual_ts(metadata, num_cpu);

- if (!etm->has_virtual_ts) + if (!etm->has_virtual_ts && !etm->sampling_mode) ui__warning("Virtual timestamps are not enabled, or not supported by the traced system.\n" "The time field of the samples will not be set accurately.\n" "For Arm CPUs prior to Armv8.4 or without support FEAT_TRF,\n" @@ -3875,6 +4093,8 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, session->auxtrace = &etm->auxtrace;

cs_etm__setup_timeless_decoding(etm); + if (etm->sampling_mode) + etm->timeless_decoding = true;

etm->tc.time_shift = tc->time_shift; etm->tc.time_mult = tc->time_mult; @@ -3889,9 +4109,11 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, etm->use_thread_stack = etm->synth_opts.thread_stack || etm->synth_opts.last_branch || etm->synth_opts.add_last_branch || + etm->synth_opts.add_callchain || etm->synth_opts.callchain;

etm->use_callchain = etm->synth_opts.thread_stack || + etm->synth_opts.add_callchain || etm->synth_opts.callchain;

if (etm->synth_opts.last_branch || etm->synth_opts.add_last_branch) { @@ -3899,13 +4121,18 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, etm->br_stack_sz_plus = etm->br_stack_sz; }

+ err = cs_etm__aux_sample_init(etm); + if (err) + goto err_free_queues; + if (etm->synth_opts.add_last_branch) { /* * Existing samples are matched to decoded trace by time, so * the trace must carry timestamps that are correlated to perf * time and the queues must be decoded in time order. */ - if (etm->timeless_decoding || !etm->has_virtual_ts) { + if (!etm->sampling_mode && + (etm->timeless_decoding || !etm->has_virtual_ts)) { pr_err("CS ETM Trace: --itrace=L requires virtual timestamped trace\n"); err = -EINVAL; goto err_free_queues; @@ -3920,7 +4147,7 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, if (err) goto err_free_queues;

- err = cs_etm__queue_aux_records(session); + err = cs_etm__prepare_auxtrace_queues(etm, session); if (err) goto err_free_queues;

@@ -3972,6 +4199,7 @@ int cs_etm__process_auxtrace_info_full(union perf_event *event, session->auxtrace = NULL; err_free_etm: zfree(&etm->br_stack); + zfree(&etm->chain); zfree(&etm); err_free_metadata: /* No need to check @metadata[j], free(NULL) is supported */

Describe recording AUX trace in PMU samples and generating callchains and branch stacks with --itrace=G and --itrace=L.

Explain the default decoding behavior, preservation of recorded stacks, context ID requirements and decoding limitations.

Assisted-by: Codex:gpt-6 Signed-off-by: Leo Yan leo.yan@arm.com --- Documentation/trace/coresight/coresight-perf.rst | 39 ++++++++++++++++++++++++ 1 file changed, 39 insertions(+)

diff --git a/Documentation/trace/coresight/coresight-perf.rst b/Documentation/trace/coresight/coresight-perf.rst index c0c82b3d26ea88dad7143afdb047c4d8f502b4a6..59eda2a5b6473ed1cb6bade5d281a12bdde76bb0 100644 --- a/Documentation/trace/coresight/coresight-perf.rst +++ b/Documentation/trace/coresight/coresight-perf.rst @@ -171,6 +171,45 @@ no branch history at all, for example samples from a thread that was never traced, or samples recorded before the first or after the last trace window. It is built and installed with perf's other dlfilters.

+Decode branch stack and callchain from AUX samples +-------------------------------------------------- + +With a kernel that supports CoreSight AUX sampling, record an AUX trace +window inside each cycle sample:: + + perf record --aux-sample=8192 -e '{cs_etm//u,cycles/period=100003/u}' \ + -- ./workload + perf script --itrace=G16L64 -F comm,pid,tid,cpu,event,ip,brstack + +``G16`` adds a reconstructed callchain and ``L64`` adds up to 64 branches. +Either option can be used alone. Existing callchains and branch stacks are +preserved. Without an explicit ``--itrace`` option, both are enabled for AUX +samples. Decoded history is available in ``perf script`` and ``perf report``. + +AUX sampling supports unformatted, per-CPU sink (TRBE) trace. Each sample +selects its CPU's decoder and starts a fresh trace window. The window already +belongs to the sample, so ETM timestamps are not required for attribution. +Samples without decodable history receive no reconstructed stack. CPU-wide +recordings must enable context IDs to distinguish tasks within a window; the +default recording setup enables them. + +Callchains cover only calls visible in the captured window; callers before +the first synchronization point cannot be recovered. + +When kernel tracing is enabled, AUX samples can include PMU interrupt-handler +execution before the trace source is paused. The late-sample helpers trim +this using instruction addresses, which may not uniquely identify the sample +boundary for kernel samples. + +Formatted sinks such as ETR are not supported. They can collect trace from +sources on multiple CPUs in a shared buffer, mixing execution from unrelated +threads into the AUX sample. Context IDs can distinguish these threads, but +trace from other CPUs can consume the limited sample window, leaving little +history for the sampled thread. + +Guest samples, synthesizing instruction events, and saving reconstructed +callchains with ``perf inject`` are also unsupported. + Perf test - Verify kernel and userspace perf CoreSight work -----------------------------------------------------------