The IDR map is now unused because only one owner can use a shared sink at a time. Replace it with a single sink local buffer, 'shared_perf_buf'.
And now that we don't have the IDR map to store buffers for different sessions, allocate a private buffer for 1:1 per-cpu sessions as well as per-thread sessions. We didn't eliminate any allocations by sharing buffers in 1:1 scenarios anyway, and we allow multiple per-cpu PERF_ATTACH_TASK events. Obviously multiple users can't use a single shared_perf_buf buffer, so use a per-event one.
Use the single-source classification saved during AUX admission so that buffer allocation follows the same sharing mode as existing sessions.
Signed-off-by: James Clark james.clark@linaro.org --- drivers/hwtracing/coresight/coresight-tmc-core.c | 4 +- drivers/hwtracing/coresight/coresight-tmc-etr.c | 103 ++++++++--------------- drivers/hwtracing/coresight/coresight-tmc.h | 10 +-- 3 files changed, 43 insertions(+), 74 deletions(-)
diff --git a/drivers/hwtracing/coresight/coresight-tmc-core.c b/drivers/hwtracing/coresight/coresight-tmc-core.c index bc5a133ada3e..864da1eea52c 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-core.c +++ b/drivers/hwtracing/coresight/coresight-tmc-core.c @@ -9,7 +9,6 @@ #include <linux/init.h> #include <linux/types.h> #include <linux/device.h> -#include <linux/idr.h> #include <linux/io.h> #include <linux/iommu.h> #include <linux/err.h> @@ -832,8 +831,7 @@ static int __tmc_probe(struct device *dev, struct resource *res) ret = tmc_etr_setup_caps(dev, devid, &desc.access); if (ret) goto out; - idr_init(&drvdata->idr); - mutex_init(&drvdata->idr_mutex); + mutex_init(&drvdata->perf_buf_mutex); dev_list = "tmc_etr"; break; case TMC_CONFIG_TYPE_ETF: diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index 361a433e6f0c..b4b186c605c6 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -8,7 +8,6 @@ #include <linux/coresight.h> #include <linux/dma-mapping.h> #include <linux/iommu.h> -#include <linux/idr.h> #include <linux/mutex.h> #include <linux/refcount.h> #include <linux/slab.h> @@ -1396,16 +1395,13 @@ alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event, return ERR_PTR(-ENOMEM); }
-static struct etr_buf * -get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata, - struct perf_event *event, int nr_pages, - void **pages, bool snapshot) +static struct etr_buf *get_perf_shared_etr_buf(struct tmc_drvdata *drvdata, + struct perf_event *event, + int nr_pages, void **pages, + bool snapshot) { - int ret; - pid_t pid = task_pid_nr(event->owner); struct etr_buf *etr_buf;
-retry: /* * An etr_perf_buffer is associated with an event and holds a reference * to the AUX ring buffer that was created for that event. In CPU-wide @@ -1424,64 +1420,45 @@ get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata, * allocated for this session. If so it is shared with this event, * otherwise it is created. */ - mutex_lock(&drvdata->idr_mutex); - etr_buf = idr_find(&drvdata->idr, pid); + mutex_lock(&drvdata->perf_buf_mutex); + etr_buf = drvdata->shared_perf_buf; if (etr_buf) { refcount_inc(&etr_buf->refcount); - mutex_unlock(&drvdata->idr_mutex); - return etr_buf; + goto out; }
/* If we made it here no buffer has been allocated, do so now. */ - mutex_unlock(&drvdata->idr_mutex); - etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot); - if (IS_ERR(etr_buf)) - return etr_buf; - - /* Now that we have a buffer, add it to the IDR. */ - mutex_lock(&drvdata->idr_mutex); - ret = idr_alloc(&drvdata->idr, etr_buf, pid, pid + 1, GFP_KERNEL); - mutex_unlock(&drvdata->idr_mutex); - - /* Another event with this session ID has allocated this buffer. */ - if (ret == -ENOSPC) { - tmc_free_etr_buf(etr_buf); - goto retry; - } - - /* The IDR can't allocate room for a new session, abandon ship. */ - if (ret == -ENOMEM) { - tmc_free_etr_buf(etr_buf); - return ERR_PTR(ret); - } - - + if (!IS_ERR(etr_buf)) + drvdata->shared_perf_buf = etr_buf; +out: + mutex_unlock(&drvdata->perf_buf_mutex); return etr_buf; }
-static struct etr_buf * -get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata, - struct perf_event *event, int nr_pages, - void **pages, bool snapshot) -{ - /* - * In per-thread mode the etr_buf isn't shared, so just go ahead - * with memory allocation. - */ - return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot); -} - static struct etr_buf * get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event, int nr_pages, void **pages, bool snapshot) { - if (event->cpu == -1) - return get_perf_etr_buf_per_thread(drvdata, event, nr_pages, - pages, snapshot); + struct coresight_perf_session_id *session = &drvdata->csdev->perf_sink_id_map.session;
- return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages, - pages, snapshot); + /* + * Migratable events and 1:1 sinks use private buffers. Don't create a + * drvdata->shared_perf_buf because these type of events support + * multiple concurrent users. The path's session reference keeps the + * saved single-source classification stable throughout this allocation. + */ + if (event->cpu == -1 || session->single_source) + return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot); + + /* + * Sharing rules defined in coresight_trace_id_perf_start() mean that + * only one user can own the sink now, so we can allocate a sink local + * bounce buffer. If that user had requested events with different aux + * sizes, we'll use the size of the first event to create the buffer. + */ + return get_perf_shared_etr_buf(drvdata, event, nr_pages, pages, + snapshot); }
static struct etr_perf_buffer * @@ -1508,7 +1485,7 @@ tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event, done: /* * Keep a reference to the ETR this buffer has been allocated for - * in order to have access to the IDR in tmc_free_etr_buffer(). + * in order to clear the shared buffer in tmc_free_etr_buffer(). */ etr_perf->drvdata = drvdata; etr_perf->etr_buf = etr_buf; @@ -1543,28 +1520,22 @@ static void tmc_free_etr_buffer(void *config) { struct etr_perf_buffer *etr_perf = config; struct tmc_drvdata *drvdata = etr_perf->drvdata; - struct etr_buf *buf, *etr_buf = etr_perf->etr_buf; + struct etr_buf *etr_buf = etr_perf->etr_buf;
if (!etr_buf) goto free_etr_perf_buffer;
- mutex_lock(&drvdata->idr_mutex); + mutex_lock(&drvdata->perf_buf_mutex); /* If we are not the last one to use the buffer, don't touch it. */ if (!refcount_dec_and_test(&etr_buf->refcount)) { - mutex_unlock(&drvdata->idr_mutex); + mutex_unlock(&drvdata->perf_buf_mutex); goto free_etr_perf_buffer; }
- /* We are the last one, remove from the IDR and free the buffer. */ - buf = idr_remove(&drvdata->idr, etr_perf->pid); - mutex_unlock(&drvdata->idr_mutex); - - /* - * Something went very wrong if the buffer associated with this ID - * is not the same in the IDR. Leak to avoid use after free. - */ - if (buf && WARN_ON(buf != etr_buf)) - goto free_etr_perf_buffer; + /* Clear the CPU-wide cache before freeing its last reference. */ + if (drvdata->shared_perf_buf == etr_buf) + drvdata->shared_perf_buf = NULL; + mutex_unlock(&drvdata->perf_buf_mutex);
tmc_free_etr_buf(etr_perf->etr_buf);
diff --git a/drivers/hwtracing/coresight/coresight-tmc.h b/drivers/hwtracing/coresight/coresight-tmc.h index 319a354ede9f..95e5623ea881 100644 --- a/drivers/hwtracing/coresight/coresight-tmc.h +++ b/drivers/hwtracing/coresight/coresight-tmc.h @@ -8,7 +8,6 @@ #define _CORESIGHT_TMC_H
#include <linux/dma-mapping.h> -#include <linux/idr.h> #include <linux/miscdevice.h> #include <linux/mutex.h> #include <linux/refcount.h> @@ -235,8 +234,8 @@ struct tmc_resrv_buf { * @etr_caps: Bitmask of capabilities of the TMC ETR, inferred from the * device configuration register (DEVID) * @etr_mode: User preferred mode of the ETR device, default auto mode. - * @idr: Holds etr_bufs allocated for this ETR. - * @idr_mutex: Access serialisation for idr. + * @shared_perf_buf: CPU-wide perf buffer shared by this ETR's owner. + * @perf_buf_mutex: Protects shared_perf_buf and its reference count. * @sysfs_buf: SYSFS buffer for ETR. * @perf_buf: PERF buffer for ETR. * @resrv_buf: Used by ETR as hardware trace buffer and for trace data @@ -269,8 +268,9 @@ struct tmc_drvdata { u32 trigger_cntr; u32 etr_caps; enum etr_mode etr_mode; - struct idr idr; - struct mutex idr_mutex; + struct etr_buf *shared_perf_buf; + /* Protects shared_perf_buf and its reference count. */ + struct mutex perf_buf_mutex; struct etr_buf *sysfs_buf; struct etr_buf *perf_buf; struct tmc_resrv_buf resrv_buf;