Hi, this is an incomplete patch for an issue with EL2 kernels, and I'm looking for feedback on how to complete it.
The background is that to support tracing multiple address spaces we get ETM to embed the context id in the trace, and we build with CONFIG_PID_IN_CONTEXTIDR to get the scheduler to put the thread id in CONTEXTIDR_EL1. This is a known technique, it's what context id tracing is designed for.
The problem is when the kernel is running not at EL1 (OS level) but at EL2 (hypervisor level), which is now becoming common. With HCR_EL2.E2H set, the kernel's writes to CONTEXTIDR_EL1 actually change a different physical register, CONTEXTIDR_EL2. However, ETM still traces CONTEXTIDR_EL1. So the context ids in the trace are zero, and trace cannot be reconstructed.
ETM 4.1 has an option VMIDOPT to cause CONTEXTIDR_EL2 to be output in trace, in the VMID field replacing the value of VTTBR.VMID. So we can use that, but the trace follower, collecting events from OpenCSD, needs to be aware it needs to check the VMID field not the CID field. OpenCSD doesn't need to change but perf does. TRCCONFIGR is already in the metadata, so perf consumers can check it to see what's going on.
The patch below does the kernel and userspace side but is not complete. The problem is that userspace perf creates the metadata copy of TRCCONFIGR based on its request (and fills in the other id registers by reading sysfs), but the detection of EL2/E2H happens in the kernel which adjusts TRCCONFIGR, and it's this config which is needed for decode. I see three ways round this:
- have userspace test to see if the kernel is EL2 (somehow) and adjust the metadata to mirror what the kernel is doing
- have the kernel pass the adjusted TRCCONFIGR back so perf can put it in the metadata
- have the perf decoder get the thread id from whichever of VMID and CONTEXTID is available in a PE_CONTEXT element
Obviously, the last is simplest, but it's a bodge, and means that OpenCSD will see VMIDs when its TRCCONFIGR says it won't. It's kind of cleanest to get the real TRCCONFIGR somehow, but how do we do that?
Al
diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c index a128b5063f46..96488a0cfdcf 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.c +++ b/drivers/hwtracing/coresight/coresight-etm4x.c @@ -353,8 +353,32 @@ static int etm4_parse_event_config(struct etmv4_drvdata *drvdata, }
if (attr->config & BIT(ETM_OPT_CTXTID)) - /* bit[6], Context ID tracing bit */ - config->cfg |= BIT(ETM4_CFG_BIT_CTXTID); + { + /* + * Enable context-id tracing. The assumption is that this + * will work with CONFIG_PID_IN_CONTEXTIDR to trace process + * id changes and support decode of multiple processes. + * But ETM's context id trace traces physical CONTEXTIDR_EL1, + * while the logical CONTEXTIDR_EL1 that is written to on + * process switch is either physical CONTEXTIDR_EL1 or + * CONTEXTIDR_EL2 depending on HCR_EL2.E2H. On principle + * we should continue to use logical CONTEXTIDR_EL1. + * In order to trace physical CONTEXTIDR_EL2, we need to + * enable VMID tracing and use the VMIDOPT flag to trace + * CONTEXTIDR_EL2 rather than VTTBR.VMID in the VMID field. + * Trace decoders will need to inspect TRCCONFIGR and use + * either the CID or the VMID field from the trace packet. + */ + if (!(is_kernel_in_hyp_mode() && + (read_sysreg(hcr_el2) & BIT(34)) != 0)) { + /* bit[6], Context ID tracing bit */ + config->cfg |= BIT(ETM4_CFG_BIT_CTXTID); + } else { + /* bits[7,15], trace CONTEXTID_EL2 in VMID field */ + config->cfg |= (BIT(ETM4_CFG_BIT_VMID) | + BIT(ETM4_CFG_BIT_VMIDOPT)); + } + }
/* return stack - enable if selected and supported */ if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack) diff --git a/include/linux/coresight-pmu.h b/include/linux/coresight-pmu.h index b0e35eec6499..c2f47b25daab 100644 --- a/include/linux/coresight-pmu.h +++ b/include/linux/coresight-pmu.h @@ -19,8 +19,10 @@ /* ETMv4 CONFIGR programming bits for the ETM OPTs */ #define ETM4_CFG_BIT_CYCACC 4 #define ETM4_CFG_BIT_CTXTID 6 +#define ETM4_CFG_BIT_VMID 7 #define ETM4_CFG_BIT_TS 11 #define ETM4_CFG_BIT_RETSTK 12 +#define ETM4_CFG_BIT_VMIDOPT 15
static inline int coresight_get_trace_id(int cpu) { diff --git a/tools/include/linux/coresight-pmu.h b/tools/include/linux/coresight-pmu.h index b0e35eec6499..c2f47b25daab 100644 --- a/tools/include/linux/coresight-pmu.h +++ b/tools/include/linux/coresight-pmu.h @@ -19,8 +19,10 @@ /* ETMv4 CONFIGR programming bits for the ETM OPTs */ #define ETM4_CFG_BIT_CYCACC 4 #define ETM4_CFG_BIT_CTXTID 6 +#define ETM4_CFG_BIT_VMID 7 #define ETM4_CFG_BIT_TS 11 #define ETM4_CFG_BIT_RETSTK 12 +#define ETM4_CFG_BIT_VMIDOPT 15
static inline int coresight_get_trace_id(int cpu) { diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c index cd92a99eb89d..a54cad778841 100644 --- a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c +++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c @@ -35,6 +35,7 @@ struct cs_etm_decoder { dcd_tree_handle_t dcd_tree; cs_etm_mem_cb_type mem_access; ocsd_datapath_resp_t prev_return; + uint32 thread_id_in_vmid:1; };
static u32 @@ -496,17 +497,24 @@ cs_etm_decoder__buffer_exception_ret(struct cs_etm_packet_queue *queue,
static ocsd_datapath_resp_t cs_etm_decoder__set_tid(struct cs_etm_queue *etmq, + struct cs_etm_decoder *decoder, struct cs_etm_packet_queue *packet_queue, const ocsd_generic_trace_elem *elem, const uint8_t trace_chan_id) { pid_t tid;
- /* Ignore PE_CONTEXT packets that don't have a valid contextID */ - if (!elem->context.ctxt_id_valid) - return OCSD_RESP_CONT; + if (!decoder->thread_id_in_vmid) { + /* Ignore PE_CONTEXT packets that don't have a valid contextID */ + if (!elem->context.ctxt_id_valid) + return OCSD_RESP_CONT; + tid = elem->context.context_id; + } else { + if (!elem->context.vmid_valid) + return OCSD_RESP_CONT; + tid = elem->context.vmid; + }
- tid = elem->context.context_id; if (cs_etm__etmq_set_tid(etmq, tid, trace_chan_id)) return OCSD_RESP_FATAL_SYS_ERR;
@@ -561,7 +569,7 @@ static ocsd_datapath_resp_t cs_etm_decoder__gen_trace_elem_printer( trace_chan_id); break; case OCSD_GEN_TRC_ELEM_PE_CONTEXT: - resp = cs_etm_decoder__set_tid(etmq, packet_queue, + resp = cs_etm_decoder__set_tid(etmq, decoder, packet_queue, elem, trace_chan_id); break; case OCSD_GEN_TRC_ELEM_ADDR_NACC: @@ -595,11 +603,15 @@ static int cs_etm_decoder__create_etm_packet_decoder( OCSD_BUILTIN_DCD_ETMV3 : OCSD_BUILTIN_DCD_PTM; trace_config = &config_etmv3; + decoder->thread_id_in_vmid = 0; break; case CS_ETM_PROTO_ETMV4i: cs_etm_decoder__gen_etmv4_config(t_params, &trace_config_etmv4); decoder_name = OCSD_BUILTIN_DCD_ETMV4I; trace_config = &trace_config_etmv4; + /* If VMID and VMIDOPT are set, thread id is in VMID not CID */ + decoder->thread_id_in_vmid = + ((trace_config_etmv4.reg.configr & 0x8080) == 0x8080); break; default: return -1; IMPORTANT NOTICE: The contents of this email and any attachments are confidential and may also be privileged. If you are not the intended recipient, please notify the sender immediately and do not disclose the contents to any other person, use it for any purpose, or store or copy the information in any medium. Thank you.