From: Kan Liang kan.liang@linux.intel.com

To optimize the cgroup context switch, the perf_event_pmu_context iteration skips the PMUs without cgroup events. A bool cgroup was introduced to indicate the case. It can work, but this way is hard to extend for other cases, e.g. skipping non-passthrough PMUs. It doesn't make sense to keep adding bool variables.

Pass the event_type instead of the specific bool variable. Check both the event_type and related pmu_ctx variables to decide whether skipping a PMU.

Event flags, e.g., EVENT_CGROUP, should be cleard in the ctx->is_active. Add EVENT_FLAGS to indicate such event flags.

No functional change.

Signed-off-by: Kan Liang kan.liang@linux.intel.com Tested-by: Yongwei Ma yongwei.ma@intel.com Signed-off-by: Mingwei Zhang mizhang@google.com --- kernel/events/core.c | 73 ++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 33 deletions(-)

diff --git a/kernel/events/core.c b/kernel/events/core.c index be623701dc48..8d3a0cc59fb4 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -163,7 +163,7 @@ enum event_type_t { /* see ctx_resched() for details */ EVENT_CPU = 0x10, EVENT_CGROUP = 0x20, - + EVENT_FLAGS = EVENT_CGROUP, /* compound helpers */ EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED, EVENT_TIME_FROZEN = EVENT_TIME | EVENT_FROZEN, @@ -733,27 +733,37 @@ do { \ ___p; \ })

-#define for_each_epc(_epc, _ctx, _pmu, _cgroup) \ +static bool perf_skip_pmu_ctx(struct perf_event_pmu_context *pmu_ctx, + enum event_type_t event_type) +{ + if ((event_type & EVENT_CGROUP) && !pmu_ctx->nr_cgroups) + return true; + return false; +} + +#define for_each_epc(_epc, _ctx, _pmu, _event_type) \ list_for_each_entry(_epc, &((_ctx)->pmu_ctx_list), pmu_ctx_entry) \ - if (_cgroup && !_epc->nr_cgroups) \ + if (perf_skip_pmu_ctx(_epc, _event_type)) \ continue; \ else if (_pmu && _epc->pmu != _pmu) \ continue; \ else

-static void perf_ctx_disable(struct perf_event_context *ctx, bool cgroup) +static void perf_ctx_disable(struct perf_event_context *ctx, + enum event_type_t event_type) { struct perf_event_pmu_context *pmu_ctx;

- for_each_epc(pmu_ctx, ctx, NULL, cgroup) + for_each_epc(pmu_ctx, ctx, NULL, event_type) perf_pmu_disable(pmu_ctx->pmu); }

-static void perf_ctx_enable(struct perf_event_context *ctx, bool cgroup) +static void perf_ctx_enable(struct perf_event_context *ctx, + enum event_type_t event_type) { struct perf_event_pmu_context *pmu_ctx;

- for_each_epc(pmu_ctx, ctx, NULL, cgroup) + for_each_epc(pmu_ctx, ctx, NULL, event_type) perf_pmu_enable(pmu_ctx->pmu); }

@@ -913,7 +923,7 @@ static void perf_cgroup_switch(struct task_struct *task) return;

perf_ctx_lock(cpuctx, cpuctx->task_ctx); - perf_ctx_disable(&cpuctx->ctx, true); + perf_ctx_disable(&cpuctx->ctx, EVENT_CGROUP);

ctx_sched_out(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP); /* @@ -929,7 +939,7 @@ static void perf_cgroup_switch(struct task_struct *task) */ ctx_sched_in(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP);

- perf_ctx_enable(&cpuctx->ctx, true); + perf_ctx_enable(&cpuctx->ctx, EVENT_CGROUP); perf_ctx_unlock(cpuctx, cpuctx->task_ctx); }

@@ -2796,11 +2806,11 @@ static void ctx_resched(struct perf_cpu_context *cpuctx,

event_type &= EVENT_ALL;

- for_each_epc(epc, &cpuctx->ctx, pmu, false) + for_each_epc(epc, &cpuctx->ctx, pmu, 0) perf_pmu_disable(epc->pmu);

if (task_ctx) { - for_each_epc(epc, task_ctx, pmu, false) + for_each_epc(epc, task_ctx, pmu, 0) perf_pmu_disable(epc->pmu);

task_ctx_sched_out(task_ctx, pmu, event_type); @@ -2820,11 +2830,11 @@ static void ctx_resched(struct perf_cpu_context *cpuctx,

perf_event_sched_in(cpuctx, task_ctx, pmu);

- for_each_epc(epc, &cpuctx->ctx, pmu, false) + for_each_epc(epc, &cpuctx->ctx, pmu, 0) perf_pmu_enable(epc->pmu);

if (task_ctx) { - for_each_epc(epc, task_ctx, pmu, false) + for_each_epc(epc, task_ctx, pmu, 0) perf_pmu_enable(epc->pmu); } } @@ -3374,11 +3384,10 @@ static void ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type) { struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); + enum event_type_t active_type = event_type & ~EVENT_FLAGS; struct perf_event_pmu_context *pmu_ctx; int is_active = ctx->is_active; - bool cgroup = event_type & EVENT_CGROUP;

- event_type &= ~EVENT_CGROUP;

lockdep_assert_held(&ctx->lock);

@@ -3409,7 +3418,7 @@ ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t * see __load_acquire() in perf_event_time_now() */ barrier(); - ctx->is_active &= ~event_type; + ctx->is_active &= ~active_type;

if (!(ctx->is_active & EVENT_ALL)) { /* @@ -3430,7 +3439,7 @@ ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t

is_active ^= ctx->is_active; /* changed bits */

- for_each_epc(pmu_ctx, ctx, pmu, cgroup) + for_each_epc(pmu_ctx, ctx, pmu, event_type) __pmu_ctx_sched_out(pmu_ctx, is_active); }

@@ -3622,7 +3631,7 @@ perf_event_context_sched_out(struct task_struct *task, struct task_struct *next) raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); if (context_equiv(ctx, next_ctx)) {

- perf_ctx_disable(ctx, false); + perf_ctx_disable(ctx, 0);

/* PMIs are disabled; ctx->nr_no_switch_fast is stable. */ if (local_read(&ctx->nr_no_switch_fast) || @@ -3647,7 +3656,7 @@ perf_event_context_sched_out(struct task_struct *task, struct task_struct *next) perf_ctx_sched_task_cb(ctx, false); perf_event_swap_task_ctx_data(ctx, next_ctx);

- perf_ctx_enable(ctx, false); + perf_ctx_enable(ctx, 0);

/* * RCU_INIT_POINTER here is safe because we've not @@ -3671,13 +3680,13 @@ perf_event_context_sched_out(struct task_struct *task, struct task_struct *next)

if (do_switch) { raw_spin_lock(&ctx->lock); - perf_ctx_disable(ctx, false); + perf_ctx_disable(ctx, 0);

inside_switch: perf_ctx_sched_task_cb(ctx, false); task_ctx_sched_out(ctx, NULL, EVENT_ALL);

- perf_ctx_enable(ctx, false); + perf_ctx_enable(ctx, 0); raw_spin_unlock(&ctx->lock); } } @@ -3981,11 +3990,9 @@ static void ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type) { struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); + enum event_type_t active_type = event_type & ~EVENT_FLAGS; struct perf_event_pmu_context *pmu_ctx; int is_active = ctx->is_active; - bool cgroup = event_type & EVENT_CGROUP; - - event_type &= ~EVENT_CGROUP;

lockdep_assert_held(&ctx->lock);

@@ -4003,7 +4010,7 @@ ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t barrier(); }

- ctx->is_active |= (event_type | EVENT_TIME); + ctx->is_active |= active_type | EVENT_TIME; if (ctx->task) { if (!(is_active & EVENT_ALL)) cpuctx->task_ctx = ctx; @@ -4018,13 +4025,13 @@ ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t * in order to give them the best chance of going on. */ if (is_active & EVENT_PINNED) { - for_each_epc(pmu_ctx, ctx, pmu, cgroup) + for_each_epc(pmu_ctx, ctx, pmu, event_type) __pmu_ctx_sched_in(pmu_ctx, EVENT_PINNED); }

/* Then walk through the lower prio flexible groups */ if (is_active & EVENT_FLEXIBLE) { - for_each_epc(pmu_ctx, ctx, pmu, cgroup) + for_each_epc(pmu_ctx, ctx, pmu, event_type) __pmu_ctx_sched_in(pmu_ctx, EVENT_FLEXIBLE); } } @@ -4041,11 +4048,11 @@ static void perf_event_context_sched_in(struct task_struct *task)

if (cpuctx->task_ctx == ctx) { perf_ctx_lock(cpuctx, ctx); - perf_ctx_disable(ctx, false); + perf_ctx_disable(ctx, 0);

perf_ctx_sched_task_cb(ctx, true);

- perf_ctx_enable(ctx, false); + perf_ctx_enable(ctx, 0); perf_ctx_unlock(cpuctx, ctx); goto rcu_unlock; } @@ -4058,7 +4065,7 @@ static void perf_event_context_sched_in(struct task_struct *task) if (!ctx->nr_events) goto unlock;

- perf_ctx_disable(ctx, false); + perf_ctx_disable(ctx, 0); /* * We want to keep the following priority order: * cpu pinned (that don't need to move), task pinned, @@ -4068,7 +4075,7 @@ static void perf_event_context_sched_in(struct task_struct *task) * events, no need to flip the cpuctx's events around. */ if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) { - perf_ctx_disable(&cpuctx->ctx, false); + perf_ctx_disable(&cpuctx->ctx, 0); ctx_sched_out(&cpuctx->ctx, NULL, EVENT_FLEXIBLE); }

@@ -4077,9 +4084,9 @@ static void perf_event_context_sched_in(struct task_struct *task) perf_ctx_sched_task_cb(cpuctx->task_ctx, true);

if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) - perf_ctx_enable(&cpuctx->ctx, false); + perf_ctx_enable(&cpuctx->ctx, 0);

- perf_ctx_enable(ctx, false); + perf_ctx_enable(ctx, 0);

unlock: perf_ctx_unlock(cpuctx, ctx);

From: Kan Liang kan.liang@linux.intel.com

Current perf doesn't explicitly schedule out all exclude_guest events while the guest is running. There is no problem with the current emulated vPMU. Because perf owns all the PMU counters. It can mask the counter which is assigned to an exclude_guest event when a guest is running (Intel way), or set the corresponding HOSTONLY bit in evsentsel (AMD way). The counter doesn't count when a guest is running.

However, either way doesn't work with the introduced passthrough vPMU. A guest owns all the PMU counters when it's running. The host should not mask any counters. The counter may be used by the guest. The evsentsel may be overwritten.

Perf should explicitly schedule out all exclude_guest events to release the PMU resources when entering a guest, and resume the counting when exiting the guest.

It's possible that an exclude_guest event is created when a guest is running. The new event should not be scheduled in as well.

The ctx time is shared among different PMUs. The time cannot be stopped when a guest is running. It is required to calculate the time for events from other PMUs, e.g., uncore events. Add timeguest to track the guest run time. For an exclude_guest event, the elapsed time equals the ctx time - guest time. Cgroup has dedicated times. Use the same method to deduct the guest time from the cgroup time as well.

Co-developed-by: Peter Zijlstra (Intel) peterz@infradead.org Signed-off-by: Peter Zijlstra (Intel) peterz@infradead.org Signed-off-by: Kan Liang kan.liang@linux.intel.com Signed-off-by: Mingwei Zhang mizhang@google.com --- include/linux/perf_event.h | 6 ++ kernel/events/core.c | 209 +++++++++++++++++++++++++++++-------- 2 files changed, 169 insertions(+), 46 deletions(-)

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index a2fd1bdc955c..7bda1e20be12 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -999,6 +999,11 @@ struct perf_event_context { */ struct perf_time_ctx time;

+ /* + * Context clock, runs when in the guest mode. + */ + struct perf_time_ctx timeguest; + /* * These fields let us detect when two contexts have both * been cloned (inherited) from a common ancestor. @@ -1089,6 +1094,7 @@ struct bpf_perf_event_data_kern { */ struct perf_cgroup_info { struct perf_time_ctx time; + struct perf_time_ctx timeguest; int active; };

diff --git a/kernel/events/core.c b/kernel/events/core.c index e38c8b5e8086..7a2115b2c5c1 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -163,7 +163,8 @@ enum event_type_t { /* see ctx_resched() for details */ EVENT_CPU = 0x10, EVENT_CGROUP = 0x20, - EVENT_FLAGS = EVENT_CGROUP, + EVENT_GUEST = 0x40, + EVENT_FLAGS = EVENT_CGROUP | EVENT_GUEST, /* compound helpers */ EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED, EVENT_TIME_FROZEN = EVENT_TIME | EVENT_FROZEN, @@ -435,6 +436,7 @@ static atomic_t nr_include_guest_events __read_mostly;

static atomic_t nr_mediated_pmu_vms; static DEFINE_MUTEX(perf_mediated_pmu_mutex); +static DEFINE_PER_CPU(bool, perf_in_guest);

/* !exclude_guest event of PMU with PERF_PMU_CAP_MEDIATED_VPMU */ static inline bool is_include_guest_event(struct perf_event *event) @@ -738,6 +740,9 @@ static bool perf_skip_pmu_ctx(struct perf_event_pmu_context *pmu_ctx, { if ((event_type & EVENT_CGROUP) && !pmu_ctx->nr_cgroups) return true; + if ((event_type & EVENT_GUEST) && + !(pmu_ctx->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU)) + return true; return false; }

@@ -788,6 +793,39 @@ static inline void update_perf_time_ctx(struct perf_time_ctx *time, u64 now, boo WRITE_ONCE(time->offset, time->time - time->stamp); }

+static_assert(offsetof(struct perf_event_context, timeguest) - + offsetof(struct perf_event_context, time) == + sizeof(struct perf_time_ctx)); + +#define T_TOTAL 0 +#define T_GUEST 1 + +static inline u64 __perf_event_time_ctx(struct perf_event *event, + struct perf_time_ctx *times) +{ + u64 time = times[T_TOTAL].time; + + if (event->attr.exclude_guest) + time -= times[T_GUEST].time; + + return time; +} + +static inline u64 __perf_event_time_ctx_now(struct perf_event *event, + struct perf_time_ctx *times, + u64 now) +{ + if (event->attr.exclude_guest && __this_cpu_read(perf_in_guest)) { + /* + * (now + times[total].offset) - (now + times[guest].offset) := + * times[total].offset - times[guest].offset + */ + return READ_ONCE(times[T_TOTAL].offset) - READ_ONCE(times[T_GUEST].offset); + } + + return now + READ_ONCE(times[T_TOTAL].offset); +} + #ifdef CONFIG_CGROUP_PERF

static inline bool @@ -824,12 +862,16 @@ static inline int is_cgroup_event(struct perf_event *event) return event->cgrp != NULL; }

+static_assert(offsetof(struct perf_cgroup_info, timeguest) - + offsetof(struct perf_cgroup_info, time) == + sizeof(struct perf_time_ctx)); + static inline u64 perf_cgroup_event_time(struct perf_event *event) { struct perf_cgroup_info *t;

t = per_cpu_ptr(event->cgrp->info, event->cpu); - return t->time.time; + return __perf_event_time_ctx(event, &t->time); }

static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now) @@ -838,9 +880,21 @@ static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)

t = per_cpu_ptr(event->cgrp->info, event->cpu); if (!__load_acquire(&t->active)) - return t->time.time; - now += READ_ONCE(t->time.offset); - return now; + return __perf_event_time_ctx(event, &t->time); + + return __perf_event_time_ctx_now(event, &t->time, now); +} + +static inline void __update_cgrp_guest_time(struct perf_cgroup_info *info, u64 now, bool adv) +{ + update_perf_time_ctx(&info->timeguest, now, adv); +} + +static inline void update_cgrp_time(struct perf_cgroup_info *info, u64 now) +{ + update_perf_time_ctx(&info->time, now, true); + if (__this_cpu_read(perf_in_guest)) + __update_cgrp_guest_time(info, now, true); }

static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final) @@ -856,7 +910,7 @@ static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, cgrp = container_of(css, struct perf_cgroup, css); info = this_cpu_ptr(cgrp->info);

- update_perf_time_ctx(&info->time, now, true); + update_cgrp_time(info, now); if (final) __store_release(&info->active, 0); } @@ -879,11 +933,11 @@ static inline void update_cgrp_time_from_event(struct perf_event *event) * Do not update time when cgroup is not active */ if (info->active) - update_perf_time_ctx(&info->time, perf_clock(), true); + update_cgrp_time(info, perf_clock()); }

static inline void -perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx) +perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx, bool guest) { struct perf_event_context *ctx = &cpuctx->ctx; struct perf_cgroup *cgrp = cpuctx->cgrp; @@ -903,8 +957,12 @@ perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx) for (css = &cgrp->css; css; css = css->parent) { cgrp = container_of(css, struct perf_cgroup, css); info = this_cpu_ptr(cgrp->info); - update_perf_time_ctx(&info->time, ctx->time.stamp, false); - __store_release(&info->active, 1); + if (guest) { + __update_cgrp_guest_time(info, ctx->time.stamp, false); + } else { + update_perf_time_ctx(&info->time, ctx->time.stamp, false); + __store_release(&info->active, 1); + } } }

@@ -1104,7 +1162,7 @@ static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event, }

static inline void -perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx) +perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx, bool guest) { }

@@ -1514,16 +1572,24 @@ static void perf_unpin_context(struct perf_event_context *ctx) */ static void __update_context_time(struct perf_event_context *ctx, bool adv) { - u64 now = perf_clock(); + lockdep_assert_held(&ctx->lock); + + update_perf_time_ctx(&ctx->time, perf_clock(), adv); +}

+static void __update_context_guest_time(struct perf_event_context *ctx, bool adv) +{ lockdep_assert_held(&ctx->lock);

- update_perf_time_ctx(&ctx->time, now, adv); + /* must be called after __update_context_time(); */ + update_perf_time_ctx(&ctx->timeguest, ctx->time.stamp, adv); }

static void update_context_time(struct perf_event_context *ctx) { __update_context_time(ctx, true); + if (__this_cpu_read(perf_in_guest)) + __update_context_guest_time(ctx, true); }

static u64 perf_event_time(struct perf_event *event) @@ -1536,7 +1602,7 @@ static u64 perf_event_time(struct perf_event *event) if (is_cgroup_event(event)) return perf_cgroup_event_time(event);

- return ctx->time.time; + return __perf_event_time_ctx(event, &ctx->time); }

static u64 perf_event_time_now(struct perf_event *event, u64 now) @@ -1550,10 +1616,9 @@ static u64 perf_event_time_now(struct perf_event *event, u64 now) return perf_cgroup_event_time_now(event, now);

if (!(__load_acquire(&ctx->is_active) & EVENT_TIME)) - return ctx->time.time; + return __perf_event_time_ctx(event, &ctx->time);

- now += READ_ONCE(ctx->time.offset); - return now; + return __perf_event_time_ctx_now(event, &ctx->time, now); }

static enum event_type_t get_event_type(struct perf_event *event) @@ -2384,20 +2449,23 @@ group_sched_out(struct perf_event *group_event, struct perf_event_context *ctx) }

static inline void -__ctx_time_update(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx, bool final) +__ctx_time_update(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx, + bool final, enum event_type_t event_type) { if (ctx->is_active & EVENT_TIME) { if (ctx->is_active & EVENT_FROZEN) return; + update_context_time(ctx); - update_cgrp_time_from_cpuctx(cpuctx, final); + /* vPMU should not stop time */ + update_cgrp_time_from_cpuctx(cpuctx, !(event_type & EVENT_GUEST) && final); } }

static inline void ctx_time_update(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx) { - __ctx_time_update(cpuctx, ctx, false); + __ctx_time_update(cpuctx, ctx, false, 0); }

/* @@ -3405,7 +3473,7 @@ ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t * * would only update time for the pinned events. */ - __ctx_time_update(cpuctx, ctx, ctx == &cpuctx->ctx); + __ctx_time_update(cpuctx, ctx, ctx == &cpuctx->ctx, event_type);

/* * CPU-release for the below ->is_active store, @@ -3431,7 +3499,18 @@ ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t cpuctx->task_ctx = NULL; }

- is_active ^= ctx->is_active; /* changed bits */ + if (event_type & EVENT_GUEST) { + /* + * Schedule out all exclude_guest events of PMU + * with PERF_PMU_CAP_MEDIATED_VPMU. + */ + is_active = EVENT_ALL; + __update_context_guest_time(ctx, false); + perf_cgroup_set_timestamp(cpuctx, true); + barrier(); + } else { + is_active ^= ctx->is_active; /* changed bits */ + }

for_each_epc(pmu_ctx, ctx, pmu, event_type) __pmu_ctx_sched_out(pmu_ctx, is_active); @@ -3926,10 +4005,15 @@ static inline void group_update_userpage(struct perf_event *group_event) event_update_userpage(event); }

+struct merge_sched_data { + int can_add_hw; + enum event_type_t event_type; +}; + static int merge_sched_in(struct perf_event *event, void *data) { struct perf_event_context *ctx = event->ctx; - int *can_add_hw = data; + struct merge_sched_data *msd = data;

if (event->state <= PERF_EVENT_STATE_OFF) return 0; @@ -3937,13 +4021,22 @@ static int merge_sched_in(struct perf_event *event, void *data) if (!event_filter_match(event)) return 0;

- if (group_can_go_on(event, *can_add_hw)) { + /* + * Don't schedule in any host events from PMU with + * PERF_PMU_CAP_MEDIATED_VPMU, while a guest is running. + */ + if (__this_cpu_read(perf_in_guest) && + event->pmu_ctx->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU && + !(msd->event_type & EVENT_GUEST)) + return 0; + + if (group_can_go_on(event, msd->can_add_hw)) { if (!group_sched_in(event, ctx)) list_add_tail(&event->active_list, get_event_list(event)); }

if (event->state == PERF_EVENT_STATE_INACTIVE) { - *can_add_hw = 0; + msd->can_add_hw = 0; if (event->attr.pinned) { perf_cgroup_event_disable(event, ctx); perf_event_set_state(event, PERF_EVENT_STATE_ERROR); @@ -3962,11 +4055,15 @@ static int merge_sched_in(struct perf_event *event, void *data)

static void pmu_groups_sched_in(struct perf_event_context *ctx, struct perf_event_groups *groups, - struct pmu *pmu) + struct pmu *pmu, + enum event_type_t event_type) { - int can_add_hw = 1; + struct merge_sched_data msd = { + .can_add_hw = 1, + .event_type = event_type, + }; visit_groups_merge(ctx, groups, smp_processor_id(), pmu, - merge_sched_in, &can_add_hw); + merge_sched_in, &msd); }

static void __pmu_ctx_sched_in(struct perf_event_pmu_context *pmu_ctx, @@ -3975,9 +4072,9 @@ static void __pmu_ctx_sched_in(struct perf_event_pmu_context *pmu_ctx, struct perf_event_context *ctx = pmu_ctx->ctx;

if (event_type & EVENT_PINNED) - pmu_groups_sched_in(ctx, &ctx->pinned_groups, pmu_ctx->pmu); + pmu_groups_sched_in(ctx, &ctx->pinned_groups, pmu_ctx->pmu, event_type); if (event_type & EVENT_FLEXIBLE) - pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu_ctx->pmu); + pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu_ctx->pmu, event_type); }

static void @@ -3994,9 +4091,11 @@ ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t return;

if (!(is_active & EVENT_TIME)) { + /* EVENT_TIME should be active while the guest runs */ + WARN_ON_ONCE(event_type & EVENT_GUEST); /* start ctx time */ __update_context_time(ctx, false); - perf_cgroup_set_timestamp(cpuctx); + perf_cgroup_set_timestamp(cpuctx, false); /* * CPU-release for the below ->is_active store, * see __load_acquire() in perf_event_time_now() @@ -4012,7 +4111,23 @@ ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t WARN_ON_ONCE(cpuctx->task_ctx != ctx); }

- is_active ^= ctx->is_active; /* changed bits */ + if (event_type & EVENT_GUEST) { + /* + * Schedule in the required exclude_guest events of PMU + * with PERF_PMU_CAP_MEDIATED_VPMU. + */ + is_active = event_type & EVENT_ALL; + + /* + * Update ctx time to set the new start time for + * the exclude_guest events. + */ + update_context_time(ctx); + update_cgrp_time_from_cpuctx(cpuctx, false); + barrier(); + } else { + is_active ^= ctx->is_active; /* changed bits */ + }

/* * First go through the list and put on any pinned groups @@ -4020,13 +4135,13 @@ ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t */ if (is_active & EVENT_PINNED) { for_each_epc(pmu_ctx, ctx, pmu, event_type) - __pmu_ctx_sched_in(pmu_ctx, EVENT_PINNED); + __pmu_ctx_sched_in(pmu_ctx, EVENT_PINNED | (event_type & EVENT_GUEST)); }

/* Then walk through the lower prio flexible groups */ if (is_active & EVENT_FLEXIBLE) { for_each_epc(pmu_ctx, ctx, pmu, event_type) - __pmu_ctx_sched_in(pmu_ctx, EVENT_FLEXIBLE); + __pmu_ctx_sched_in(pmu_ctx, EVENT_FLEXIBLE | (event_type & EVENT_GUEST)); } }

@@ -6285,23 +6400,25 @@ void perf_event_update_userpage(struct perf_event *event) if (!rb) goto unlock;

- /* - * compute total_time_enabled, total_time_running - * based on snapshot values taken when the event - * was last scheduled in. - * - * we cannot simply called update_context_time() - * because of locking issue as we can be called in - * NMI context - */ - calc_timer_values(event, &now, &enabled, &running); - - userpg = rb->user_page; /* * Disable preemption to guarantee consistent time stamps are stored to * the user page. */ preempt_disable(); + + /* + * compute total_time_enabled, total_time_running + * based on snapshot values taken when the event + * was last scheduled in. + * + * we cannot simply called update_context_time() + * because of locking issue as we can be called in + * NMI context + */ + calc_timer_values(event, &now, &enabled, &running); + + userpg = rb->user_page; + ++userpg->lock; barrier(); userpg->index = perf_event_index(event);

From: Xiong Zhang xiong.y.zhang@linux.intel.com

Create a new vector in the host IDT for kvm guest PMI handling within mediated passthrough vPMU. In addition, guest PMI handler registration is added into x86_set_kvm_irq_handler().

This is the preparation work to support mediated passthrough vPMU to handle kvm guest PMIs without interference from PMI handler of the host PMU.

Signed-off-by: Xiong Zhang xiong.y.zhang@linux.intel.com Signed-off-by: Dapeng Mi dapeng1.mi@linux.intel.com Signed-off-by: Mingwei Zhang mizhang@google.com --- arch/x86/include/asm/hardirq.h | 1 + arch/x86/include/asm/idtentry.h | 1 + arch/x86/include/asm/irq_vectors.h | 5 ++++- arch/x86/kernel/idt.c | 1 + arch/x86/kernel/irq.c | 21 +++++++++++++++++++ .../beauty/arch/x86/include/asm/irq_vectors.h | 5 ++++- 6 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h index 6ffa8b75f4cd..25fac35b9a29 100644 --- a/arch/x86/include/asm/hardirq.h +++ b/arch/x86/include/asm/hardirq.h @@ -19,6 +19,7 @@ typedef struct { unsigned int kvm_posted_intr_ipis; unsigned int kvm_posted_intr_wakeup_ipis; unsigned int kvm_posted_intr_nested_ipis; + unsigned int kvm_guest_pmis; #endif unsigned int x86_platform_ipis; /* arch dependent */ unsigned int apic_perf_irqs; diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index ad5c68f0509d..b0cb3220e1bb 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -745,6 +745,7 @@ DECLARE_IDTENTRY_SYSVEC(IRQ_WORK_VECTOR, sysvec_irq_work); DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_VECTOR, sysvec_kvm_posted_intr_ipi); DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_WAKEUP_VECTOR, sysvec_kvm_posted_intr_wakeup_ipi); DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_NESTED_VECTOR, sysvec_kvm_posted_intr_nested_ipi); +DECLARE_IDTENTRY_SYSVEC(KVM_GUEST_PMI_VECTOR, sysvec_kvm_guest_pmi_handler); #else # define fred_sysvec_kvm_posted_intr_ipi NULL # define fred_sysvec_kvm_posted_intr_wakeup_ipi NULL diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h index 47051871b436..250cdab11306 100644 --- a/arch/x86/include/asm/irq_vectors.h +++ b/arch/x86/include/asm/irq_vectors.h @@ -77,7 +77,10 @@ */ #define IRQ_WORK_VECTOR 0xf6

-/* 0xf5 - unused, was UV_BAU_MESSAGE */ +#if IS_ENABLED(CONFIG_KVM) +#define KVM_GUEST_PMI_VECTOR 0xf5 +#endif + #define DEFERRED_ERROR_VECTOR 0xf4

/* Vector on which hypervisor callbacks will be delivered */ diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c index f445bec516a0..0bec4c7e2308 100644 --- a/arch/x86/kernel/idt.c +++ b/arch/x86/kernel/idt.c @@ -157,6 +157,7 @@ static const __initconst struct idt_data apic_idts[] = { INTG(POSTED_INTR_VECTOR, asm_sysvec_kvm_posted_intr_ipi), INTG(POSTED_INTR_WAKEUP_VECTOR, asm_sysvec_kvm_posted_intr_wakeup_ipi), INTG(POSTED_INTR_NESTED_VECTOR, asm_sysvec_kvm_posted_intr_nested_ipi), + INTG(KVM_GUEST_PMI_VECTOR, asm_sysvec_kvm_guest_pmi_handler), # endif # ifdef CONFIG_IRQ_WORK INTG(IRQ_WORK_VECTOR, asm_sysvec_irq_work), diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index 18cd418fe106..b29714e23fc4 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -183,6 +183,12 @@ int arch_show_interrupts(struct seq_file *p, int prec) seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_wakeup_ipis); seq_puts(p, " Posted-interrupt wakeup event\n"); + + seq_printf(p, "%*s: ", prec, "VPMU"); + for_each_online_cpu(j) + seq_printf(p, "%10u ", + irq_stats(j)->kvm_guest_pmis); + seq_puts(p, " KVM GUEST PMI\n"); #endif #ifdef CONFIG_X86_POSTED_MSI seq_printf(p, "%*s: ", prec, "PMN"); @@ -311,6 +317,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_x86_platform_ipi) #if IS_ENABLED(CONFIG_KVM) static void dummy_handler(void) {} static void (*kvm_posted_intr_wakeup_handler)(void) = dummy_handler; +static void (*kvm_guest_pmi_handler)(void) = dummy_handler;

void x86_set_kvm_irq_handler(u8 vector, void (*handler)(void)) { @@ -321,6 +328,10 @@ void x86_set_kvm_irq_handler(u8 vector, void (*handler)(void)) (handler == dummy_handler || kvm_posted_intr_wakeup_handler == dummy_handler)) kvm_posted_intr_wakeup_handler = handler; + else if (vector == KVM_GUEST_PMI_VECTOR && + (handler == dummy_handler || + kvm_guest_pmi_handler == dummy_handler)) + kvm_guest_pmi_handler = handler; else WARN_ON_ONCE(1);

@@ -356,6 +367,16 @@ DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_nested_ipi) apic_eoi(); inc_irq_stat(kvm_posted_intr_nested_ipis); } + +/* + * Handler for KVM_GUEST_PMI_VECTOR. + */ +DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_guest_pmi_handler) +{ + apic_eoi(); + inc_irq_stat(kvm_guest_pmis); + kvm_guest_pmi_handler(); +} #endif

#ifdef CONFIG_X86_POSTED_MSI diff --git a/tools/perf/trace/beauty/arch/x86/include/asm/irq_vectors.h b/tools/perf/trace/beauty/arch/x86/include/asm/irq_vectors.h index 47051871b436..250cdab11306 100644 --- a/tools/perf/trace/beauty/arch/x86/include/asm/irq_vectors.h +++ b/tools/perf/trace/beauty/arch/x86/include/asm/irq_vectors.h @@ -77,7 +77,10 @@ */ #define IRQ_WORK_VECTOR 0xf6

-/* 0xf5 - unused, was UV_BAU_MESSAGE */ +#if IS_ENABLED(CONFIG_KVM) +#define KVM_GUEST_PMI_VECTOR 0xf5 +#endif + #define DEFERRED_ERROR_VECTOR 0xf4

/* Vector on which hypervisor callbacks will be delivered */

From: Kan Liang kan.liang@linux.intel.com

When entering/exiting a guest, some contexts for a guest have to be switched. For examples, there is a dedicated interrupt vector for guests on Intel platforms.

When PMI switch into a new guest vector, guest_lvtpc value need to be reflected onto HW, e,g., guest clear PMI mask bit, the HW PMI mask bit should be cleared also, then PMI can be generated continuously for guest. So guest_lvtpc parameter is added into perf_guest_enter() and switch_guest_ctx().

Add a dedicated list to track all the pmus with the PASSTHROUGH cap, which may require switching the guest context. It can avoid going through the huge pmus list.

Suggested-by: Peter Zijlstra (Intel) peterz@infradead.org Signed-off-by: Kan Liang kan.liang@linux.intel.com Signed-off-by: Mingwei Zhang mizhang@google.com --- include/linux/perf_event.h | 17 +++++++++++-- kernel/events/core.c | 51 +++++++++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-)

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 37187ee8e226..58c1cf6939bf 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -584,6 +584,11 @@ struct pmu { * Check period value for PERF_EVENT_IOC_PERIOD ioctl. */ int (*check_period) (struct perf_event *event, u64 value); /* optional */ + + /* + * Switch guest context when a guest enter/exit, e.g., interrupt vectors. + */ + void (*switch_guest_ctx) (bool enter, void *data); /* optional */ };

enum perf_addr_filter_action_t { @@ -1030,6 +1035,11 @@ struct perf_event_context { local_t nr_no_switch_fast; };

+struct mediated_pmus_list { + raw_spinlock_t lock; + struct list_head list; +}; + struct perf_cpu_pmu_context { struct perf_event_pmu_context epc; struct perf_event_pmu_context *task_epc; @@ -1044,6 +1054,9 @@ struct perf_cpu_pmu_context { struct hrtimer hrtimer; ktime_t hrtimer_interval; unsigned int hrtimer_active; + + /* Track the PMU with PERF_PMU_CAP_MEDIATED_VPMU cap */ + struct list_head mediated_entry; };

/** @@ -1822,7 +1835,7 @@ extern int perf_event_period(struct perf_event *event, u64 value); extern u64 perf_event_pause(struct perf_event *event, bool reset); int perf_get_mediated_pmu(void); void perf_put_mediated_pmu(void); -void perf_guest_enter(void); +void perf_guest_enter(u32 guest_lvtpc); void perf_guest_exit(void); #else /* !CONFIG_PERF_EVENTS: */ static inline void * @@ -1921,7 +1934,7 @@ static inline int perf_get_mediated_pmu(void) }

static inline void perf_put_mediated_pmu(void) { } -static inline void perf_guest_enter(void) { } +static inline void perf_guest_enter(u32 guest_lvtpc) { } static inline void perf_guest_exit(void) { } #endif

diff --git a/kernel/events/core.c b/kernel/events/core.c index d05487d465c9..406b86641f02 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -451,6 +451,7 @@ static inline bool is_include_guest_event(struct perf_event *event) static LIST_HEAD(pmus); static DEFINE_MUTEX(pmus_lock); static struct srcu_struct pmus_srcu; +static DEFINE_PER_CPU(struct mediated_pmus_list, mediated_pmus); static cpumask_var_t perf_online_mask; static cpumask_var_t perf_online_core_mask; static cpumask_var_t perf_online_die_mask; @@ -6053,8 +6054,26 @@ static inline void perf_host_exit(struct perf_cpu_context *cpuctx) } }

+static void perf_switch_guest_ctx(bool enter, u32 guest_lvtpc) +{ + struct mediated_pmus_list *pmus = this_cpu_ptr(&mediated_pmus); + struct perf_cpu_pmu_context *cpc; + struct pmu *pmu; + + lockdep_assert_irqs_disabled(); + + rcu_read_lock(); + list_for_each_entry_rcu(cpc, &pmus->list, mediated_entry) { + pmu = cpc->epc.pmu; + + if (pmu->switch_guest_ctx) + pmu->switch_guest_ctx(enter, (void *)&guest_lvtpc); + } + rcu_read_unlock(); +} + /* When entering a guest, schedule out all exclude_guest events. */ -void perf_guest_enter(void) +void perf_guest_enter(u32 guest_lvtpc) { struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);

@@ -6067,6 +6086,8 @@ void perf_guest_enter(void)

perf_host_exit(cpuctx);

+ perf_switch_guest_ctx(true, guest_lvtpc); + __this_cpu_write(perf_in_guest, true);

unlock: @@ -6098,6 +6119,8 @@ void perf_guest_exit(void) if (WARN_ON_ONCE(!__this_cpu_read(perf_in_guest))) goto unlock;

+ perf_switch_guest_ctx(false, 0); + perf_host_enter(cpuctx);

__this_cpu_write(perf_in_guest, false); @@ -12104,6 +12127,15 @@ int perf_pmu_register(struct pmu *pmu, const char *name, int type) cpc = per_cpu_ptr(pmu->cpu_pmu_context, cpu); __perf_init_event_pmu_context(&cpc->epc, pmu); __perf_mux_hrtimer_init(cpc, cpu); + + if (pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU) { + struct mediated_pmus_list *pmus; + + pmus = per_cpu_ptr(&mediated_pmus, cpu); + raw_spin_lock(&pmus->lock); + list_add_rcu(&cpc->mediated_entry, &pmus->list); + raw_spin_unlock(&pmus->lock); + } }

if (!pmu->start_txn) { @@ -12162,6 +12194,20 @@ void perf_pmu_unregister(struct pmu *pmu) mutex_lock(&pmus_lock); list_del_rcu(&pmu->entry);

+ if (pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU) { + struct mediated_pmus_list *pmus; + struct perf_cpu_pmu_context *cpc; + int cpu; + + for_each_possible_cpu(cpu) { + cpc = per_cpu_ptr(pmu->cpu_pmu_context, cpu); + pmus = per_cpu_ptr(&mediated_pmus, cpu); + raw_spin_lock(&pmus->lock); + list_del_rcu(&cpc->mediated_entry); + raw_spin_unlock(&pmus->lock); + } + } + /* * We dereference the pmu list under both SRCU and regular RCU, so * synchronize against both of those. @@ -14252,6 +14298,9 @@ static void __init perf_event_init_all_cpus(void)

INIT_LIST_HEAD(&per_cpu(sched_cb_list, cpu));

+ INIT_LIST_HEAD(&per_cpu(mediated_pmus.list, cpu)); + raw_spin_lock_init(&per_cpu(mediated_pmus.lock, cpu)); + cpuctx = per_cpu_ptr(&perf_cpu_context, cpu); __perf_event_init_context(&cpuctx->ctx); lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);

From: Kan Liang kan.liang@linux.intel.com

Implement switch_guest_ctx interface for x86 PMU, switch PMI to dedicated KVM_GUEST_PMI_VECTOR at perf guest enter, and switch PMI back to NMI at perf guest exit.

Signed-off-by: Xiong Zhang xiong.y.zhang@linux.intel.com Signed-off-by: Kan Liang kan.liang@linux.intel.com Tested-by: Yongwei Ma yongwei.ma@intel.com Signed-off-by: Mingwei Zhang mizhang@google.com --- arch/x86/events/core.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)

diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 8f218ac0d445..28161d6ff26d 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2677,6 +2677,16 @@ static bool x86_pmu_filter(struct pmu *pmu, int cpu) return ret; }

+static void x86_pmu_switch_guest_ctx(bool enter, void *data) +{ + u32 guest_lvtpc = *(u32 *)data; + + if (enter) + apic_write(APIC_LVTPC, guest_lvtpc); + else + apic_write(APIC_LVTPC, APIC_DM_NMI); +} + static struct pmu pmu = { .pmu_enable = x86_pmu_enable, .pmu_disable = x86_pmu_disable, @@ -2706,6 +2716,8 @@ static struct pmu pmu = { .aux_output_match = x86_pmu_aux_output_match,

.filter = x86_pmu_filter, + + .switch_guest_ctx = x86_pmu_switch_guest_ctx, };

void arch_perf_update_userpage(struct perf_event *event,

From: Sandipan Das sandipan.das@amd.com

Not all x86 processors have fixed counters. It may also be the case that a processor has only fixed counters and no general-purpose counters. Set the bit widths corresponding to each counter type only if such counters are available.

Fixes: b3d9468a8bd2 ("perf, x86: Expose perf capability to other modules") Signed-off-by: Sandipan Das sandipan.das@amd.com Co-developed-by: Dapeng Mi dapeng1.mi@linux.intel.com Signed-off-by: Dapeng Mi dapeng1.mi@linux.intel.com Signed-off-by: Mingwei Zhang mizhang@google.com --- arch/x86/events/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 96a173bbbec2..7c852ee3e217 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -3107,8 +3107,8 @@ void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) cap->version = x86_pmu.version; cap->num_counters_gp = x86_pmu_num_counters(NULL); cap->num_counters_fixed = x86_pmu_num_counters_fixed(NULL); - cap->bit_width_gp = x86_pmu.cntval_bits; - cap->bit_width_fixed = x86_pmu.cntval_bits; + cap->bit_width_gp = cap->num_counters_gp ? x86_pmu.cntval_bits : 0; + cap->bit_width_fixed = cap->num_counters_fixed ? x86_pmu.cntval_bits : 0; cap->events_mask = (unsigned int)x86_pmu.events_maskl; cap->events_mask_len = x86_pmu.events_mask_len; cap->pebs_ept = x86_pmu.pebs_ept;

From: Dapeng Mi dapeng1.mi@linux.intel.com

Introduce enable_mediated_pmu global parameter to control if mediated vPMU can be enabled on KVM level. Even enable_mediated_pmu is set to true in KVM, user space hypervisor still need to enable mediated vPMU explicitly by calling KVM_CAP_PMU_CAPABILITY ioctl. This gives hypervisor flexibility to enable or disable mediated vPMU for each VM.

Mediated vPMU depends on some PMU features on higher PMU version, like PERF_GLOBAL_STATUS_SET MSR in v4+ for Intel PMU. Thus introduce a pmu_ops variable MIN_MEDIATED_PMU_VERSION to indicates the minimum host PMU version which mediated vPMU needs.

Currently enable_mediated_pmu is not exposed to user space as a module parameter until all mediated vPMU code are in place.

Suggested-by: Sean Christopherson seanjc@google.com Co-developed-by: Mingwei Zhang mizhang@google.com Signed-off-by: Mingwei Zhang mizhang@google.com Signed-off-by: Dapeng Mi dapeng1.mi@linux.intel.com --- arch/x86/kvm/pmu.c | 3 ++- arch/x86/kvm/pmu.h | 11 +++++++++ arch/x86/kvm/svm/pmu.c | 1 + arch/x86/kvm/vmx/capabilities.h | 3 ++- arch/x86/kvm/vmx/pmu_intel.c | 5 ++++ arch/x86/kvm/vmx/vmx.c | 3 ++- arch/x86/kvm/x86.c | 44 ++++++++++++++++++++++++++++++--- arch/x86/kvm/x86.h | 1 + 8 files changed, 64 insertions(+), 7 deletions(-)

diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 75e9cfc689f8..4f455afe4009 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -775,7 +775,8 @@ void kvm_pmu_refresh(struct kvm_vcpu *vcpu) pmu->pebs_data_cfg_rsvd = ~0ull; bitmap_zero(pmu->all_valid_pmc_idx, X86_PMC_IDX_MAX);

- if (!vcpu->kvm->arch.enable_pmu) + if (!vcpu->kvm->arch.enable_pmu || + (!lapic_in_kernel(vcpu) && enable_mediated_pmu)) return;

kvm_pmu_call(refresh)(vcpu); diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index ad89d0bd6005..dd45a0c6be74 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -45,6 +45,7 @@ struct kvm_pmu_ops { const u64 EVENTSEL_EVENT; const int MAX_NR_GP_COUNTERS; const int MIN_NR_GP_COUNTERS; + const int MIN_MEDIATED_PMU_VERSION; };

void kvm_pmu_ops_update(const struct kvm_pmu_ops *pmu_ops); @@ -63,6 +64,12 @@ static inline bool kvm_pmu_has_perf_global_ctrl(struct kvm_pmu *pmu) return pmu->version > 1; }

+static inline bool kvm_mediated_pmu_enabled(struct kvm_vcpu *vcpu) +{ + return vcpu->kvm->arch.enable_pmu && + enable_mediated_pmu && vcpu_to_pmu(vcpu)->version; +} + /* * KVM tracks all counters in 64-bit bitmaps, with general purpose counters * mapped to bits 31:0 and fixed counters mapped to 63:32, e.g. fixed counter 0 @@ -210,6 +217,10 @@ static inline void kvm_init_pmu_capability(const struct kvm_pmu_ops *pmu_ops) enable_pmu = false; }

+ if (!enable_pmu || !kvm_pmu_cap.mediated || + pmu_ops->MIN_MEDIATED_PMU_VERSION > kvm_pmu_cap.version) + enable_mediated_pmu = false; + if (!enable_pmu) { memset(&kvm_pmu_cap, 0, sizeof(kvm_pmu_cap)); return; diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index 288f7f2a46f2..c8b9fd9b5350 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -239,4 +239,5 @@ struct kvm_pmu_ops amd_pmu_ops __initdata = { .EVENTSEL_EVENT = AMD64_EVENTSEL_EVENT, .MAX_NR_GP_COUNTERS = KVM_MAX_NR_AMD_GP_COUNTERS, .MIN_NR_GP_COUNTERS = AMD64_NUM_COUNTERS, + .MIN_MEDIATED_PMU_VERSION = 2, }; diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index cb6588238f46..fac2c80ddbab 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -390,7 +390,8 @@ static inline bool vmx_pt_mode_is_host_guest(void)

static inline bool vmx_pebs_supported(void) { - return boot_cpu_has(X86_FEATURE_PEBS) && kvm_pmu_cap.pebs_ept; + return boot_cpu_has(X86_FEATURE_PEBS) && + !enable_mediated_pmu && kvm_pmu_cap.pebs_ept; }

static inline bool cpu_has_notify_vmexit(void) diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 77012b2eca0e..425e93d4b1c6 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -739,4 +739,9 @@ struct kvm_pmu_ops intel_pmu_ops __initdata = { .EVENTSEL_EVENT = ARCH_PERFMON_EVENTSEL_EVENT, .MAX_NR_GP_COUNTERS = KVM_MAX_NR_INTEL_GP_COUNTERS, .MIN_NR_GP_COUNTERS = 1, + /* + * Intel mediated vPMU support depends on + * MSR_CORE_PERF_GLOBAL_STATUS_SET which is supported from 4+. + */ + .MIN_MEDIATED_PMU_VERSION = 4, }; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 00ac94535c21..a4b5b6455c7b 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7916,7 +7916,8 @@ static __init u64 vmx_get_perf_capabilities(void) if (boot_cpu_has(X86_FEATURE_PDCM)) rdmsrl(MSR_IA32_PERF_CAPABILITIES, host_perf_cap);

- if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR)) { + if (!cpu_feature_enabled(X86_FEATURE_ARCH_LBR) && + !enable_mediated_pmu) { x86_perf_get_lbr(&vmx_lbr_caps);

/* diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 72995952978a..1ebe169b88b6 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -188,6 +188,14 @@ bool __read_mostly enable_pmu = true; EXPORT_SYMBOL_GPL(enable_pmu); module_param(enable_pmu, bool, 0444);

+/* + * Enable/disable mediated passthrough PMU virtualization. + * Don't expose it to userspace as a module paramerter until + * all mediated vPMU code is in place. + */ +bool __read_mostly enable_mediated_pmu; +EXPORT_SYMBOL_GPL(enable_mediated_pmu); + bool __read_mostly eager_page_split = true; module_param(eager_page_split, bool, 0644);

@@ -6643,9 +6651,28 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, break;

mutex_lock(&kvm->lock); - if (!kvm->created_vcpus) { - kvm->arch.enable_pmu = !(cap->args[0] & KVM_PMU_CAP_DISABLE); - r = 0; + /* + * To keep PMU configuration "simple", setting vPMU support is + * disallowed if vCPUs are created, or if mediated PMU support + * was already enabled for the VM. + */ + if (!kvm->created_vcpus && + (!enable_mediated_pmu || !kvm->arch.enable_pmu)) { + bool pmu_enable = !(cap->args[0] & KVM_PMU_CAP_DISABLE); + + if (enable_mediated_pmu && pmu_enable) { + char *err_msg = "Fail to enable mediated vPMU, " \ + "please disable system wide perf events or nmi_watchdog " \ + "(echo 0 > /proc/sys/kernel/nmi_watchdog).\n"; + + r = perf_get_mediated_pmu(); + if (r) + kvm_err("%s", err_msg); + } else + r = 0; + + if (!r) + kvm->arch.enable_pmu = pmu_enable; } mutex_unlock(&kvm->lock); break; @@ -12723,7 +12750,14 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm->arch.default_tsc_khz = max_tsc_khz ? : tsc_khz; kvm->arch.apic_bus_cycle_ns = APIC_BUS_CYCLE_NS_DEFAULT; kvm->arch.guest_can_read_msr_platform_info = true; - kvm->arch.enable_pmu = enable_pmu; + + /* + * PMU virtualization is opt-in when mediated PMU support is enabled. + * KVM_CAP_PMU_CAPABILITY ioctl must be called explicitly to enable + * mediated vPMU. For legacy perf-based vPMU, its behavior isn't changed, + * KVM_CAP_PMU_CAPABILITY ioctl is optional. + */ + kvm->arch.enable_pmu = enable_pmu && !enable_mediated_pmu;

#if IS_ENABLED(CONFIG_HYPERV) spin_lock_init(&kvm->arch.hv_root_tdp_lock); @@ -12876,6 +12910,8 @@ void kvm_arch_destroy_vm(struct kvm *kvm) __x86_set_memory_region(kvm, TSS_PRIVATE_MEMSLOT, 0, 0); mutex_unlock(&kvm->slots_lock); } + if (kvm->arch.enable_pmu && enable_mediated_pmu) + perf_put_mediated_pmu(); kvm_unload_vcpu_mmus(kvm); kvm_x86_call(vm_destroy)(kvm); kvm_free_msr_filter(srcu_dereference_check(kvm->arch.msr_filter, &kvm->srcu, 1)); diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 91e50a513100..dbf9973b3d09 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -391,6 +391,7 @@ extern struct kvm_caps kvm_caps; extern struct kvm_host_values kvm_host;

extern bool enable_pmu; +extern bool enable_mediated_pmu;

/* * Get a filtered version of KVM's supported XCR0 that strips out dynamic

From: Dapeng Mi dapeng1.mi@linux.intel.com

Rename the two helpers vmx_vmentry/vmexit_ctrl() to vmx_get_initial_vmentry/vmexit_ctrl() to represent their real meaning.

No functional change intended.

Suggested-by: Sean Christopherson seanjc@google.com Signed-off-by: Dapeng Mi dapeng1.mi@linux.intel.com Signed-off-by: Mingwei Zhang mizhang@google.com --- arch/x86/kvm/vmx/vmx.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index a4b5b6455c7b..acd3582874b9 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4424,7 +4424,7 @@ static u32 vmx_pin_based_exec_ctrl(struct vcpu_vmx *vmx) return pin_based_exec_ctrl; }

-static u32 vmx_vmentry_ctrl(void) +static u32 vmx_get_initial_vmentry_ctrl(void) { u32 vmentry_ctrl = vmcs_config.vmentry_ctrl;

@@ -4441,7 +4441,7 @@ static u32 vmx_vmentry_ctrl(void) return vmentry_ctrl; }

-static u32 vmx_vmexit_ctrl(void) +static u32 vmx_get_initial_vmexit_ctrl(void) { u32 vmexit_ctrl = vmcs_config.vmexit_ctrl;

@@ -4806,10 +4806,10 @@ static void init_vmcs(struct vcpu_vmx *vmx) if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);

- vm_exit_controls_set(vmx, vmx_vmexit_ctrl()); + vm_exit_controls_set(vmx, vmx_get_initial_vmexit_ctrl());

/* 22.2.1, 20.8.1 */ - vm_entry_controls_set(vmx, vmx_vmentry_ctrl()); + vm_entry_controls_set(vmx, vmx_get_initial_vmentry_ctrl());

vmx->vcpu.arch.cr0_guest_owned_bits = vmx_l1_guest_owned_cr0_bits(); vmcs_writel(CR0_GUEST_HOST_MASK, ~vmx->vcpu.arch.cr0_guest_owned_bits);

From: Dapeng Mi dapeng1.mi@linux.intel.com

Move PMU_CAP_{FW_WRITES,LBR_FMT} into msr-index.h and rename them with PERF_CAP prefix to keep consistent with other perf capabilities macros.

No functional change intended.

Signed-off-by: Dapeng Mi dapeng1.mi@linux.intel.com Signed-off-by: Mingwei Zhang mizhang@google.com --- arch/x86/include/asm/msr-index.h | 15 +++++++++------ arch/x86/kvm/vmx/capabilities.h | 3 --- arch/x86/kvm/vmx/pmu_intel.c | 4 ++-- arch/x86/kvm/vmx/vmx.c | 12 ++++++------ 4 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 72765b2fe0d8..ca70846ffd55 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -305,12 +305,15 @@ #define PERF_CAP_PT_IDX 16

#define MSR_PEBS_LD_LAT_THRESHOLD 0x000003f6 -#define PERF_CAP_PEBS_TRAP BIT_ULL(6) -#define PERF_CAP_ARCH_REG BIT_ULL(7) -#define PERF_CAP_PEBS_FORMAT 0xf00 -#define PERF_CAP_PEBS_BASELINE BIT_ULL(14) -#define PERF_CAP_PEBS_MASK (PERF_CAP_PEBS_TRAP | PERF_CAP_ARCH_REG | \ - PERF_CAP_PEBS_FORMAT | PERF_CAP_PEBS_BASELINE) + +#define PERF_CAP_LBR_FMT 0x3f +#define PERF_CAP_PEBS_TRAP BIT_ULL(6) +#define PERF_CAP_ARCH_REG BIT_ULL(7) +#define PERF_CAP_PEBS_FORMAT 0xf00 +#define PERF_CAP_FW_WRITES BIT_ULL(13) +#define PERF_CAP_PEBS_BASELINE BIT_ULL(14) +#define PERF_CAP_PEBS_MASK (PERF_CAP_PEBS_TRAP | PERF_CAP_ARCH_REG | \ + PERF_CAP_PEBS_FORMAT | PERF_CAP_PEBS_BASELINE)

#define MSR_IA32_RTIT_CTL 0x00000570 #define RTIT_CTL_TRACEEN BIT(0) diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index fac2c80ddbab..013536fde10b 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -21,9 +21,6 @@ extern int __read_mostly pt_mode; #define PT_MODE_SYSTEM 0 #define PT_MODE_HOST_GUEST 1

-#define PMU_CAP_FW_WRITES (1ULL << 13) -#define PMU_CAP_LBR_FMT 0x3f - struct nested_vmx_msrs { /* * We only store the "true" versions of the VMX capability MSRs. We diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 425e93d4b1c6..fc017e9a6a0c 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -118,7 +118,7 @@ static inline u64 vcpu_get_perf_capabilities(struct kvm_vcpu *vcpu)

static inline bool fw_writes_is_enabled(struct kvm_vcpu *vcpu) { - return (vcpu_get_perf_capabilities(vcpu) & PMU_CAP_FW_WRITES) != 0; + return (vcpu_get_perf_capabilities(vcpu) & PERF_CAP_FW_WRITES) != 0; }

static inline struct kvm_pmc *get_fw_gp_pmc(struct kvm_pmu *pmu, u32 msr) @@ -543,7 +543,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu)

perf_capabilities = vcpu_get_perf_capabilities(vcpu); if (cpuid_model_is_consistent(vcpu) && - (perf_capabilities & PMU_CAP_LBR_FMT)) + (perf_capabilities & PERF_CAP_LBR_FMT)) memcpy(&lbr_desc->records, &vmx_lbr_caps, sizeof(vmx_lbr_caps)); else lbr_desc->records.nr = 0; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index ca1c53f855e0..9c4b3c2b1d65 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2188,7 +2188,7 @@ static u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated (host_initiated || guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))) debugctl |= DEBUGCTLMSR_BUS_LOCK_DETECT;

- if ((kvm_caps.supported_perf_cap & PMU_CAP_LBR_FMT) && + if ((kvm_caps.supported_perf_cap & PERF_CAP_LBR_FMT) && (host_initiated || intel_pmu_lbr_is_enabled(vcpu))) debugctl |= DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI;

@@ -2464,9 +2464,9 @@ int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) vmx->pt_desc.guest.addr_a[index / 2] = data; break; case MSR_IA32_PERF_CAPABILITIES: - if (data & PMU_CAP_LBR_FMT) { - if ((data & PMU_CAP_LBR_FMT) != - (kvm_caps.supported_perf_cap & PMU_CAP_LBR_FMT)) + if (data & PERF_CAP_LBR_FMT) { + if ((data & PERF_CAP_LBR_FMT) != + (kvm_caps.supported_perf_cap & PERF_CAP_LBR_FMT)) return 1; if (!cpuid_model_is_consistent(vcpu)) return 1; @@ -7907,7 +7907,7 @@ void vmx_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu)

static __init u64 vmx_get_perf_capabilities(void) { - u64 perf_cap = PMU_CAP_FW_WRITES; + u64 perf_cap = PERF_CAP_FW_WRITES;

if (!enable_pmu) return 0; @@ -7924,7 +7924,7 @@ static __init u64 vmx_get_perf_capabilities(void) if (!vmx_lbr_caps.has_callstack) memset(&vmx_lbr_caps, 0, sizeof(vmx_lbr_caps)); else if (vmx_lbr_caps.nr) - perf_cap |= kvm_host.perf_capabilities & PMU_CAP_LBR_FMT; + perf_cap |= kvm_host.perf_capabilities & PERF_CAP_LBR_FMT; }

if (vmx_pebs_supported()) {

From: Dapeng Mi dapeng1.mi@linux.intel.com

Check if rdpmc can be intercepted for mediated vPMU. Simply speaking, if guest own all PMU counters in mediated vPMU, then rdpmc interception should be disabled to mitigate the performance impact, otherwise rdpmc has to be intercepted to avoid guest obtain host counter's data via rdpmc instruction.

Co-developed-by: Mingwei Zhang mizhang@google.com Signed-off-by: Mingwei Zhang mizhang@google.com Co-developed-by: Sandipan Das sandipan.das@amd.com Signed-off-by: Sandipan Das sandipan.das@amd.com Signed-off-by: Dapeng Mi dapeng1.mi@linux.intel.com --- arch/x86/include/asm/msr-index.h | 1 + arch/x86/kvm/pmu.c | 34 ++++++++++++++++++++++++++++++++ arch/x86/kvm/pmu.h | 19 ++++++++++++++++++ arch/x86/kvm/svm/pmu.c | 14 ++++++++++++- arch/x86/kvm/vmx/pmu_intel.c | 18 ++++++++--------- 5 files changed, 76 insertions(+), 10 deletions(-)

diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index ca70846ffd55..337f4b0a2998 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -312,6 +312,7 @@ #define PERF_CAP_PEBS_FORMAT 0xf00 #define PERF_CAP_FW_WRITES BIT_ULL(13) #define PERF_CAP_PEBS_BASELINE BIT_ULL(14) +#define PERF_CAP_PERF_METRICS BIT_ULL(15) #define PERF_CAP_PEBS_MASK (PERF_CAP_PEBS_TRAP | PERF_CAP_ARCH_REG | \ PERF_CAP_PEBS_FORMAT | PERF_CAP_PEBS_BASELINE)

diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 92c742ead663..6ad71752be4b 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -604,6 +604,40 @@ int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned idx, u64 *data) return 0; }

+inline bool kvm_rdpmc_in_guest(struct kvm_vcpu *vcpu) +{ + struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); + + if (!kvm_mediated_pmu_enabled(vcpu)) + return false; + + /* + * VMware allows access to these Pseduo-PMCs even when read via RDPMC + * in Ring3 when CR4.PCE=0. + */ + if (enable_vmware_backdoor) + return false; + + /* + * FIXME: In theory, perf metrics is always combined with fixed + * counter 3. it's fair enough to compare the guest and host + * fixed counter number and don't need to check perf metrics + * explicitly. However kvm_pmu_cap.num_counters_fixed is limited + * KVM_MAX_NR_FIXED_COUNTERS (3) as fixed counter 3 is not + * supported now. perf metrics is still needed to be checked + * explicitly here. Once fixed counter 3 is supported, the perf + * metrics checking can be removed. + */ + return pmu->nr_arch_gp_counters == kvm_pmu_cap.num_counters_gp && + pmu->nr_arch_fixed_counters == kvm_pmu_cap.num_counters_fixed && + vcpu_has_perf_metrics(vcpu) == kvm_host_has_perf_metrics() && + pmu->counter_bitmask[KVM_PMC_GP] == + (BIT_ULL(kvm_pmu_cap.bit_width_gp) - 1) && + pmu->counter_bitmask[KVM_PMC_FIXED] == + (BIT_ULL(kvm_pmu_cap.bit_width_fixed) - 1); +} +EXPORT_SYMBOL_GPL(kvm_rdpmc_in_guest); + void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu) { if (lapic_in_kernel(vcpu)) { diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index e1d0096f249b..509c995b7871 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -271,6 +271,24 @@ static inline bool pmc_is_globally_enabled(struct kvm_pmc *pmc) return test_bit(pmc->idx, (unsigned long *)&pmu->global_ctrl); }

+static inline u64 vcpu_get_perf_capabilities(struct kvm_vcpu *vcpu) +{ + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM)) + return 0; + + return vcpu->arch.perf_capabilities; +} + +static inline bool vcpu_has_perf_metrics(struct kvm_vcpu *vcpu) +{ + return !!(vcpu_get_perf_capabilities(vcpu) & PERF_CAP_PERF_METRICS); +} + +static inline bool kvm_host_has_perf_metrics(void) +{ + return !!(kvm_host.perf_capabilities & PERF_CAP_PERF_METRICS); +} + void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu); void kvm_pmu_handle_event(struct kvm_vcpu *vcpu); int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned pmc, u64 *data); @@ -287,6 +305,7 @@ void kvm_pmu_trigger_event(struct kvm_vcpu *vcpu, u64 eventsel); bool vcpu_pmu_can_enable(struct kvm_vcpu *vcpu);

bool is_vmware_backdoor_pmc(u32 pmc_idx); +bool kvm_rdpmc_in_guest(struct kvm_vcpu *vcpu);

extern struct kvm_pmu_ops intel_pmu_ops; extern struct kvm_pmu_ops amd_pmu_ops; diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index c8b9fd9b5350..153972e944eb 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -173,7 +173,7 @@ static int amd_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return 1; }

-static void amd_pmu_refresh(struct kvm_vcpu *vcpu) +static void __amd_pmu_refresh(struct kvm_vcpu *vcpu) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); union cpuid_0x80000022_ebx ebx; @@ -212,6 +212,18 @@ static void amd_pmu_refresh(struct kvm_vcpu *vcpu) bitmap_set(pmu->all_valid_pmc_idx, 0, pmu->nr_arch_gp_counters); }

+static void amd_pmu_refresh(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + __amd_pmu_refresh(vcpu); + + if (kvm_rdpmc_in_guest(vcpu)) + svm_clr_intercept(svm, INTERCEPT_RDPMC); + else + svm_set_intercept(svm, INTERCEPT_RDPMC); +} + static void amd_pmu_init(struct kvm_vcpu *vcpu) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index fc017e9a6a0c..2a5f79206b02 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -108,14 +108,6 @@ static struct kvm_pmc *intel_rdpmc_ecx_to_pmc(struct kvm_vcpu *vcpu, return &counters[array_index_nospec(idx, num_counters)]; }

-static inline u64 vcpu_get_perf_capabilities(struct kvm_vcpu *vcpu) -{ - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_PDCM)) - return 0; - - return vcpu->arch.perf_capabilities; -} - static inline bool fw_writes_is_enabled(struct kvm_vcpu *vcpu) { return (vcpu_get_perf_capabilities(vcpu) & PERF_CAP_FW_WRITES) != 0; @@ -456,7 +448,7 @@ static void intel_pmu_enable_fixed_counter_bits(struct kvm_pmu *pmu, u64 bits) pmu->fixed_ctr_ctrl_rsvd &= ~intel_fixed_bits_by_idx(i, bits); }

-static void intel_pmu_refresh(struct kvm_vcpu *vcpu) +static void __intel_pmu_refresh(struct kvm_vcpu *vcpu) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); @@ -564,6 +556,14 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) } }

+static void intel_pmu_refresh(struct kvm_vcpu *vcpu) +{ + __intel_pmu_refresh(vcpu); + + exec_controls_changebit(to_vmx(vcpu), CPU_BASED_RDPMC_EXITING, + !kvm_rdpmc_in_guest(vcpu)); +} + static void intel_pmu_init(struct kvm_vcpu *vcpu) { int i;

From: Dapeng Mi dapeng1.mi@linux.intel.com

Currently pmu->global_ctrl is initialized in the common kvm_pmu_refresh() helper since both Intel and AMD CPUs set enable bits for all GP counters for PERF_GLOBAL_CTRL MSR. But it may be not the best place to initialize pmu->global_ctrl. Strictly speaking, pmu->global_ctrl is vendor specific and there are lots of global_ctrl related processing in intel/amd_pmu_refresh() helpers, so better handle them in same place. Thus move pmu->global_ctrl initialization into intel/amd_pmu_refresh() helpers.

Besides, intel_pmu_refresh() doesn't handle global_ctrl_rsvd and global_status_rsvd properly and fix it.

Signed-off-by: Dapeng Mi dapeng1.mi@linux.intel.com Signed-off-by: Mingwei Zhang mizhang@google.com --- arch/x86/kvm/pmu.c | 10 ------- arch/x86/kvm/svm/pmu.c | 14 +++++++-- arch/x86/kvm/vmx/pmu_intel.c | 55 ++++++++++++++++++------------------ 3 files changed, 39 insertions(+), 40 deletions(-)

diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 4e8cefcce7ab..2ac4c039de8b 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -843,16 +843,6 @@ void kvm_pmu_refresh(struct kvm_vcpu *vcpu) return;

kvm_pmu_call(refresh)(vcpu); - - /* - * At RESET, both Intel and AMD CPUs set all enable bits for general - * purpose counters in IA32_PERF_GLOBAL_CTRL (so that software that - * was written for v1 PMUs don't unknowingly leave GP counters disabled - * in the global controls). Emulate that behavior when refreshing the - * PMU so that userspace doesn't need to manually set PERF_GLOBAL_CTRL. - */ - if (kvm_pmu_has_perf_global_ctrl(pmu) && pmu->nr_arch_gp_counters) - pmu->global_ctrl = GENMASK_ULL(pmu->nr_arch_gp_counters - 1, 0); }

void kvm_pmu_init(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index 153972e944eb..eba086ef5eca 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -198,12 +198,20 @@ static void __amd_pmu_refresh(struct kvm_vcpu *vcpu) pmu->nr_arch_gp_counters = min_t(unsigned int, pmu->nr_arch_gp_counters, kvm_pmu_cap.num_counters_gp);

- if (pmu->version > 1) { - pmu->global_ctrl_rsvd = ~((1ull << pmu->nr_arch_gp_counters) - 1); + if (kvm_pmu_cap.version > 1) { + /* + * At RESET, AMD CPUs set all enable bits for general purpose counters in + * IA32_PERF_GLOBAL_CTRL (so that software that was written for v1 PMUs + * don't unknowingly leave GP counters disabled in the global controls). + * Emulate that behavior when refreshing the PMU so that userspace doesn't + * need to manually set PERF_GLOBAL_CTRL. + */ + pmu->global_ctrl = BIT_ULL(pmu->nr_arch_gp_counters) - 1; + pmu->global_ctrl_rsvd = ~pmu->global_ctrl; pmu->global_status_rsvd = pmu->global_ctrl_rsvd; }

- pmu->counter_bitmask[KVM_PMC_GP] = ((u64)1 << 48) - 1; + pmu->counter_bitmask[KVM_PMC_GP] = BIT_ULL(48) - 1; pmu->reserved_bits = 0xfffffff000280000ull; pmu->raw_event_mask = AMD64_RAW_EVENT_MASK; /* not applicable to AMD; but clean them to prevent any fall out */ diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 04a893e56135..c30c6c5e36c8 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -466,7 +466,6 @@ static void __intel_pmu_refresh(struct kvm_vcpu *vcpu) union cpuid10_eax eax; union cpuid10_edx edx; u64 perf_capabilities; - u64 counter_rsvd;

memset(&lbr_desc->records, 0, sizeof(lbr_desc->records));

@@ -493,11 +492,10 @@ static void __intel_pmu_refresh(struct kvm_vcpu *vcpu) kvm_pmu_cap.num_counters_gp); eax.split.bit_width = min_t(int, eax.split.bit_width, kvm_pmu_cap.bit_width_gp); - pmu->counter_bitmask[KVM_PMC_GP] = ((u64)1 << eax.split.bit_width) - 1; + pmu->counter_bitmask[KVM_PMC_GP] = BIT_ULL(eax.split.bit_width) - 1; eax.split.mask_length = min_t(int, eax.split.mask_length, kvm_pmu_cap.events_mask_len); - pmu->available_event_types = ~entry->ebx & - ((1ull << eax.split.mask_length) - 1); + pmu->available_event_types = ~entry->ebx & (BIT_ULL(eax.split.mask_length) - 1);

if (pmu->version == 1) { pmu->nr_arch_fixed_counters = 0; @@ -506,29 +504,34 @@ static void __intel_pmu_refresh(struct kvm_vcpu *vcpu) kvm_pmu_cap.num_counters_fixed); edx.split.bit_width_fixed = min_t(int, edx.split.bit_width_fixed, kvm_pmu_cap.bit_width_fixed); - pmu->counter_bitmask[KVM_PMC_FIXED] = - ((u64)1 << edx.split.bit_width_fixed) - 1; + pmu->counter_bitmask[KVM_PMC_FIXED] = BIT_ULL(edx.split.bit_width_fixed) - 1; }

intel_pmu_enable_fixed_counter_bits(pmu, INTEL_FIXED_0_KERNEL | INTEL_FIXED_0_USER | INTEL_FIXED_0_ENABLE_PMI);

- counter_rsvd = ~(((1ull << pmu->nr_arch_gp_counters) - 1) | - (((1ull << pmu->nr_arch_fixed_counters) - 1) << KVM_FIXED_PMC_BASE_IDX)); - pmu->global_ctrl_rsvd = counter_rsvd; + if (kvm_pmu_has_perf_global_ctrl(pmu)) { + /* + * At RESET, Intel CPUs set all enable bits for general purpose counters + * in IA32_PERF_GLOBAL_CTRL. Emulate this behavior. + */ + pmu->global_ctrl = BIT_ULL(pmu->nr_arch_gp_counters) - 1; + pmu->global_ctrl_rsvd = ~((BIT_ULL(pmu->nr_arch_gp_counters) - 1) | + ((BIT_ULL(pmu->nr_arch_fixed_counters) - 1) << + KVM_FIXED_PMC_BASE_IDX));

- /* - * GLOBAL_STATUS and GLOBAL_OVF_CONTROL (a.k.a. GLOBAL_STATUS_RESET) - * share reserved bit definitions. The kernel just happens to use - * OVF_CTRL for the names. - */ - pmu->global_status_rsvd = pmu->global_ctrl_rsvd - & ~(MSR_CORE_PERF_GLOBAL_OVF_CTRL_OVF_BUF | - MSR_CORE_PERF_GLOBAL_OVF_CTRL_COND_CHGD); - if (vmx_pt_mode_is_host_guest()) - pmu->global_status_rsvd &= - ~MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI; + /* + * GLOBAL_STATUS and GLOBAL_OVF_CONTROL (a.k.a. GLOBAL_STATUS_RESET) + * share reserved bit definitions. The kernel just happens to use + * OVF_CTRL for the names. + */ + pmu->global_status_rsvd = pmu->global_ctrl_rsvd & + ~(MSR_CORE_PERF_GLOBAL_OVF_CTRL_OVF_BUF | + MSR_CORE_PERF_GLOBAL_OVF_CTRL_COND_CHGD); + if (vmx_pt_mode_is_host_guest()) + pmu->global_status_rsvd &= ~MSR_CORE_PERF_GLOBAL_OVF_CTRL_TRACE_TOPA_PMI; + }

entry = kvm_find_cpuid_entry_index(vcpu, 7, 0); if (entry && @@ -538,10 +541,9 @@ static void __intel_pmu_refresh(struct kvm_vcpu *vcpu) pmu->raw_event_mask |= (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED); }

- bitmap_set(pmu->all_valid_pmc_idx, - 0, pmu->nr_arch_gp_counters); - bitmap_set(pmu->all_valid_pmc_idx, - INTEL_PMC_MAX_GENERIC, pmu->nr_arch_fixed_counters); + bitmap_set(pmu->all_valid_pmc_idx, 0, pmu->nr_arch_gp_counters); + bitmap_set(pmu->all_valid_pmc_idx, INTEL_PMC_MAX_GENERIC, + pmu->nr_arch_fixed_counters);

perf_capabilities = vcpu_get_perf_capabilities(vcpu); if (cpuid_model_is_consistent(vcpu) && @@ -555,13 +557,12 @@ static void __intel_pmu_refresh(struct kvm_vcpu *vcpu)

if (perf_capabilities & PERF_CAP_PEBS_FORMAT) { if (perf_capabilities & PERF_CAP_PEBS_BASELINE) { - pmu->pebs_enable_rsvd = counter_rsvd; + pmu->pebs_enable_rsvd = pmu->global_ctrl_rsvd; pmu->reserved_bits &= ~ICL_EVENTSEL_ADAPTIVE; pmu->pebs_data_cfg_rsvd = ~0xff00000full; intel_pmu_enable_fixed_counter_bits(pmu, ICL_FIXED_0_ADAPTIVE); } else { - pmu->pebs_enable_rsvd = - ~((1ull << pmu->nr_arch_gp_counters) - 1); + pmu->pebs_enable_rsvd = ~(BIT_ULL(pmu->nr_arch_gp_counters) - 1); } } }

Reject PMU MSRs interception explicitly in vmx_get_passthrough_msr_slot() since interception of PMU MSRs are specially handled in intel_passthrough_pmu_msrs().

Signed-off-by: Mingwei Zhang mizhang@google.com Co-developed-by: Dapeng Mi dapeng1.mi@linux.intel.com Signed-off-by: Dapeng Mi dapeng1.mi@linux.intel.com --- arch/x86/kvm/vmx/vmx.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 38ecf3c116bd..7bb16bed08da 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -165,7 +165,7 @@ module_param(allow_smaller_maxphyaddr, bool, S_IRUGO);

/* * List of MSRs that can be directly passed to the guest. - * In addition to these x2apic, PT and LBR MSRs are handled specially. + * In addition to these x2apic, PMU, PT and LBR MSRs are handled specially. */ static u32 vmx_possible_passthrough_msrs[MAX_POSSIBLE_PASSTHROUGH_MSRS] = { MSR_IA32_SPEC_CTRL, @@ -691,6 +691,16 @@ static int vmx_get_passthrough_msr_slot(u32 msr) case MSR_LBR_CORE_FROM ... MSR_LBR_CORE_FROM + 8: case MSR_LBR_CORE_TO ... MSR_LBR_CORE_TO + 8: /* LBR MSRs. These are handled in vmx_update_intercept_for_lbr_msrs() */ + case MSR_IA32_PMC0 ... + MSR_IA32_PMC0 + KVM_MAX_NR_GP_COUNTERS - 1: + case MSR_IA32_PERFCTR0 ... + MSR_IA32_PERFCTR0 + KVM_MAX_NR_GP_COUNTERS - 1: + case MSR_CORE_PERF_FIXED_CTR0 ... + MSR_CORE_PERF_FIXED_CTR0 + KVM_MAX_NR_FIXED_COUNTERS - 1: + case MSR_CORE_PERF_GLOBAL_STATUS: + case MSR_CORE_PERF_GLOBAL_CTRL: + case MSR_CORE_PERF_GLOBAL_OVF_CTRL: + /* PMU MSRs. These are handled in intel_passthrough_pmu_msrs() */ return -ENOENT; }

Introduce eventsel_hw and fixed_ctr_ctrl_hw to store the actual HW value in PMU event selector MSRs. In mediated PMU checks events before allowing the event values written to the PMU MSRs. However, to match the HW behavior, when PMU event checks fails, KVM should allow guest to read the value back.

This essentially requires an extra variable to separate the guest requested value from actual PMU MSR value. Note this only applies to event selectors.

Signed-off-by: Mingwei Zhang mizhang@google.com Co-developed-by: Dapeng Mi dapeng1.mi@linux.intel.com Signed-off-by: Dapeng Mi dapeng1.mi@linux.intel.com --- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/pmu.c | 7 +++++-- arch/x86/kvm/svm/pmu.c | 1 + arch/x86/kvm/vmx/pmu_intel.c | 2 ++ 4 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 4b3bfefc2d05..7ee74bbbb0aa 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -524,6 +524,7 @@ struct kvm_pmc { */ u64 emulated_counter; u64 eventsel; + u64 eventsel_hw; struct perf_event *perf_event; struct kvm_vcpu *vcpu; /* @@ -552,6 +553,7 @@ struct kvm_pmu { unsigned nr_arch_fixed_counters; unsigned available_event_types; u64 fixed_ctr_ctrl; + u64 fixed_ctr_ctrl_hw; u64 fixed_ctr_ctrl_rsvd; /* * kvm_pmu_sync_global_ctrl_from_vmcs() must be called to update diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 2ac4c039de8b..63143eeb5c44 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -794,11 +794,14 @@ static void kvm_pmu_reset(struct kvm_vcpu *vcpu) pmc->counter = 0; pmc->emulated_counter = 0;

- if (pmc_is_gp(pmc)) + if (pmc_is_gp(pmc)) { pmc->eventsel = 0; + pmc->eventsel_hw = 0; + } }

- pmu->fixed_ctr_ctrl = pmu->global_ctrl = pmu->global_status = 0; + pmu->fixed_ctr_ctrl = pmu->fixed_ctr_ctrl_hw = 0; + pmu->global_ctrl = pmu->global_status = 0;

kvm_pmu_call(reset)(vcpu); } diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index 4fc809c74ba8..9feaca739b96 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -165,6 +165,7 @@ static int amd_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) data &= ~pmu->reserved_bits; if (data != pmc->eventsel) { pmc->eventsel = data; + pmc->eventsel_hw = data; kvm_pmu_request_counter_reprogram(pmc); } return 0; diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 450f9e5b9e40..796b7bc4affe 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -41,6 +41,7 @@ static void reprogram_fixed_counters(struct kvm_pmu *pmu, u64 data) int i;

pmu->fixed_ctr_ctrl = data; + pmu->fixed_ctr_ctrl_hw = data; for (i = 0; i < pmu->nr_arch_fixed_counters; i++) { u8 new_ctrl = fixed_ctrl_field(data, i); u8 old_ctrl = fixed_ctrl_field(old_fixed_ctr_ctrl, i); @@ -403,6 +404,7 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)

if (data != pmc->eventsel) { pmc->eventsel = data; + pmc->eventsel_hw = data; kvm_pmu_request_counter_reprogram(pmc); } break;

From: Sandipan Das sandipan.das@amd.com

On AMD platforms, there is no way to restore PerfCntrGlobalCtl at VM-Entry or clear it at VM-Exit. Since the register states will be restored before entering and saved after exiting guest context, the counters can keep ticking and even overflow leading to chaos while still in host context.

To avoid this, intecept event selectors, which is already done by mediated PMU. In addition, always set the GuestOnly bit and clear the HostOnly bit for PMU selectors on AMD. Doing so allows the counters run only in guest context even if their enable bits are still set after VM exit and before host/guest PMU context switch.

Signed-off-by: Sandipan Das sandipan.das@amd.com Signed-off-by: Mingwei Zhang mizhang@google.com --- arch/x86/kvm/svm/pmu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index 9feaca739b96..1a7e3a897fdf 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -165,7 +165,8 @@ static int amd_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) data &= ~pmu->reserved_bits; if (data != pmc->eventsel) { pmc->eventsel = data; - pmc->eventsel_hw = data; + pmc->eventsel_hw = (data & ~AMD64_EVENTSEL_HOSTONLY) | + AMD64_EVENTSEL_GUESTONLY; kvm_pmu_request_counter_reprogram(pmc); } return 0;

From: Dapeng Mi dapeng1.mi@linux.intel.com

Mediated vPMU needs to accumulate the emulated instructions into counter and load the counter into HW at vm-entry.

Moreover, if the accumulation leads to counter overflow, KVM needs to update GLOBAL_STATUS and inject PMI into guest as well.

Suggested-by: Sean Christopherson seanjc@google.com Signed-off-by: Dapeng Mi dapeng1.mi@linux.intel.com Signed-off-by: Mingwei Zhang mizhang@google.com --- arch/x86/kvm/pmu.c | 44 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-)

diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index 68f203454bbc..f71009ec92cf 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -911,10 +911,50 @@ void kvm_pmu_destroy(struct kvm_vcpu *vcpu) kvm_pmu_reset(vcpu); }

+static bool pmc_pmi_enabled(struct kvm_pmc *pmc) +{ + struct kvm_pmu *pmu = pmc_to_pmu(pmc); + u8 fixed_ctr_ctrl; + bool pmi_enabled; + + if (pmc_is_gp(pmc)) { + pmi_enabled = pmc->eventsel & ARCH_PERFMON_EVENTSEL_INT; + } else { + fixed_ctr_ctrl = fixed_ctrl_field(pmu->fixed_ctr_ctrl, + pmc->idx - KVM_FIXED_PMC_BASE_IDX); + pmi_enabled = fixed_ctr_ctrl & INTEL_FIXED_0_ENABLE_PMI; + } + + return pmi_enabled; +} + static void kvm_pmu_incr_counter(struct kvm_pmc *pmc) { - pmc->emulated_counter++; - kvm_pmu_request_counter_reprogram(pmc); + struct kvm_vcpu *vcpu = pmc->vcpu; + + /* + * For perf-based PMUs, accumulate software-emulated events separately + * from pmc->counter, as pmc->counter is offset by the count of the + * associated perf event. Request reprogramming, which will consult + * both emulated and hardware-generated events to detect overflow. + */ + if (!kvm_mediated_pmu_enabled(vcpu)) { + pmc->emulated_counter++; + kvm_pmu_request_counter_reprogram(pmc); + return; + } + + /* + * For mediated PMUs, pmc->counter is updated when the vCPU's PMU is + * put, and will be loaded into hardware when the PMU is loaded. Simply + * increment the counter and signal overflow if it wraps to zero. + */ + pmc->counter = (pmc->counter + 1) & pmc_bitmask(pmc); + if (!pmc->counter) { + pmc_to_pmu(pmc)->global_status |= BIT_ULL(pmc->idx); + if (pmc_pmi_enabled(pmc)) + kvm_make_request(KVM_REQ_PMI, vcpu); + } }

static inline bool cpl_is_matched(struct kvm_pmc *pmc)

Add nested virtualization support for mediated PMU by combining the MSR interception bitmaps of vmcs01 and vmcs12. Readers may argue even without this patch, nested virtualization works for mediated PMU because L1 will see Perfmon v2 and will have to use legacy vPMU implementation if it is Linux. However, any assumption made on L1 may be invalid, e.g., L1 may not even be Linux.

If both L0 and L1 pass through PMU MSRs, the correct behavior is to allow MSR access from L2 directly touch HW MSRs, since both L0 and L1 passthrough the access.

However, in current implementation, if without adding anything for nested, KVM always set MSR interception bits in vmcs02. This leads to the fact that L0 will emulate all MSR read/writes for L2, leading to errors, since the current mediated vPMU never implements set_msr() and get_msr() for any counter access except counter accesses from the VMM side.

So fix the issue by setting up the correct MSR interception for PMU MSRs.

Signed-off-by: Mingwei Zhang mizhang@google.com Co-developed-by: Dapeng Mi dapeng1.mi@linux.intel.com Signed-off-by: Dapeng Mi dapeng1.mi@linux.intel.com --- arch/x86/kvm/vmx/nested.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+)

diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index cf557acf91f8..dbec40cb55bc 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -626,6 +626,36 @@ static inline void nested_vmx_set_intercept_for_msr(struct vcpu_vmx *vmx, #define nested_vmx_merge_msr_bitmaps_rw(msr) \ nested_vmx_merge_msr_bitmaps(msr, MSR_TYPE_RW)

+/* + * Disable PMU MSRs interception for nested VM if L0 and L1 are + * both mediated vPMU. + */ +static void nested_vmx_merge_pmu_msr_bitmaps(struct kvm_vcpu *vcpu, + unsigned long *msr_bitmap_l1, + unsigned long *msr_bitmap_l0) +{ + struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); + struct vcpu_vmx *vmx = to_vmx(vcpu); + int i; + + if (!kvm_mediated_pmu_enabled(vcpu)) + return; + + for (i = 0; i < pmu->nr_arch_gp_counters; i++) { + nested_vmx_merge_msr_bitmaps_rw(MSR_ARCH_PERFMON_EVENTSEL0 + i); + nested_vmx_merge_msr_bitmaps_rw(MSR_IA32_PERFCTR0 + i); + nested_vmx_merge_msr_bitmaps_rw(MSR_IA32_PMC0 + i); + } + + for (i = 0; i < pmu->nr_arch_fixed_counters; i++) + nested_vmx_merge_msr_bitmaps_rw(MSR_CORE_PERF_FIXED_CTR0 + i); + + nested_vmx_merge_msr_bitmaps_rw(MSR_CORE_PERF_FIXED_CTR_CTRL); + nested_vmx_merge_msr_bitmaps_rw(MSR_CORE_PERF_GLOBAL_CTRL); + nested_vmx_merge_msr_bitmaps_read(MSR_CORE_PERF_GLOBAL_STATUS); + nested_vmx_merge_msr_bitmaps_write(MSR_CORE_PERF_GLOBAL_OVF_CTRL); +} + /* * Merge L0's and L1's MSR bitmap, return false to indicate that * we do not use the hardware. @@ -717,6 +747,8 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, nested_vmx_merge_msr_bitmaps_write(MSR_IA32_PRED_CMD); nested_vmx_merge_msr_bitmaps_write(MSR_IA32_FLUSH_CMD);

+ nested_vmx_merge_pmu_msr_bitmaps(vcpu, msr_bitmap_l1, msr_bitmap_l0); + kvm_vcpu_unmap(vcpu, &map);

vmx->nested.force_msr_bitmap_recalc = false;

From: Sandipan Das sandipan.das@amd.com

Apply the PERF_PMU_CAP_MEDIATED_VPMU flag for version 2 and later implementations of the core PMU. Aside from having Global Control and Status registers, virtualizing the PMU using the passthrough model requires an interface to set or clear the overflow bits in the Global Status MSRs while restoring or saving the PMU context of a vCPU.

PerfMonV2-capable hardware has additional MSRs for this purpose namely, PerfCntrGlobalStatusSet and PerfCntrGlobalStatusClr, thereby making it suitable for use with mediated vPMU.

Signed-off-by: Sandipan Das sandipan.das@amd.com Signed-off-by: Mingwei Zhang mizhang@google.com --- arch/x86/events/amd/core.c | 2 ++ 1 file changed, 2 insertions(+)

diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index 30d6ceb4c8ad..a8b537dd2ddb 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -1433,6 +1433,8 @@ static int __init amd_core_pmu_init(void)

amd_pmu_global_cntr_mask = x86_pmu.cntr_mask64;

+ x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_MEDIATED_VPMU; + /* Update PMC handling functions */ x86_pmu.enable_all = amd_pmu_v2_enable_all; x86_pmu.disable_all = amd_pmu_v2_disable_all;

From: Dapeng Mi dapeng1.mi@linux.intel.com

Mediated vPMU needs to call KVM_CAP_PMU_CAPABILITY ioctl to enable it. Thus add a helper vm_create_with_one_vcpu_with_pmu() to create PMU enabled VM and replace vm_create_with_one_vcpu() helper with this new helper in pmu tests.

Signed-off-by: Dapeng Mi dapeng1.mi@linux.intel.com Signed-off-by: Mingwei Zhang mizhang@google.com --- .../testing/selftests/kvm/include/kvm_util.h | 3 +++ tools/testing/selftests/kvm/lib/kvm_util.c | 23 +++++++++++++++++++ .../selftests/kvm/x86/pmu_counters_test.c | 4 +++- .../selftests/kvm/x86/pmu_event_filter_test.c | 8 ++++--- 4 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 4c4e5a847f67..a73b0b98be5e 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -961,6 +961,9 @@ static inline struct kvm_vm *vm_create_shape_with_one_vcpu(struct vm_shape shape return __vm_create_shape_with_one_vcpu(shape, vcpu, 0, guest_code); }

+struct kvm_vm *vm_create_with_one_vcpu_with_pmu(struct kvm_vcpu **vcpu, + void *guest_code); + struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm);

void kvm_pin_this_task_to_pcpu(uint32_t pcpu); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 33fefeb3ca44..18143ec2e751 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -545,6 +545,29 @@ struct kvm_vcpu *vm_recreate_with_one_vcpu(struct kvm_vm *vm) return vm_vcpu_recreate(vm, 0); }

+struct kvm_vm *vm_create_with_one_vcpu_with_pmu(struct kvm_vcpu **vcpu, + void *guest_code) +{ + struct kvm_vm *vm; + int r; + + r = kvm_check_cap(KVM_CAP_PMU_CAPABILITY); + if (!(r & KVM_PMU_CAP_DISABLE)) + return NULL; + + vm = vm_create(1); + + /* + * KVM_CAP_PMU_CAPABILITY ioctl must be explicitly called to enable + * mediated vPMU. + */ + vm_enable_cap(vm, KVM_CAP_PMU_CAPABILITY, !KVM_PMU_CAP_DISABLE); + + *vcpu = vm_vcpu_add(vm, 0, guest_code); + + return vm; +} + void kvm_pin_this_task_to_pcpu(uint32_t pcpu) { cpu_set_t mask; diff --git a/tools/testing/selftests/kvm/x86/pmu_counters_test.c b/tools/testing/selftests/kvm/x86/pmu_counters_test.c index 698cb36989db..441c66f314fb 100644 --- a/tools/testing/selftests/kvm/x86/pmu_counters_test.c +++ b/tools/testing/selftests/kvm/x86/pmu_counters_test.c @@ -40,7 +40,9 @@ static struct kvm_vm *pmu_vm_create_with_one_vcpu(struct kvm_vcpu **vcpu, { struct kvm_vm *vm;

- vm = vm_create_with_one_vcpu(vcpu, guest_code); + vm = vm_create_with_one_vcpu_with_pmu(vcpu, guest_code); + assert(vm); + sync_global_to_guest(vm, kvm_pmu_version);

/* diff --git a/tools/testing/selftests/kvm/x86/pmu_event_filter_test.c b/tools/testing/selftests/kvm/x86/pmu_event_filter_test.c index c15513cd74d1..1c7d265a0003 100644 --- a/tools/testing/selftests/kvm/x86/pmu_event_filter_test.c +++ b/tools/testing/selftests/kvm/x86/pmu_event_filter_test.c @@ -822,8 +822,9 @@ static void test_fixed_counter_bitmap(void) * fixed performance counters. */ for (idx = 0; idx < nr_fixed_counters; idx++) { - vm = vm_create_with_one_vcpu(&vcpu, - intel_run_fixed_counter_guest_code); + vm = vm_create_with_one_vcpu_with_pmu(&vcpu, + intel_run_fixed_counter_guest_code); + assert(vm); vcpu_args_set(vcpu, 1, idx); __test_fixed_counter_bitmap(vcpu, idx, nr_fixed_counters); kvm_vm_free(vm); @@ -843,7 +844,8 @@ int main(int argc, char *argv[]) TEST_REQUIRE(use_intel_pmu() || use_amd_pmu()); guest_code = use_intel_pmu() ? intel_guest_code : amd_guest_code;

- vm = vm_create_with_one_vcpu(&vcpu, guest_code); + vm = vm_create_with_one_vcpu_with_pmu(&vcpu, guest_code); + assert(vm);

TEST_REQUIRE(sanity_check_pmu(vcpu));

From: Dapeng Mi dapeng1.mi@linux.intel.com

As previous patch commit 'f8905c638eb7 ("KVM: x86/pmu: Check PMU cpuid configuration from user space")', KVM would check if user space configured pmu version is larger than KVM supported maximum pmu version for mediated vPMU, or if fixed counter bitmap is configured incorrectly, if so, KVM would return an error.

This enhanced check would lead to pmu_counters_test fails, thus limit pmu_counters_test only validate KVM supported pmu versions for mediated vPMU and only validate 0 fixed counter bitmap if pmu version is less than 5.

Signed-off-by: Dapeng Mi dapeng1.mi@linux.intel.com Signed-off-by: Mingwei Zhang mizhang@google.com --- .../selftests/kvm/include/x86/processor.h | 8 ++++++++ .../selftests/kvm/x86/pmu_counters_test.c | 20 ++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index d60da8966772..7db34f48427a 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -1311,6 +1311,14 @@ static inline bool kvm_is_pmu_enabled(void) return get_kvm_param_bool("enable_pmu"); }

+static inline bool kvm_is_mediated_pmu_enabled(void) +{ + if (host_cpu_is_intel) + return get_kvm_intel_param_bool("enable_mediated_pmu"); + else + return get_kvm_amd_param_bool("enable_mediated_pmu"); +} + static inline bool kvm_is_forced_emulation_enabled(void) { return !!get_kvm_param_integer("force_emulation_prefix"); diff --git a/tools/testing/selftests/kvm/x86/pmu_counters_test.c b/tools/testing/selftests/kvm/x86/pmu_counters_test.c index 441c66f314fb..4745f82ce860 100644 --- a/tools/testing/selftests/kvm/x86/pmu_counters_test.c +++ b/tools/testing/selftests/kvm/x86/pmu_counters_test.c @@ -564,8 +564,14 @@ static void test_intel_counters(void) * Test up to PMU v5, which is the current maximum version defined by * Intel, i.e. is the last version that is guaranteed to be backwards * compatible with KVM's existing behavior. + * + * Whereas for mediated vPMU, limit max_pmu_version to KVM supported + * maximum pmu version since KVM rejects PMU versions larger than KVM + * supported maximum PMU version to avoid guest to manipulate unsupported + * or unallowed PMU MSRs directly. */ - uint8_t max_pmu_version = max_t(typeof(pmu_version), pmu_version, 5); + uint8_t max_pmu_version = kvm_is_mediated_pmu_enabled() ? + pmu_version : max_t(typeof(pmu_version), pmu_version, 5);

/* * Detect the existence of events that aren't supported by selftests. @@ -622,8 +628,16 @@ static void test_intel_counters(void) pr_info("Testing fixed counters, PMU version %u, perf_caps = %lx\n", v, perf_caps[i]); for (j = 0; j <= nr_fixed_counters; j++) { - for (k = 0; k <= (BIT(nr_fixed_counters) - 1); k++) - test_fixed_counters(v, perf_caps[i], j, k); + /* + * pmu version less than 5 doesn't support fixed counter + * bitmap, so only set fixed counter bitamp to 0. + */ + if (v < 5) { + test_fixed_counters(v, perf_caps[i], j, 0); + } else { + for (k = 0; k <= (BIT(nr_fixed_counters) - 1); k++) + test_fixed_counters(v, perf_caps[i], j, k); + } } } }