With the addition of the chg parameter, vma_has_reserves() no longer just determines whether the vma has reserves.

The comment in the vma->vm_flags & VM_NORESERVE block indicates that this function actually computes whether or not the reserved count should be decremented.

This refactoring also takes into account the allocation's request parameter avoid_reserve, which helps to further simplify the calling function alloc_hugetlb_folio().

Signed-off-by: Ackerley Tng ackerleytng@google.com --- mm/hugetlb.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/mm/hugetlb.c b/mm/hugetlb.c index af5c6bbc9ff0..597102ed224b 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1245,9 +1245,19 @@ void clear_vma_resv_huge_pages(struct vm_area_struct *vma) hugetlb_dup_vma_private(vma); }

-/* Returns true if the VMA has associated reserve pages */ -static bool vma_has_reserves(struct vm_area_struct *vma, long chg) +/* + * Returns true if this allocation should use (debit) hstate reservations, based on + * + * @vma: VMA config + * @chg: Whether the page requirement can be satisfied using subpool reservations + * @avoid_reserve: Whether allocation was requested to avoid using reservations + */ +static bool should_use_hstate_resv(struct vm_area_struct *vma, long chg, + bool avoid_reserve) { + if (avoid_reserve) + return false; + if (vma->vm_flags & VM_NORESERVE) { /* * This address is already reserved by other process(chg == 0), @@ -3182,7 +3192,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, if (ret) goto out_uncharge_cgroup_reservation;

- use_hstate_resv = !avoid_reserve && vma_has_reserves(vma, gbl_chg); + use_hstate_resv = should_use_hstate_resv(vma, gbl_chg, avoid_reserve);

spin_lock_irq(&hugetlb_lock); folio = dequeue_hugetlb_folio_vma(h, vma, addr, use_hstate_resv);

This was refactored out of huge_node().

huge_node()'s interpretation of vma for order assumes the hugetlb-specific storage of the hstate information in the inode. policy_node_nodemask() does not assume that, and can be used more generically.

This refactoring also enforces that nid default to the current node id, which was not previously enforced.

alloc_pages_mpol_noprof() is the last remaining direct user of policy_nodemask(). All its callers begin with nid being the current node id as well. More refactoring is required for to simplify that.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- include/linux/mempolicy.h | 2 ++ mm/mempolicy.c | 36 ++++++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h index 1add16f21612..a49631e47421 100644 --- a/include/linux/mempolicy.h +++ b/include/linux/mempolicy.h @@ -138,6 +138,8 @@ extern void numa_policy_init(void); extern void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new); extern void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new);

+extern int policy_node_nodemask(struct mempolicy *mpol, gfp_t gfp_flags, + pgoff_t ilx, nodemask_t **nodemask); extern int huge_node(struct vm_area_struct *vma, unsigned long addr, gfp_t gfp_flags, struct mempolicy **mpol, nodemask_t **nodemask); diff --git a/mm/mempolicy.c b/mm/mempolicy.c index b858e22b259d..f3e572e17775 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -1212,7 +1212,6 @@ static struct folio *alloc_migration_target_by_mpol(struct folio *src, struct mempolicy *pol = mmpol->pol; pgoff_t ilx = mmpol->ilx; unsigned int order; - int nid = numa_node_id(); gfp_t gfp;

order = folio_order(src); @@ -1221,10 +1220,11 @@ static struct folio *alloc_migration_target_by_mpol(struct folio *src, if (folio_test_hugetlb(src)) { nodemask_t *nodemask; struct hstate *h; + int nid;

h = folio_hstate(src); gfp = htlb_alloc_mask(h); - nodemask = policy_nodemask(gfp, pol, ilx, &nid); + nid = policy_node_nodemask(pol, gfp, ilx, &nodemask); return alloc_hugetlb_folio_nodemask(h, nid, nodemask, gfp, htlb_allow_alloc_fallback(MR_MEMPOLICY_MBIND)); } @@ -1234,7 +1234,7 @@ static struct folio *alloc_migration_target_by_mpol(struct folio *src, else gfp = GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL | __GFP_COMP;

- return folio_alloc_mpol(gfp, order, pol, ilx, nid); + return folio_alloc_mpol(gfp, order, pol, ilx, numa_node_id()); } #else

@@ -2084,6 +2084,27 @@ static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *pol, return nodemask; }

+/** + * policy_node_nodemask(@mpol, @gfp_flags, @ilx, @nodemask) + * @mpol: the memory policy to interpret. Reference must be taken. + * @gfp_flags: for this request + * @ilx: interleave index, for use only when MPOL_INTERLEAVE or + * MPOL_WEIGHTED_INTERLEAVE + * @nodemask: (output) pointer to nodemask pointer for 'bind' and 'prefer-many' + * policy + * + * Returns a nid suitable for a page allocation and a pointer. If the effective + * policy is 'bind' or 'prefer-many', returns a pointer to the mempolicy's + * @nodemask for filtering the zonelist. + */ +int policy_node_nodemask(struct mempolicy *mpol, gfp_t gfp_flags, + pgoff_t ilx, nodemask_t **nodemask) +{ + int nid = numa_node_id(); + *nodemask = policy_nodemask(gfp_flags, mpol, ilx, &nid); + return nid; +} + #ifdef CONFIG_HUGETLBFS /* * huge_node(@vma, @addr, @gfp_flags, @mpol) @@ -2102,12 +2123,8 @@ int huge_node(struct vm_area_struct *vma, unsigned long addr, gfp_t gfp_flags, struct mempolicy **mpol, nodemask_t **nodemask) { pgoff_t ilx; - int nid; - - nid = numa_node_id(); *mpol = get_vma_policy(vma, addr, hstate_vma(vma)->order, &ilx); - *nodemask = policy_nodemask(gfp_flags, *mpol, ilx, &nid); - return nid; + return policy_node_nodemask(*mpol, gfp_flags, ilx, nodemask); }

/* @@ -2549,8 +2566,7 @@ unsigned long alloc_pages_bulk_array_mempolicy_noprof(gfp_t gfp, return alloc_pages_bulk_array_preferred_many(gfp, numa_node_id(), pol, nr_pages, page_array);

- nid = numa_node_id(); - nodemask = policy_nodemask(gfp, pol, NO_INTERLEAVE_INDEX, &nid); + nid = policy_node_nodemask(pol, gfp, NO_INTERLEAVE_INDEX, &nodemask); return alloc_pages_bulk_noprof(gfp, nid, nodemask, nr_pages, NULL, page_array); }

Reducing dependence on vma avoids the hugetlb-specific assumption of where the mempolicy is stored. This will open up other ways of using hugetlb.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- mm/hugetlb.c | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 5cf7fb117e9d..2f2bd2444ae2 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2536,32 +2536,31 @@ static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mas }

/* - * Use the VMA's mpolicy to allocate a huge page from the buddy. + * Allocate a huge page from the buddy allocator, given memory policy, node id + * and nodemask. */ -static -struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h, - struct vm_area_struct *vma, unsigned long addr) +static struct folio *alloc_buddy_hugetlb_folio_from_node(struct hstate *h, + struct mempolicy *mpol, + int nid, + nodemask_t *nodemask) { - struct folio *folio = NULL; - struct mempolicy *mpol; gfp_t gfp_mask = htlb_alloc_mask(h); - int nid; - nodemask_t *nodemask; + struct folio *folio = NULL;

- nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask); if (mpol_is_preferred_many(mpol)) { gfp_t gfp = gfp_mask | __GFP_NOWARN;

gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); folio = alloc_surplus_hugetlb_folio(h, gfp, nid, nodemask); + }

- /* Fallback to all nodes if page==NULL */ + if (!folio) { + /* Fallback to all nodes if earlier allocation failed */ nodemask = NULL; - }

- if (!folio) folio = alloc_surplus_hugetlb_folio(h, gfp_mask, nid, nodemask); - mpol_cond_put(mpol); + } + return folio; }

@@ -3187,8 +3186,18 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, spin_lock_irq(&hugetlb_lock); folio = dequeue_hugetlb_folio_vma(h, vma, addr, use_hstate_resv); if (!folio) { + struct mempolicy *mpol; + nodemask_t *nodemask; + pgoff_t ilx; + int nid; + spin_unlock_irq(&hugetlb_lock); - folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr); + + mpol = get_vma_policy(vma, addr, hstate_vma(vma)->order, &ilx); + nid = policy_node_nodemask(mpol, htlb_alloc_mask(h), ilx, &nodemask); + folio = alloc_buddy_hugetlb_folio_from_node(h, mpol, nid, nodemask); + mpol_cond_put(mpol); + if (!folio) goto out_uncharge_cgroup; spin_lock_irq(&hugetlb_lock);

hugetlb_alloc_folio() allocates a hugetlb folio without handling reservations in the vma and subpool, since some of that reservation concepts are hugetlbfs specific.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- include/linux/hugetlb.h | 12 ++++ mm/hugetlb.c | 144 ++++++++++++++++++++++++---------------- 2 files changed, 98 insertions(+), 58 deletions(-)

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index c9bf68c239a0..e4a05a421623 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -690,6 +690,10 @@ struct huge_bootmem_page { };

int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list); +struct folio *hugetlb_alloc_folio(struct hstate *h, struct mempolicy *mpol, + int nid, nodemask_t *nodemask, + bool charge_cgroup_reservation, + bool use_hstate_resv); struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, int avoid_reserve); struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, @@ -1027,6 +1031,14 @@ static inline int isolate_or_dissolve_huge_page(struct page *page, return -ENOMEM; }

+static inline struct folio * +hugetlb_alloc_folio(struct hstate *h, struct mempolicy *mpol, int nid, + nodemask_t *nodemask, bool charge_cgroup_reservation, + bool use_hstate_resv) +{ + return NULL; +} + static inline struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, int avoid_reserve) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index e341bc0eb49a..7e73ebcc0f26 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3106,6 +3106,75 @@ int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list) return ret; }

+/** + * Allocates a hugetlb folio either by dequeueing or from buddy allocator. + */ +struct folio *hugetlb_alloc_folio(struct hstate *h, struct mempolicy *mpol, + int nid, nodemask_t *nodemask, + bool charge_cgroup_reservation, + bool use_hstate_resv) +{ + struct hugetlb_cgroup *h_cg = NULL; + struct folio *folio; + int ret; + int idx; + + idx = hstate_index(h); + + if (charge_cgroup_reservation) { + ret = hugetlb_cgroup_charge_cgroup_rsvd( + idx, pages_per_huge_page(h), &h_cg); + if (ret) + return NULL; + } + + ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); + if (ret) + goto err_uncharge_cgroup_reservation; + + spin_lock_irq(&hugetlb_lock); + + folio = dequeue_hugetlb_folio(h, mpol, nid, nodemask, use_hstate_resv); + if (!folio) { + spin_unlock_irq(&hugetlb_lock); + + folio = alloc_buddy_hugetlb_folio_from_node(h, mpol, nid, nodemask); + if (!folio) + goto err_uncharge_cgroup; + + spin_lock_irq(&hugetlb_lock); + if (use_hstate_resv) { + folio_set_hugetlb_restore_reserve(folio); + h->resv_huge_pages--; + } + list_add(&folio->lru, &h->hugepage_activelist); + folio_ref_unfreeze(folio, 1); + /* Fall through */ + } + + hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, folio); + + if (charge_cgroup_reservation) { + hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h), + h_cg, folio); + } + + spin_unlock_irq(&hugetlb_lock); + + return folio; + +err_uncharge_cgroup: + hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); + +err_uncharge_cgroup_reservation: + if (charge_cgroup_reservation) { + hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h), + h_cg); + } + + return NULL; +} + struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, int avoid_reserve) { @@ -3114,11 +3183,10 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, struct folio *folio; long map_chg, map_commit, nr_pages = pages_per_huge_page(h); long gbl_chg; - int memcg_charge_ret, ret, idx; - struct hugetlb_cgroup *h_cg = NULL; + int memcg_charge_ret; struct mem_cgroup *memcg; - bool deferred_reserve; - gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL; + bool charge_cgroup_reservation; + gfp_t gfp = htlb_alloc_mask(h); bool use_hstate_resv; struct mempolicy *mpol; nodemask_t *nodemask; @@ -3126,13 +3194,14 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, int nid;

memcg = get_mem_cgroup_from_current(); - memcg_charge_ret = mem_cgroup_hugetlb_try_charge(memcg, gfp, nr_pages); + memcg_charge_ret = + mem_cgroup_hugetlb_try_charge(memcg, gfp | __GFP_RETRY_MAYFAIL, + nr_pages); if (memcg_charge_ret == -ENOMEM) { mem_cgroup_put(memcg); return ERR_PTR(-ENOMEM); }

- idx = hstate_index(h); /* * Examine the region/reserve map to determine if the process * has a reservation for the page to be allocated. A return @@ -3160,57 +3229,22 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,

}

- /* If this allocation is not consuming a reservation, charge it now. - */ - deferred_reserve = map_chg || avoid_reserve; - if (deferred_reserve) { - ret = hugetlb_cgroup_charge_cgroup_rsvd( - idx, pages_per_huge_page(h), &h_cg); - if (ret) - goto out_subpool_put; - } - - ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); - if (ret) - goto out_uncharge_cgroup_reservation; - use_hstate_resv = should_use_hstate_resv(vma, gbl_chg, avoid_reserve);

- spin_lock_irq(&hugetlb_lock); + /* + * charge_cgroup_reservation if this allocation is not consuming a + * reservation + */ + charge_cgroup_reservation = map_chg || avoid_reserve;

mpol = get_vma_policy(vma, addr, hstate_vma(vma)->order, &ilx); - nid = policy_node_nodemask(mpol, htlb_alloc_mask(h), ilx, &nodemask); - folio = dequeue_hugetlb_folio(h, mpol, nid, nodemask, use_hstate_resv); - if (!folio) { - spin_unlock_irq(&hugetlb_lock); - - folio = alloc_buddy_hugetlb_folio_from_node(h, mpol, nid, nodemask); - if (!folio) { - mpol_cond_put(mpol); - goto out_uncharge_cgroup; - } - - spin_lock_irq(&hugetlb_lock); - if (use_hstate_resv) { - folio_set_hugetlb_restore_reserve(folio); - h->resv_huge_pages--; - } - list_add(&folio->lru, &h->hugepage_activelist); - folio_ref_unfreeze(folio, 1); - /* Fall through */ - } + nid = policy_node_nodemask(mpol, gfp, ilx, &nodemask); + folio = hugetlb_alloc_folio(h, mpol, nid, nodemask, + charge_cgroup_reservation, use_hstate_resv); mpol_cond_put(mpol);

- hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, folio); - /* If allocation is not consuming a reservation, also store the - * hugetlb_cgroup pointer on the page. - */ - if (deferred_reserve) { - hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h), - h_cg, folio); - } - - spin_unlock_irq(&hugetlb_lock); + if (!folio) + goto out_subpool_put;

hugetlb_set_folio_subpool(folio, spool);

@@ -3229,7 +3263,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,

rsv_adjust = hugepage_subpool_put_pages(spool, 1); hugetlb_acct_memory(h, -rsv_adjust); - if (deferred_reserve) { + if (charge_cgroup_reservation) { spin_lock_irq(&hugetlb_lock); hugetlb_cgroup_uncharge_folio_rsvd(hstate_index(h), pages_per_huge_page(h), folio); @@ -3243,12 +3277,6 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,

return folio;

-out_uncharge_cgroup: - hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); -out_uncharge_cgroup_reservation: - if (deferred_reserve) - hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h), - h_cg); out_subpool_put: if (map_chg || avoid_reserve) hugepage_subpool_put_pages(spool, 1);

This will allow hugetlb subpools to be used by guest_memfd.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- include/linux/hugetlb.h | 3 +++ mm/hugetlb.c | 6 ++---- 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index e4a05a421623..907cfbbd9e24 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -119,6 +119,9 @@ struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, long min_hpages); void hugepage_put_subpool(struct hugepage_subpool *spool);

+long hugepage_subpool_get_pages(struct hugepage_subpool *spool, long delta); +long hugepage_subpool_put_pages(struct hugepage_subpool *spool, long delta); + void hugetlb_dup_vma_private(struct vm_area_struct *vma); void clear_vma_resv_huge_pages(struct vm_area_struct *vma); int move_hugetlb_page_tables(struct vm_area_struct *vma, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 7e73ebcc0f26..808915108126 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -170,8 +170,7 @@ void hugepage_put_subpool(struct hugepage_subpool *spool) * only be different than the passed value (delta) in the case where * a subpool minimum size must be maintained. */ -static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, - long delta) +long hugepage_subpool_get_pages(struct hugepage_subpool *spool, long delta) { long ret = delta;

@@ -215,8 +214,7 @@ static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, * The return value may only be different than the passed value (delta) * in the case where a subpool minimum size must be maintained. */ -static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, - long delta) +long hugepage_subpool_put_pages(struct hugepage_subpool *spool, long delta) { long ret = delta; unsigned long flags;

This will used by guest_memfd in a later patch.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- include/linux/hugetlb.h | 2 ++ mm/hugetlb.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 9ef1adbd3207..4d47bf94c211 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -122,6 +122,8 @@ void hugepage_put_subpool(struct hugepage_subpool *spool); long hugepage_subpool_get_pages(struct hugepage_subpool *spool, long delta); long hugepage_subpool_put_pages(struct hugepage_subpool *spool, long delta);

+int hugetlb_acct_memory(struct hstate *h, long delta); + void hugetlb_dup_vma_private(struct vm_area_struct *vma); void clear_vma_resv_huge_pages(struct vm_area_struct *vma); int move_hugetlb_page_tables(struct vm_area_struct *vma, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index efdb5772b367..5a37b03e1361 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -93,7 +93,7 @@ struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;

/* Forward declaration */ static int __hugetlb_acct_memory(struct hstate *h, long delta, bool use_surplus); -static int hugetlb_acct_memory(struct hstate *h, long delta); +int hugetlb_acct_memory(struct hstate *h, long delta); static void hugetlb_vma_lock_free(struct vm_area_struct *vma); static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma); static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma); @@ -5170,7 +5170,7 @@ static int __hugetlb_acct_memory(struct hstate *h, long delta, bool use_surplus) return ret; }

-static int hugetlb_acct_memory(struct hstate *h, long delta) +int hugetlb_acct_memory(struct hstate *h, long delta) { return __hugetlb_acct_memory(h, delta, true); }

Using guest mem inodes allows us to store metadata for the backing memory on the inode. Metadata will be added in a later patch to support HugeTLB pages.

Metadata about backing memory should not be stored on the file, since the file represents a guest_memfd's binding with a struct kvm, and metadata about backing memory is not unique to a specific binding and struct kvm.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- include/uapi/linux/magic.h | 1 + virt/kvm/guest_memfd.c | 119 ++++++++++++++++++++++++++++++------- 2 files changed, 100 insertions(+), 20 deletions(-)

diff --git a/include/uapi/linux/magic.h b/include/uapi/linux/magic.h index bb575f3ab45e..169dba2a6920 100644 --- a/include/uapi/linux/magic.h +++ b/include/uapi/linux/magic.h @@ -103,5 +103,6 @@ #define DEVMEM_MAGIC 0x454d444d /* "DMEM" */ #define SECRETMEM_MAGIC 0x5345434d /* "SECM" */ #define PID_FS_MAGIC 0x50494446 /* "PIDF" */ +#define GUEST_MEMORY_MAGIC 0x474d454d /* "GMEM" */

#endif /* __LINUX_MAGIC_H__ */ diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 8f079a61a56d..5d7fd1f708a6 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -1,12 +1,17 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/fs.h> +#include <linux/mount.h> #include <linux/backing-dev.h> #include <linux/falloc.h> #include <linux/kvm_host.h> +#include <linux/pseudo_fs.h> #include <linux/pagemap.h> #include <linux/anon_inodes.h>

#include "kvm_mm.h"

+static struct vfsmount *kvm_gmem_mnt; + struct kvm_gmem { struct kvm *kvm; struct xarray bindings; @@ -302,6 +307,38 @@ static inline struct file *kvm_gmem_get_file(struct kvm_memory_slot *slot) return get_file_active(&slot->gmem.file); }

+static const struct super_operations kvm_gmem_super_operations = { + .statfs = simple_statfs, +}; + +static int kvm_gmem_init_fs_context(struct fs_context *fc) +{ + struct pseudo_fs_context *ctx; + + if (!init_pseudo(fc, GUEST_MEMORY_MAGIC)) + return -ENOMEM; + + ctx = fc->fs_private; + ctx->ops = &kvm_gmem_super_operations; + + return 0; +} + +static struct file_system_type kvm_gmem_fs = { + .name = "kvm_guest_memory", + .init_fs_context = kvm_gmem_init_fs_context, + .kill_sb = kill_anon_super, +}; + +static void kvm_gmem_init_mount(void) +{ + kvm_gmem_mnt = kern_mount(&kvm_gmem_fs); + BUG_ON(IS_ERR(kvm_gmem_mnt)); + + /* For giggles. Userspace can never map this anyways. */ + kvm_gmem_mnt->mnt_flags |= MNT_NOEXEC; +} + static struct file_operations kvm_gmem_fops = { .open = generic_file_open, .release = kvm_gmem_release, @@ -311,6 +348,8 @@ static struct file_operations kvm_gmem_fops = { void kvm_gmem_init(struct module *module) { kvm_gmem_fops.owner = module; + + kvm_gmem_init_mount(); }

static int kvm_gmem_migrate_folio(struct address_space *mapping, @@ -392,11 +431,67 @@ static const struct inode_operations kvm_gmem_iops = { .setattr = kvm_gmem_setattr, };

+static struct inode *kvm_gmem_inode_make_secure_inode(const char *name, + loff_t size, u64 flags) +{ + const struct qstr qname = QSTR_INIT(name, strlen(name)); + struct inode *inode; + int err; + + inode = alloc_anon_inode(kvm_gmem_mnt->mnt_sb); + if (IS_ERR(inode)) + return inode; + + err = security_inode_init_security_anon(inode, &qname, NULL); + if (err) { + iput(inode); + return ERR_PTR(err); + } + + inode->i_private = (void *)(unsigned long)flags; + inode->i_op = &kvm_gmem_iops; + inode->i_mapping->a_ops = &kvm_gmem_aops; + inode->i_mode |= S_IFREG; + inode->i_size = size; + mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); + mapping_set_inaccessible(inode->i_mapping); + /* Unmovable mappings are supposed to be marked unevictable as well. */ + WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping)); + + return inode; +} + +static struct file *kvm_gmem_inode_create_getfile(void *priv, loff_t size, + u64 flags) +{ + static const char *name = "[kvm-gmem]"; + struct inode *inode; + struct file *file; + + if (kvm_gmem_fops.owner && !try_module_get(kvm_gmem_fops.owner)) + return ERR_PTR(-ENOENT); + + inode = kvm_gmem_inode_make_secure_inode(name, size, flags); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + file = alloc_file_pseudo(inode, kvm_gmem_mnt, name, O_RDWR, + &kvm_gmem_fops); + if (IS_ERR(file)) { + iput(inode); + return file; + } + + file->f_mapping = inode->i_mapping; + file->f_flags |= O_LARGEFILE; + file->private_data = priv; + + return file; +} + static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) { - const char *anon_name = "[kvm-gmem]"; struct kvm_gmem *gmem; - struct inode *inode; struct file *file; int fd, err;

@@ -410,32 +505,16 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) goto err_fd; }

- file = anon_inode_create_getfile(anon_name, &kvm_gmem_fops, gmem, - O_RDWR, NULL); + file = kvm_gmem_inode_create_getfile(gmem, size, flags); if (IS_ERR(file)) { err = PTR_ERR(file); goto err_gmem; }

- file->f_flags |= O_LARGEFILE; - - inode = file->f_inode; - WARN_ON(file->f_mapping != inode->i_mapping); - - inode->i_private = (void *)(unsigned long)flags; - inode->i_op = &kvm_gmem_iops; - inode->i_mapping->a_ops = &kvm_gmem_aops; - inode->i_mode |= S_IFREG; - inode->i_size = size; - mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); - mapping_set_inaccessible(inode->i_mapping); - /* Unmovable mappings are supposed to be marked unevictable as well. */ - WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping)); - kvm_get_kvm(kvm); gmem->kvm = kvm; xa_init(&gmem->bindings); - list_add(&gmem->entry, &inode->i_mapping->i_private_list); + list_add(&gmem->entry, &file_inode(file)->i_mapping->i_private_list);

fd_install(fd, file); return fd;

If HugeTLB is requested at guest_memfd creation time, HugeTLB pages will be used to back guest_memfd.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- virt/kvm/guest_memfd.c | 252 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 239 insertions(+), 13 deletions(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 31e1115273e1..2e6f12e2bac8 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -8,6 +8,8 @@ #include <linux/pseudo_fs.h> #include <linux/pagemap.h> #include <linux/anon_inodes.h> +#include <linux/memcontrol.h> +#include <linux/mempolicy.h>

#include "kvm_mm.h"

@@ -29,6 +31,13 @@ static struct kvm_gmem_hugetlb *kvm_gmem_hgmem(struct inode *inode) return inode->i_mapping->i_private_data; }

+static bool is_kvm_gmem_hugetlb(struct inode *inode) +{ + u64 flags = (u64)inode->i_private; + + return flags & KVM_GUEST_MEMFD_HUGETLB; +} + /** * folio_file_pfn - like folio_file_page, but return a pfn. * @folio: The folio which contains this index. @@ -58,6 +67,9 @@ static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slo return 0; }

+/** + * Use the uptodate flag to indicate that the folio is prepared for KVM's usage. + */ static inline void kvm_gmem_mark_prepared(struct folio *folio) { folio_mark_uptodate(folio); @@ -72,13 +84,18 @@ static inline void kvm_gmem_mark_prepared(struct folio *folio) static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, struct folio *folio) { - unsigned long nr_pages, i; pgoff_t index; int r;

- nr_pages = folio_nr_pages(folio); - for (i = 0; i < nr_pages; i++) - clear_highpage(folio_page(folio, i)); + if (folio_test_hugetlb(folio)) { + folio_zero_user(folio, folio->index << PAGE_SHIFT); + } else { + unsigned long nr_pages, i; + + nr_pages = folio_nr_pages(folio); + for (i = 0; i < nr_pages; i++) + clear_highpage(folio_page(folio, i)); + }

/* * Preparing huge folios should always be safe, since it should @@ -103,6 +120,174 @@ static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, return r; }

+static int kvm_gmem_get_mpol_node_nodemask(gfp_t gfp_mask, + struct mempolicy **mpol, + nodemask_t **nodemask) +{ + /* + * TODO: mempolicy would probably have to be stored on the inode, use + * task policy for now. + */ + *mpol = get_task_policy(current); + + /* TODO: ignore interleaving (set ilx to 0) for now. */ + return policy_node_nodemask(*mpol, gfp_mask, 0, nodemask); +} + +static struct folio *kvm_gmem_hugetlb_alloc_folio(struct hstate *h, + struct hugepage_subpool *spool) +{ + bool memcg_charge_was_prepared; + struct mem_cgroup *memcg; + struct mempolicy *mpol; + nodemask_t *nodemask; + struct folio *folio; + gfp_t gfp_mask; + int ret; + int nid; + + gfp_mask = htlb_alloc_mask(h); + + memcg = get_mem_cgroup_from_current(); + ret = mem_cgroup_hugetlb_try_charge(memcg, + gfp_mask | __GFP_RETRY_MAYFAIL, + pages_per_huge_page(h)); + if (ret == -ENOMEM) + goto err; + + memcg_charge_was_prepared = ret != -EOPNOTSUPP; + + /* Pages are only to be taken from guest_memfd subpool and nowhere else. */ + if (hugepage_subpool_get_pages(spool, 1)) + goto err_cancel_charge; + + nid = kvm_gmem_get_mpol_node_nodemask(htlb_alloc_mask(h), &mpol, + &nodemask); + /* + * charge_cgroup_reservation is false because we didn't make any cgroup + * reservations when creating the guest_memfd subpool. + * + * use_hstate_resv is true because we reserved from global hstate when + * creating the guest_memfd subpool. + */ + folio = hugetlb_alloc_folio(h, mpol, nid, nodemask, false, true); + mpol_cond_put(mpol); + + if (!folio) + goto err_put_pages; + + hugetlb_set_folio_subpool(folio, spool); + + if (memcg_charge_was_prepared) + mem_cgroup_commit_charge(folio, memcg); + +out: + mem_cgroup_put(memcg); + + return folio; + +err_put_pages: + hugepage_subpool_put_pages(spool, 1); + +err_cancel_charge: + if (memcg_charge_was_prepared) + mem_cgroup_cancel_charge(memcg, pages_per_huge_page(h)); + +err: + folio = ERR_PTR(-ENOMEM); + goto out; +} + +static int kvm_gmem_hugetlb_filemap_add_folio(struct address_space *mapping, + struct folio *folio, pgoff_t index, + gfp_t gfp) +{ + int ret; + + __folio_set_locked(folio); + ret = __filemap_add_folio(mapping, folio, index, gfp, NULL); + if (unlikely(ret)) { + __folio_clear_locked(folio); + return ret; + } + + /* + * In hugetlb_add_to_page_cache(), there is a call to + * folio_clear_hugetlb_restore_reserve(). This is handled when the pages + * are removed from the page cache in unmap_hugepage_range() -> + * __unmap_hugepage_range() by conditionally calling + * folio_set_hugetlb_restore_reserve(). In kvm_gmem_hugetlb's usage of + * hugetlb, there are no VMAs involved, and pages are never taken from + * the surplus, so when pages are freed, the hstate reserve must be + * restored. Hence, this function makes no call to + * folio_clear_hugetlb_restore_reserve(). + */ + + /* mark folio dirty so that it will not be removed from cache/inode */ + folio_mark_dirty(folio); + + return 0; +} + +static struct folio *kvm_gmem_hugetlb_alloc_and_cache_folio(struct inode *inode, + pgoff_t index) +{ + struct kvm_gmem_hugetlb *hgmem; + struct folio *folio; + int ret; + + hgmem = kvm_gmem_hgmem(inode); + folio = kvm_gmem_hugetlb_alloc_folio(hgmem->h, hgmem->spool); + if (IS_ERR(folio)) + return folio; + + /* TODO: Fix index here to be aligned to huge page size. */ + ret = kvm_gmem_hugetlb_filemap_add_folio( + inode->i_mapping, folio, index, htlb_alloc_mask(hgmem->h)); + if (ret) { + folio_put(folio); + return ERR_PTR(ret); + } + + spin_lock(&inode->i_lock); + inode->i_blocks += blocks_per_huge_page(hgmem->h); + spin_unlock(&inode->i_lock); + + return folio; +} + +static struct folio *kvm_gmem_get_hugetlb_folio(struct inode *inode, + pgoff_t index) +{ + struct address_space *mapping; + struct folio *folio; + struct hstate *h; + pgoff_t hindex; + u32 hash; + + h = kvm_gmem_hgmem(inode)->h; + hindex = index >> huge_page_order(h); + mapping = inode->i_mapping; + + /* To lock, we calculate the hash using the hindex and not index. */ + hash = hugetlb_fault_mutex_hash(mapping, hindex); + mutex_lock(&hugetlb_fault_mutex_table[hash]); + + /* + * The filemap is indexed with index and not hindex. Taking lock on + * folio to align with kvm_gmem_get_regular_folio() + */ + folio = filemap_lock_folio(mapping, index); + if (!IS_ERR(folio)) + goto out; + + folio = kvm_gmem_hugetlb_alloc_and_cache_folio(inode, index); +out: + mutex_unlock(&hugetlb_fault_mutex_table[hash]); + + return folio; +} + /* * Returns a locked folio on success. The caller is responsible for * setting the up-to-date flag before the memory is mapped into the guest. @@ -114,8 +299,10 @@ static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, */ static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index) { - /* TODO: Support huge pages. */ - return filemap_grab_folio(inode->i_mapping, index); + if (is_kvm_gmem_hugetlb(inode)) + return kvm_gmem_get_hugetlb_folio(inode, index); + else + return filemap_grab_folio(inode->i_mapping, index); }

static void kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start, @@ -240,6 +427,35 @@ static void kvm_gmem_hugetlb_truncate_folios_range(struct inode *inode, spin_unlock(&inode->i_lock); }

+static void kvm_gmem_hugetlb_truncate_range(struct inode *inode, loff_t lstart, + loff_t lend) +{ + loff_t full_hpage_start; + loff_t full_hpage_end; + unsigned long hsize; + struct hstate *h; + + h = kvm_gmem_hgmem(inode)->h; + hsize = huge_page_size(h); + + full_hpage_start = round_up(lstart, hsize); + full_hpage_end = round_down(lend, hsize); + + if (lstart < full_hpage_start) { + hugetlb_zero_partial_page(h, inode->i_mapping, lstart, + full_hpage_start); + } + + if (full_hpage_end > full_hpage_start) { + kvm_gmem_hugetlb_truncate_folios_range(inode, full_hpage_start, + full_hpage_end); + } + + if (lend > full_hpage_end) { + hugetlb_zero_partial_page(h, inode->i_mapping, full_hpage_end, + lend); + } +}

static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) { @@ -257,7 +473,12 @@ static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) list_for_each_entry(gmem, gmem_list, entry) kvm_gmem_invalidate_begin(gmem, start, end);

- truncate_inode_pages_range(inode->i_mapping, offset, offset + len - 1); + if (is_kvm_gmem_hugetlb(inode)) { + kvm_gmem_hugetlb_truncate_range(inode, offset, offset + len); + } else { + truncate_inode_pages_range(inode->i_mapping, offset, + offset + len - 1); + }

list_for_each_entry(gmem, gmem_list, entry) kvm_gmem_invalidate_end(gmem, start, end); @@ -279,8 +500,15 @@ static long kvm_gmem_allocate(struct inode *inode, loff_t offset, loff_t len)

filemap_invalidate_lock_shared(mapping);

- start = offset >> PAGE_SHIFT; - end = (offset + len) >> PAGE_SHIFT; + if (is_kvm_gmem_hugetlb(inode)) { + unsigned long hsize = huge_page_size(kvm_gmem_hgmem(inode)->h); + + start = round_down(offset, hsize) >> PAGE_SHIFT; + end = round_down(offset + len, hsize) >> PAGE_SHIFT; + } else { + start = offset >> PAGE_SHIFT; + end = (offset + len) >> PAGE_SHIFT; + }

r = 0; for (index = start; index < end; ) { @@ -408,9 +636,7 @@ static void kvm_gmem_hugetlb_teardown(struct inode *inode)

static void kvm_gmem_evict_inode(struct inode *inode) { - u64 flags = (u64)inode->i_private; - - if (flags & KVM_GUEST_MEMFD_HUGETLB) + if (is_kvm_gmem_hugetlb(inode)) kvm_gmem_hugetlb_teardown(inode); else truncate_inode_pages_final(inode->i_mapping); @@ -827,7 +1053,7 @@ __kvm_gmem_get_pfn(struct file *file, struct kvm_memory_slot *slot,

*pfn = folio_file_pfn(folio, index); if (max_order) - *max_order = 0; + *max_order = folio_order(folio);

*is_prepared = folio_test_uptodate(folio); return folio;

When a hugetlb guest_memfd is requested, the requested size should be aligned to the size of the hugetlb page requested.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- virt/kvm/guest_memfd.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 2e6f12e2bac8..eacbfdb950d1 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -909,6 +909,13 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) return err; }

+static inline bool kvm_gmem_hugetlb_page_aligned(u32 flags, u64 value) +{ + int page_size_log = (flags >> KVM_GUEST_MEMFD_HUGE_SHIFT) & KVM_GUEST_MEMFD_HUGE_MASK; + u64 page_size = 1ULL << page_size_log; + return IS_ALIGNED(value, page_size); +} + #define KVM_GUEST_MEMFD_ALL_FLAGS KVM_GUEST_MEMFD_HUGETLB

int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args) @@ -921,12 +928,18 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args) if (flags & ~(KVM_GUEST_MEMFD_ALL_FLAGS | (KVM_GUEST_MEMFD_HUGE_MASK << KVM_GUEST_MEMFD_HUGE_SHIFT))) return -EINVAL; + + if (!kvm_gmem_hugetlb_page_aligned(flags, size)) + return -EINVAL; } else { if (flags & ~KVM_GUEST_MEMFD_ALL_FLAGS) return -EINVAL; + + if (!PAGE_ALIGNED(size)) + return -EINVAL; }

- if (size <= 0 || !PAGE_ALIGNED(size)) + if (size <= 0) return -EINVAL;

return __kvm_gmem_create(kvm, size, flags);

Adds support for various type of backing sources for private memory (in the sense of confidential computing), similar to the backing sources available for shared memory.

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- .../testing/selftests/kvm/include/test_util.h | 16 ++++ tools/testing/selftests/kvm/lib/test_util.c | 74 +++++++++++++++++++ 2 files changed, 90 insertions(+)

diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index 3e473058849f..011e757d4e2c 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -142,6 +142,16 @@ struct vm_mem_backing_src_alias { uint32_t flag; };

+enum vm_private_mem_backing_src_type { + VM_PRIVATE_MEM_SRC_GUEST_MEM, /* Use default page size */ + VM_PRIVATE_MEM_SRC_HUGETLB, /* Use kernel default page size for hugetlb pages */ + VM_PRIVATE_MEM_SRC_HUGETLB_2MB, + VM_PRIVATE_MEM_SRC_HUGETLB_1GB, + NUM_PRIVATE_MEM_SRC_TYPES, +}; + +#define DEFAULT_VM_PRIVATE_MEM_SRC VM_PRIVATE_MEM_SRC_GUEST_MEM + #define MIN_RUN_DELAY_NS 200000UL

bool thp_configured(void); @@ -152,6 +162,12 @@ size_t get_backing_src_pagesz(uint32_t i); bool is_backing_src_hugetlb(uint32_t i); void backing_src_help(const char *flag); enum vm_mem_backing_src_type parse_backing_src_type(const char *type_name); + +void private_mem_backing_src_help(const char *flag); +enum vm_private_mem_backing_src_type parse_private_mem_backing_src_type(const char *type_name); +const struct vm_mem_backing_src_alias *vm_private_mem_backing_src_alias(uint32_t i); +size_t get_private_mem_backing_src_pagesz(uint32_t i); + long get_run_delay(void);

/* diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c index 8ed0b74ae837..d0a9b5ee0c01 100644 --- a/tools/testing/selftests/kvm/lib/test_util.c +++ b/tools/testing/selftests/kvm/lib/test_util.c @@ -15,6 +15,7 @@ #include <sys/syscall.h> #include <linux/mman.h> #include "linux/kernel.h" +#include <linux/kvm.h>

#include "test_util.h"

@@ -288,6 +289,34 @@ const struct vm_mem_backing_src_alias *vm_mem_backing_src_alias(uint32_t i) return &aliases[i]; }

+const struct vm_mem_backing_src_alias *vm_private_mem_backing_src_alias(uint32_t i) +{ + static const struct vm_mem_backing_src_alias aliases[] = { + [VM_PRIVATE_MEM_SRC_GUEST_MEM] = { + .name = "private_mem_guest_mem", + .flag = 0, + }, + [VM_PRIVATE_MEM_SRC_HUGETLB] = { + .name = "private_mem_hugetlb", + .flag = KVM_GUEST_MEMFD_HUGETLB, + }, + [VM_PRIVATE_MEM_SRC_HUGETLB_2MB] = { + .name = "private_mem_hugetlb_2mb", + .flag = KVM_GUEST_MEMFD_HUGETLB | KVM_GUEST_MEMFD_HUGE_2MB, + }, + [VM_PRIVATE_MEM_SRC_HUGETLB_1GB] = { + .name = "private_mem_hugetlb_1gb", + .flag = KVM_GUEST_MEMFD_HUGETLB | KVM_GUEST_MEMFD_HUGE_1GB, + }, + }; + _Static_assert(ARRAY_SIZE(aliases) == NUM_PRIVATE_MEM_SRC_TYPES, + "Missing new backing private mem src types?"); + + TEST_ASSERT(i < NUM_PRIVATE_MEM_SRC_TYPES, "Private mem backing src type ID %d too big", i); + + return &aliases[i]; +} + #define MAP_HUGE_PAGE_SIZE(x) (1ULL << ((x >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK))

size_t get_backing_src_pagesz(uint32_t i) @@ -308,6 +337,20 @@ size_t get_backing_src_pagesz(uint32_t i) } }

+size_t get_private_mem_backing_src_pagesz(uint32_t i) +{ + uint32_t flag = vm_private_mem_backing_src_alias(i)->flag; + + switch (i) { + case VM_PRIVATE_MEM_SRC_GUEST_MEM: + return getpagesize(); + case VM_PRIVATE_MEM_SRC_HUGETLB: + return get_def_hugetlb_pagesz(); + default: + return MAP_HUGE_PAGE_SIZE(flag); + } +} + bool is_backing_src_hugetlb(uint32_t i) { return !!(vm_mem_backing_src_alias(i)->flag & MAP_HUGETLB); @@ -344,6 +387,37 @@ enum vm_mem_backing_src_type parse_backing_src_type(const char *type_name) return -1; }

+static void print_available_private_mem_backing_src_types(const char *prefix) +{ + int i; + + printf("%sAvailable private mem backing src types:\n", prefix); + + for (i = 0; i < NUM_PRIVATE_MEM_SRC_TYPES; i++) + printf("%s %s\n", prefix, vm_private_mem_backing_src_alias(i)->name); +} + +void private_mem_backing_src_help(const char *flag) +{ + printf(" %s: specify the type of memory that should be used to\n" + " back guest private memory. (default: %s)\n", + flag, vm_private_mem_backing_src_alias(DEFAULT_VM_PRIVATE_MEM_SRC)->name); + print_available_private_mem_backing_src_types(" "); +} + +enum vm_private_mem_backing_src_type parse_private_mem_backing_src_type(const char *type_name) +{ + int i; + + for (i = 0; i < NUM_PRIVATE_MEM_SRC_TYPES; i++) + if (!strcmp(type_name, vm_private_mem_backing_src_alias(i)->name)) + return i; + + print_available_private_mem_backing_src_types(""); + TEST_FAIL("Unknown private mem backing src type: %s", type_name); + return -1; +} + long get_run_delay(void) { char path[64];

Add private_mem_conversions_test.sh to automate testing of different combinations of private_mem_conversions_test.

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- .../x86_64/private_mem_conversions_test.sh | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100755 tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.sh

diff --git a/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.sh b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.sh new file mode 100755 index 000000000000..fb6705fef466 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-only */ +# +# Wrapper script which runs different test setups of +# private_mem_conversions_test. +# +# tools/testing/selftests/kvm/private_mem_conversions_test.sh +# Copyright (C) 2023, Google LLC. + +set -e + +num_vcpus_to_test=4 +num_memslots_to_test=$num_vcpus_to_test + +get_default_hugepage_size_in_kB() { + grep "Hugepagesize:" /proc/meminfo | grep -o '[[:digit:]]+' +} + +# Required pages are based on the test setup (see computation for memfd_size) in +# test_mem_conversions() in private_mem_migrate_tests.c) + +# These static requirements are set to the maximum required for +# num_vcpus_to_test, over all the hugetlb-related tests +required_num_2m_hugepages=$(( 1024 * num_vcpus_to_test )) +required_num_1g_hugepages=$(( 2 * num_vcpus_to_test )) + +# The other hugetlb sizes are not supported on x86_64 +[ "$(cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages 2>/dev/null || echo 0)" -ge "$required_num_2m_hugepages" ] && hugepage_2mb_enabled=1 +[ "$(cat /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages 2>/dev/null || echo 0)" -ge "$required_num_1g_hugepages" ] && hugepage_1gb_enabled=1 + +case $(get_default_hugepage_size_in_kB) in + 2048) + hugepage_default_enabled=$hugepage_2mb_enabled + ;; + 1048576) + hugepage_default_enabled=$hugepage_1gb_enabled + ;; + *) + hugepage_default_enabled=0 + ;; +esac + +backing_src_types=( anonymous ) +backing_src_types+=( anonymous_thp ) +[ -n "$hugepage_default_enabled" ] && \ + backing_src_types+=( anonymous_hugetlb ) || echo "skipping anonymous_hugetlb backing source type" +[ -n "$hugepage_2mb_enabled" ] && \ + backing_src_types+=( anonymous_hugetlb_2mb ) || echo "skipping anonymous_hugetlb_2mb backing source type" +[ -n "$hugepage_1gb_enabled" ] && \ + backing_src_types+=( anonymous_hugetlb_1gb ) || echo "skipping anonymous_hugetlb_1gb backing source type" +backing_src_types+=( shmem ) +[ -n "$hugepage_default_enabled" ] && \ + backing_src_types+=( shared_hugetlb ) || echo "skipping shared_hugetlb backing source type" + +private_mem_backing_src_types=( private_mem_guest_mem ) +[ -n "$hugepage_default_enabled" ] && \ + private_mem_backing_src_types+=( private_mem_hugetlb ) || echo "skipping private_mem_hugetlb backing source type" +[ -n "$hugepage_2mb_enabled" ] && \ + private_mem_backing_src_types+=( private_mem_hugetlb_2mb ) || echo "skipping private_mem_hugetlb_2mb backing source type" +[ -n "$hugepage_1gb_enabled" ] && \ + private_mem_backing_src_types+=( private_mem_hugetlb_1gb ) || echo "skipping private_mem_hugetlb_1gb backing source type" + +set +e + +TEST_EXECUTABLE="$(dirname "$0")/private_mem_conversions_test" + +( + set -e + + for src_type in "${backing_src_types[@]}"; do + + for private_mem_src_type in "${private_mem_backing_src_types[@]}"; do + set -x + + $TEST_EXECUTABLE -s "$src_type" -p "$private_mem_src_type" -n $num_vcpus_to_test + $TEST_EXECUTABLE -s "$src_type" -p "$private_mem_src_type" -n $num_vcpus_to_test -m $num_memslots_to_test + + { set +x; } 2>/dev/null + + echo + + done + + done +) +RET=$? + +exit $RET

These functions will need to be used by guest_memfd when splitting/reconstructing HugeTLB pages.

Co-developed-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com Co-developed-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Vishal Annapurve vannapurve@google.com --- include/linux/hugetlb.h | 14 ++++++++++++++ mm/hugetlb_vmemmap.h | 11 ----------- 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 752062044b0b..7ba4ed9e0001 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -284,6 +284,20 @@ bool is_hugetlb_entry_migration(pte_t pte); bool is_hugetlb_entry_hwpoisoned(pte_t pte); void hugetlb_unshare_all_pmds(struct vm_area_struct *vma);

+#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio); +void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio); +#else +static inline int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio) +{ + return 0; +} + +static inline void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio) +{ +} +#endif + #else /* !CONFIG_HUGETLB_PAGE */

static inline void hugetlb_dup_vma_private(struct vm_area_struct *vma) diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h index 2fcae92d3359..e702ace3b42f 100644 --- a/mm/hugetlb_vmemmap.h +++ b/mm/hugetlb_vmemmap.h @@ -18,11 +18,9 @@ #define HUGETLB_VMEMMAP_RESERVE_PAGES (HUGETLB_VMEMMAP_RESERVE_SIZE / sizeof(struct page))

#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP -int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio); long hugetlb_vmemmap_restore_folios(const struct hstate *h, struct list_head *folio_list, struct list_head *non_hvo_folios); -void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio); void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list);

static inline unsigned int hugetlb_vmemmap_size(const struct hstate *h) @@ -43,11 +41,6 @@ static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate return size > 0 ? size : 0; } #else -static inline int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio) -{ - return 0; -} - static long hugetlb_vmemmap_restore_folios(const struct hstate *h, struct list_head *folio_list, struct list_head *non_hvo_folios) @@ -56,10 +49,6 @@ static long hugetlb_vmemmap_restore_folios(const struct hstate *h, return 0; }

-static inline void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio) -{ -} - static inline void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list) { }

These functions are introduced in hugetlb.c so the private hugetlb_lock can be accessed.

hugetlb_lock is reused for this PoC, but a separate lock should be used in a future revision to avoid interference due to hash collisions with HugeTLB's usage of this lock.

Co-developed-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com Co-developed-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Vishal Annapurve vannapurve@google.com

--- include/linux/hugetlb.h | 3 +++ mm/hugetlb.c | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+)

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index ac9d4ada52bd..0f3f920ad608 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -164,6 +164,9 @@ bool hugetlb_reserve_pages(struct inode *inode, long from, long to, vm_flags_t vm_flags); long hugetlb_unreserve_pages(struct inode *inode, long start, long end, long freed); +void hugetlb_folio_list_add(struct folio *folio, struct list_head *list); +void hugetlb_folio_list_move(struct folio *folio, struct list_head *list); +void hugetlb_folio_list_del(struct folio *folio); bool isolate_hugetlb(struct folio *folio, struct list_head *list); int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison); int get_huge_page_for_hwpoison(unsigned long pfn, int flags, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 8f2b7b411b60..60e72214d5bf 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -7264,6 +7264,27 @@ long hugetlb_unreserve_pages(struct inode *inode, long start, long end, return 0; }

+void hugetlb_folio_list_add(struct folio *folio, struct list_head *list) +{ + spin_lock_irq(&hugetlb_lock); + list_add(&folio->lru, list); + spin_unlock_irq(&hugetlb_lock); +} + +void hugetlb_folio_list_move(struct folio *folio, struct list_head *list) +{ + spin_lock_irq(&hugetlb_lock); + list_move_tail(&folio->lru, list); + spin_unlock_irq(&hugetlb_lock); +} + +void hugetlb_folio_list_del(struct folio *folio) +{ + spin_lock_irq(&hugetlb_lock); + list_del(&folio->lru); + spin_unlock_irq(&hugetlb_lock); +} + #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE static unsigned long page_table_shareable(struct vm_area_struct *svma, struct vm_area_struct *vma,

The faultability xarray is stored on the inode since faultability is a property of the guest_memfd's memory contents.

In this RFC, presence of an entry in the xarray indicates faultable, but this could be flipped so that presence indicates unfaultable. For flexibility, a special value "FAULT" is used instead of a simple boolean.

However, at some stages of a VM's lifecycle there could be more private pages, and at other stages there could be more shared pages.

This is likely to be replaced by a better data structure in a future revision to better support ranges.

Also store struct kvm_gmem_hugetlb in struct kvm_gmem_hugetlb as a pointer. inode->i_mapping->i_private_data.

Co-developed-by: Fuad Tabba tabba@google.com Signed-off-by: Fuad Tabba tabba@google.com Co-developed-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com Co-developed-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Vishal Annapurve vannapurve@google.com

--- virt/kvm/guest_memfd.c | 105 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 94 insertions(+), 11 deletions(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 8151df2c03e5..b603518f7b62 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -26,11 +26,21 @@ struct kvm_gmem_hugetlb { struct hugepage_subpool *spool; };

-static struct kvm_gmem_hugetlb *kvm_gmem_hgmem(struct inode *inode) +struct kvm_gmem_inode_private { + struct xarray faultability; + struct kvm_gmem_hugetlb *hgmem; +}; + +static struct kvm_gmem_inode_private *kvm_gmem_private(struct inode *inode) { return inode->i_mapping->i_private_data; }

+static struct kvm_gmem_hugetlb *kvm_gmem_hgmem(struct inode *inode) +{ + return kvm_gmem_private(inode)->hgmem; +} + static bool is_kvm_gmem_hugetlb(struct inode *inode) { u64 flags = (u64)inode->i_private; @@ -38,6 +48,57 @@ static bool is_kvm_gmem_hugetlb(struct inode *inode) return flags & KVM_GUEST_MEMFD_HUGETLB; }

+#define KVM_GMEM_FAULTABILITY_VALUE 0x4641554c54 /* FAULT */ + +/** + * Set faultability of given range of inode indices [@start, @end) to + * @faultable. Return 0 if attributes were successfully updated or negative + * errno on error. + */ +static int kvm_gmem_set_faultable(struct inode *inode, pgoff_t start, pgoff_t end, + bool faultable) +{ + struct xarray *faultability; + void *val; + pgoff_t i; + + /* + * The expectation is that fewer pages are faultable, hence save memory + * entries are created for faultable pages as opposed to creating + * entries for non-faultable pages. + */ + val = faultable ? xa_mk_value(KVM_GMEM_FAULTABILITY_VALUE) : NULL; + faultability = &kvm_gmem_private(inode)->faultability; + + /* + * TODO replace this with something else (maybe interval + * tree?). store_range doesn't quite do what we expect if overlapping + * ranges are specified: if we store_range(5, 10, val) and then + * store_range(7, 12, NULL), the entire range [5, 12] will be NULL. For + * now, use the slower xa_store() to store individual entries on indices + * to avoid this. + */ + for (i = start; i < end; i++) { + int r; + + r = xa_err(xa_store(faultability, i, val, GFP_KERNEL_ACCOUNT)); + if (r) + return r; + } + + return 0; +} + +/** + * Return true if the page at @index is allowed to be faulted in. + */ +static bool kvm_gmem_is_faultable(struct inode *inode, pgoff_t index) +{ + struct xarray *faultability = &kvm_gmem_private(inode)->faultability; + + return xa_to_value(xa_load(faultability, index)) == KVM_GMEM_FAULTABILITY_VALUE; +} + /** * folio_file_pfn - like folio_file_page, but return a pfn. * @folio: The folio which contains this index. @@ -895,11 +956,21 @@ static void kvm_gmem_hugetlb_teardown(struct inode *inode)

static void kvm_gmem_evict_inode(struct inode *inode) { + struct kvm_gmem_inode_private *private = kvm_gmem_private(inode); + + /* + * .evict_inode can be called before faultability is set up if there are + * issues during inode creation. + */ + if (private) + xa_destroy(&private->faultability); + if (is_kvm_gmem_hugetlb(inode)) kvm_gmem_hugetlb_teardown(inode); else truncate_inode_pages_final(inode->i_mapping);

+ kfree(private); clear_inode(inode); }

@@ -1028,7 +1099,9 @@ static const struct inode_operations kvm_gmem_iops = { .setattr = kvm_gmem_setattr, };

-static int kvm_gmem_hugetlb_setup(struct inode *inode, loff_t size, u64 flags) +static int kvm_gmem_hugetlb_setup(struct inode *inode, + struct kvm_gmem_inode_private *private, + loff_t size, u64 flags) { struct kvm_gmem_hugetlb *hgmem; struct hugepage_subpool *spool; @@ -1036,6 +1109,10 @@ static int kvm_gmem_hugetlb_setup(struct inode *inode, loff_t size, u64 flags) struct hstate *h; long hpages;

+ hgmem = kzalloc(sizeof(*hgmem), GFP_KERNEL); + if (!hgmem) + return -ENOMEM; + page_size_log = (flags >> KVM_GUEST_MEMFD_HUGE_SHIFT) & KVM_GUEST_MEMFD_HUGE_MASK; h = hstate_sizelog(page_size_log);

@@ -1046,21 +1123,16 @@ static int kvm_gmem_hugetlb_setup(struct inode *inode, loff_t size, u64 flags) if (!spool) goto err;

- hgmem = kzalloc(sizeof(*hgmem), GFP_KERNEL); - if (!hgmem) - goto err_subpool; - inode->i_blkbits = huge_page_shift(h);

hgmem->h = h; hgmem->spool = spool; - inode->i_mapping->i_private_data = hgmem;

+ private->hgmem = hgmem; return 0;

-err_subpool: - kfree(spool); err: + kfree(hgmem); return -ENOMEM; }

@@ -1068,6 +1140,7 @@ static struct inode *kvm_gmem_inode_make_secure_inode(const char *name, loff_t size, u64 flags) { const struct qstr qname = QSTR_INIT(name, strlen(name)); + struct kvm_gmem_inode_private *private; struct inode *inode; int err;

@@ -1079,12 +1152,20 @@ static struct inode *kvm_gmem_inode_make_secure_inode(const char *name, if (err) goto out;

+ err = -ENOMEM; + private = kzalloc(sizeof(*private), GFP_KERNEL); + if (!private) + goto out; + if (flags & KVM_GUEST_MEMFD_HUGETLB) { - err = kvm_gmem_hugetlb_setup(inode, size, flags); + err = kvm_gmem_hugetlb_setup(inode, private, size, flags); if (err) - goto out; + goto free_private; }

+ xa_init(&private->faultability); + inode->i_mapping->i_private_data = private; + inode->i_private = (void *)(unsigned long)flags; inode->i_op = &kvm_gmem_iops; inode->i_mapping->a_ops = &kvm_gmem_aops; @@ -1097,6 +1178,8 @@ static struct inode *kvm_gmem_inode_make_secure_inode(const char *name,

return inode;

+free_private: + kfree(private); out: iput(inode);

guest_memfd files can always be mmap()ed to userspace, but faultability is controlled by an attribute on the inode.

Co-developed-by: Fuad Tabba tabba@google.com Signed-off-by: Fuad Tabba tabba@google.com Co-developed-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com

--- virt/kvm/guest_memfd.c | 46 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index b603518f7b62..fc2483e35876 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -781,7 +781,8 @@ static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) { struct list_head *gmem_list = &inode->i_mapping->i_private_list; pgoff_t start = offset >> PAGE_SHIFT; - pgoff_t end = (offset + len) >> PAGE_SHIFT; + pgoff_t nr = len >> PAGE_SHIFT; + pgoff_t end = start + nr; struct kvm_gmem *gmem;

/* @@ -790,6 +791,9 @@ static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) */ filemap_invalidate_lock(inode->i_mapping);

+ /* TODO: Check if even_cows should be 0 or 1 */ + unmap_mapping_range(inode->i_mapping, start, len, 0); + list_for_each_entry(gmem, gmem_list, entry) kvm_gmem_invalidate_begin(gmem, start, end);

@@ -946,6 +950,9 @@ static void kvm_gmem_hugetlb_teardown(struct inode *inode) { struct kvm_gmem_hugetlb *hgmem;

+ /* TODO: Check if even_cows should be 0 or 1 */ + unmap_mapping_range(inode->i_mapping, 0, LLONG_MAX, 0); + truncate_inode_pages_final_prepare(inode->i_mapping); kvm_gmem_hugetlb_truncate_folios_range(inode, 0, LLONG_MAX);

@@ -1003,11 +1010,46 @@ static void kvm_gmem_init_mount(void) kvm_gmem_mnt = kern_mount(&kvm_gmem_fs); BUG_ON(IS_ERR(kvm_gmem_mnt));

- /* For giggles. Userspace can never map this anyways. */ kvm_gmem_mnt->mnt_flags |= MNT_NOEXEC; }

+static vm_fault_t kvm_gmem_fault(struct vm_fault *vmf) +{ + struct inode *inode; + struct folio *folio; + + inode = file_inode(vmf->vma->vm_file); + if (!kvm_gmem_is_faultable(inode, vmf->pgoff)) + return VM_FAULT_SIGBUS; + + folio = kvm_gmem_get_folio(inode, vmf->pgoff); + if (!folio) + return VM_FAULT_SIGBUS; + + vmf->page = folio_file_page(folio, vmf->pgoff); + return VM_FAULT_LOCKED; +} + +static const struct vm_operations_struct kvm_gmem_vm_ops = { + .fault = kvm_gmem_fault, +}; + +static int kvm_gmem_mmap(struct file *file, struct vm_area_struct *vma) +{ + if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) != + (VM_SHARED | VM_MAYSHARE)) { + return -EINVAL; + } + + file_accessed(file); + vm_flags_set(vma, VM_DONTDUMP); + vma->vm_ops = &kvm_gmem_vm_ops; + + return 0; +} + static struct file_operations kvm_gmem_fops = { + .mmap = kvm_gmem_mmap, .open = generic_file_open, .release = kvm_gmem_release, .fallocate = kvm_gmem_fallocate,

The key steps for a private to shared conversion are:

1. Unmap from guest page tables 2. Set pages associated with requested range in memslot to be faultable 3. Update kvm->mem_attr_array

The key steps for a shared to private conversion are:

1. Check and disallow set_memory_attributes if any page in the range is still mapped or pinned, by a. Updating guest_memfd's faultability to prevent future faulting b. Returning -EINVAL if any pages are still pinned. 2. Update kvm->mem_attr_array

Userspace VMM must ensure shared pages are not in use, since any faults racing with this call will get a SIGBUS.

Co-developed-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com Co-developed-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Vishal Annapurve vannapurve@google.com

--- include/linux/kvm_host.h | 1 + virt/kvm/guest_memfd.c | 207 +++++++++++++++++++++++++++++++++++++++ virt/kvm/kvm_main.c | 15 +++ virt/kvm/kvm_mm.h | 9 ++ 4 files changed, 232 insertions(+)

diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 79a6b1a63027..10993cd33e34 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2476,6 +2476,7 @@ typedef int (*kvm_gmem_populate_cb)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,

long kvm_gmem_populate(struct kvm *kvm, gfn_t gfn, void __user *src, long npages, kvm_gmem_populate_cb post_populate, void *opaque); + #endif

#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 1d4dfe0660ad..110c4bbb004b 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -1592,4 +1592,211 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long return ret && !i ? ret : i; } EXPORT_SYMBOL_GPL(kvm_gmem_populate); + +/** + * Returns true if pages in range [@start, @end) in inode @inode have no + * userspace mappings. + */ +static bool kvm_gmem_no_mappings_range(struct inode *inode, pgoff_t start, pgoff_t end) +{ + pgoff_t index; + bool checked_indices_unmapped; + + filemap_invalidate_lock_shared(inode->i_mapping); + + /* TODO: replace iteration with filemap_get_folios() for efficiency. */ + checked_indices_unmapped = true; + for (index = start; checked_indices_unmapped && index < end;) { + struct folio *folio; + + /* Don't use kvm_gmem_get_folio to avoid allocating */ + folio = filemap_lock_folio(inode->i_mapping, index); + if (IS_ERR(folio)) { + ++index; + continue; + } + + if (folio_mapped(folio) || folio_maybe_dma_pinned(folio)) + checked_indices_unmapped = false; + else + index = folio_next_index(folio); + + folio_unlock(folio); + folio_put(folio); + } + + filemap_invalidate_unlock_shared(inode->i_mapping); + return checked_indices_unmapped; +} + +/** + * Returns true if pages in range [@start, @end) in memslot @slot have no + * userspace mappings. + */ +static bool kvm_gmem_no_mappings_slot(struct kvm_memory_slot *slot, + gfn_t start, gfn_t end) +{ + pgoff_t offset_start; + pgoff_t offset_end; + struct file *file; + bool ret; + + offset_start = start - slot->base_gfn + slot->gmem.pgoff; + offset_end = end - slot->base_gfn + slot->gmem.pgoff; + + file = kvm_gmem_get_file(slot); + if (!file) + return false; + + ret = kvm_gmem_no_mappings_range(file_inode(file), offset_start, offset_end); + + fput(file); + + return ret; +} + +/** + * Returns true if pages in range [@start, @end) have no host userspace mappings. + */ +static bool kvm_gmem_no_mappings(struct kvm *kvm, gfn_t start, gfn_t end) +{ + int i; + + lockdep_assert_held(&kvm->slots_lock); + + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { + struct kvm_memslot_iter iter; + struct kvm_memslots *slots; + + slots = __kvm_memslots(kvm, i); + kvm_for_each_memslot_in_gfn_range(&iter, slots, start, end) { + struct kvm_memory_slot *slot; + gfn_t gfn_start; + gfn_t gfn_end; + + slot = iter.slot; + gfn_start = max(start, slot->base_gfn); + gfn_end = min(end, slot->base_gfn + slot->npages); + + if (iter.slot->flags & KVM_MEM_GUEST_MEMFD && + !kvm_gmem_no_mappings_slot(iter.slot, gfn_start, gfn_end)) + return false; + } + } + + return true; +} + +/** + * Set faultability of given range of gfns [@start, @end) in memslot @slot to + * @faultable. + */ +static void kvm_gmem_set_faultable_slot(struct kvm_memory_slot *slot, gfn_t start, + gfn_t end, bool faultable) +{ + pgoff_t start_offset; + pgoff_t end_offset; + struct file *file; + + file = kvm_gmem_get_file(slot); + if (!file) + return; + + start_offset = start - slot->base_gfn + slot->gmem.pgoff; + end_offset = end - slot->base_gfn + slot->gmem.pgoff; + + WARN_ON(kvm_gmem_set_faultable(file_inode(file), start_offset, end_offset, + faultable)); + + fput(file); +} + +/** + * Set faultability of given range of gfns [@start, @end) in memslot @slot to + * @faultable. + */ +static void kvm_gmem_set_faultable_vm(struct kvm *kvm, gfn_t start, gfn_t end, + bool faultable) +{ + int i; + + lockdep_assert_held(&kvm->slots_lock); + + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { + struct kvm_memslot_iter iter; + struct kvm_memslots *slots; + + slots = __kvm_memslots(kvm, i); + kvm_for_each_memslot_in_gfn_range(&iter, slots, start, end) { + struct kvm_memory_slot *slot; + gfn_t gfn_start; + gfn_t gfn_end; + + slot = iter.slot; + gfn_start = max(start, slot->base_gfn); + gfn_end = min(end, slot->base_gfn + slot->npages); + + if (iter.slot->flags & KVM_MEM_GUEST_MEMFD) { + kvm_gmem_set_faultable_slot(slot, gfn_start, + gfn_end, faultable); + } + } + } +} + +/** + * Returns true if guest_memfd permits setting range [@start, @end) to PRIVATE. + * + * If memory is faulted in to host userspace and a request was made to set the + * memory to PRIVATE, the faulted in pages must not be pinned for the request to + * be permitted. + */ +static int kvm_gmem_should_set_attributes_private(struct kvm *kvm, gfn_t start, + gfn_t end) +{ + kvm_gmem_set_faultable_vm(kvm, start, end, false); + + if (kvm_gmem_no_mappings(kvm, start, end)) + return 0; + + kvm_gmem_set_faultable_vm(kvm, start, end, true); + return -EINVAL; +} + +/** + * Returns true if guest_memfd permits setting range [@start, @end) to SHARED. + * + * Because this allows pages to be faulted in to userspace, this must only be + * called after the pages have been invalidated from guest page tables. + */ +static int kvm_gmem_should_set_attributes_shared(struct kvm *kvm, gfn_t start, + gfn_t end) +{ + /* Always okay to set shared, hence set range faultable here. */ + kvm_gmem_set_faultable_vm(kvm, start, end, true); + + return 0; +} + +/** + * Returns 0 if guest_memfd permits setting attributes @attrs for range [@start, + * @end) or negative error otherwise. + * + * If memory is faulted in to host userspace and a request was made to set the + * memory to PRIVATE, the faulted in pages must not be pinned for the request to + * be permitted. + * + * Because this may allow pages to be faulted in to userspace when requested to + * set attributes to shared, this must only be called after the pages have been + * invalidated from guest page tables. + */ +int kvm_gmem_should_set_attributes(struct kvm *kvm, gfn_t start, gfn_t end, + unsigned long attrs) +{ + if (attrs & KVM_MEMORY_ATTRIBUTE_PRIVATE) + return kvm_gmem_should_set_attributes_private(kvm, start, end); + else + return kvm_gmem_should_set_attributes_shared(kvm, start, end); +} + #endif diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 92901656a0d4..1a7bbcc31b7e 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2524,6 +2524,13 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, .on_lock = kvm_mmu_invalidate_end, .may_block = true, }; + struct kvm_mmu_notifier_range error_set_range = { + .start = start, + .end = end, + .handler = (void *)kvm_null_fn, + .on_lock = kvm_mmu_invalidate_end, + .may_block = true, + }; unsigned long i; void *entry; int r = 0; @@ -2548,6 +2555,10 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end,

kvm_handle_gfn_range(kvm, &pre_set_range);

+ r = kvm_gmem_should_set_attributes(kvm, start, end, attributes); + if (r) + goto err; + for (i = start; i < end; i++) { r = xa_err(xa_store(&kvm->mem_attr_array, i, entry, GFP_KERNEL_ACCOUNT)); @@ -2560,6 +2571,10 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, mutex_unlock(&kvm->slots_lock);

return r; + +err: + kvm_handle_gfn_range(kvm, &error_set_range); + goto out_unlock; } static int kvm_vm_ioctl_set_mem_attributes(struct kvm *kvm, struct kvm_memory_attributes *attrs) diff --git a/virt/kvm/kvm_mm.h b/virt/kvm/kvm_mm.h index 715f19669d01..d8ff2b380d0e 100644 --- a/virt/kvm/kvm_mm.h +++ b/virt/kvm/kvm_mm.h @@ -41,6 +41,8 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args); int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, unsigned int fd, loff_t offset); void kvm_gmem_unbind(struct kvm_memory_slot *slot); +int kvm_gmem_should_set_attributes(struct kvm *kvm, gfn_t start, gfn_t end, + unsigned long attrs); #else static inline void kvm_gmem_init(struct module *module) { @@ -59,6 +61,13 @@ static inline void kvm_gmem_unbind(struct kvm_memory_slot *slot) { WARN_ON_ONCE(1); } + +static inline int kvm_gmem_should_set_attributes(struct kvm *kvm, gfn_t start, + gfn_t end, unsigned long attrs) +{ + return 0; +} + #endif /* CONFIG_KVM_PRIVATE_MEM */

#endif /* __KVM_MM_H__ */

Test using guest_memfd from userspace, since guest_memfd now has mmap() support.

Tests:

1. mmap() should now always return a valid address 2. Test that madvise() doesn't give any issues when pages are not faulted in. 3. Test that pages should not be faultable before association with a memslot, and that faults result in SIGBUS. 4. Test that pages can be faulted if marked faultable, and the flow of setting a memory range as private, which is: a. madvise(MADV_DONTNEED) to request kernel to unmap pages b. Set memory attributes of VM to private Also test that if pages are still mapped, setting memory attributes will fail. 5. Test that madvise(MADV_REMOVE) can be used to remove pages from guest_memfd, forcing zeroing of those pages before the next time the pages are faulted in.

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- .../testing/selftests/kvm/guest_memfd_test.c | 195 +++++++++++++++++- 1 file changed, 189 insertions(+), 6 deletions(-)

diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c index 3618ce06663e..b6f3c3e6d0dd 100644 --- a/tools/testing/selftests/kvm/guest_memfd_test.c +++ b/tools/testing/selftests/kvm/guest_memfd_test.c @@ -6,6 +6,7 @@ */ #include <stdlib.h> #include <string.h> +#include <sys/wait.h> #include <unistd.h> #include <errno.h> #include <stdio.h> @@ -35,12 +36,192 @@ static void test_file_read_write(int fd) "pwrite on a guest_mem fd should fail"); }

-static void test_mmap(int fd, size_t page_size) +static void test_mmap_should_map_pages_into_userspace(int fd, size_t page_size) { char *mem;

mem = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - TEST_ASSERT_EQ(mem, MAP_FAILED); + TEST_ASSERT(mem != MAP_FAILED, "mmap should return valid address"); + + TEST_ASSERT_EQ(munmap(mem, page_size), 0); +} + +static void test_madvise_no_error_when_pages_not_faulted(int fd, size_t page_size) +{ + char *mem; + + mem = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + TEST_ASSERT(mem != MAP_FAILED, "mmap should return valid address"); + + TEST_ASSERT_EQ(madvise(mem, page_size, MADV_DONTNEED), 0); + + TEST_ASSERT_EQ(munmap(mem, page_size), 0); +} + +static void assert_not_faultable(char *address) +{ + pid_t child_pid; + + child_pid = fork(); + TEST_ASSERT(child_pid != -1, "fork failed"); + + if (child_pid == 0) { + *address = 'A'; + } else { + int status; + waitpid(child_pid, &status, 0); + + TEST_ASSERT(WIFSIGNALED(status), + "Child should have exited with a signal"); + TEST_ASSERT_EQ(WTERMSIG(status), SIGBUS); + } +} + +/* + * Pages should not be faultable before association with memslot because pages + * (in a KVM_X86_SW_PROTECTED_VM) only default to faultable at memslot + * association time. + */ +static void test_pages_not_faultable_if_not_associated_with_memslot(int fd, + size_t page_size) +{ + char *mem = mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + TEST_ASSERT(mem != MAP_FAILED, "mmap should return valid address"); + + assert_not_faultable(mem); + + TEST_ASSERT_EQ(munmap(mem, page_size), 0); +} + +static void test_pages_faultable_if_marked_faultable(struct kvm_vm *vm, int fd, + size_t page_size) +{ + char *mem; + uint64_t gpa = 0; + uint64_t guest_memfd_offset = 0; + + /* + * This test uses KVM_X86_SW_PROTECTED_VM is required to set + * arch.has_private_mem, to add a memslot with guest_memfd to a VM. + */ + if (!(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM))) { + printf("Faultability test skipped since KVM_X86_SW_PROTECTED_VM is not supported."); + return; + } + + mem = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + guest_memfd_offset); + TEST_ASSERT(mem != MAP_FAILED, "mmap should return valid address"); + + /* + * Setting up this memslot with a KVM_X86_SW_PROTECTED_VM marks all + * offsets in the file as shared, allowing pages to be faulted in. + */ + vm_set_user_memory_region2(vm, 0, KVM_MEM_GUEST_MEMFD, gpa, page_size, + mem, fd, guest_memfd_offset); + + *mem = 'A'; + TEST_ASSERT_EQ(*mem, 'A'); + + /* Should fail since the page is still faulted in. */ + TEST_ASSERT_EQ(__vm_set_memory_attributes(vm, gpa, page_size, + KVM_MEMORY_ATTRIBUTE_PRIVATE), + -1); + TEST_ASSERT_EQ(errno, EINVAL); + + /* + * Use madvise() to remove the pages from userspace page tables, then + * test that the page is still faultable, and that page contents remain + * the same. + */ + madvise(mem, page_size, MADV_DONTNEED); + TEST_ASSERT_EQ(*mem, 'A'); + + /* Tell kernel to unmap the page from userspace. */ + madvise(mem, page_size, MADV_DONTNEED); + + /* Now kernel can set this page to private. */ + vm_mem_set_private(vm, gpa, page_size); + assert_not_faultable(mem); + + /* + * Should be able to fault again after setting this back to shared, and + * memory contents should be cleared since pages must be re-prepared for + * SHARED use. + */ + vm_mem_set_shared(vm, gpa, page_size); + TEST_ASSERT_EQ(*mem, 0); + + /* Cleanup */ + vm_set_user_memory_region2(vm, 0, KVM_MEM_GUEST_MEMFD, gpa, 0, mem, fd, + guest_memfd_offset); + + TEST_ASSERT_EQ(munmap(mem, page_size), 0); +} + +static void test_madvise_remove_releases_pages(struct kvm_vm *vm, int fd, + size_t page_size) +{ + char *mem; + uint64_t gpa = 0; + uint64_t guest_memfd_offset = 0; + + /* + * This test uses KVM_X86_SW_PROTECTED_VM is required to set + * arch.has_private_mem, to add a memslot with guest_memfd to a VM. + */ + if (!(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM))) { + printf("madvise test skipped since KVM_X86_SW_PROTECTED_VM is not supported."); + return; + } + + mem = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + TEST_ASSERT(mem != MAP_FAILED, "mmap should return valid address"); + + /* + * Setting up this memslot with a KVM_X86_SW_PROTECTED_VM marks all + * offsets in the file as shared, allowing pages to be faulted in. + */ + vm_set_user_memory_region2(vm, 0, KVM_MEM_GUEST_MEMFD, gpa, page_size, + mem, fd, guest_memfd_offset); + + *mem = 'A'; + TEST_ASSERT_EQ(*mem, 'A'); + + /* + * MADV_DONTNEED causes pages to be removed from userspace page tables + * but should not release pages, hence page contents are kept. + */ + TEST_ASSERT_EQ(madvise(mem, page_size, MADV_DONTNEED), 0); + TEST_ASSERT_EQ(*mem, 'A'); + + /* + * MADV_REMOVE causes pages to be released. Pages are then zeroed when + * prepared for shared use, hence 0 is expected on next fault. + */ + TEST_ASSERT_EQ(madvise(mem, page_size, MADV_REMOVE), 0); + TEST_ASSERT_EQ(*mem, 0); + + TEST_ASSERT_EQ(munmap(mem, page_size), 0); + + /* Cleanup */ + vm_set_user_memory_region2(vm, 0, KVM_MEM_GUEST_MEMFD, gpa, 0, mem, fd, + guest_memfd_offset); +} + +static void test_using_memory_directly_from_userspace(struct kvm_vm *vm, + int fd, size_t page_size) +{ + test_mmap_should_map_pages_into_userspace(fd, page_size); + + test_madvise_no_error_when_pages_not_faulted(fd, page_size); + + test_pages_not_faultable_if_not_associated_with_memslot(fd, page_size); + + test_pages_faultable_if_marked_faultable(vm, fd, page_size); + + test_madvise_remove_releases_pages(vm, fd, page_size); }

static void test_file_size(int fd, size_t page_size, size_t total_size) @@ -180,18 +361,17 @@ static void test_guest_memfd(struct kvm_vm *vm, uint32_t flags, size_t page_size size_t total_size; int fd;

- TEST_REQUIRE(kvm_has_cap(KVM_CAP_GUEST_MEMFD)); - total_size = page_size * 4;

fd = vm_create_guest_memfd(vm, total_size, flags);

test_file_read_write(fd); - test_mmap(fd, page_size); test_file_size(fd, page_size, total_size); test_fallocate(fd, page_size, total_size); test_invalid_punch_hole(fd, page_size, total_size);

+ test_using_memory_directly_from_userspace(vm, fd, page_size); + close(fd); }

@@ -201,7 +381,10 @@ int main(int argc, char *argv[])

TEST_REQUIRE(kvm_has_cap(KVM_CAP_GUEST_MEMFD));

- vm = vm_create_barebones(); + if ((kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM))) + vm = vm_create_barebones_type(KVM_X86_SW_PROTECTED_VM); + else + vm = vm_create_barebones();

test_create_guest_memfd_invalid(vm); test_create_guest_memfd_multiple(vm);

Note in comments why madvise() is not needed before setting memory to private.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- .../selftests/kvm/x86_64/private_mem_kvm_exits_test.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tools/testing/selftests/kvm/x86_64/private_mem_kvm_exits_test.c b/tools/testing/selftests/kvm/x86_64/private_mem_kvm_exits_test.c index 13e72fcec8dd..f8bcfc897f6a 100644 --- a/tools/testing/selftests/kvm/x86_64/private_mem_kvm_exits_test.c +++ b/tools/testing/selftests/kvm/x86_64/private_mem_kvm_exits_test.c @@ -62,7 +62,11 @@ static void test_private_access_memslot_deleted(void)

virt_map(vm, EXITS_TEST_GVA, EXITS_TEST_GPA, EXITS_TEST_NPAGES);

- /* Request to access page privately */ + /* + * Request to access page privately. madvise(MADV_DONTNEED) not required + * since memory was never mmap()-ed from guest_memfd. Anonymous memory + * was used instead for this memslot's userspace_addr. + */ vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE);

pthread_create(&vm_thread, NULL, @@ -98,7 +102,10 @@ static void test_private_access_memslot_not_private(void)

virt_map(vm, EXITS_TEST_GVA, EXITS_TEST_GPA, EXITS_TEST_NPAGES);

- /* Request to access page privately */ + /* + * Request to access page privately. madvise(MADV_DONTNEED) not required + * since the affected memslot doesn't use guest_memfd. + */ vm_mem_set_private(vm, EXITS_TEST_GPA, EXITS_TEST_SIZE);

exit_reason = run_vcpu_get_exit_reason(vcpu);

enum vm_mem_backing_src_type is encoding too many different possibilities on different axes of (1) whether to mmap from an fd, (2) granularity of mapping for THP, (3) size of hugetlb mapping, and has yet to be extended to support guest_memfd.

When guest_memfd supports mmap() and we also want to support testing with mmap()ing from guest_memfd, the number of combinations make enumeration in vm_mem_backing_src_type difficult.

This refactor separates out vm_mem_backing_src_type from userspace_mem_region. For now, vm_mem_backing_src_type remains a possible way for tests to specify, on the command line, the combination of backing memory to test.

vm_mem_add() is now the last place where vm_mem_backing_src_type is interpreted, to

1. Check validity of requested guest_paddr 2. Align mmap_size appropriately based on the mapping's page_size and architecture 3. Install memory appropriately according to mapping's page size

mmap()ing an alias seems to be specific to userfaultfd tests and could be refactored out of struct userspace_mem_region and localized in userfaultfd tests in future.

This paves the way for replacing vm_mem_backing_src_type with multiple command line flags that would specify backing memory more flexibly. Future tests are expected to use vm_mem_region_alloc() to allocate a struct userspace_mem_region, then use more fundamental functions like vm_mem_region_mmap(), vm_mem_region_madvise_thp(), kvm_memfd_create(), vm_create_guest_memfd(), and other functions in vm_mem_add() to flexibly build up struct userspace_mem_region before finally adding the region to the vm with vm_mem_region_add().

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- .../testing/selftests/kvm/include/kvm_util.h | 29 +- .../testing/selftests/kvm/include/test_util.h | 2 + tools/testing/selftests/kvm/lib/kvm_util.c | 413 +++++++++++------- tools/testing/selftests/kvm/lib/test_util.c | 25 ++ 4 files changed, 319 insertions(+), 150 deletions(-)

diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index d336cd0c8f19..1576e7e4aefe 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -35,11 +35,26 @@ struct userspace_mem_region { struct sparsebit *protected_phy_pages; int fd; off_t offset; - enum vm_mem_backing_src_type backing_src_type; + /* + * host_mem is mmap_start aligned upwards to an address suitable for the + * architecture. In most cases, host_mem and mmap_start are the same, + * except for s390x, where the host address must be aligned to 1M (due + * to PGSTEs). + */ +#ifdef __s390x__ +#define S390X_HOST_ADDRESS_ALIGNMENT 0x100000 +#endif void *host_mem; + /* host_alias is to mmap_alias as host_mem is to mmap_start */ void *host_alias; void *mmap_start; void *mmap_alias; + /* + * mmap_size is possibly larger than region.memory_size because in some + * cases, host_mem has to be adjusted upwards (see comment for host_mem + * above). In those cases, mmap_size has to be adjusted upwards so that + * enough memory is available in this memslot. + */ size_t mmap_size; struct rb_node gpa_node; struct rb_node hva_node; @@ -559,6 +574,18 @@ int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flag uint64_t gpa, uint64_t size, void *hva, uint32_t guest_memfd, uint64_t guest_memfd_offset);

+struct userspace_mem_region *vm_mem_region_alloc(struct kvm_vm *vm); +void *vm_mem_region_mmap(struct userspace_mem_region *region, size_t length, + int flags, int fd, off_t offset); +void vm_mem_region_install_memory(struct userspace_mem_region *region, + size_t memslot_size, size_t alignment); +void vm_mem_region_madvise_thp(struct userspace_mem_region *region, int advice); +int vm_mem_region_install_guest_memfd(struct userspace_mem_region *region, + int guest_memfd); +void *vm_mem_region_mmap_alias(struct userspace_mem_region *region, int flags, + size_t alignment); +void vm_mem_region_add(struct kvm_vm *vm, struct userspace_mem_region *region); + void vm_userspace_mem_region_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, uint64_t guest_paddr, uint32_t slot, uint64_t npages, diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index 011e757d4e2c..983adeb54c0e 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -159,6 +159,8 @@ size_t get_trans_hugepagesz(void); size_t get_def_hugetlb_pagesz(void); const struct vm_mem_backing_src_alias *vm_mem_backing_src_alias(uint32_t i); size_t get_backing_src_pagesz(uint32_t i); +int backing_src_should_madvise(uint32_t i); +int get_backing_src_madvise_advice(uint32_t i); bool is_backing_src_hugetlb(uint32_t i); void backing_src_help(const char *flag); enum vm_mem_backing_src_type parse_backing_src_type(const char *type_name); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 56b170b725b3..9bdd03a5da90 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -774,15 +774,12 @@ void kvm_vm_free(struct kvm_vm *vmp) free(vmp); }

-int kvm_memfd_alloc(size_t size, bool hugepages) +int kvm_create_memfd(size_t size, unsigned int flags) { - int memfd_flags = MFD_CLOEXEC; - int fd, r; - - if (hugepages) - memfd_flags |= MFD_HUGETLB; + int fd; + int r;

- fd = memfd_create("kvm_selftest", memfd_flags); + fd = memfd_create("kvm_selftest", flags); TEST_ASSERT(fd != -1, __KVM_SYSCALL_ERROR("memfd_create()", fd));

r = ftruncate(fd, size); @@ -794,6 +791,16 @@ int kvm_memfd_alloc(size_t size, bool hugepages) return fd; }

+int kvm_memfd_alloc(size_t size, bool hugepages) +{ + int memfd_flags = MFD_CLOEXEC; + + if (hugepages) + memfd_flags |= MFD_HUGETLB; + + return kvm_create_memfd(size, memfd_flags); +} + /* * Memory Compare, host virtual to guest virtual * @@ -973,185 +980,293 @@ void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags errno, strerror(errno)); }

+/** + * Allocates and returns a struct userspace_mem_region. + */ +struct userspace_mem_region *vm_mem_region_alloc(struct kvm_vm *vm) +{ + struct userspace_mem_region *region;

-/* FIXME: This thing needs to be ripped apart and rewritten. */ -void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, - uint64_t guest_paddr, uint32_t slot, uint64_t npages, - uint32_t flags, int guest_memfd, uint64_t guest_memfd_offset) + /* Allocate and initialize new mem region structure. */ + region = calloc(1, sizeof(*region)); + TEST_ASSERT(region != NULL, "Insufficient Memory"); + + region->unused_phy_pages = sparsebit_alloc(); + if (vm_arch_has_protected_memory(vm)) + region->protected_phy_pages = sparsebit_alloc(); + + region->fd = -1; + region->region.guest_memfd = -1; + + return region; +} + +static size_t compute_page_size(int mmap_flags, int madvise_advice) +{ + if (mmap_flags & MAP_HUGETLB) { + int size_flags = (mmap_flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK; + if (!size_flags) + return get_def_hugetlb_pagesz(); + + return 1ULL << size_flags; + } + + return madvise_advice == MADV_HUGEPAGE ? get_trans_hugepagesz() : getpagesize(); +} + +/** + * Calls mmap() with @length, @flags, @fd, @offset for @region. + * + * Think of this as the struct userspace_mem_region wrapper for the mmap() + * syscall. + */ +void *vm_mem_region_mmap(struct userspace_mem_region *region, size_t length, + int flags, int fd, off_t offset) +{ + void *mem; + + if (flags & MAP_SHARED) { + TEST_ASSERT(fd != -1, + "Ensure that fd is provided for shared mappings."); + TEST_ASSERT( + region->fd == fd || region->region.guest_memfd == fd, + "Ensure that fd is opened before mmap, and is either " + "set up in region->fd or region->region.guest_memfd."); + } + + mem = mmap(NULL, length, PROT_READ | PROT_WRITE, flags, fd, offset); + TEST_ASSERT(mem != MAP_FAILED, "Couldn't mmap anonymous memory"); + + region->mmap_start = mem; + region->mmap_size = length; + region->offset = offset; + + return mem; +} + +/** + * Installs mmap()ed memory in @region->mmap_start as @region->host_mem, + * checking constraints. + */ +void vm_mem_region_install_memory(struct userspace_mem_region *region, + size_t memslot_size, size_t alignment) +{ + TEST_ASSERT(region->mmap_size >= memslot_size, + "mmap()ed memory insufficient for memslot"); + + region->host_mem = align_ptr_up(region->mmap_start, alignment); + region->region.userspace_addr = (uint64_t)region->host_mem; + region->region.memory_size = memslot_size; +} + + +/** + * Calls madvise with @advice for @region. + * + * Think of this as the struct userspace_mem_region wrapper for the madvise() + * syscall. + */ +void vm_mem_region_madvise_thp(struct userspace_mem_region *region, int advice) { int ret; - struct userspace_mem_region *region; - size_t backing_src_pagesz = get_backing_src_pagesz(src_type); - size_t mem_size = npages * vm->page_size; - size_t alignment;

- TEST_REQUIRE_SET_USER_MEMORY_REGION2(); + TEST_ASSERT( + region->host_mem && region->mmap_size, + "vm_mem_region_madvise_thp() must be called after vm_mem_region_mmap()");

- TEST_ASSERT(vm_adjust_num_guest_pages(vm->mode, npages) == npages, - "Number of guest pages is not compatible with the host. " - "Try npages=%d", vm_adjust_num_guest_pages(vm->mode, npages)); - - TEST_ASSERT((guest_paddr % vm->page_size) == 0, "Guest physical " - "address not on a page boundary.\n" - " guest_paddr: 0x%lx vm->page_size: 0x%x", - guest_paddr, vm->page_size); - TEST_ASSERT((((guest_paddr >> vm->page_shift) + npages) - 1) - <= vm->max_gfn, "Physical range beyond maximum " - "supported physical address,\n" - " guest_paddr: 0x%lx npages: 0x%lx\n" - " vm->max_gfn: 0x%lx vm->page_size: 0x%x", - guest_paddr, npages, vm->max_gfn, vm->page_size); + ret = madvise(region->host_mem, region->mmap_size, advice); + TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx", + region->host_mem, region->mmap_size); +} + +/** + * Installs guest_memfd by setting it up in @region. + * + * Returns the guest_memfd that was installed in the @region. + */ +int vm_mem_region_install_guest_memfd(struct userspace_mem_region *region, + int guest_memfd) +{ + /* + * Install a unique fd for each memslot so that the fd can be closed + * when the region is deleted without needing to track if the fd is + * owned by the framework or by the caller. + */ + guest_memfd = dup(guest_memfd); + TEST_ASSERT(guest_memfd >= 0, __KVM_SYSCALL_ERROR("dup()", guest_memfd)); + region->region.guest_memfd = guest_memfd; + + return guest_memfd; +} + +/** + * Calls mmap() to create an alias for mmap()ed memory at region->host_mem, + * exactly the same size the was mmap()ed. + * + * This is used mainly for userfaultfd tests. + */ +void *vm_mem_region_mmap_alias(struct userspace_mem_region *region, int flags, + size_t alignment) +{ + region->mmap_alias = mmap(NULL, region->mmap_size, + PROT_READ | PROT_WRITE, flags, region->fd, 0); + TEST_ASSERT(region->mmap_alias != MAP_FAILED, + __KVM_SYSCALL_ERROR("mmap()", (int)(unsigned long)MAP_FAILED)); + + region->host_alias = align_ptr_up(region->mmap_alias, alignment); + + return region->host_alias; +} + +static void vm_mem_region_assert_no_duplicate(struct kvm_vm *vm, uint32_t slot, + uint64_t gpa, size_t size) +{ + struct userspace_mem_region *region;

/* * Confirm a mem region with an overlapping address doesn't * already exist. */ - region = (struct userspace_mem_region *) userspace_mem_region_find( - vm, guest_paddr, (guest_paddr + npages * vm->page_size) - 1); - if (region != NULL) - TEST_FAIL("overlapping userspace_mem_region already " - "exists\n" - " requested guest_paddr: 0x%lx npages: 0x%lx " - "page_size: 0x%x\n" - " existing guest_paddr: 0x%lx size: 0x%lx", - guest_paddr, npages, vm->page_size, - (uint64_t) region->region.guest_phys_addr, - (uint64_t) region->region.memory_size); + region = userspace_mem_region_find(vm, gpa, gpa + size - 1); + if (region != NULL) { + TEST_FAIL("overlapping userspace_mem_region already exists\n" + " requested gpa: 0x%lx size: 0x%lx" + " existing gpa: 0x%lx size: 0x%lx", + gpa, size, + (uint64_t) region->region.guest_phys_addr, + (uint64_t) region->region.memory_size); + }

/* Confirm no region with the requested slot already exists. */ - hash_for_each_possible(vm->regions.slot_hash, region, slot_node, - slot) { + hash_for_each_possible(vm->regions.slot_hash, region, slot_node, slot) { if (region->region.slot != slot) continue;

- TEST_FAIL("A mem region with the requested slot " - "already exists.\n" - " requested slot: %u paddr: 0x%lx npages: 0x%lx\n" - " existing slot: %u paddr: 0x%lx size: 0x%lx", - slot, guest_paddr, npages, - region->region.slot, - (uint64_t) region->region.guest_phys_addr, - (uint64_t) region->region.memory_size); + TEST_FAIL("A mem region with the requested slot already exists.\n" + " requested slot: %u paddr: 0x%lx size: 0x%lx\n" + " existing slot: %u paddr: 0x%lx size: 0x%lx", + slot, gpa, size, + region->region.slot, + (uint64_t) region->region.guest_phys_addr, + (uint64_t) region->region.memory_size); } +}

- /* Allocate and initialize new mem region structure. */ - region = calloc(1, sizeof(*region)); - TEST_ASSERT(region != NULL, "Insufficient Memory"); - region->mmap_size = mem_size; +/** + * Add a @region to @vm. All necessary fields in region->region should already + * be populated. + * + * Think of this as the struct userspace_mem_region wrapper for the + * KVM_SET_USER_MEMORY_REGION2 ioctl. + */ +void vm_mem_region_add(struct kvm_vm *vm, struct userspace_mem_region *region) +{ + uint64_t npages; + uint64_t gpa; + int ret;

-#ifdef __s390x__ - /* On s390x, the host address must be aligned to 1M (due to PGSTEs) */ - alignment = 0x100000; -#else - alignment = 1; -#endif + TEST_REQUIRE_SET_USER_MEMORY_REGION2();

- /* - * When using THP mmap is not guaranteed to returned a hugepage aligned - * address so we have to pad the mmap. Padding is not needed for HugeTLB - * because mmap will always return an address aligned to the HugeTLB - * page size. - */ - if (src_type == VM_MEM_SRC_ANONYMOUS_THP) - alignment = max(backing_src_pagesz, alignment); + npages = region->region.memory_size / vm->page_size; + TEST_ASSERT(vm_adjust_num_guest_pages(vm->mode, npages) == npages, + "Number of guest pages is not compatible with the host. " + "Try npages=%d", vm_adjust_num_guest_pages(vm->mode, npages)); + + gpa = region->region.guest_phys_addr; + TEST_ASSERT((gpa % vm->page_size) == 0, + "Guest physical address not on a page boundary.\n" + " gpa: 0x%lx vm->page_size: 0x%x", + gpa, vm->page_size); + TEST_ASSERT((((gpa >> vm->page_shift) + npages) - 1) <= vm->max_gfn, + "Physical range beyond maximum supported physical address,\n" + " gpa: 0x%lx npages: 0x%lx\n" + " vm->max_gfn: 0x%lx vm->page_size: 0x%x", + gpa, npages, vm->max_gfn, vm->page_size); + + vm_mem_region_assert_no_duplicate(vm, region->region.slot, gpa, + region->mmap_size);

- TEST_ASSERT_EQ(guest_paddr, align_up(guest_paddr, backing_src_pagesz)); + ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region); + TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n" + " rc: %i errno: %i\n" + " slot: %u flags: 0x%x\n" + " guest_phys_addr: 0x%lx size: 0x%llx guest_memfd: %d", + ret, errno, region->region.slot, region->region.flags, + gpa, region->region.memory_size, + region->region.guest_memfd);

- /* Add enough memory to align up if necessary */ - if (alignment > 1) - region->mmap_size += alignment; + sparsebit_set_num(region->unused_phy_pages, gpa >> vm->page_shift, npages);

- region->fd = -1; - if (backing_src_is_shared(src_type)) - region->fd = kvm_memfd_alloc(region->mmap_size, - src_type == VM_MEM_SRC_SHARED_HUGETLB); - - region->mmap_start = mmap(NULL, region->mmap_size, - PROT_READ | PROT_WRITE, - vm_mem_backing_src_alias(src_type)->flag, - region->fd, 0); - TEST_ASSERT(region->mmap_start != MAP_FAILED, - __KVM_SYSCALL_ERROR("mmap()", (int)(unsigned long)MAP_FAILED)); + /* Add to quick lookup data structures */ + vm_userspace_mem_region_gpa_insert(&vm->regions.gpa_tree, region); + vm_userspace_mem_region_hva_insert(&vm->regions.hva_tree, region); + hash_add(vm->regions.slot_hash, &region->slot_node, region->region.slot); +}

- TEST_ASSERT(!is_backing_src_hugetlb(src_type) || - region->mmap_start == align_ptr_up(region->mmap_start, backing_src_pagesz), - "mmap_start %p is not aligned to HugeTLB page size 0x%lx", - region->mmap_start, backing_src_pagesz); +void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, + uint64_t guest_paddr, uint32_t slot, uint64_t npages, + uint32_t flags, int guest_memfd, uint64_t guest_memfd_offset) +{ + struct userspace_mem_region *region; + size_t mapping_page_size; + size_t memslot_size; + int madvise_advice; + size_t mmap_size; + size_t alignment; + int mmap_flags; + int memfd;

- /* Align host address */ - region->host_mem = align_ptr_up(region->mmap_start, alignment); + memslot_size = npages * vm->page_size; + + mmap_flags = vm_mem_backing_src_alias(src_type)->flag; + madvise_advice = get_backing_src_madvise_advice(src_type); + mapping_page_size = compute_page_size(mmap_flags, madvise_advice); + + TEST_ASSERT_EQ(guest_paddr, align_up(guest_paddr, mapping_page_size)); + + alignment = mapping_page_size; +#ifdef __s390x__ + alignment = max(alignment, S390X_HOST_ADDRESS_ALIGNMENT); +#endif

- /* As needed perform madvise */ - if ((src_type == VM_MEM_SRC_ANONYMOUS || - src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) { - ret = madvise(region->host_mem, mem_size, - src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE); - TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s", - region->host_mem, mem_size, - vm_mem_backing_src_alias(src_type)->name); + region = vm_mem_region_alloc(vm); + + memfd = -1; + if (backing_src_is_shared(src_type)) { + unsigned int memfd_flags = MFD_CLOEXEC; + if (src_type == VM_MEM_SRC_SHARED_HUGETLB) + memfd_flags |= MFD_HUGETLB; + + memfd = kvm_create_memfd(memslot_size, memfd_flags); } + region->fd = memfd; + + mmap_size = align_up(memslot_size, alignment); + vm_mem_region_mmap(region, mmap_size, mmap_flags, memfd, 0); + vm_mem_region_install_memory(region, memslot_size, alignment);

- region->backing_src_type = src_type; + if (backing_src_should_madvise(src_type)) + vm_mem_region_madvise_thp(region, madvise_advice); + + if (backing_src_is_shared(src_type)) + vm_mem_region_mmap_alias(region, mmap_flags, alignment);

if (flags & KVM_MEM_GUEST_MEMFD) { if (guest_memfd < 0) { - uint32_t guest_memfd_flags = 0; - TEST_ASSERT(!guest_memfd_offset, - "Offset must be zero when creating new guest_memfd"); - guest_memfd = vm_create_guest_memfd(vm, mem_size, guest_memfd_flags); - } else { - /* - * Install a unique fd for each memslot so that the fd - * can be closed when the region is deleted without - * needing to track if the fd is owned by the framework - * or by the caller. - */ - guest_memfd = dup(guest_memfd); - TEST_ASSERT(guest_memfd >= 0, __KVM_SYSCALL_ERROR("dup()", guest_memfd)); + TEST_ASSERT( + guest_memfd_offset == 0, + "Offset must be zero when creating new guest_memfd"); + guest_memfd = vm_create_guest_memfd(vm, memslot_size, 0); }

- region->region.guest_memfd = guest_memfd; - region->region.guest_memfd_offset = guest_memfd_offset; - } else { - region->region.guest_memfd = -1; + vm_mem_region_install_guest_memfd(region, guest_memfd); }

- region->unused_phy_pages = sparsebit_alloc(); - if (vm_arch_has_protected_memory(vm)) - region->protected_phy_pages = sparsebit_alloc(); - sparsebit_set_num(region->unused_phy_pages, - guest_paddr >> vm->page_shift, npages); region->region.slot = slot; region->region.flags = flags; region->region.guest_phys_addr = guest_paddr; - region->region.memory_size = npages * vm->page_size; - region->region.userspace_addr = (uintptr_t) region->host_mem; - ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, &region->region); - TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n" - " rc: %i errno: %i\n" - " slot: %u flags: 0x%x\n" - " guest_phys_addr: 0x%lx size: 0x%lx guest_memfd: %d", - ret, errno, slot, flags, - guest_paddr, (uint64_t) region->region.memory_size, - region->region.guest_memfd); - - /* Add to quick lookup data structures */ - vm_userspace_mem_region_gpa_insert(&vm->regions.gpa_tree, region); - vm_userspace_mem_region_hva_insert(&vm->regions.hva_tree, region); - hash_add(vm->regions.slot_hash, &region->slot_node, slot); - - /* If shared memory, create an alias. */ - if (region->fd >= 0) { - region->mmap_alias = mmap(NULL, region->mmap_size, - PROT_READ | PROT_WRITE, - vm_mem_backing_src_alias(src_type)->flag, - region->fd, 0); - TEST_ASSERT(region->mmap_alias != MAP_FAILED, - __KVM_SYSCALL_ERROR("mmap()", (int)(unsigned long)MAP_FAILED)); - - /* Align host alias address */ - region->host_alias = align_ptr_up(region->mmap_alias, alignment); - } + region->region.guest_memfd_offset = guest_memfd_offset; + vm_mem_region_add(vm, region); }

void vm_userspace_mem_region_add(struct kvm_vm *vm, diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c index d0a9b5ee0c01..cbcc1e7ad578 100644 --- a/tools/testing/selftests/kvm/lib/test_util.c +++ b/tools/testing/selftests/kvm/lib/test_util.c @@ -351,6 +351,31 @@ size_t get_private_mem_backing_src_pagesz(uint32_t i) } }

+int backing_src_should_madvise(uint32_t i) +{ + switch (i) { + case VM_MEM_SRC_ANONYMOUS: + case VM_MEM_SRC_SHMEM: + case VM_MEM_SRC_ANONYMOUS_THP: + return true; + default: + return false; + } +} + +int get_backing_src_madvise_advice(uint32_t i) +{ + switch (i) { + case VM_MEM_SRC_ANONYMOUS: + case VM_MEM_SRC_SHMEM: + return MADV_NOHUGEPAGE; + case VM_MEM_SRC_ANONYMOUS_THP: + return MADV_NOHUGEPAGE; + default: + return 0; + } +} + bool is_backing_src_hugetlb(uint32_t i) { return !!(vm_mem_backing_src_alias(i)->flag & MAP_HUGETLB);

Signed-off-by: Ackerley Tng ackerleytng@google.com --- .../kvm/x86_64/private_mem_conversions_test.c | 146 +++++++++++++++--- .../x86_64/private_mem_conversions_test.sh | 3 + 2 files changed, 124 insertions(+), 25 deletions(-)

diff --git a/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c index 71f480c19f92..6524ef398584 100644 --- a/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c +++ b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c @@ -11,6 +11,8 @@ #include <stdlib.h> #include <string.h> #include <sys/ioctl.h> +#include <sys/mman.h> +#include <sys/wait.h>

#include <linux/compiler.h> #include <linux/kernel.h> @@ -202,15 +204,19 @@ static void guest_test_explicit_conversion(uint64_t base_gpa, bool do_fallocate) guest_sync_shared(gpa, size, p3, p4); memcmp_g(gpa, p4, size);

- /* Reset the shared memory back to the initial pattern. */ - memset((void *)gpa, init_p, size); - /* * Free (via PUNCH_HOLE) *all* private memory so that the next * iteration starts from a clean slate, e.g. with respect to * whether or not there are pages/folios in guest_mem. */ guest_map_shared(base_gpa, PER_CPU_DATA_SIZE, true); + + /* + * Reset the entire block back to the initial pattern. Do this + * after fallocate(PUNCH_HOLE) because hole-punching zeroes + * memory. + */ + memset((void *)base_gpa, init_p, PER_CPU_DATA_SIZE); } }

@@ -286,7 +292,8 @@ static void guest_code(uint64_t base_gpa) GUEST_DONE(); }

-static void handle_exit_hypercall(struct kvm_vcpu *vcpu) +static void handle_exit_hypercall(struct kvm_vcpu *vcpu, + bool back_shared_memory_with_guest_memfd) { struct kvm_run *run = vcpu->run; uint64_t gpa = run->hypercall.args[0]; @@ -303,17 +310,46 @@ static void handle_exit_hypercall(struct kvm_vcpu *vcpu) if (do_fallocate) vm_guest_mem_fallocate(vm, gpa, size, map_shared);

- if (set_attributes) + if (set_attributes) { + if (back_shared_memory_with_guest_memfd && !map_shared) + vm_guest_mem_madvise(vm, gpa, size, MADV_DONTNEED); vm_set_memory_attributes(vm, gpa, size, map_shared ? 0 : KVM_MEMORY_ATTRIBUTE_PRIVATE); + } run->hypercall.ret = 0; }

+static void assert_not_faultable(uint8_t *address) +{ + pid_t child_pid; + + child_pid = fork(); + TEST_ASSERT(child_pid != -1, "fork failed"); + + if (child_pid == 0) { + *address = 'A'; + } else { + int status; + waitpid(child_pid, &status, 0); + + TEST_ASSERT(WIFSIGNALED(status), + "Child should have exited with a signal"); + TEST_ASSERT_EQ(WTERMSIG(status), SIGBUS); + } +} + static bool run_vcpus;

-static void *__test_mem_conversions(void *__vcpu) +struct test_thread_args { - struct kvm_vcpu *vcpu = __vcpu; + struct kvm_vcpu *vcpu; + bool back_shared_memory_with_guest_memfd; +}; + +static void *__test_mem_conversions(void *params) +{ + struct test_thread_args *args = params; + struct kvm_vcpu *vcpu = args->vcpu; struct kvm_run *run = vcpu->run; struct kvm_vm *vm = vcpu->vm; struct ucall uc; @@ -325,7 +361,8 @@ static void *__test_mem_conversions(void *__vcpu) vcpu_run(vcpu);

if (run->exit_reason == KVM_EXIT_HYPERCALL) { - handle_exit_hypercall(vcpu); + handle_exit_hypercall(vcpu, + args->back_shared_memory_with_guest_memfd); continue; }

@@ -349,8 +386,18 @@ static void *__test_mem_conversions(void *__vcpu) size_t nr_bytes = min_t(size_t, vm->page_size, size - i); uint8_t *hva = addr_gpa2hva(vm, gpa + i);

- /* In all cases, the host should observe the shared data. */ - memcmp_h(hva, gpa + i, uc.args[3], nr_bytes); + /* Check contents of memory */ + if (args->back_shared_memory_with_guest_memfd && + uc.args[0] == SYNC_PRIVATE) { + assert_not_faultable(hva); + } else { + /* + * If shared and private memory use + * separate backing memory, the host + * should always observe shared data. + */ + memcmp_h(hva, gpa + i, uc.args[3], nr_bytes); + }

/* For shared, write the new pattern to guest memory. */ if (uc.args[0] == SYNC_SHARED) @@ -366,11 +413,41 @@ static void *__test_mem_conversions(void *__vcpu) } }

-static void -test_mem_conversions(enum vm_mem_backing_src_type src_type, - enum vm_private_mem_backing_src_type private_mem_src_type, - uint32_t nr_vcpus, - uint32_t nr_memslots) +static void add_memslot(struct kvm_vm *vm, uint64_t gpa, uint32_t slot, + uint64_t size, int guest_memfd, + uint64_t guest_memfd_offset, + enum vm_mem_backing_src_type src_type, + bool back_shared_memory_with_guest_memfd) +{ + struct userspace_mem_region *region; + + if (!back_shared_memory_with_guest_memfd) { + vm_mem_add(vm, src_type, gpa, slot, size / vm->page_size, + KVM_MEM_GUEST_MEMFD, guest_memfd, + guest_memfd_offset); + return; + } + + region = vm_mem_region_alloc(vm); + + guest_memfd = vm_mem_region_install_guest_memfd(region, guest_memfd); + + vm_mem_region_mmap(region, size, MAP_SHARED, guest_memfd, guest_memfd_offset); + vm_mem_region_install_memory(region, size, getpagesize()); + + region->region.slot = slot; + region->region.flags = KVM_MEM_GUEST_MEMFD; + region->region.guest_phys_addr = gpa; + region->region.guest_memfd_offset = guest_memfd_offset; + + vm_mem_region_add(vm, region); +} + +static void test_mem_conversions(enum vm_mem_backing_src_type src_type, + enum vm_private_mem_backing_src_type private_mem_src_type, + uint32_t nr_vcpus, + uint32_t nr_memslots, + bool back_shared_memory_with_guest_memfd) { /* * Allocate enough memory so that each vCPU's chunk of memory can be @@ -381,6 +458,7 @@ test_mem_conversions(enum vm_mem_backing_src_type src_type, get_private_mem_backing_src_pagesz(private_mem_src_type), get_backing_src_pagesz(src_type))); const size_t per_cpu_size = align_up(PER_CPU_DATA_SIZE, alignment); + struct test_thread_args *thread_args[KVM_MAX_VCPUS]; const size_t memfd_size = per_cpu_size * nr_vcpus; const size_t slot_size = memfd_size / nr_memslots; struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; @@ -404,13 +482,14 @@ test_mem_conversions(enum vm_mem_backing_src_type src_type, vm, memfd_size, vm_private_mem_backing_src_alias(private_mem_src_type)->flag);

- for (i = 0; i < nr_memslots; i++) - vm_mem_add(vm, src_type, BASE_DATA_GPA + slot_size * i, - BASE_DATA_SLOT + i, slot_size / vm->page_size, - KVM_MEM_GUEST_MEMFD, memfd, slot_size * i); + for (i = 0; i < nr_memslots; i++) { + add_memslot(vm, BASE_DATA_GPA + slot_size * i, + BASE_DATA_SLOT + i, slot_size, memfd, slot_size * i, + src_type, back_shared_memory_with_guest_memfd); + }

for (i = 0; i < nr_vcpus; i++) { - uint64_t gpa = BASE_DATA_GPA + i * per_cpu_size; + uint64_t gpa = BASE_DATA_GPA + i * per_cpu_size;

vcpu_args_set(vcpus[i], 1, gpa);

@@ -420,13 +499,23 @@ test_mem_conversions(enum vm_mem_backing_src_type src_type, */ virt_map(vm, gpa, gpa, PER_CPU_DATA_SIZE / vm->page_size);

- pthread_create(&threads[i], NULL, __test_mem_conversions, vcpus[i]); + thread_args[i] = malloc(sizeof(struct test_thread_args)); + TEST_ASSERT(thread_args[i] != NULL, + "Could not allocate memory for thread parameters"); + thread_args[i]->vcpu = vcpus[i]; + thread_args[i]->back_shared_memory_with_guest_memfd = + back_shared_memory_with_guest_memfd; + + pthread_create(&threads[i], NULL, __test_mem_conversions, + (void *)thread_args[i]); }

WRITE_ONCE(run_vcpus, true);

- for (i = 0; i < nr_vcpus; i++) + for (i = 0; i < nr_vcpus; i++) { pthread_join(threads[i], NULL); + free(thread_args[i]); + }

kvm_vm_free(vm);

@@ -448,7 +537,7 @@ test_mem_conversions(enum vm_mem_backing_src_type src_type, static void usage(const char *cmd) { puts(""); - printf("usage: %s [-h] [-m nr_memslots] [-s mem_type] [-p private_mem_type] [-n nr_vcpus]\n", cmd); + printf("usage: %s [-h] [-m nr_memslots] [-s mem_type] [-p private_mem_type] [-n nr_vcpus] [-g]\n", cmd); puts(""); backing_src_help("-s"); puts(""); @@ -458,19 +547,22 @@ static void usage(const char *cmd) puts(""); puts(" -m: specify the number of memslots (default: 1)"); puts(""); + puts(" -g: back shared memory with guest_memfd (default: false)"); + puts(""); }

int main(int argc, char *argv[]) { enum vm_mem_backing_src_type src_type = DEFAULT_VM_MEM_SRC; enum vm_private_mem_backing_src_type private_mem_src_type = DEFAULT_VM_PRIVATE_MEM_SRC; + bool back_shared_memory_with_guest_memfd = false; uint32_t nr_memslots = 1; uint32_t nr_vcpus = 1; int opt;

TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM));

- while ((opt = getopt(argc, argv, "hm:s:p:n:")) != -1) { + while ((opt = getopt(argc, argv, "hgm:s:p:n:")) != -1) { switch (opt) { case 's': src_type = parse_backing_src_type(optarg); @@ -484,6 +576,9 @@ int main(int argc, char *argv[]) case 'm': nr_memslots = atoi_positive("nr_memslots", optarg); break; + case 'g': + back_shared_memory_with_guest_memfd = true; + break; case 'h': default: usage(argv[0]); @@ -491,7 +586,8 @@ int main(int argc, char *argv[]) } }

- test_mem_conversions(src_type, private_mem_src_type, nr_vcpus, nr_memslots); + test_mem_conversions(src_type, private_mem_src_type, nr_vcpus, nr_memslots, + back_shared_memory_with_guest_memfd);

return 0; } diff --git a/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.sh b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.sh index fb6705fef466..c7f3dfee0336 100755 --- a/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.sh +++ b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.sh @@ -75,6 +75,9 @@ TEST_EXECUTABLE="$(dirname "$0")/private_mem_conversions_test" $TEST_EXECUTABLE -s "$src_type" -p "$private_mem_src_type" -n $num_vcpus_to_test $TEST_EXECUTABLE -s "$src_type" -p "$private_mem_src_type" -n $num_vcpus_to_test -m $num_memslots_to_test

+ $TEST_EXECUTABLE -s "$src_type" -p "$private_mem_src_type" -n $num_vcpus_to_test -g + $TEST_EXECUTABLE -s "$src_type" -p "$private_mem_src_type" -n $num_vcpus_to_test -m $num_memslots_to_test -g + { set +x; } 2>/dev/null

echo