With the addition of the chg parameter, vma_has_reserves() no longer just determines whether the vma has reserves.

The comment in the vma->vm_flags & VM_NORESERVE block indicates that this function actually computes whether or not the reserved count should be decremented.

This refactoring also takes into account the allocation's request parameter avoid_reserve, which helps to further simplify the calling function alloc_hugetlb_folio().

Signed-off-by: Ackerley Tng ackerleytng@google.com --- mm/hugetlb.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/mm/hugetlb.c b/mm/hugetlb.c index af5c6bbc9ff0..597102ed224b 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1245,9 +1245,19 @@ void clear_vma_resv_huge_pages(struct vm_area_struct *vma) hugetlb_dup_vma_private(vma); }

-/* Returns true if the VMA has associated reserve pages */ -static bool vma_has_reserves(struct vm_area_struct *vma, long chg) +/* + * Returns true if this allocation should use (debit) hstate reservations, based on + * + * @vma: VMA config + * @chg: Whether the page requirement can be satisfied using subpool reservations + * @avoid_reserve: Whether allocation was requested to avoid using reservations + */ +static bool should_use_hstate_resv(struct vm_area_struct *vma, long chg, + bool avoid_reserve) { + if (avoid_reserve) + return false; + if (vma->vm_flags & VM_NORESERVE) { /* * This address is already reserved by other process(chg == 0), @@ -3182,7 +3192,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, if (ret) goto out_uncharge_cgroup_reservation;

- use_hstate_resv = !avoid_reserve && vma_has_reserves(vma, gbl_chg); + use_hstate_resv = should_use_hstate_resv(vma, gbl_chg, avoid_reserve);

spin_lock_irq(&hugetlb_lock); folio = dequeue_hugetlb_folio_vma(h, vma, addr, use_hstate_resv);

This was refactored out of huge_node().

huge_node()'s interpretation of vma for order assumes the hugetlb-specific storage of the hstate information in the inode. policy_node_nodemask() does not assume that, and can be used more generically.

This refactoring also enforces that nid default to the current node id, which was not previously enforced.

alloc_pages_mpol_noprof() is the last remaining direct user of policy_nodemask(). All its callers begin with nid being the current node id as well. More refactoring is required for to simplify that.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- include/linux/mempolicy.h | 2 ++ mm/mempolicy.c | 36 ++++++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h index 1add16f21612..a49631e47421 100644 --- a/include/linux/mempolicy.h +++ b/include/linux/mempolicy.h @@ -138,6 +138,8 @@ extern void numa_policy_init(void); extern void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new); extern void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new);

+extern int policy_node_nodemask(struct mempolicy *mpol, gfp_t gfp_flags, + pgoff_t ilx, nodemask_t **nodemask); extern int huge_node(struct vm_area_struct *vma, unsigned long addr, gfp_t gfp_flags, struct mempolicy **mpol, nodemask_t **nodemask); diff --git a/mm/mempolicy.c b/mm/mempolicy.c index b858e22b259d..f3e572e17775 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -1212,7 +1212,6 @@ static struct folio *alloc_migration_target_by_mpol(struct folio *src, struct mempolicy *pol = mmpol->pol; pgoff_t ilx = mmpol->ilx; unsigned int order; - int nid = numa_node_id(); gfp_t gfp;

order = folio_order(src); @@ -1221,10 +1220,11 @@ static struct folio *alloc_migration_target_by_mpol(struct folio *src, if (folio_test_hugetlb(src)) { nodemask_t *nodemask; struct hstate *h; + int nid;

h = folio_hstate(src); gfp = htlb_alloc_mask(h); - nodemask = policy_nodemask(gfp, pol, ilx, &nid); + nid = policy_node_nodemask(pol, gfp, ilx, &nodemask); return alloc_hugetlb_folio_nodemask(h, nid, nodemask, gfp, htlb_allow_alloc_fallback(MR_MEMPOLICY_MBIND)); } @@ -1234,7 +1234,7 @@ static struct folio *alloc_migration_target_by_mpol(struct folio *src, else gfp = GFP_HIGHUSER_MOVABLE | __GFP_RETRY_MAYFAIL | __GFP_COMP;

- return folio_alloc_mpol(gfp, order, pol, ilx, nid); + return folio_alloc_mpol(gfp, order, pol, ilx, numa_node_id()); } #else

@@ -2084,6 +2084,27 @@ static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *pol, return nodemask; }

+/** + * policy_node_nodemask(@mpol, @gfp_flags, @ilx, @nodemask) + * @mpol: the memory policy to interpret. Reference must be taken. + * @gfp_flags: for this request + * @ilx: interleave index, for use only when MPOL_INTERLEAVE or + * MPOL_WEIGHTED_INTERLEAVE + * @nodemask: (output) pointer to nodemask pointer for 'bind' and 'prefer-many' + * policy + * + * Returns a nid suitable for a page allocation and a pointer. If the effective + * policy is 'bind' or 'prefer-many', returns a pointer to the mempolicy's + * @nodemask for filtering the zonelist. + */ +int policy_node_nodemask(struct mempolicy *mpol, gfp_t gfp_flags, + pgoff_t ilx, nodemask_t **nodemask) +{ + int nid = numa_node_id(); + *nodemask = policy_nodemask(gfp_flags, mpol, ilx, &nid); + return nid; +} + #ifdef CONFIG_HUGETLBFS /* * huge_node(@vma, @addr, @gfp_flags, @mpol) @@ -2102,12 +2123,8 @@ int huge_node(struct vm_area_struct *vma, unsigned long addr, gfp_t gfp_flags, struct mempolicy **mpol, nodemask_t **nodemask) { pgoff_t ilx; - int nid; - - nid = numa_node_id(); *mpol = get_vma_policy(vma, addr, hstate_vma(vma)->order, &ilx); - *nodemask = policy_nodemask(gfp_flags, *mpol, ilx, &nid); - return nid; + return policy_node_nodemask(*mpol, gfp_flags, ilx, nodemask); }

/* @@ -2549,8 +2566,7 @@ unsigned long alloc_pages_bulk_array_mempolicy_noprof(gfp_t gfp, return alloc_pages_bulk_array_preferred_many(gfp, numa_node_id(), pol, nr_pages, page_array);

- nid = numa_node_id(); - nodemask = policy_nodemask(gfp, pol, NO_INTERLEAVE_INDEX, &nid); + nid = policy_node_nodemask(pol, gfp, NO_INTERLEAVE_INDEX, &nodemask); return alloc_pages_bulk_noprof(gfp, nid, nodemask, nr_pages, NULL, page_array); }

Reducing dependence on vma avoids the hugetlb-specific assumption of where the mempolicy is stored. This will open up other ways of using hugetlb.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- mm/hugetlb.c | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-)

diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 5cf7fb117e9d..2f2bd2444ae2 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -2536,32 +2536,31 @@ static struct folio *alloc_migrate_hugetlb_folio(struct hstate *h, gfp_t gfp_mas }

/* - * Use the VMA's mpolicy to allocate a huge page from the buddy. + * Allocate a huge page from the buddy allocator, given memory policy, node id + * and nodemask. */ -static -struct folio *alloc_buddy_hugetlb_folio_with_mpol(struct hstate *h, - struct vm_area_struct *vma, unsigned long addr) +static struct folio *alloc_buddy_hugetlb_folio_from_node(struct hstate *h, + struct mempolicy *mpol, + int nid, + nodemask_t *nodemask) { - struct folio *folio = NULL; - struct mempolicy *mpol; gfp_t gfp_mask = htlb_alloc_mask(h); - int nid; - nodemask_t *nodemask; + struct folio *folio = NULL;

- nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask); if (mpol_is_preferred_many(mpol)) { gfp_t gfp = gfp_mask | __GFP_NOWARN;

gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); folio = alloc_surplus_hugetlb_folio(h, gfp, nid, nodemask); + }

- /* Fallback to all nodes if page==NULL */ + if (!folio) { + /* Fallback to all nodes if earlier allocation failed */ nodemask = NULL; - }

- if (!folio) folio = alloc_surplus_hugetlb_folio(h, gfp_mask, nid, nodemask); - mpol_cond_put(mpol); + } + return folio; }

@@ -3187,8 +3186,18 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, spin_lock_irq(&hugetlb_lock); folio = dequeue_hugetlb_folio_vma(h, vma, addr, use_hstate_resv); if (!folio) { + struct mempolicy *mpol; + nodemask_t *nodemask; + pgoff_t ilx; + int nid; + spin_unlock_irq(&hugetlb_lock); - folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr); + + mpol = get_vma_policy(vma, addr, hstate_vma(vma)->order, &ilx); + nid = policy_node_nodemask(mpol, htlb_alloc_mask(h), ilx, &nodemask); + folio = alloc_buddy_hugetlb_folio_from_node(h, mpol, nid, nodemask); + mpol_cond_put(mpol); + if (!folio) goto out_uncharge_cgroup; spin_lock_irq(&hugetlb_lock);

hugetlb_alloc_folio() allocates a hugetlb folio without handling reservations in the vma and subpool, since some of that reservation concepts are hugetlbfs specific.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- include/linux/hugetlb.h | 12 ++++ mm/hugetlb.c | 144 ++++++++++++++++++++++++---------------- 2 files changed, 98 insertions(+), 58 deletions(-)

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index c9bf68c239a0..e4a05a421623 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -690,6 +690,10 @@ struct huge_bootmem_page { };

int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list); +struct folio *hugetlb_alloc_folio(struct hstate *h, struct mempolicy *mpol, + int nid, nodemask_t *nodemask, + bool charge_cgroup_reservation, + bool use_hstate_resv); struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, int avoid_reserve); struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, @@ -1027,6 +1031,14 @@ static inline int isolate_or_dissolve_huge_page(struct page *page, return -ENOMEM; }

+static inline struct folio * +hugetlb_alloc_folio(struct hstate *h, struct mempolicy *mpol, int nid, + nodemask_t *nodemask, bool charge_cgroup_reservation, + bool use_hstate_resv) +{ + return NULL; +} + static inline struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, int avoid_reserve) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index e341bc0eb49a..7e73ebcc0f26 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3106,6 +3106,75 @@ int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list) return ret; }

+/** + * Allocates a hugetlb folio either by dequeueing or from buddy allocator. + */ +struct folio *hugetlb_alloc_folio(struct hstate *h, struct mempolicy *mpol, + int nid, nodemask_t *nodemask, + bool charge_cgroup_reservation, + bool use_hstate_resv) +{ + struct hugetlb_cgroup *h_cg = NULL; + struct folio *folio; + int ret; + int idx; + + idx = hstate_index(h); + + if (charge_cgroup_reservation) { + ret = hugetlb_cgroup_charge_cgroup_rsvd( + idx, pages_per_huge_page(h), &h_cg); + if (ret) + return NULL; + } + + ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); + if (ret) + goto err_uncharge_cgroup_reservation; + + spin_lock_irq(&hugetlb_lock); + + folio = dequeue_hugetlb_folio(h, mpol, nid, nodemask, use_hstate_resv); + if (!folio) { + spin_unlock_irq(&hugetlb_lock); + + folio = alloc_buddy_hugetlb_folio_from_node(h, mpol, nid, nodemask); + if (!folio) + goto err_uncharge_cgroup; + + spin_lock_irq(&hugetlb_lock); + if (use_hstate_resv) { + folio_set_hugetlb_restore_reserve(folio); + h->resv_huge_pages--; + } + list_add(&folio->lru, &h->hugepage_activelist); + folio_ref_unfreeze(folio, 1); + /* Fall through */ + } + + hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, folio); + + if (charge_cgroup_reservation) { + hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h), + h_cg, folio); + } + + spin_unlock_irq(&hugetlb_lock); + + return folio; + +err_uncharge_cgroup: + hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); + +err_uncharge_cgroup_reservation: + if (charge_cgroup_reservation) { + hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h), + h_cg); + } + + return NULL; +} + struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, int avoid_reserve) { @@ -3114,11 +3183,10 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, struct folio *folio; long map_chg, map_commit, nr_pages = pages_per_huge_page(h); long gbl_chg; - int memcg_charge_ret, ret, idx; - struct hugetlb_cgroup *h_cg = NULL; + int memcg_charge_ret; struct mem_cgroup *memcg; - bool deferred_reserve; - gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL; + bool charge_cgroup_reservation; + gfp_t gfp = htlb_alloc_mask(h); bool use_hstate_resv; struct mempolicy *mpol; nodemask_t *nodemask; @@ -3126,13 +3194,14 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, int nid;

memcg = get_mem_cgroup_from_current(); - memcg_charge_ret = mem_cgroup_hugetlb_try_charge(memcg, gfp, nr_pages); + memcg_charge_ret = + mem_cgroup_hugetlb_try_charge(memcg, gfp | __GFP_RETRY_MAYFAIL, + nr_pages); if (memcg_charge_ret == -ENOMEM) { mem_cgroup_put(memcg); return ERR_PTR(-ENOMEM); }

- idx = hstate_index(h); /* * Examine the region/reserve map to determine if the process * has a reservation for the page to be allocated. A return @@ -3160,57 +3229,22 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,

}

- /* If this allocation is not consuming a reservation, charge it now. - */ - deferred_reserve = map_chg || avoid_reserve; - if (deferred_reserve) { - ret = hugetlb_cgroup_charge_cgroup_rsvd( - idx, pages_per_huge_page(h), &h_cg); - if (ret) - goto out_subpool_put; - } - - ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg); - if (ret) - goto out_uncharge_cgroup_reservation; - use_hstate_resv = should_use_hstate_resv(vma, gbl_chg, avoid_reserve);

- spin_lock_irq(&hugetlb_lock); + /* + * charge_cgroup_reservation if this allocation is not consuming a + * reservation + */ + charge_cgroup_reservation = map_chg || avoid_reserve;

mpol = get_vma_policy(vma, addr, hstate_vma(vma)->order, &ilx); - nid = policy_node_nodemask(mpol, htlb_alloc_mask(h), ilx, &nodemask); - folio = dequeue_hugetlb_folio(h, mpol, nid, nodemask, use_hstate_resv); - if (!folio) { - spin_unlock_irq(&hugetlb_lock); - - folio = alloc_buddy_hugetlb_folio_from_node(h, mpol, nid, nodemask); - if (!folio) { - mpol_cond_put(mpol); - goto out_uncharge_cgroup; - } - - spin_lock_irq(&hugetlb_lock); - if (use_hstate_resv) { - folio_set_hugetlb_restore_reserve(folio); - h->resv_huge_pages--; - } - list_add(&folio->lru, &h->hugepage_activelist); - folio_ref_unfreeze(folio, 1); - /* Fall through */ - } + nid = policy_node_nodemask(mpol, gfp, ilx, &nodemask); + folio = hugetlb_alloc_folio(h, mpol, nid, nodemask, + charge_cgroup_reservation, use_hstate_resv); mpol_cond_put(mpol);

- hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, folio); - /* If allocation is not consuming a reservation, also store the - * hugetlb_cgroup pointer on the page. - */ - if (deferred_reserve) { - hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h), - h_cg, folio); - } - - spin_unlock_irq(&hugetlb_lock); + if (!folio) + goto out_subpool_put;

hugetlb_set_folio_subpool(folio, spool);

@@ -3229,7 +3263,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,

rsv_adjust = hugepage_subpool_put_pages(spool, 1); hugetlb_acct_memory(h, -rsv_adjust); - if (deferred_reserve) { + if (charge_cgroup_reservation) { spin_lock_irq(&hugetlb_lock); hugetlb_cgroup_uncharge_folio_rsvd(hstate_index(h), pages_per_huge_page(h), folio); @@ -3243,12 +3277,6 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,

return folio;

-out_uncharge_cgroup: - hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg); -out_uncharge_cgroup_reservation: - if (deferred_reserve) - hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h), - h_cg); out_subpool_put: if (map_chg || avoid_reserve) hugepage_subpool_put_pages(spool, 1);

This will allow hugetlb subpools to be used by guest_memfd.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- include/linux/hugetlb.h | 3 +++ mm/hugetlb.c | 6 ++---- 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index e4a05a421623..907cfbbd9e24 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -119,6 +119,9 @@ struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, long min_hpages); void hugepage_put_subpool(struct hugepage_subpool *spool);

+long hugepage_subpool_get_pages(struct hugepage_subpool *spool, long delta); +long hugepage_subpool_put_pages(struct hugepage_subpool *spool, long delta); + void hugetlb_dup_vma_private(struct vm_area_struct *vma); void clear_vma_resv_huge_pages(struct vm_area_struct *vma); int move_hugetlb_page_tables(struct vm_area_struct *vma, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 7e73ebcc0f26..808915108126 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -170,8 +170,7 @@ void hugepage_put_subpool(struct hugepage_subpool *spool) * only be different than the passed value (delta) in the case where * a subpool minimum size must be maintained. */ -static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, - long delta) +long hugepage_subpool_get_pages(struct hugepage_subpool *spool, long delta) { long ret = delta;

@@ -215,8 +214,7 @@ static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, * The return value may only be different than the passed value (delta) * in the case where a subpool minimum size must be maintained. */ -static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, - long delta) +long hugepage_subpool_put_pages(struct hugepage_subpool *spool, long delta) { long ret = delta; unsigned long flags;

This will used by guest_memfd in a later patch.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- include/linux/hugetlb.h | 2 ++ mm/hugetlb.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 9ef1adbd3207..4d47bf94c211 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -122,6 +122,8 @@ void hugepage_put_subpool(struct hugepage_subpool *spool); long hugepage_subpool_get_pages(struct hugepage_subpool *spool, long delta); long hugepage_subpool_put_pages(struct hugepage_subpool *spool, long delta);

+int hugetlb_acct_memory(struct hstate *h, long delta); + void hugetlb_dup_vma_private(struct vm_area_struct *vma); void clear_vma_resv_huge_pages(struct vm_area_struct *vma); int move_hugetlb_page_tables(struct vm_area_struct *vma, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index efdb5772b367..5a37b03e1361 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -93,7 +93,7 @@ struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;

/* Forward declaration */ static int __hugetlb_acct_memory(struct hstate *h, long delta, bool use_surplus); -static int hugetlb_acct_memory(struct hstate *h, long delta); +int hugetlb_acct_memory(struct hstate *h, long delta); static void hugetlb_vma_lock_free(struct vm_area_struct *vma); static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma); static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma); @@ -5170,7 +5170,7 @@ static int __hugetlb_acct_memory(struct hstate *h, long delta, bool use_surplus) return ret; }

-static int hugetlb_acct_memory(struct hstate *h, long delta) +int hugetlb_acct_memory(struct hstate *h, long delta) { return __hugetlb_acct_memory(h, delta, true); }

Using guest mem inodes allows us to store metadata for the backing memory on the inode. Metadata will be added in a later patch to support HugeTLB pages.

Metadata about backing memory should not be stored on the file, since the file represents a guest_memfd's binding with a struct kvm, and metadata about backing memory is not unique to a specific binding and struct kvm.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- include/uapi/linux/magic.h | 1 + virt/kvm/guest_memfd.c | 119 ++++++++++++++++++++++++++++++------- 2 files changed, 100 insertions(+), 20 deletions(-)

diff --git a/include/uapi/linux/magic.h b/include/uapi/linux/magic.h index bb575f3ab45e..169dba2a6920 100644 --- a/include/uapi/linux/magic.h +++ b/include/uapi/linux/magic.h @@ -103,5 +103,6 @@ #define DEVMEM_MAGIC 0x454d444d /* "DMEM" */ #define SECRETMEM_MAGIC 0x5345434d /* "SECM" */ #define PID_FS_MAGIC 0x50494446 /* "PIDF" */ +#define GUEST_MEMORY_MAGIC 0x474d454d /* "GMEM" */

#endif /* __LINUX_MAGIC_H__ */ diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 8f079a61a56d..5d7fd1f708a6 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -1,12 +1,17 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/fs.h> +#include <linux/mount.h> #include <linux/backing-dev.h> #include <linux/falloc.h> #include <linux/kvm_host.h> +#include <linux/pseudo_fs.h> #include <linux/pagemap.h> #include <linux/anon_inodes.h>

#include "kvm_mm.h"

+static struct vfsmount *kvm_gmem_mnt; + struct kvm_gmem { struct kvm *kvm; struct xarray bindings; @@ -302,6 +307,38 @@ static inline struct file *kvm_gmem_get_file(struct kvm_memory_slot *slot) return get_file_active(&slot->gmem.file); }

+static const struct super_operations kvm_gmem_super_operations = { + .statfs = simple_statfs, +}; + +static int kvm_gmem_init_fs_context(struct fs_context *fc) +{ + struct pseudo_fs_context *ctx; + + if (!init_pseudo(fc, GUEST_MEMORY_MAGIC)) + return -ENOMEM; + + ctx = fc->fs_private; + ctx->ops = &kvm_gmem_super_operations; + + return 0; +} + +static struct file_system_type kvm_gmem_fs = { + .name = "kvm_guest_memory", + .init_fs_context = kvm_gmem_init_fs_context, + .kill_sb = kill_anon_super, +}; + +static void kvm_gmem_init_mount(void) +{ + kvm_gmem_mnt = kern_mount(&kvm_gmem_fs); + BUG_ON(IS_ERR(kvm_gmem_mnt)); + + /* For giggles. Userspace can never map this anyways. */ + kvm_gmem_mnt->mnt_flags |= MNT_NOEXEC; +} + static struct file_operations kvm_gmem_fops = { .open = generic_file_open, .release = kvm_gmem_release, @@ -311,6 +348,8 @@ static struct file_operations kvm_gmem_fops = { void kvm_gmem_init(struct module *module) { kvm_gmem_fops.owner = module; + + kvm_gmem_init_mount(); }

static int kvm_gmem_migrate_folio(struct address_space *mapping, @@ -392,11 +431,67 @@ static const struct inode_operations kvm_gmem_iops = { .setattr = kvm_gmem_setattr, };

+static struct inode *kvm_gmem_inode_make_secure_inode(const char *name, + loff_t size, u64 flags) +{ + const struct qstr qname = QSTR_INIT(name, strlen(name)); + struct inode *inode; + int err; + + inode = alloc_anon_inode(kvm_gmem_mnt->mnt_sb); + if (IS_ERR(inode)) + return inode; + + err = security_inode_init_security_anon(inode, &qname, NULL); + if (err) { + iput(inode); + return ERR_PTR(err); + } + + inode->i_private = (void *)(unsigned long)flags; + inode->i_op = &kvm_gmem_iops; + inode->i_mapping->a_ops = &kvm_gmem_aops; + inode->i_mode |= S_IFREG; + inode->i_size = size; + mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); + mapping_set_inaccessible(inode->i_mapping); + /* Unmovable mappings are supposed to be marked unevictable as well. */ + WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping)); + + return inode; +} + +static struct file *kvm_gmem_inode_create_getfile(void *priv, loff_t size, + u64 flags) +{ + static const char *name = "[kvm-gmem]"; + struct inode *inode; + struct file *file; + + if (kvm_gmem_fops.owner && !try_module_get(kvm_gmem_fops.owner)) + return ERR_PTR(-ENOENT); + + inode = kvm_gmem_inode_make_secure_inode(name, size, flags); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + file = alloc_file_pseudo(inode, kvm_gmem_mnt, name, O_RDWR, + &kvm_gmem_fops); + if (IS_ERR(file)) { + iput(inode); + return file; + } + + file->f_mapping = inode->i_mapping; + file->f_flags |= O_LARGEFILE; + file->private_data = priv; + + return file; +} + static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) { - const char *anon_name = "[kvm-gmem]"; struct kvm_gmem *gmem; - struct inode *inode; struct file *file; int fd, err;

@@ -410,32 +505,16 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) goto err_fd; }

- file = anon_inode_create_getfile(anon_name, &kvm_gmem_fops, gmem, - O_RDWR, NULL); + file = kvm_gmem_inode_create_getfile(gmem, size, flags); if (IS_ERR(file)) { err = PTR_ERR(file); goto err_gmem; }

- file->f_flags |= O_LARGEFILE; - - inode = file->f_inode; - WARN_ON(file->f_mapping != inode->i_mapping); - - inode->i_private = (void *)(unsigned long)flags; - inode->i_op = &kvm_gmem_iops; - inode->i_mapping->a_ops = &kvm_gmem_aops; - inode->i_mode |= S_IFREG; - inode->i_size = size; - mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); - mapping_set_inaccessible(inode->i_mapping); - /* Unmovable mappings are supposed to be marked unevictable as well. */ - WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping)); - kvm_get_kvm(kvm); gmem->kvm = kvm; xa_init(&gmem->bindings); - list_add(&gmem->entry, &inode->i_mapping->i_private_list); + list_add(&gmem->entry, &file_inode(file)->i_mapping->i_private_list);

fd_install(fd, file); return fd;

If HugeTLB is requested at guest_memfd creation time, HugeTLB pages will be used to back guest_memfd.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- virt/kvm/guest_memfd.c | 252 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 239 insertions(+), 13 deletions(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 31e1115273e1..2e6f12e2bac8 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -8,6 +8,8 @@ #include <linux/pseudo_fs.h> #include <linux/pagemap.h> #include <linux/anon_inodes.h> +#include <linux/memcontrol.h> +#include <linux/mempolicy.h>

#include "kvm_mm.h"

@@ -29,6 +31,13 @@ static struct kvm_gmem_hugetlb *kvm_gmem_hgmem(struct inode *inode) return inode->i_mapping->i_private_data; }

+static bool is_kvm_gmem_hugetlb(struct inode *inode) +{ + u64 flags = (u64)inode->i_private; + + return flags & KVM_GUEST_MEMFD_HUGETLB; +} + /** * folio_file_pfn - like folio_file_page, but return a pfn. * @folio: The folio which contains this index. @@ -58,6 +67,9 @@ static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slo return 0; }

+/** + * Use the uptodate flag to indicate that the folio is prepared for KVM's usage. + */ static inline void kvm_gmem_mark_prepared(struct folio *folio) { folio_mark_uptodate(folio); @@ -72,13 +84,18 @@ static inline void kvm_gmem_mark_prepared(struct folio *folio) static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, struct folio *folio) { - unsigned long nr_pages, i; pgoff_t index; int r;

- nr_pages = folio_nr_pages(folio); - for (i = 0; i < nr_pages; i++) - clear_highpage(folio_page(folio, i)); + if (folio_test_hugetlb(folio)) { + folio_zero_user(folio, folio->index << PAGE_SHIFT); + } else { + unsigned long nr_pages, i; + + nr_pages = folio_nr_pages(folio); + for (i = 0; i < nr_pages; i++) + clear_highpage(folio_page(folio, i)); + }

/* * Preparing huge folios should always be safe, since it should @@ -103,6 +120,174 @@ static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, return r; }

+static int kvm_gmem_get_mpol_node_nodemask(gfp_t gfp_mask, + struct mempolicy **mpol, + nodemask_t **nodemask) +{ + /* + * TODO: mempolicy would probably have to be stored on the inode, use + * task policy for now. + */ + *mpol = get_task_policy(current); + + /* TODO: ignore interleaving (set ilx to 0) for now. */ + return policy_node_nodemask(*mpol, gfp_mask, 0, nodemask); +} + +static struct folio *kvm_gmem_hugetlb_alloc_folio(struct hstate *h, + struct hugepage_subpool *spool) +{ + bool memcg_charge_was_prepared; + struct mem_cgroup *memcg; + struct mempolicy *mpol; + nodemask_t *nodemask; + struct folio *folio; + gfp_t gfp_mask; + int ret; + int nid; + + gfp_mask = htlb_alloc_mask(h); + + memcg = get_mem_cgroup_from_current(); + ret = mem_cgroup_hugetlb_try_charge(memcg, + gfp_mask | __GFP_RETRY_MAYFAIL, + pages_per_huge_page(h)); + if (ret == -ENOMEM) + goto err; + + memcg_charge_was_prepared = ret != -EOPNOTSUPP; + + /* Pages are only to be taken from guest_memfd subpool and nowhere else. */ + if (hugepage_subpool_get_pages(spool, 1)) + goto err_cancel_charge; + + nid = kvm_gmem_get_mpol_node_nodemask(htlb_alloc_mask(h), &mpol, + &nodemask); + /* + * charge_cgroup_reservation is false because we didn't make any cgroup + * reservations when creating the guest_memfd subpool. + * + * use_hstate_resv is true because we reserved from global hstate when + * creating the guest_memfd subpool. + */ + folio = hugetlb_alloc_folio(h, mpol, nid, nodemask, false, true); + mpol_cond_put(mpol); + + if (!folio) + goto err_put_pages; + + hugetlb_set_folio_subpool(folio, spool); + + if (memcg_charge_was_prepared) + mem_cgroup_commit_charge(folio, memcg); + +out: + mem_cgroup_put(memcg); + + return folio; + +err_put_pages: + hugepage_subpool_put_pages(spool, 1); + +err_cancel_charge: + if (memcg_charge_was_prepared) + mem_cgroup_cancel_charge(memcg, pages_per_huge_page(h)); + +err: + folio = ERR_PTR(-ENOMEM); + goto out; +} + +static int kvm_gmem_hugetlb_filemap_add_folio(struct address_space *mapping, + struct folio *folio, pgoff_t index, + gfp_t gfp) +{ + int ret; + + __folio_set_locked(folio); + ret = __filemap_add_folio(mapping, folio, index, gfp, NULL); + if (unlikely(ret)) { + __folio_clear_locked(folio); + return ret; + } + + /* + * In hugetlb_add_to_page_cache(), there is a call to + * folio_clear_hugetlb_restore_reserve(). This is handled when the pages + * are removed from the page cache in unmap_hugepage_range() -> + * __unmap_hugepage_range() by conditionally calling + * folio_set_hugetlb_restore_reserve(). In kvm_gmem_hugetlb's usage of + * hugetlb, there are no VMAs involved, and pages are never taken from + * the surplus, so when pages are freed, the hstate reserve must be + * restored. Hence, this function makes no call to + * folio_clear_hugetlb_restore_reserve(). + */ + + /* mark folio dirty so that it will not be removed from cache/inode */ + folio_mark_dirty(folio); + + return 0; +} + +static struct folio *kvm_gmem_hugetlb_alloc_and_cache_folio(struct inode *inode, + pgoff_t index) +{ + struct kvm_gmem_hugetlb *hgmem; + struct folio *folio; + int ret; + + hgmem = kvm_gmem_hgmem(inode); + folio = kvm_gmem_hugetlb_alloc_folio(hgmem->h, hgmem->spool); + if (IS_ERR(folio)) + return folio; + + /* TODO: Fix index here to be aligned to huge page size. */ + ret = kvm_gmem_hugetlb_filemap_add_folio( + inode->i_mapping, folio, index, htlb_alloc_mask(hgmem->h)); + if (ret) { + folio_put(folio); + return ERR_PTR(ret); + } + + spin_lock(&inode->i_lock); + inode->i_blocks += blocks_per_huge_page(hgmem->h); + spin_unlock(&inode->i_lock); + + return folio; +} + +static struct folio *kvm_gmem_get_hugetlb_folio(struct inode *inode, + pgoff_t index) +{ + struct address_space *mapping; + struct folio *folio; + struct hstate *h; + pgoff_t hindex; + u32 hash; + + h = kvm_gmem_hgmem(inode)->h; + hindex = index >> huge_page_order(h); + mapping = inode->i_mapping; + + /* To lock, we calculate the hash using the hindex and not index. */ + hash = hugetlb_fault_mutex_hash(mapping, hindex); + mutex_lock(&hugetlb_fault_mutex_table[hash]); + + /* + * The filemap is indexed with index and not hindex. Taking lock on + * folio to align with kvm_gmem_get_regular_folio() + */ + folio = filemap_lock_folio(mapping, index); + if (!IS_ERR(folio)) + goto out; + + folio = kvm_gmem_hugetlb_alloc_and_cache_folio(inode, index); +out: + mutex_unlock(&hugetlb_fault_mutex_table[hash]); + + return folio; +} + /* * Returns a locked folio on success. The caller is responsible for * setting the up-to-date flag before the memory is mapped into the guest. @@ -114,8 +299,10 @@ static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, */ static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index) { - /* TODO: Support huge pages. */ - return filemap_grab_folio(inode->i_mapping, index); + if (is_kvm_gmem_hugetlb(inode)) + return kvm_gmem_get_hugetlb_folio(inode, index); + else + return filemap_grab_folio(inode->i_mapping, index); }

static void kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start, @@ -240,6 +427,35 @@ static void kvm_gmem_hugetlb_truncate_folios_range(struct inode *inode, spin_unlock(&inode->i_lock); }

+static void kvm_gmem_hugetlb_truncate_range(struct inode *inode, loff_t lstart, + loff_t lend) +{ + loff_t full_hpage_start; + loff_t full_hpage_end; + unsigned long hsize; + struct hstate *h; + + h = kvm_gmem_hgmem(inode)->h; + hsize = huge_page_size(h); + + full_hpage_start = round_up(lstart, hsize); + full_hpage_end = round_down(lend, hsize); + + if (lstart < full_hpage_start) { + hugetlb_zero_partial_page(h, inode->i_mapping, lstart, + full_hpage_start); + } + + if (full_hpage_end > full_hpage_start) { + kvm_gmem_hugetlb_truncate_folios_range(inode, full_hpage_start, + full_hpage_end); + } + + if (lend > full_hpage_end) { + hugetlb_zero_partial_page(h, inode->i_mapping, full_hpage_end, + lend); + } +}

static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) { @@ -257,7 +473,12 @@ static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) list_for_each_entry(gmem, gmem_list, entry) kvm_gmem_invalidate_begin(gmem, start, end);

- truncate_inode_pages_range(inode->i_mapping, offset, offset + len - 1); + if (is_kvm_gmem_hugetlb(inode)) { + kvm_gmem_hugetlb_truncate_range(inode, offset, offset + len); + } else { + truncate_inode_pages_range(inode->i_mapping, offset, + offset + len - 1); + }

list_for_each_entry(gmem, gmem_list, entry) kvm_gmem_invalidate_end(gmem, start, end); @@ -279,8 +500,15 @@ static long kvm_gmem_allocate(struct inode *inode, loff_t offset, loff_t len)

filemap_invalidate_lock_shared(mapping);

- start = offset >> PAGE_SHIFT; - end = (offset + len) >> PAGE_SHIFT; + if (is_kvm_gmem_hugetlb(inode)) { + unsigned long hsize = huge_page_size(kvm_gmem_hgmem(inode)->h); + + start = round_down(offset, hsize) >> PAGE_SHIFT; + end = round_down(offset + len, hsize) >> PAGE_SHIFT; + } else { + start = offset >> PAGE_SHIFT; + end = (offset + len) >> PAGE_SHIFT; + }

r = 0; for (index = start; index < end; ) { @@ -408,9 +636,7 @@ static void kvm_gmem_hugetlb_teardown(struct inode *inode)

static void kvm_gmem_evict_inode(struct inode *inode) { - u64 flags = (u64)inode->i_private; - - if (flags & KVM_GUEST_MEMFD_HUGETLB) + if (is_kvm_gmem_hugetlb(inode)) kvm_gmem_hugetlb_teardown(inode); else truncate_inode_pages_final(inode->i_mapping); @@ -827,7 +1053,7 @@ __kvm_gmem_get_pfn(struct file *file, struct kvm_memory_slot *slot,

*pfn = folio_file_pfn(folio, index); if (max_order) - *max_order = 0; + *max_order = folio_order(folio);

*is_prepared = folio_test_uptodate(folio); return folio;

When a hugetlb guest_memfd is requested, the requested size should be aligned to the size of the hugetlb page requested.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- virt/kvm/guest_memfd.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 2e6f12e2bac8..eacbfdb950d1 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -909,6 +909,13 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) return err; }

+static inline bool kvm_gmem_hugetlb_page_aligned(u32 flags, u64 value) +{ + int page_size_log = (flags >> KVM_GUEST_MEMFD_HUGE_SHIFT) & KVM_GUEST_MEMFD_HUGE_MASK; + u64 page_size = 1ULL << page_size_log; + return IS_ALIGNED(value, page_size); +} + #define KVM_GUEST_MEMFD_ALL_FLAGS KVM_GUEST_MEMFD_HUGETLB

int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args) @@ -921,12 +928,18 @@ int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args) if (flags & ~(KVM_GUEST_MEMFD_ALL_FLAGS | (KVM_GUEST_MEMFD_HUGE_MASK << KVM_GUEST_MEMFD_HUGE_SHIFT))) return -EINVAL; + + if (!kvm_gmem_hugetlb_page_aligned(flags, size)) + return -EINVAL; } else { if (flags & ~KVM_GUEST_MEMFD_ALL_FLAGS) return -EINVAL; + + if (!PAGE_ALIGNED(size)) + return -EINVAL; }

- if (size <= 0 || !PAGE_ALIGNED(size)) + if (size <= 0) return -EINVAL;

return __kvm_gmem_create(kvm, size, flags);

Adds support for various type of backing sources for private memory (in the sense of confidential computing), similar to the backing sources available for shared memory.

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- .../testing/selftests/kvm/include/test_util.h | 16 ++++ tools/testing/selftests/kvm/lib/test_util.c | 74 +++++++++++++++++++ 2 files changed, 90 insertions(+)

diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index 3e473058849f..011e757d4e2c 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -142,6 +142,16 @@ struct vm_mem_backing_src_alias { uint32_t flag; };

+enum vm_private_mem_backing_src_type { + VM_PRIVATE_MEM_SRC_GUEST_MEM, /* Use default page size */ + VM_PRIVATE_MEM_SRC_HUGETLB, /* Use kernel default page size for hugetlb pages */ + VM_PRIVATE_MEM_SRC_HUGETLB_2MB, + VM_PRIVATE_MEM_SRC_HUGETLB_1GB, + NUM_PRIVATE_MEM_SRC_TYPES, +}; + +#define DEFAULT_VM_PRIVATE_MEM_SRC VM_PRIVATE_MEM_SRC_GUEST_MEM + #define MIN_RUN_DELAY_NS 200000UL

bool thp_configured(void); @@ -152,6 +162,12 @@ size_t get_backing_src_pagesz(uint32_t i); bool is_backing_src_hugetlb(uint32_t i); void backing_src_help(const char *flag); enum vm_mem_backing_src_type parse_backing_src_type(const char *type_name); + +void private_mem_backing_src_help(const char *flag); +enum vm_private_mem_backing_src_type parse_private_mem_backing_src_type(const char *type_name); +const struct vm_mem_backing_src_alias *vm_private_mem_backing_src_alias(uint32_t i); +size_t get_private_mem_backing_src_pagesz(uint32_t i); + long get_run_delay(void);

/* diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c index 8ed0b74ae837..d0a9b5ee0c01 100644 --- a/tools/testing/selftests/kvm/lib/test_util.c +++ b/tools/testing/selftests/kvm/lib/test_util.c @@ -15,6 +15,7 @@ #include <sys/syscall.h> #include <linux/mman.h> #include "linux/kernel.h" +#include <linux/kvm.h>

#include "test_util.h"

@@ -288,6 +289,34 @@ const struct vm_mem_backing_src_alias *vm_mem_backing_src_alias(uint32_t i) return &aliases[i]; }

+const struct vm_mem_backing_src_alias *vm_private_mem_backing_src_alias(uint32_t i) +{ + static const struct vm_mem_backing_src_alias aliases[] = { + [VM_PRIVATE_MEM_SRC_GUEST_MEM] = { + .name = "private_mem_guest_mem", + .flag = 0, + }, + [VM_PRIVATE_MEM_SRC_HUGETLB] = { + .name = "private_mem_hugetlb", + .flag = KVM_GUEST_MEMFD_HUGETLB, + }, + [VM_PRIVATE_MEM_SRC_HUGETLB_2MB] = { + .name = "private_mem_hugetlb_2mb", + .flag = KVM_GUEST_MEMFD_HUGETLB | KVM_GUEST_MEMFD_HUGE_2MB, + }, + [VM_PRIVATE_MEM_SRC_HUGETLB_1GB] = { + .name = "private_mem_hugetlb_1gb", + .flag = KVM_GUEST_MEMFD_HUGETLB | KVM_GUEST_MEMFD_HUGE_1GB, + }, + }; + _Static_assert(ARRAY_SIZE(aliases) == NUM_PRIVATE_MEM_SRC_TYPES, + "Missing new backing private mem src types?"); + + TEST_ASSERT(i < NUM_PRIVATE_MEM_SRC_TYPES, "Private mem backing src type ID %d too big", i); + + return &aliases[i]; +} + #define MAP_HUGE_PAGE_SIZE(x) (1ULL << ((x >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK))

size_t get_backing_src_pagesz(uint32_t i) @@ -308,6 +337,20 @@ size_t get_backing_src_pagesz(uint32_t i) } }

+size_t get_private_mem_backing_src_pagesz(uint32_t i) +{ + uint32_t flag = vm_private_mem_backing_src_alias(i)->flag; + + switch (i) { + case VM_PRIVATE_MEM_SRC_GUEST_MEM: + return getpagesize(); + case VM_PRIVATE_MEM_SRC_HUGETLB: + return get_def_hugetlb_pagesz(); + default: + return MAP_HUGE_PAGE_SIZE(flag); + } +} + bool is_backing_src_hugetlb(uint32_t i) { return !!(vm_mem_backing_src_alias(i)->flag & MAP_HUGETLB); @@ -344,6 +387,37 @@ enum vm_mem_backing_src_type parse_backing_src_type(const char *type_name) return -1; }

+static void print_available_private_mem_backing_src_types(const char *prefix) +{ + int i; + + printf("%sAvailable private mem backing src types:\n", prefix); + + for (i = 0; i < NUM_PRIVATE_MEM_SRC_TYPES; i++) + printf("%s %s\n", prefix, vm_private_mem_backing_src_alias(i)->name); +} + +void private_mem_backing_src_help(const char *flag) +{ + printf(" %s: specify the type of memory that should be used to\n" + " back guest private memory. (default: %s)\n", + flag, vm_private_mem_backing_src_alias(DEFAULT_VM_PRIVATE_MEM_SRC)->name); + print_available_private_mem_backing_src_types(" "); +} + +enum vm_private_mem_backing_src_type parse_private_mem_backing_src_type(const char *type_name) +{ + int i; + + for (i = 0; i < NUM_PRIVATE_MEM_SRC_TYPES; i++) + if (!strcmp(type_name, vm_private_mem_backing_src_alias(i)->name)) + return i; + + print_available_private_mem_backing_src_types(""); + TEST_FAIL("Unknown private mem backing src type: %s", type_name); + return -1; +} + long get_run_delay(void) { char path[64];

Add private_mem_conversions_test.sh to automate testing of different combinations of private_mem_conversions_test.

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- .../x86_64/private_mem_conversions_test.sh | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100755 tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.sh

diff --git a/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.sh b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.sh new file mode 100755 index 000000000000..fb6705fef466 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-only */ +# +# Wrapper script which runs different test setups of +# private_mem_conversions_test. +# +# tools/testing/selftests/kvm/private_mem_conversions_test.sh +# Copyright (C) 2023, Google LLC. + +set -e + +num_vcpus_to_test=4 +num_memslots_to_test=$num_vcpus_to_test + +get_default_hugepage_size_in_kB() { + grep "Hugepagesize:" /proc/meminfo | grep -o '[[:digit:]]+' +} + +# Required pages are based on the test setup (see computation for memfd_size) in +# test_mem_conversions() in private_mem_migrate_tests.c) + +# These static requirements are set to the maximum required for +# num_vcpus_to_test, over all the hugetlb-related tests +required_num_2m_hugepages=$(( 1024 * num_vcpus_to_test )) +required_num_1g_hugepages=$(( 2 * num_vcpus_to_test )) + +# The other hugetlb sizes are not supported on x86_64 +[ "$(cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages 2>/dev/null || echo 0)" -ge "$required_num_2m_hugepages" ] && hugepage_2mb_enabled=1 +[ "$(cat /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages 2>/dev/null || echo 0)" -ge "$required_num_1g_hugepages" ] && hugepage_1gb_enabled=1 + +case $(get_default_hugepage_size_in_kB) in + 2048) + hugepage_default_enabled=$hugepage_2mb_enabled + ;; + 1048576) + hugepage_default_enabled=$hugepage_1gb_enabled + ;; + *) + hugepage_default_enabled=0 + ;; +esac + +backing_src_types=( anonymous ) +backing_src_types+=( anonymous_thp ) +[ -n "$hugepage_default_enabled" ] && \ + backing_src_types+=( anonymous_hugetlb ) || echo "skipping anonymous_hugetlb backing source type" +[ -n "$hugepage_2mb_enabled" ] && \ + backing_src_types+=( anonymous_hugetlb_2mb ) || echo "skipping anonymous_hugetlb_2mb backing source type" +[ -n "$hugepage_1gb_enabled" ] && \ + backing_src_types+=( anonymous_hugetlb_1gb ) || echo "skipping anonymous_hugetlb_1gb backing source type" +backing_src_types+=( shmem ) +[ -n "$hugepage_default_enabled" ] && \ + backing_src_types+=( shared_hugetlb ) || echo "skipping shared_hugetlb backing source type" + +private_mem_backing_src_types=( private_mem_guest_mem ) +[ -n "$hugepage_default_enabled" ] && \ + private_mem_backing_src_types+=( private_mem_hugetlb ) || echo "skipping private_mem_hugetlb backing source type" +[ -n "$hugepage_2mb_enabled" ] && \ + private_mem_backing_src_types+=( private_mem_hugetlb_2mb ) || echo "skipping private_mem_hugetlb_2mb backing source type" +[ -n "$hugepage_1gb_enabled" ] && \ + private_mem_backing_src_types+=( private_mem_hugetlb_1gb ) || echo "skipping private_mem_hugetlb_1gb backing source type" + +set +e + +TEST_EXECUTABLE="$(dirname "$0")/private_mem_conversions_test" + +( + set -e + + for src_type in "${backing_src_types[@]}"; do + + for private_mem_src_type in "${private_mem_backing_src_types[@]}"; do + set -x + + $TEST_EXECUTABLE -s "$src_type" -p "$private_mem_src_type" -n $num_vcpus_to_test + $TEST_EXECUTABLE -s "$src_type" -p "$private_mem_src_type" -n $num_vcpus_to_test -m $num_memslots_to_test + + { set +x; } 2>/dev/null + + echo + + done + + done +) +RET=$? + +exit $RET

These functions will need to be used by guest_memfd when splitting/reconstructing HugeTLB pages.

Co-developed-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com Co-developed-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Vishal Annapurve vannapurve@google.com --- include/linux/hugetlb.h | 14 ++++++++++++++ mm/hugetlb_vmemmap.h | 11 ----------- 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 752062044b0b..7ba4ed9e0001 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -284,6 +284,20 @@ bool is_hugetlb_entry_migration(pte_t pte); bool is_hugetlb_entry_hwpoisoned(pte_t pte); void hugetlb_unshare_all_pmds(struct vm_area_struct *vma);

+#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP +int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio); +void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio); +#else +static inline int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio) +{ + return 0; +} + +static inline void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio) +{ +} +#endif + #else /* !CONFIG_HUGETLB_PAGE */

static inline void hugetlb_dup_vma_private(struct vm_area_struct *vma) diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h index 2fcae92d3359..e702ace3b42f 100644 --- a/mm/hugetlb_vmemmap.h +++ b/mm/hugetlb_vmemmap.h @@ -18,11 +18,9 @@ #define HUGETLB_VMEMMAP_RESERVE_PAGES (HUGETLB_VMEMMAP_RESERVE_SIZE / sizeof(struct page))

#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP -int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio); long hugetlb_vmemmap_restore_folios(const struct hstate *h, struct list_head *folio_list, struct list_head *non_hvo_folios); -void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio); void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list);

static inline unsigned int hugetlb_vmemmap_size(const struct hstate *h) @@ -43,11 +41,6 @@ static inline unsigned int hugetlb_vmemmap_optimizable_size(const struct hstate return size > 0 ? size : 0; } #else -static inline int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio) -{ - return 0; -} - static long hugetlb_vmemmap_restore_folios(const struct hstate *h, struct list_head *folio_list, struct list_head *non_hvo_folios) @@ -56,10 +49,6 @@ static long hugetlb_vmemmap_restore_folios(const struct hstate *h, return 0; }

-static inline void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio) -{ -} - static inline void hugetlb_vmemmap_optimize_folios(struct hstate *h, struct list_head *folio_list) { }

These functions are introduced in hugetlb.c so the private hugetlb_lock can be accessed.

hugetlb_lock is reused for this PoC, but a separate lock should be used in a future revision to avoid interference due to hash collisions with HugeTLB's usage of this lock.

Co-developed-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com Co-developed-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Vishal Annapurve vannapurve@google.com

--- include/linux/hugetlb.h | 3 +++ mm/hugetlb.c | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+)

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index ac9d4ada52bd..0f3f920ad608 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -164,6 +164,9 @@ bool hugetlb_reserve_pages(struct inode *inode, long from, long to, vm_flags_t vm_flags); long hugetlb_unreserve_pages(struct inode *inode, long start, long end, long freed); +void hugetlb_folio_list_add(struct folio *folio, struct list_head *list); +void hugetlb_folio_list_move(struct folio *folio, struct list_head *list); +void hugetlb_folio_list_del(struct folio *folio); bool isolate_hugetlb(struct folio *folio, struct list_head *list); int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison); int get_huge_page_for_hwpoison(unsigned long pfn, int flags, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 8f2b7b411b60..60e72214d5bf 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -7264,6 +7264,27 @@ long hugetlb_unreserve_pages(struct inode *inode, long start, long end, return 0; }

+void hugetlb_folio_list_add(struct folio *folio, struct list_head *list) +{ + spin_lock_irq(&hugetlb_lock); + list_add(&folio->lru, list); + spin_unlock_irq(&hugetlb_lock); +} + +void hugetlb_folio_list_move(struct folio *folio, struct list_head *list) +{ + spin_lock_irq(&hugetlb_lock); + list_move_tail(&folio->lru, list); + spin_unlock_irq(&hugetlb_lock); +} + +void hugetlb_folio_list_del(struct folio *folio) +{ + spin_lock_irq(&hugetlb_lock); + list_del(&folio->lru); + spin_unlock_irq(&hugetlb_lock); +} + #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE static unsigned long page_table_shareable(struct vm_area_struct *svma, struct vm_area_struct *vma,

The faultability xarray is stored on the inode since faultability is a property of the guest_memfd's memory contents.

In this RFC, presence of an entry in the xarray indicates faultable, but this could be flipped so that presence indicates unfaultable. For flexibility, a special value "FAULT" is used instead of a simple boolean.

However, at some stages of a VM's lifecycle there could be more private pages, and at other stages there could be more shared pages.

This is likely to be replaced by a better data structure in a future revision to better support ranges.

Also store struct kvm_gmem_hugetlb in struct kvm_gmem_hugetlb as a pointer. inode->i_mapping->i_private_data.

Co-developed-by: Fuad Tabba tabba@google.com Signed-off-by: Fuad Tabba tabba@google.com Co-developed-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com Co-developed-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Vishal Annapurve vannapurve@google.com

--- virt/kvm/guest_memfd.c | 105 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 94 insertions(+), 11 deletions(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 8151df2c03e5..b603518f7b62 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -26,11 +26,21 @@ struct kvm_gmem_hugetlb { struct hugepage_subpool *spool; };

-static struct kvm_gmem_hugetlb *kvm_gmem_hgmem(struct inode *inode) +struct kvm_gmem_inode_private { + struct xarray faultability; + struct kvm_gmem_hugetlb *hgmem; +}; + +static struct kvm_gmem_inode_private *kvm_gmem_private(struct inode *inode) { return inode->i_mapping->i_private_data; }

+static struct kvm_gmem_hugetlb *kvm_gmem_hgmem(struct inode *inode) +{ + return kvm_gmem_private(inode)->hgmem; +} + static bool is_kvm_gmem_hugetlb(struct inode *inode) { u64 flags = (u64)inode->i_private; @@ -38,6 +48,57 @@ static bool is_kvm_gmem_hugetlb(struct inode *inode) return flags & KVM_GUEST_MEMFD_HUGETLB; }

+#define KVM_GMEM_FAULTABILITY_VALUE 0x4641554c54 /* FAULT */ + +/** + * Set faultability of given range of inode indices [@start, @end) to + * @faultable. Return 0 if attributes were successfully updated or negative + * errno on error. + */ +static int kvm_gmem_set_faultable(struct inode *inode, pgoff_t start, pgoff_t end, + bool faultable) +{ + struct xarray *faultability; + void *val; + pgoff_t i; + + /* + * The expectation is that fewer pages are faultable, hence save memory + * entries are created for faultable pages as opposed to creating + * entries for non-faultable pages. + */ + val = faultable ? xa_mk_value(KVM_GMEM_FAULTABILITY_VALUE) : NULL; + faultability = &kvm_gmem_private(inode)->faultability; + + /* + * TODO replace this with something else (maybe interval + * tree?). store_range doesn't quite do what we expect if overlapping + * ranges are specified: if we store_range(5, 10, val) and then + * store_range(7, 12, NULL), the entire range [5, 12] will be NULL. For + * now, use the slower xa_store() to store individual entries on indices + * to avoid this. + */ + for (i = start; i < end; i++) { + int r; + + r = xa_err(xa_store(faultability, i, val, GFP_KERNEL_ACCOUNT)); + if (r) + return r; + } + + return 0; +} + +/** + * Return true if the page at @index is allowed to be faulted in. + */ +static bool kvm_gmem_is_faultable(struct inode *inode, pgoff_t index) +{ + struct xarray *faultability = &kvm_gmem_private(inode)->faultability; + + return xa_to_value(xa_load(faultability, index)) == KVM_GMEM_FAULTABILITY_VALUE; +} + /** * folio_file_pfn - like folio_file_page, but return a pfn. * @folio: The folio which contains this index. @@ -895,11 +956,21 @@ static void kvm_gmem_hugetlb_teardown(struct inode *inode)

static void kvm_gmem_evict_inode(struct inode *inode) { + struct kvm_gmem_inode_private *private = kvm_gmem_private(inode); + + /* + * .evict_inode can be called before faultability is set up if there are + * issues during inode creation. + */ + if (private) + xa_destroy(&private->faultability); + if (is_kvm_gmem_hugetlb(inode)) kvm_gmem_hugetlb_teardown(inode); else truncate_inode_pages_final(inode->i_mapping);

+ kfree(private); clear_inode(inode); }

@@ -1028,7 +1099,9 @@ static const struct inode_operations kvm_gmem_iops = { .setattr = kvm_gmem_setattr, };

-static int kvm_gmem_hugetlb_setup(struct inode *inode, loff_t size, u64 flags) +static int kvm_gmem_hugetlb_setup(struct inode *inode, + struct kvm_gmem_inode_private *private, + loff_t size, u64 flags) { struct kvm_gmem_hugetlb *hgmem; struct hugepage_subpool *spool; @@ -1036,6 +1109,10 @@ static int kvm_gmem_hugetlb_setup(struct inode *inode, loff_t size, u64 flags) struct hstate *h; long hpages;

+ hgmem = kzalloc(sizeof(*hgmem), GFP_KERNEL); + if (!hgmem) + return -ENOMEM; + page_size_log = (flags >> KVM_GUEST_MEMFD_HUGE_SHIFT) & KVM_GUEST_MEMFD_HUGE_MASK; h = hstate_sizelog(page_size_log);

@@ -1046,21 +1123,16 @@ static int kvm_gmem_hugetlb_setup(struct inode *inode, loff_t size, u64 flags) if (!spool) goto err;

- hgmem = kzalloc(sizeof(*hgmem), GFP_KERNEL); - if (!hgmem) - goto err_subpool; - inode->i_blkbits = huge_page_shift(h);

hgmem->h = h; hgmem->spool = spool; - inode->i_mapping->i_private_data = hgmem;

+ private->hgmem = hgmem; return 0;

-err_subpool: - kfree(spool); err: + kfree(hgmem); return -ENOMEM; }

@@ -1068,6 +1140,7 @@ static struct inode *kvm_gmem_inode_make_secure_inode(const char *name, loff_t size, u64 flags) { const struct qstr qname = QSTR_INIT(name, strlen(name)); + struct kvm_gmem_inode_private *private; struct inode *inode; int err;

@@ -1079,12 +1152,20 @@ static struct inode *kvm_gmem_inode_make_secure_inode(const char *name, if (err) goto out;

+ err = -ENOMEM; + private = kzalloc(sizeof(*private), GFP_KERNEL); + if (!private) + goto out; + if (flags & KVM_GUEST_MEMFD_HUGETLB) { - err = kvm_gmem_hugetlb_setup(inode, size, flags); + err = kvm_gmem_hugetlb_setup(inode, private, size, flags); if (err) - goto out; + goto free_private; }

+ xa_init(&private->faultability); + inode->i_mapping->i_private_data = private; + inode->i_private = (void *)(unsigned long)flags; inode->i_op = &kvm_gmem_iops; inode->i_mapping->a_ops = &kvm_gmem_aops; @@ -1097,6 +1178,8 @@ static struct inode *kvm_gmem_inode_make_secure_inode(const char *name,

return inode;

+free_private: + kfree(private); out: iput(inode);

guest_memfd files can always be mmap()ed to userspace, but faultability is controlled by an attribute on the inode.

Co-developed-by: Fuad Tabba tabba@google.com Signed-off-by: Fuad Tabba tabba@google.com Co-developed-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com

--- virt/kvm/guest_memfd.c | 46 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index b603518f7b62..fc2483e35876 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -781,7 +781,8 @@ static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) { struct list_head *gmem_list = &inode->i_mapping->i_private_list; pgoff_t start = offset >> PAGE_SHIFT; - pgoff_t end = (offset + len) >> PAGE_SHIFT; + pgoff_t nr = len >> PAGE_SHIFT; + pgoff_t end = start + nr; struct kvm_gmem *gmem;

/* @@ -790,6 +791,9 @@ static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) */ filemap_invalidate_lock(inode->i_mapping);

+ /* TODO: Check if even_cows should be 0 or 1 */ + unmap_mapping_range(inode->i_mapping, start, len, 0); + list_for_each_entry(gmem, gmem_list, entry) kvm_gmem_invalidate_begin(gmem, start, end);

@@ -946,6 +950,9 @@ static void kvm_gmem_hugetlb_teardown(struct inode *inode) { struct kvm_gmem_hugetlb *hgmem;

+ /* TODO: Check if even_cows should be 0 or 1 */ + unmap_mapping_range(inode->i_mapping, 0, LLONG_MAX, 0); + truncate_inode_pages_final_prepare(inode->i_mapping); kvm_gmem_hugetlb_truncate_folios_range(inode, 0, LLONG_MAX);

@@ -1003,11 +1010,46 @@ static void kvm_gmem_init_mount(void) kvm_gmem_mnt = kern_mount(&kvm_gmem_fs); BUG_ON(IS_ERR(kvm_gmem_mnt));

- /* For giggles. Userspace can never map this anyways. */ kvm_gmem_mnt->mnt_flags |= MNT_NOEXEC; }

+static vm_fault_t kvm_gmem_fault(struct vm_fault *vmf) +{ + struct inode *inode; + struct folio *folio; + + inode = file_inode(vmf->vma->vm_file); + if (!kvm_gmem_is_faultable(inode, vmf->pgoff)) + return VM_FAULT_SIGBUS; + + folio = kvm_gmem_get_folio(inode, vmf->pgoff); + if (!folio) + return VM_FAULT_SIGBUS; + + vmf->page = folio_file_page(folio, vmf->pgoff); + return VM_FAULT_LOCKED; +} + +static const struct vm_operations_struct kvm_gmem_vm_ops = { + .fault = kvm_gmem_fault, +}; + +static int kvm_gmem_mmap(struct file *file, struct vm_area_struct *vma) +{ + if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) != + (VM_SHARED | VM_MAYSHARE)) { + return -EINVAL; + } + + file_accessed(file); + vm_flags_set(vma, VM_DONTDUMP); + vma->vm_ops = &kvm_gmem_vm_ops; + + return 0; +} + static struct file_operations kvm_gmem_fops = { + .mmap = kvm_gmem_mmap, .open = generic_file_open, .release = kvm_gmem_release, .fallocate = kvm_gmem_fallocate,

Memory of a KVM_X86_SW_PROTECTED_VM defaults to faultable to align with the default in kvm->mem_attr_array.

For this RFC, determine default faultability when associating a range with a memslot.

Another option is to determine default faultability at guest_memfd creation time. guest_memfd is created for a specific VM, hence we can set default faultability based on the VM type.

In future, if different struct kvms are bound to the same guest_memfd inode, all the struct kvms must be of the same vm_type.

TODO: Perhaps faultability should be based on kvm->mem_attr_array?

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- virt/kvm/guest_memfd.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index fc2483e35876..1d4dfe0660ad 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -1256,6 +1256,23 @@ static struct file *kvm_gmem_inode_create_getfile(void *priv, loff_t size, return file; }

+static void kvm_gmem_set_default_faultability_by_vm_type(struct inode *inode, + u8 vm_type, + loff_t start, loff_t end) +{ + bool faultable; + + switch (vm_type) { + case KVM_X86_SW_PROTECTED_VM: + faultable = true; + break; + default: + faultable = false; + } + + WARN_ON(kvm_gmem_set_faultable(inode, start, end, faultable)); +} + static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) { struct kvm_gmem *gmem; @@ -1378,6 +1395,11 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, slot->gmem.pgoff = start;

xa_store_range(&gmem->bindings, start, end - 1, slot, GFP_KERNEL); + + kvm_gmem_set_default_faultability_by_vm_type(file_inode(file), + kvm->arch.vm_type, + start, end); + filemap_invalidate_unlock(inode->i_mapping);

/*

Since guest_memfd now supports mmap(), folios have to be prepared before they are faulted into userspace.

When memory attributes are switched between shared and private, the up-to-date flags will be cleared.

Use the folio's up-to-date flag to indicate being ready for the guest usage and can be used to mark whether the folio is ready for shared OR private use.

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- virt/kvm/guest_memfd.c | 131 ++++++++++++++++++++++++++++++++++++++++- virt/kvm/kvm_main.c | 2 + virt/kvm/kvm_mm.h | 7 +++ 3 files changed, 139 insertions(+), 1 deletion(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 110c4bbb004b..fb292e542381 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -129,13 +129,29 @@ static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slo }

/** - * Use the uptodate flag to indicate that the folio is prepared for KVM's usage. + * Use folio's up-to-date flag to indicate that this folio is prepared for usage + * by the guest. + * + * This flag can be used whether the folio is prepared for PRIVATE or SHARED + * usage. */ static inline void kvm_gmem_mark_prepared(struct folio *folio) { folio_mark_uptodate(folio); }

+/** + * Use folio's up-to-date flag to indicate that this folio is not yet prepared for + * usage by the guest. + * + * This flag can be used whether the folio is prepared for PRIVATE or SHARED + * usage. + */ +static inline void kvm_gmem_clear_prepared(struct folio *folio) +{ + folio_clear_uptodate(folio); +} + /* * Process @folio, which contains @gfn, so that the guest can use it. * The folio must be locked and the gfn must be contained in @slot. @@ -148,6 +164,12 @@ static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, pgoff_t index; int r;

+ /* + * Defensively zero folio to avoid leaking kernel memory in + * uninitialized pages. This is important since pages can now be mapped + * into userspace, where hardware (e.g. TDX) won't be clearing those + * pages. + */ if (folio_test_hugetlb(folio)) { folio_zero_user(folio, folio->index << PAGE_SHIFT); } else { @@ -1017,6 +1039,7 @@ static vm_fault_t kvm_gmem_fault(struct vm_fault *vmf) { struct inode *inode; struct folio *folio; + bool is_prepared;

inode = file_inode(vmf->vma->vm_file); if (!kvm_gmem_is_faultable(inode, vmf->pgoff)) @@ -1026,6 +1049,31 @@ static vm_fault_t kvm_gmem_fault(struct vm_fault *vmf) if (!folio) return VM_FAULT_SIGBUS;

+ is_prepared = folio_test_uptodate(folio); + if (!is_prepared) { + unsigned long nr_pages; + unsigned long i; + + if (folio_test_hugetlb(folio)) { + folio_zero_user(folio, folio->index << PAGE_SHIFT); + } else { + /* + * Defensively zero folio to avoid leaking kernel memory in + * uninitialized pages. This is important since pages can now be + * mapped into userspace, where hardware (e.g. TDX) won't be + * clearing those pages. + * + * Will probably need a version of kvm_gmem_prepare_folio() to + * prepare the page for SHARED use. + */ + nr_pages = folio_nr_pages(folio); + for (i = 0; i < nr_pages; i++) + clear_highpage(folio_page(folio, i)); + } + + kvm_gmem_mark_prepared(folio); + } + vmf->page = folio_file_page(folio, vmf->pgoff); return VM_FAULT_LOCKED; } @@ -1593,6 +1641,87 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long } EXPORT_SYMBOL_GPL(kvm_gmem_populate);

+static void kvm_gmem_clear_prepared_range(struct inode *inode, pgoff_t start, + pgoff_t end) +{ + pgoff_t index; + + filemap_invalidate_lock_shared(inode->i_mapping); + + /* TODO: replace iteration with filemap_get_folios() for efficiency. */ + for (index = start; index < end;) { + struct folio *folio; + + /* Don't use kvm_gmem_get_folio to avoid allocating */ + folio = filemap_lock_folio(inode->i_mapping, index); + if (IS_ERR(folio)) { + ++index; + continue; + } + + kvm_gmem_clear_prepared(folio); + + index = folio_next_index(folio); + folio_unlock(folio); + folio_put(folio); + } + + filemap_invalidate_unlock_shared(inode->i_mapping); +} + +/** + * Clear the prepared flag for all folios in gfn range [@start, @end) in memslot + * @slot. + */ +static void kvm_gmem_clear_prepared_slot(struct kvm_memory_slot *slot, gfn_t start, + gfn_t end) +{ + pgoff_t start_offset; + pgoff_t end_offset; + struct file *file; + + file = kvm_gmem_get_file(slot); + if (!file) + return; + + start_offset = start - slot->base_gfn + slot->gmem.pgoff; + end_offset = end - slot->base_gfn + slot->gmem.pgoff; + + kvm_gmem_clear_prepared_range(file_inode(file), start_offset, end_offset); + + fput(file); +} + +/** + * Clear the prepared flag for all folios for any slot in gfn range + * [@start, @end) in @kvm. + */ +void kvm_gmem_clear_prepared_vm(struct kvm *kvm, gfn_t start, gfn_t end) +{ + int i; + + lockdep_assert_held(&kvm->slots_lock); + + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { + struct kvm_memslot_iter iter; + struct kvm_memslots *slots; + + slots = __kvm_memslots(kvm, i); + kvm_for_each_memslot_in_gfn_range(&iter, slots, start, end) { + struct kvm_memory_slot *slot; + gfn_t gfn_start; + gfn_t gfn_end; + + slot = iter.slot; + gfn_start = max(start, slot->base_gfn); + gfn_end = min(end, slot->base_gfn + slot->npages); + + if (iter.slot->flags & KVM_MEM_GUEST_MEMFD) + kvm_gmem_clear_prepared_slot(iter.slot, gfn_start, gfn_end); + } + } +} + /** * Returns true if pages in range [@start, @end) in inode @inode have no * userspace mappings. diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 1a7bbcc31b7e..255d27df7f5c 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2565,6 +2565,8 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, KVM_BUG_ON(r, kvm); }

+ kvm_gmem_clear_prepared_vm(kvm, start, end); + kvm_handle_gfn_range(kvm, &post_set_range);

out_unlock: diff --git a/virt/kvm/kvm_mm.h b/virt/kvm/kvm_mm.h index d8ff2b380d0e..25fd0d9f66cc 100644 --- a/virt/kvm/kvm_mm.h +++ b/virt/kvm/kvm_mm.h @@ -43,6 +43,7 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, void kvm_gmem_unbind(struct kvm_memory_slot *slot); int kvm_gmem_should_set_attributes(struct kvm *kvm, gfn_t start, gfn_t end, unsigned long attrs); +void kvm_gmem_clear_prepared_vm(struct kvm *kvm, gfn_t start, gfn_t end); #else static inline void kvm_gmem_init(struct module *module) { @@ -68,6 +69,12 @@ static inline int kvm_gmem_should_set_attributes(struct kvm *kvm, gfn_t start, return 0; }

+static inline void kvm_gmem_clear_prepared_slots(struct kvm *kvm, + gfn_t start, gfn_t end) +{ + WARN_ON_ONCE(1); +} + #endif /* CONFIG_KVM_PRIVATE_MEM */

#endif /* __KVM_MM_H__ */

No functional change intended.

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- tools/testing/selftests/kvm/include/kvm_util.h | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 63c2aaae51f3..d336cd0c8f19 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -374,8 +374,8 @@ static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0) vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); }

-static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa, - uint64_t size, uint64_t attributes) +static inline int __vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa, + uint64_t size, uint64_t attributes) { struct kvm_memory_attributes attr = { .attributes = attributes, @@ -391,7 +391,15 @@ static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa, TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE, "Update me to support multiple attributes!");

- vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); + return __vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); +} + +static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa, + uint64_t size, uint64_t attributes) +{ + int ret = __vm_set_memory_attributes(vm, gpa, size, attributes); + + __TEST_ASSERT_VM_VCPU_IOCTL(!ret, "KVM_SET_MEMORY_ATTRIBUTES", ret, vm); }

Minimal test for guest_memfd to test that when memory is marked shared in a VM, the host can read and write to it via an mmap()ed address, and the guest can also read and write to it.

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- tools/testing/selftests/kvm/Makefile | 1 + .../selftests/kvm/guest_memfd_sharing_test.c | 160 ++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 tools/testing/selftests/kvm/guest_memfd_sharing_test.c

diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index b3b7e83f39fc..3c1f35456bfc 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -135,6 +135,7 @@ TEST_GEN_PROGS_x86_64 += dirty_log_test TEST_GEN_PROGS_x86_64 += dirty_log_perf_test TEST_GEN_PROGS_x86_64 += guest_memfd_test TEST_GEN_PROGS_x86_64 += guest_memfd_hugetlb_reporting_test +TEST_GEN_PROGS_x86_64 += guest_memfd_sharing_test TEST_GEN_PROGS_x86_64 += guest_print_test TEST_GEN_PROGS_x86_64 += hardware_disable_test TEST_GEN_PROGS_x86_64 += kvm_create_max_vcpus diff --git a/tools/testing/selftests/kvm/guest_memfd_sharing_test.c b/tools/testing/selftests/kvm/guest_memfd_sharing_test.c new file mode 100644 index 000000000000..fef5a73e5053 --- /dev/null +++ b/tools/testing/selftests/kvm/guest_memfd_sharing_test.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Minimal test for guest_memfd to test that when memory is marked shared in a + * VM, the host can read and write to it via an mmap()ed address, and the guest + * can also read and write to it. + * + * Copyright (c) 2024, Google LLC. + */ +#include <string.h> +#include <sys/mman.h> +#include <unistd.h> + +#include "test_util.h" +#include "kvm_util.h" +#include "ucall_common.h" + +#define GUEST_MEMFD_SHARING_TEST_SLOT 10 +#define GUEST_MEMFD_SHARING_TEST_GPA 0x50000000ULL +#define GUEST_MEMFD_SHARING_TEST_GVA 0x90000000ULL +#define GUEST_MEMFD_SHARING_TEST_OFFSET 0 +#define GUEST_MEMFD_SHARING_TEST_GUEST_TO_HOST_VALUE 0x11 +#define GUEST_MEMFD_SHARING_TEST_HOST_TO_GUEST_VALUE 0x22 + +static void guest_code(int page_size) +{ + char *mem; + int i; + + mem = (char *)GUEST_MEMFD_SHARING_TEST_GVA; + + for (i = 0; i < page_size; ++i) { + GUEST_ASSERT_EQ(mem[i], GUEST_MEMFD_SHARING_TEST_HOST_TO_GUEST_VALUE); + } + + memset(mem, GUEST_MEMFD_SHARING_TEST_GUEST_TO_HOST_VALUE, page_size); + + GUEST_DONE(); +} + +int run_test(struct kvm_vcpu *vcpu, void *hva, int page_size) +{ + struct ucall uc; + uint64_t uc_cmd; + + memset(hva, GUEST_MEMFD_SHARING_TEST_HOST_TO_GUEST_VALUE, page_size); + vcpu_args_set(vcpu, 1, page_size); + + /* Reset vCPU to guest_code every time run_test is called. */ + vcpu_arch_set_entry_point(vcpu, guest_code); + + vcpu_run(vcpu); + uc_cmd = get_ucall(vcpu, &uc); + + if (uc_cmd == UCALL_ABORT) { + REPORT_GUEST_ASSERT(uc); + return 1; + } else if (uc_cmd == UCALL_DONE) { + char *mem; + int i; + + mem = hva; + for (i = 0; i < page_size; ++i) + TEST_ASSERT_EQ(mem[i], GUEST_MEMFD_SHARING_TEST_GUEST_TO_HOST_VALUE); + + return 0; + } else { + TEST_FAIL("Unknown ucall 0x%lx.", uc.cmd); + return 1; + } +} + +void *add_memslot(struct kvm_vm *vm, int guest_memfd, size_t page_size, + bool back_shared_memory_with_guest_memfd) +{ + void *mem; + + if (back_shared_memory_with_guest_memfd) { + mem = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, + guest_memfd, GUEST_MEMFD_SHARING_TEST_OFFSET); + } else { + mem = mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + } + TEST_ASSERT(mem != MAP_FAILED, "mmap should return valid address"); + + /* + * Setting up this memslot with a KVM_X86_SW_PROTECTED_VM marks all + * offsets in the file as shared. + */ + vm_set_user_memory_region2(vm, GUEST_MEMFD_SHARING_TEST_SLOT, + KVM_MEM_GUEST_MEMFD, + GUEST_MEMFD_SHARING_TEST_GPA, page_size, mem, + guest_memfd, GUEST_MEMFD_SHARING_TEST_OFFSET); + + return mem; +} + +void test_sharing(bool back_shared_memory_with_guest_memfd) +{ + const struct vm_shape shape = { + .mode = VM_MODE_DEFAULT, + .type = KVM_X86_SW_PROTECTED_VM, + }; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + size_t page_size; + int guest_memfd; + void *mem; + + TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); + + vm = vm_create_shape_with_one_vcpu(shape, &vcpu, &guest_code); + + page_size = getpagesize(); + + guest_memfd = vm_create_guest_memfd(vm, page_size, 0); + + mem = add_memslot(vm, guest_memfd, page_size, back_shared_memory_with_guest_memfd); + + virt_map(vm, GUEST_MEMFD_SHARING_TEST_GVA, GUEST_MEMFD_SHARING_TEST_GPA, 1); + + run_test(vcpu, mem, page_size); + + /* Toggle private flag of memory attributes and run the test again. */ + if (back_shared_memory_with_guest_memfd) { + /* + * Use MADV_REMOVE to release the backing guest_memfd memory + * back to the system before it is used again. Test that this is + * only necessary when guest_memfd is used to back shared + * memory. + */ + madvise(mem, page_size, MADV_REMOVE); + } + vm_mem_set_private(vm, GUEST_MEMFD_SHARING_TEST_GPA, page_size); + vm_mem_set_shared(vm, GUEST_MEMFD_SHARING_TEST_GPA, page_size); + + run_test(vcpu, mem, page_size); + + kvm_vm_free(vm); + munmap(mem, page_size); + close(guest_memfd); +} + +int main(int argc, char *argv[]) +{ + /* + * Confidence check that when guest_memfd is associated with a memslot + * but only anonymous memory is used to back shared memory, sharing + * memory between guest and host works as expected. + */ + test_sharing(false); + + /* + * Memory sharing should work as expected when shared memory is backed + * with guest_memfd. + */ + test_sharing(true); + + return 0; +}

CONFIG_GUP_TEST provides userspace with an ioctl to invoke pin_user_pages(), and this test uses the ioctl to pin pages, to check that memory attributes cannot be set to private if shared pages are pinned.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- tools/testing/selftests/kvm/Makefile | 1 + .../selftests/kvm/guest_memfd_pin_test.c | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 tools/testing/selftests/kvm/guest_memfd_pin_test.c

diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 3c1f35456bfc..c5a1c8c7125a 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -136,6 +136,7 @@ TEST_GEN_PROGS_x86_64 += dirty_log_perf_test TEST_GEN_PROGS_x86_64 += guest_memfd_test TEST_GEN_PROGS_x86_64 += guest_memfd_hugetlb_reporting_test TEST_GEN_PROGS_x86_64 += guest_memfd_sharing_test +TEST_GEN_PROGS_x86_64 += guest_memfd_pin_test TEST_GEN_PROGS_x86_64 += guest_print_test TEST_GEN_PROGS_x86_64 += hardware_disable_test TEST_GEN_PROGS_x86_64 += kvm_create_max_vcpus diff --git a/tools/testing/selftests/kvm/guest_memfd_pin_test.c b/tools/testing/selftests/kvm/guest_memfd_pin_test.c new file mode 100644 index 000000000000..b45fb8024970 --- /dev/null +++ b/tools/testing/selftests/kvm/guest_memfd_pin_test.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Test that pinned pages block KVM from setting memory attributes to PRIVATE. + * + * Copyright (c) 2024, Google LLC. + */ +#include <string.h> +#include <sys/mman.h> +#include <unistd.h> + +#include "test_util.h" +#include "kvm_util.h" +#include "../../../../mm/gup_test.h" + +#define GUEST_MEMFD_PIN_TEST_SLOT 10 +#define GUEST_MEMFD_PIN_TEST_GPA 0x50000000ULL +#define GUEST_MEMFD_PIN_TEST_OFFSET 0 + +static int gup_test_fd; + +void pin_pages(void *vaddr, uint64_t size) +{ + const struct pin_longterm_test args = { + .addr = (uint64_t)vaddr, + .size = size, + .flags = PIN_LONGTERM_TEST_FLAG_USE_WRITE, + }; + + TEST_ASSERT_EQ(ioctl(gup_test_fd, PIN_LONGTERM_TEST_START, &args), 0); +} + +void unpin_pages(void) +{ + TEST_ASSERT_EQ(ioctl(gup_test_fd, PIN_LONGTERM_TEST_STOP), 0); +} + +void run_test(void) +{ + struct kvm_vm *vm; + size_t page_size; + void *mem; + int fd; + + vm = vm_create_barebones_type(KVM_X86_SW_PROTECTED_VM); + + page_size = getpagesize(); + fd = vm_create_guest_memfd(vm, page_size, 0); + + mem = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + GUEST_MEMFD_PIN_TEST_OFFSET); + TEST_ASSERT(mem != MAP_FAILED, "mmap should return valid address"); + + /* + * Setting up this memslot with a KVM_X86_SW_PROTECTED_VM marks all + * offsets in the file as shared. + */ + vm_set_user_memory_region2(vm, GUEST_MEMFD_PIN_TEST_SLOT, + KVM_MEM_GUEST_MEMFD, + GUEST_MEMFD_PIN_TEST_GPA, page_size, mem, fd, + GUEST_MEMFD_PIN_TEST_OFFSET); + + /* Before pinning pages, toggling memory attributes should be fine. */ + vm_mem_set_private(vm, GUEST_MEMFD_PIN_TEST_GPA, page_size); + vm_mem_set_shared(vm, GUEST_MEMFD_PIN_TEST_GPA, page_size); + + pin_pages(mem, page_size); + + /* + * Pinning also faults pages in, so remove these pages from userspace + * page tables to properly test that pinning blocks setting memory + * attributes to private. + */ + TEST_ASSERT_EQ(madvise(mem, page_size, MADV_DONTNEED), 0); + + /* Should fail since the page is still faulted in. */ + TEST_ASSERT_EQ(__vm_set_memory_attributes(vm, GUEST_MEMFD_PIN_TEST_GPA, + page_size, + KVM_MEMORY_ATTRIBUTE_PRIVATE), + -1); + TEST_ASSERT_EQ(errno, EINVAL); + + unpin_pages(); + + /* With the pages unpinned, kvm can set this page to private. */ + vm_mem_set_private(vm, GUEST_MEMFD_PIN_TEST_GPA, page_size); + + kvm_vm_free(vm); + close(fd); +} + +int main(int argc, char *argv[]) +{ + gup_test_fd = open("/sys/kernel/debug/gup_test", O_RDWR); + /* + * This test depends on CONFIG_GUP_TEST to provide a kernel module that + * exposes pin_user_pages() to userspace. + */ + TEST_REQUIRE(gup_test_fd != -1); + TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); + + run_test(); + + return 0; +}

A contiguous GPA range may not be contiguous in HVA.

This helper performs madvise, given a GPA range, by madvising in blocks according to memslot configuration.

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- tools/include/linux/kernel.h | 4 +-- .../testing/selftests/kvm/include/kvm_util.h | 2 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 30 +++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/tools/include/linux/kernel.h b/tools/include/linux/kernel.h index 07cfad817d53..5454cd3272ed 100644 --- a/tools/include/linux/kernel.h +++ b/tools/include/linux/kernel.h @@ -54,8 +54,8 @@ _min1 < _min2 ? _min1 : _min2; }) #endif

-#define max_t(type, x, y) max((type)x, (type)y) -#define min_t(type, x, y) min((type)x, (type)y) +#define max_t(type, x, y) max((type)(x), (type)(y)) +#define min_t(type, x, y) min((type)(x), (type)(y)) #define clamp(val, lo, hi) min((typeof(val))max(val, lo), hi)

#ifndef BUG_ON diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 1576e7e4aefe..58b516c23574 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -433,6 +433,8 @@ static inline void vm_mem_set_shared(struct kvm_vm *vm, uint64_t gpa, void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t gpa, uint64_t size, bool punch_hole);

+void vm_guest_mem_madvise(struct kvm_vm *vm, vm_paddr_t gpa_start, uint64_t size, + int advice); static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, uint64_t gpa, uint64_t size) { diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 9bdd03a5da90..21ea6616124c 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1416,6 +1416,36 @@ void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t base, uint64_t size, } }

+void vm_guest_mem_madvise(struct kvm_vm *vm, vm_paddr_t gpa_start, uint64_t size, + int advice) +{ + size_t madvise_len; + vm_paddr_t gpa_end; + vm_paddr_t gpa; + + gpa_end = gpa_start + size; + for (gpa = gpa_start; gpa < gpa_end; gpa += madvise_len) { + struct userspace_mem_region *region; + void *hva_start; + uint64_t memslot_end; + int ret; + + region = userspace_mem_region_find(vm, gpa, gpa); + TEST_ASSERT(region, "Memory region not found for GPA 0x%lx", gpa); + + hva_start = addr_gpa2hva(vm, gpa); + memslot_end = region->region.userspace_addr + + region->region.memory_size; + madvise_len = min_t(size_t, memslot_end - (uint64_t)hva_start, + gpa_end - gpa); + + ret = madvise(hva_start, madvise_len, advice); + TEST_ASSERT(!ret, "madvise(addr=%p, len=%lx, advice=%x) failed\n", + hva_start, madvise_len, advice); + } +} + + /* Returns the size of a vCPU's kvm_run structure. */ static int vcpu_mmap_sz(void) {

From: Vishal Annapurve vannapurve@google.com

The faultability of a page is used to determine whether to split or reconstruct a page.

If there is any page in a folio that is faultable, split the folio. If all pages in a folio are not faultable, reconstruct the folio.

On truncation, always reconstruct and free regardless of faultability (as long as a HugeTLB page's worth of pages is truncated).

Co-developed-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Vishal Annapurve vannapurve@google.com Co-developed-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com

--- virt/kvm/guest_memfd.c | 678 +++++++++++++++++++++++++++-------------- 1 file changed, 456 insertions(+), 222 deletions(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index fb292e542381..0afc111099c0 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -99,6 +99,23 @@ static bool kvm_gmem_is_faultable(struct inode *inode, pgoff_t index) return xa_to_value(xa_load(faultability, index)) == KVM_GMEM_FAULTABILITY_VALUE; }

+/** + * Return true if any of the @nr_pages beginning at @index is allowed to be + * faulted in. + */ +static bool kvm_gmem_is_any_faultable(struct inode *inode, pgoff_t index, + int nr_pages) +{ + pgoff_t i; + + for (i = index; i < index + nr_pages; ++i) { + if (kvm_gmem_is_faultable(inode, i)) + return true; + } + + return false; +} + /** * folio_file_pfn - like folio_file_page, but return a pfn. * @folio: The folio which contains this index. @@ -312,6 +329,40 @@ static int kvm_gmem_hugetlb_filemap_add_folio(struct address_space *mapping, return 0; }

+static inline void kvm_gmem_hugetlb_filemap_remove_folio(struct folio *folio) +{ + folio_lock(folio); + + folio_clear_dirty(folio); + folio_clear_uptodate(folio); + filemap_remove_folio(folio); + + folio_unlock(folio); +} + +/* + * Locks a block of nr_pages (1 << huge_page_order(h)) pages within @mapping + * beginning at @index. Take either this or filemap_invalidate_lock() whenever + * the filemap is accessed. + */ +static u32 hugetlb_fault_mutex_lock(struct address_space *mapping, pgoff_t index) +{ + pgoff_t hindex; + u32 hash; + + hindex = index >> huge_page_order(kvm_gmem_hgmem(mapping->host)->h); + hash = hugetlb_fault_mutex_hash(mapping, hindex); + + mutex_lock(&hugetlb_fault_mutex_table[hash]); + + return hash; +} + +static void hugetlb_fault_mutex_unlock(u32 hash) +{ + mutex_unlock(&hugetlb_fault_mutex_table[hash]); +} + struct kvm_gmem_split_stash { struct { unsigned long _flags_2; @@ -394,15 +445,136 @@ static int kvm_gmem_hugetlb_reconstruct_folio(struct hstate *h, struct folio *fo }

__folio_set_hugetlb(folio); - - folio_set_count(folio, 1); + hugetlb_folio_list_add(folio, &h->hugepage_activelist);

hugetlb_vmemmap_optimize_folio(h, folio);

+ folio_set_count(folio, 1); + return 0; }

-/* Basically folio_set_order(folio, 1) without the checks. */ +/** + * Reconstruct a HugeTLB folio out of folio_nr_pages(@first_folio) pages. Will + * clean up subfolios from filemap and add back the reconstructed folio. Folios + * to be reconstructed must not be locked, and reconstructed folio will not be + * locked. Return 0 on success or negative error otherwise. + * + * hugetlb_fault_mutex_lock() has to be held when calling this function. + * + * Expects that before this call, the filemap's refcounts are the only refcounts + * for the folios in the filemap. After this function returns, the filemap's + * refcount will be the only refcount on the reconstructed folio. + */ +static int kvm_gmem_reconstruct_folio_in_filemap(struct hstate *h, + struct folio *first_folio) +{ + struct address_space *mapping; + struct folio_batch fbatch; + unsigned long end; + pgoff_t index; + pgoff_t next; + int ret; + int i; + + if (folio_order(first_folio) == huge_page_order(h)) + return 0; + + index = first_folio->index; + mapping = first_folio->mapping; + + next = index; + end = index + (1UL << huge_page_order(h)); + folio_batch_init(&fbatch); + while (filemap_get_folios(mapping, &next, end - 1, &fbatch)) { + for (i = 0; i < folio_batch_count(&fbatch); ++i) { + struct folio *folio; + + folio = fbatch.folios[i]; + + /* + * Before removing from filemap, take a reference so + * sub-folios don't get freed when removing from + * filemap. + */ + folio_get(folio); + + kvm_gmem_hugetlb_filemap_remove_folio(folio); + } + folio_batch_release(&fbatch); + } + + ret = kvm_gmem_hugetlb_reconstruct_folio(h, first_folio); + if (ret) { + /* TODO: handle cleanup properly. */ + WARN_ON(ret); + return ret; + } + + kvm_gmem_hugetlb_filemap_add_folio(mapping, first_folio, index, + htlb_alloc_mask(h)); + + folio_unlock(first_folio); + folio_put(first_folio); + + return ret; +} + +/** + * Reconstruct any HugeTLB folios in range [@start, @end), if all the subfolios + * are not faultable. Return 0 on success or negative error otherwise. + * + * Will skip any folios that are already reconstructed. + */ +static int kvm_gmem_try_reconstruct_folios_range(struct inode *inode, + pgoff_t start, pgoff_t end) +{ + unsigned int nr_pages; + pgoff_t aligned_start; + pgoff_t aligned_end; + struct hstate *h; + pgoff_t index; + int ret; + + if (!is_kvm_gmem_hugetlb(inode)) + return 0; + + h = kvm_gmem_hgmem(inode)->h; + nr_pages = 1UL << huge_page_order(h); + + aligned_start = round_up(start, nr_pages); + aligned_end = round_down(end, nr_pages); + + ret = 0; + for (index = aligned_start; !ret && index < aligned_end; index += nr_pages) { + struct folio *folio; + u32 hash; + + hash = hugetlb_fault_mutex_lock(inode->i_mapping, index); + + folio = filemap_get_folio(inode->i_mapping, index); + if (!IS_ERR(folio)) { + /* + * Drop refcount because reconstruction expects an equal number + * of refcounts for all subfolios - just keep the refcount taken + * by the filemap. + */ + folio_put(folio); + + /* Merge only when the entire block of nr_pages is not faultable. */ + if (!kvm_gmem_is_any_faultable(inode, index, nr_pages)) { + ret = kvm_gmem_reconstruct_folio_in_filemap(h, folio); + WARN_ON(ret); + } + } + + hugetlb_fault_mutex_unlock(hash); + } + + return ret; +} + +/* Basically folio_set_order() without the checks. */ static inline void kvm_gmem_folio_set_order(struct folio *folio, unsigned int order) { folio->_flags_1 = (folio->_flags_1 & ~0xffUL) | order; @@ -414,8 +586,8 @@ static inline void kvm_gmem_folio_set_order(struct folio *folio, unsigned int or /** * Split a HugeTLB @folio of size huge_page_size(@h). * - * After splitting, each split folio has a refcount of 1. There are no checks on - * refcounts before splitting. + * Folio must have refcount of 1 when this function is called. After splitting, + * each split folio has a refcount of 1. * * Return 0 on success and negative error otherwise. */ @@ -423,14 +595,18 @@ static int kvm_gmem_hugetlb_split_folio(struct hstate *h, struct folio *folio) { int ret;

+ VM_WARN_ON_ONCE_FOLIO(folio_ref_count(folio) != 1, folio); + + folio_set_count(folio, 0); + ret = hugetlb_vmemmap_restore_folio(h, folio); if (ret) - return ret; + goto out;

ret = kvm_gmem_hugetlb_stash_metadata(folio); if (ret) { hugetlb_vmemmap_optimize_folio(h, folio); - return ret; + goto out; }

kvm_gmem_folio_set_order(folio, 0); @@ -439,109 +615,183 @@ static int kvm_gmem_hugetlb_split_folio(struct hstate *h, struct folio *folio) __folio_clear_hugetlb(folio);

/* - * Remove the first folio from h->hugepage_activelist since it is no + * Remove the original folio from h->hugepage_activelist since it is no * longer a HugeTLB page. The other split pages should not be on any * lists. */ hugetlb_folio_list_del(folio);

- return 0; + ret = 0; +out: + folio_set_count(folio, 1); + return ret; }

-static struct folio *kvm_gmem_hugetlb_alloc_and_cache_folio(struct inode *inode, - pgoff_t index) +/** + * Split a HugeTLB folio into folio_nr_pages(@folio) pages. Will clean up folio + * from filemap and add back the split folios. @folio must not be locked, and + * all split folios will not be locked. Return 0 on success or negative error + * otherwise. + * + * hugetlb_fault_mutex_lock() has to be held when calling this function. + * + * Expects that before this call, the filemap's refcounts are the only refcounts + * for the folio. After this function returns, the filemap's refcounts will be + * the only refcounts on the split folios. + */ +static int kvm_gmem_split_folio_in_filemap(struct hstate *h, struct folio *folio) { - struct folio *allocated_hugetlb_folio; - pgoff_t hugetlb_first_subpage_index; - struct page *hugetlb_first_subpage; - struct kvm_gmem_hugetlb *hgmem; - struct page *requested_page; + struct address_space *mapping; + struct page *first_subpage; + pgoff_t index; int ret; int i;

- hgmem = kvm_gmem_hgmem(inode); - allocated_hugetlb_folio = kvm_gmem_hugetlb_alloc_folio(hgmem->h, hgmem->spool); - if (IS_ERR(allocated_hugetlb_folio)) - return allocated_hugetlb_folio; + if (folio_order(folio) == 0) + return 0;

- requested_page = folio_file_page(allocated_hugetlb_folio, index); - hugetlb_first_subpage = folio_file_page(allocated_hugetlb_folio, 0); - hugetlb_first_subpage_index = index & (huge_page_mask(hgmem->h) >> PAGE_SHIFT); + index = folio->index; + mapping = folio->mapping;

- ret = kvm_gmem_hugetlb_split_folio(hgmem->h, allocated_hugetlb_folio); + first_subpage = folio_page(folio, 0); + + /* + * Take reference so that folio will not be released when removed from + * filemap. + */ + folio_get(folio); + + kvm_gmem_hugetlb_filemap_remove_folio(folio); + + ret = kvm_gmem_hugetlb_split_folio(h, folio); if (ret) { - folio_put(allocated_hugetlb_folio); - return ERR_PTR(ret); + WARN_ON(ret); + kvm_gmem_hugetlb_filemap_add_folio(mapping, folio, index, + htlb_alloc_mask(h)); + folio_put(folio); + return ret; }

- for (i = 0; i < pages_per_huge_page(hgmem->h); ++i) { - struct folio *folio = page_folio(nth_page(hugetlb_first_subpage, i)); + for (i = 0; i < pages_per_huge_page(h); ++i) { + struct folio *folio = page_folio(nth_page(first_subpage, i));

- ret = kvm_gmem_hugetlb_filemap_add_folio(inode->i_mapping, - folio, - hugetlb_first_subpage_index + i, - htlb_alloc_mask(hgmem->h)); + ret = kvm_gmem_hugetlb_filemap_add_folio(mapping, folio, + index + i, + htlb_alloc_mask(h)); if (ret) { /* TODO: handle cleanup properly. */ - pr_err("Handle cleanup properly index=%lx, ret=%d\n", - hugetlb_first_subpage_index + i, ret); - dump_page(nth_page(hugetlb_first_subpage, i), "check"); - return ERR_PTR(ret); + WARN_ON(ret); + return ret; }

+ folio_unlock(folio); + /* - * Skip unlocking for the requested index since - * kvm_gmem_get_folio() returns a locked folio. - * - * Do folio_put() to drop the refcount that came with the folio, - * from splitting the folio. Splitting the folio has a refcount - * to be in line with hugetlb_alloc_folio(), which returns a - * folio with refcount 1. - * - * Skip folio_put() for requested index since - * kvm_gmem_get_folio() returns a folio with refcount 1. + * Drop reference so that the only remaining reference is the + * one held by the filemap. */ - if (hugetlb_first_subpage_index + i != index) { - folio_unlock(folio); - folio_put(folio); - } + folio_put(folio); }

+ return ret; +} + +/* + * Allocates and then caches a folio in the filemap. Returns a folio with + * refcount of 2: 1 after allocation, and 1 taken by the filemap. + */ +static struct folio *kvm_gmem_hugetlb_alloc_and_cache_folio(struct inode *inode, + pgoff_t index) +{ + struct kvm_gmem_hugetlb *hgmem; + pgoff_t aligned_index; + struct folio *folio; + int nr_pages; + int ret; + + hgmem = kvm_gmem_hgmem(inode); + folio = kvm_gmem_hugetlb_alloc_folio(hgmem->h, hgmem->spool); + if (IS_ERR(folio)) + return folio; + + nr_pages = 1UL << huge_page_order(hgmem->h); + aligned_index = round_down(index, nr_pages); + + ret = kvm_gmem_hugetlb_filemap_add_folio(inode->i_mapping, folio, + aligned_index, + htlb_alloc_mask(hgmem->h)); + WARN_ON(ret); + spin_lock(&inode->i_lock); inode->i_blocks += blocks_per_huge_page(hgmem->h); spin_unlock(&inode->i_lock);

- return page_folio(requested_page); + return folio; +} + +/** + * Split @folio if any of the subfolios are faultable. Returns the split + * (locked, refcount=2) folio at @index. + * + * Expects a locked folio with 1 refcount in addition to filemap's refcounts. + * + * After splitting, the subfolios in the filemap will be unlocked and have + * refcount 1 (other than the returned folio, which will be locked and have + * refcount 2). + */ +static struct folio *kvm_gmem_maybe_split_folio(struct folio *folio, pgoff_t index) +{ + pgoff_t aligned_index; + struct inode *inode; + struct hstate *h; + int nr_pages; + int ret; + + inode = folio->mapping->host; + h = kvm_gmem_hgmem(inode)->h; + nr_pages = 1UL << huge_page_order(h); + aligned_index = round_down(index, nr_pages); + + if (!kvm_gmem_is_any_faultable(inode, aligned_index, nr_pages)) + return folio; + + /* Drop lock and refcount in preparation for splitting. */ + folio_unlock(folio); + folio_put(folio); + + ret = kvm_gmem_split_folio_in_filemap(h, folio); + if (ret) { + kvm_gmem_hugetlb_filemap_remove_folio(folio); + return ERR_PTR(ret); + } + + /* + * At this point, the filemap has the only reference on the folio. Take + * lock and refcount on folio to align with kvm_gmem_get_folio(). + */ + return filemap_lock_folio(inode->i_mapping, index); }

static struct folio *kvm_gmem_get_hugetlb_folio(struct inode *inode, pgoff_t index) { - struct address_space *mapping; struct folio *folio; - struct hstate *h; - pgoff_t hindex; u32 hash;

- h = kvm_gmem_hgmem(inode)->h; - hindex = index >> huge_page_order(h); - mapping = inode->i_mapping; - - /* To lock, we calculate the hash using the hindex and not index. */ - hash = hugetlb_fault_mutex_hash(mapping, hindex); - mutex_lock(&hugetlb_fault_mutex_table[hash]); + hash = hugetlb_fault_mutex_lock(inode->i_mapping, index);

/* - * The filemap is indexed with index and not hindex. Taking lock on - * folio to align with kvm_gmem_get_regular_folio() + * The filemap is indexed with index and not hindex. Take lock on folio + * to align with kvm_gmem_get_regular_folio() */ - folio = filemap_lock_folio(mapping, index); + folio = filemap_lock_folio(inode->i_mapping, index); + if (IS_ERR(folio)) + folio = kvm_gmem_hugetlb_alloc_and_cache_folio(inode, index); + if (!IS_ERR(folio)) - goto out; + folio = kvm_gmem_maybe_split_folio(folio, index);

- folio = kvm_gmem_hugetlb_alloc_and_cache_folio(inode, index); -out: - mutex_unlock(&hugetlb_fault_mutex_table[hash]); + hugetlb_fault_mutex_unlock(hash);

return folio; } @@ -610,17 +860,6 @@ static void kvm_gmem_invalidate_end(struct kvm_gmem *gmem, pgoff_t start, } }

-static inline void kvm_gmem_hugetlb_filemap_remove_folio(struct folio *folio) -{ - folio_lock(folio); - - folio_clear_dirty(folio); - folio_clear_uptodate(folio); - filemap_remove_folio(folio); - - folio_unlock(folio); -} - /** * Removes folios in range [@lstart, @lend) from page cache/filemap (@mapping), * returning the number of HugeTLB pages freed. @@ -631,61 +870,30 @@ static int kvm_gmem_hugetlb_filemap_remove_folios(struct address_space *mapping, struct hstate *h, loff_t lstart, loff_t lend) { - const pgoff_t end = lend >> PAGE_SHIFT; - pgoff_t next = lstart >> PAGE_SHIFT; - LIST_HEAD(folios_to_reconstruct); - struct folio_batch fbatch; - struct folio *folio, *tmp; - int num_freed = 0; - int i; - - /* - * TODO: Iterate over huge_page_size(h) blocks to avoid taking and - * releasing hugetlb_fault_mutex_table[hash] lock so often. When - * truncating, lstart and lend should be clipped to the size of this - * guest_memfd file, otherwise there would be too many iterations. - */ - folio_batch_init(&fbatch); - while (filemap_get_folios(mapping, &next, end - 1, &fbatch)) { - for (i = 0; i < folio_batch_count(&fbatch); ++i) { - struct folio *folio; - pgoff_t hindex; - u32 hash; - - folio = fbatch.folios[i]; + loff_t offset; + int num_freed;

- hindex = folio->index >> huge_page_order(h); - hash = hugetlb_fault_mutex_hash(mapping, hindex); - mutex_lock(&hugetlb_fault_mutex_table[hash]); + num_freed = 0; + for (offset = lstart; offset < lend; offset += huge_page_size(h)) { + struct folio *folio; + pgoff_t index; + u32 hash;

- /* - * Collect first pages of HugeTLB folios for - * reconstruction later. - */ - if ((folio->index & ~(huge_page_mask(h) >> PAGE_SHIFT)) == 0) - list_add(&folio->lru, &folios_to_reconstruct); + index = offset >> PAGE_SHIFT; + hash = hugetlb_fault_mutex_lock(mapping, index);

- /* - * Before removing from filemap, take a reference so - * sub-folios don't get freed. Don't free the sub-folios - * until after reconstruction. - */ - folio_get(folio); + folio = filemap_get_folio(mapping, index); + if (!IS_ERR(folio)) { + /* Drop refcount so that filemap holds only reference. */ + folio_put(folio);

+ kvm_gmem_reconstruct_folio_in_filemap(h, folio); kvm_gmem_hugetlb_filemap_remove_folio(folio);

- mutex_unlock(&hugetlb_fault_mutex_table[hash]); + num_freed++; } - folio_batch_release(&fbatch); - cond_resched(); - } - - list_for_each_entry_safe(folio, tmp, &folios_to_reconstruct, lru) { - kvm_gmem_hugetlb_reconstruct_folio(h, folio); - hugetlb_folio_list_move(folio, &h->hugepage_activelist);

- folio_put(folio); - num_freed++; + hugetlb_fault_mutex_unlock(hash); }

return num_freed; @@ -705,6 +913,10 @@ static void kvm_gmem_hugetlb_truncate_folios_range(struct inode *inode, int gbl_reserve; int num_freed;

+ /* No point truncating more than inode size. */ + lstart = min(lstart, inode->i_size); + lend = min(lend, inode->i_size); + hgmem = kvm_gmem_hgmem(inode); h = hgmem->h;

@@ -1042,13 +1254,27 @@ static vm_fault_t kvm_gmem_fault(struct vm_fault *vmf) bool is_prepared;

inode = file_inode(vmf->vma->vm_file); - if (!kvm_gmem_is_faultable(inode, vmf->pgoff)) + + /* + * Use filemap_invalidate_lock_shared() to make sure + * kvm_gmem_get_folio() doesn't race with faultability updates. + */ + filemap_invalidate_lock_shared(inode->i_mapping); + + if (!kvm_gmem_is_faultable(inode, vmf->pgoff)) { + filemap_invalidate_unlock_shared(inode->i_mapping); return VM_FAULT_SIGBUS; + }

folio = kvm_gmem_get_folio(inode, vmf->pgoff); + + filemap_invalidate_unlock_shared(inode->i_mapping); + if (!folio) return VM_FAULT_SIGBUS;

+ WARN(folio_test_hugetlb(folio), "should not be faulting in hugetlb folio=%p\n", folio); + is_prepared = folio_test_uptodate(folio); if (!is_prepared) { unsigned long nr_pages; @@ -1731,8 +1957,6 @@ static bool kvm_gmem_no_mappings_range(struct inode *inode, pgoff_t start, pgoff pgoff_t index; bool checked_indices_unmapped;

- filemap_invalidate_lock_shared(inode->i_mapping); - /* TODO: replace iteration with filemap_get_folios() for efficiency. */ checked_indices_unmapped = true; for (index = start; checked_indices_unmapped && index < end;) { @@ -1754,98 +1978,130 @@ static bool kvm_gmem_no_mappings_range(struct inode *inode, pgoff_t start, pgoff folio_put(folio); }

- filemap_invalidate_unlock_shared(inode->i_mapping); return checked_indices_unmapped; }

/** - * Returns true if pages in range [@start, @end) in memslot @slot have no - * userspace mappings. + * Split any HugeTLB folios in range [@start, @end), if any of the offsets in + * the folio are faultable. Return 0 on success or negative error otherwise. + * + * Will skip any folios that are already split. */ -static bool kvm_gmem_no_mappings_slot(struct kvm_memory_slot *slot, - gfn_t start, gfn_t end) +static int kvm_gmem_try_split_folios_range(struct inode *inode, + pgoff_t start, pgoff_t end) { - pgoff_t offset_start; - pgoff_t offset_end; - struct file *file; - bool ret; - - offset_start = start - slot->base_gfn + slot->gmem.pgoff; - offset_end = end - slot->base_gfn + slot->gmem.pgoff; - - file = kvm_gmem_get_file(slot); - if (!file) - return false; - - ret = kvm_gmem_no_mappings_range(file_inode(file), offset_start, offset_end); + unsigned int nr_pages; + pgoff_t aligned_start; + pgoff_t aligned_end; + struct hstate *h; + pgoff_t index; + int ret;

- fput(file); + if (!is_kvm_gmem_hugetlb(inode)) + return 0;

- return ret; -} + h = kvm_gmem_hgmem(inode)->h; + nr_pages = 1UL << huge_page_order(h);

-/** - * Returns true if pages in range [@start, @end) have no host userspace mappings. - */ -static bool kvm_gmem_no_mappings(struct kvm *kvm, gfn_t start, gfn_t end) -{ - int i; + aligned_start = round_down(start, nr_pages); + aligned_end = round_up(end, nr_pages);

- lockdep_assert_held(&kvm->slots_lock); + ret = 0; + for (index = aligned_start; !ret && index < aligned_end; index += nr_pages) { + struct folio *folio; + u32 hash;

- for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { - struct kvm_memslot_iter iter; - struct kvm_memslots *slots; + hash = hugetlb_fault_mutex_lock(inode->i_mapping, index);

- slots = __kvm_memslots(kvm, i); - kvm_for_each_memslot_in_gfn_range(&iter, slots, start, end) { - struct kvm_memory_slot *slot; - gfn_t gfn_start; - gfn_t gfn_end; - - slot = iter.slot; - gfn_start = max(start, slot->base_gfn); - gfn_end = min(end, slot->base_gfn + slot->npages); + folio = filemap_get_folio(inode->i_mapping, index); + if (!IS_ERR(folio)) { + /* + * Drop refcount so that the only references held are refcounts + * from the filemap. + */ + folio_put(folio);

- if (iter.slot->flags & KVM_MEM_GUEST_MEMFD && - !kvm_gmem_no_mappings_slot(iter.slot, gfn_start, gfn_end)) - return false; + if (kvm_gmem_is_any_faultable(inode, index, nr_pages)) { + ret = kvm_gmem_split_folio_in_filemap(h, folio); + if (ret) { + /* TODO cleanup properly. */ + WARN_ON(ret); + } + } } + + hugetlb_fault_mutex_unlock(hash); }

- return true; + return ret; }

/** - * Set faultability of given range of gfns [@start, @end) in memslot @slot to - * @faultable. + * Returns 0 if guest_memfd permits setting range [@start, @end) with + * faultability @faultable within memslot @slot, or negative error otherwise. + * + * If a request was made to set the memory to PRIVATE (not faultable), the pages + * in the range must not be pinned or mapped for the request to be permitted. + * + * Because this may allow pages to be faulted in to userspace when requested to + * set attributes to shared, this must only be called after the pages have been + * invalidated from guest page tables. */ -static void kvm_gmem_set_faultable_slot(struct kvm_memory_slot *slot, gfn_t start, - gfn_t end, bool faultable) +static int kvm_gmem_try_set_faultable_slot(struct kvm_memory_slot *slot, + gfn_t start, gfn_t end, + bool faultable) { pgoff_t start_offset; + struct inode *inode; pgoff_t end_offset; struct file *file; + int ret;

file = kvm_gmem_get_file(slot); if (!file) - return; + return 0;

start_offset = start - slot->base_gfn + slot->gmem.pgoff; end_offset = end - slot->base_gfn + slot->gmem.pgoff;

- WARN_ON(kvm_gmem_set_faultable(file_inode(file), start_offset, end_offset, - faultable)); + inode = file_inode(file); + + /* + * Use filemap_invalidate_lock_shared() to make sure + * splitting/reconstruction doesn't race with faultability updates. + */ + filemap_invalidate_lock(inode->i_mapping); + + kvm_gmem_set_faultable(inode, start_offset, end_offset, faultable); + + if (faultable) { + ret = kvm_gmem_try_split_folios_range(inode, start_offset, + end_offset); + } else { + if (kvm_gmem_no_mappings_range(inode, start_offset, end_offset)) { + ret = kvm_gmem_try_reconstruct_folios_range(inode, + start_offset, + end_offset); + } else { + ret = -EINVAL; + } + } + + filemap_invalidate_unlock(inode->i_mapping);

fput(file); + + return ret; }

/** - * Set faultability of given range of gfns [@start, @end) in memslot @slot to - * @faultable. + * Returns 0 if guest_memfd permits setting range [@start, @end) with + * faultability @faultable within VM @kvm, or negative error otherwise. + * + * See kvm_gmem_try_set_faultable_slot() for details. */ -static void kvm_gmem_set_faultable_vm(struct kvm *kvm, gfn_t start, gfn_t end, - bool faultable) +static int kvm_gmem_try_set_faultable_vm(struct kvm *kvm, gfn_t start, gfn_t end, + bool faultable) { int i;

@@ -1866,43 +2122,15 @@ static void kvm_gmem_set_faultable_vm(struct kvm *kvm, gfn_t start, gfn_t end, gfn_end = min(end, slot->base_gfn + slot->npages);

if (iter.slot->flags & KVM_MEM_GUEST_MEMFD) { - kvm_gmem_set_faultable_slot(slot, gfn_start, - gfn_end, faultable); + int ret; + + ret = kvm_gmem_try_set_faultable_slot(slot, gfn_start, + gfn_end, faultable); + if (ret) + return ret; } } } -} - -/** - * Returns true if guest_memfd permits setting range [@start, @end) to PRIVATE. - * - * If memory is faulted in to host userspace and a request was made to set the - * memory to PRIVATE, the faulted in pages must not be pinned for the request to - * be permitted. - */ -static int kvm_gmem_should_set_attributes_private(struct kvm *kvm, gfn_t start, - gfn_t end) -{ - kvm_gmem_set_faultable_vm(kvm, start, end, false); - - if (kvm_gmem_no_mappings(kvm, start, end)) - return 0; - - kvm_gmem_set_faultable_vm(kvm, start, end, true); - return -EINVAL; -} - -/** - * Returns true if guest_memfd permits setting range [@start, @end) to SHARED. - * - * Because this allows pages to be faulted in to userspace, this must only be - * called after the pages have been invalidated from guest page tables. - */ -static int kvm_gmem_should_set_attributes_shared(struct kvm *kvm, gfn_t start, - gfn_t end) -{ - /* Always okay to set shared, hence set range faultable here. */ - kvm_gmem_set_faultable_vm(kvm, start, end, true);

return 0; } @@ -1922,10 +2150,16 @@ static int kvm_gmem_should_set_attributes_shared(struct kvm *kvm, gfn_t start, int kvm_gmem_should_set_attributes(struct kvm *kvm, gfn_t start, gfn_t end, unsigned long attrs) { - if (attrs & KVM_MEMORY_ATTRIBUTE_PRIVATE) - return kvm_gmem_should_set_attributes_private(kvm, start, end); - else - return kvm_gmem_should_set_attributes_shared(kvm, start, end); + bool faultable; + int ret; + + faultable = !(attrs & KVM_MEMORY_ATTRIBUTE_PRIVATE); + + ret = kvm_gmem_try_set_faultable_vm(kvm, start, end, faultable); + if (ret) + WARN_ON(kvm_gmem_try_set_faultable_vm(kvm, start, end, !faultable)); + + return ret; }

#endif