Replace arguments avoid_reserve and chg in dequeue_hugetlb_folio_vma() so dequeue_hugetlb_folio_vma() is more understandable.

The new argument, use_hstate_resv, indicates whether the folio to be dequeued should be taken from reservations in hstate.

If use_hstate_resv is true, the folio to be dequeued should be taken from reservations in hstate and hence h->resv_huge_pages is decremented, and the folio is marked so that the reservation is restored.

If use_hstate_resv is false, then a folio needs to be taken from the pool and hence there must exist available_huge_pages(h), failing which, goto err.

The bool use_hstate_resv can be reused within dequeue_hugetlb_folio_vma()'s caller, alloc_hugetlb_folio().

No functional changes are intended.

As proof, the original two if conditions

!vma_has_reserves(vma, chg) && !available_huge_pages(h)

and

avoid_reserve && !available_huge_pages(h)

can be combined into

(avoid_reserve || !vma_has_reserves(vma, chg)) && !available_huge_pages(h).

Applying de Morgan's theorem on

avoid_reserve || !vma_has_reserves(vma, chg)

yields

!avoid_reserve && vma_has_reserves(vma, chg),

hence the simplification is correct.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- mm/hugetlb.c | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-)

diff --git a/mm/hugetlb.c b/mm/hugetlb.c index aaf508be0a2b..af5c6bbc9ff0 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1280,8 +1280,9 @@ static bool vma_has_reserves(struct vm_area_struct *vma, long chg) }

/* - * Only the process that called mmap() has reserves for - * private mappings. + * Only the process that called mmap() has reserves for private + * mappings. A child process with MAP_PRIVATE mappings created by their + * parent have no page reserves. */ if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) { /* @@ -1393,8 +1394,7 @@ static unsigned long available_huge_pages(struct hstate *h)

static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, struct vm_area_struct *vma, - unsigned long address, int avoid_reserve, - long chg) + unsigned long address, bool use_hstate_resv) { struct folio *folio = NULL; struct mempolicy *mpol; @@ -1402,16 +1402,7 @@ static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, nodemask_t *nodemask; int nid;

- /* - * A child process with MAP_PRIVATE mappings created by their parent - * have no page reserves. This check ensures that reservations are - * not "stolen". The child may still get SIGKILLed - */ - if (!vma_has_reserves(vma, chg) && !available_huge_pages(h)) - goto err; - - /* If reserves cannot be used, ensure enough pages are in the pool */ - if (avoid_reserve && !available_huge_pages(h)) + if (!use_hstate_resv && !available_huge_pages(h)) goto err;

gfp_mask = htlb_alloc_mask(h); @@ -1429,7 +1420,7 @@ static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, nid, nodemask);

- if (folio && !avoid_reserve && vma_has_reserves(vma, chg)) { + if (folio && use_hstate_resv) { folio_set_hugetlb_restore_reserve(folio); h->resv_huge_pages--; } @@ -3130,6 +3121,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, struct mem_cgroup *memcg; bool deferred_reserve; gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL; + bool use_hstate_resv;

memcg = get_mem_cgroup_from_current(); memcg_charge_ret = mem_cgroup_hugetlb_try_charge(memcg, gfp, nr_pages); @@ -3190,20 +3182,17 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, if (ret) goto out_uncharge_cgroup_reservation;

+ use_hstate_resv = !avoid_reserve && vma_has_reserves(vma, gbl_chg); + spin_lock_irq(&hugetlb_lock); - /* - * glb_chg is passed to indicate whether or not a page must be taken - * from the global free pool (global change). gbl_chg == 0 indicates - * a reservation exists for the allocation. - */ - folio = dequeue_hugetlb_folio_vma(h, vma, addr, avoid_reserve, gbl_chg); + folio = dequeue_hugetlb_folio_vma(h, vma, addr, use_hstate_resv); if (!folio) { spin_unlock_irq(&hugetlb_lock); folio = alloc_buddy_hugetlb_folio_with_mpol(h, vma, addr); if (!folio) goto out_uncharge_cgroup; spin_lock_irq(&hugetlb_lock); - if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) { + if (use_hstate_resv) { folio_set_hugetlb_restore_reserve(folio); h->resv_huge_pages--; }

If avoid_reserve is true, gbl_chg is not used anyway, so there is no point in setting gbl_chg.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- mm/hugetlb.c | 10 ---------- 1 file changed, 10 deletions(-)

diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 597102ed224b..5cf7fb117e9d 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3166,16 +3166,6 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, if (gbl_chg < 0) goto out_end_reservation;

- /* - * Even though there was no reservation in the region/reserve - * map, there could be reservations associated with the - * subpool that can be used. This would be indicated if the - * return value of hugepage_subpool_get_pages() is zero. - * However, if avoid_reserve is specified we still avoid even - * the subpool reservations. - */ - if (avoid_reserve) - gbl_chg = 1; }

/* If this allocation is not consuming a reservation, charge it now.

Reduce dependence on vma since the use of huge_node() assumes that the mempolicy is stored in a specific place in the inode, accessed via the vma.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- mm/hugetlb.c | 55 ++++++++++++++++++++++------------------------------ 1 file changed, 23 insertions(+), 32 deletions(-)

diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 2f2bd2444ae2..e341bc0eb49a 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1402,44 +1402,33 @@ static unsigned long available_huge_pages(struct hstate *h) return h->free_huge_pages - h->resv_huge_pages; }

-static struct folio *dequeue_hugetlb_folio_vma(struct hstate *h, - struct vm_area_struct *vma, - unsigned long address, bool use_hstate_resv) +static struct folio *dequeue_hugetlb_folio(struct hstate *h, + struct mempolicy *mpol, int nid, + nodemask_t *nodemask, + bool use_hstate_resv) { struct folio *folio = NULL; - struct mempolicy *mpol; gfp_t gfp_mask; - nodemask_t *nodemask; - int nid;

if (!use_hstate_resv && !available_huge_pages(h)) - goto err; + return NULL;

gfp_mask = htlb_alloc_mask(h); - nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);

- if (mpol_is_preferred_many(mpol)) { - folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, - nid, nodemask); + if (mpol_is_preferred_many(mpol)) + folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, nid, nodemask);

- /* Fallback to all nodes if page==NULL */ - nodemask = NULL; + if (!folio) { + /* Fallback to all nodes if earlier allocation failed */ + folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, nid, NULL); }

- if (!folio) - folio = dequeue_hugetlb_folio_nodemask(h, gfp_mask, - nid, nodemask); - if (folio && use_hstate_resv) { folio_set_hugetlb_restore_reserve(folio); h->resv_huge_pages--; }

- mpol_cond_put(mpol); return folio; - -err: - return NULL; }

/* @@ -3131,6 +3120,10 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, bool deferred_reserve; gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL; bool use_hstate_resv; + struct mempolicy *mpol; + nodemask_t *nodemask; + pgoff_t ilx; + int nid;

memcg = get_mem_cgroup_from_current(); memcg_charge_ret = mem_cgroup_hugetlb_try_charge(memcg, gfp, nr_pages); @@ -3184,22 +3177,19 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, use_hstate_resv = should_use_hstate_resv(vma, gbl_chg, avoid_reserve);

spin_lock_irq(&hugetlb_lock); - folio = dequeue_hugetlb_folio_vma(h, vma, addr, use_hstate_resv); - if (!folio) { - struct mempolicy *mpol; - nodemask_t *nodemask; - pgoff_t ilx; - int nid;

+ mpol = get_vma_policy(vma, addr, hstate_vma(vma)->order, &ilx); + nid = policy_node_nodemask(mpol, htlb_alloc_mask(h), ilx, &nodemask); + folio = dequeue_hugetlb_folio(h, mpol, nid, nodemask, use_hstate_resv); + if (!folio) { spin_unlock_irq(&hugetlb_lock);

- mpol = get_vma_policy(vma, addr, hstate_vma(vma)->order, &ilx); - nid = policy_node_nodemask(mpol, htlb_alloc_mask(h), ilx, &nodemask); folio = alloc_buddy_hugetlb_folio_from_node(h, mpol, nid, nodemask); - mpol_cond_put(mpol); - - if (!folio) + if (!folio) { + mpol_cond_put(mpol); goto out_uncharge_cgroup; + } + spin_lock_irq(&hugetlb_lock); if (use_hstate_resv) { folio_set_hugetlb_restore_reserve(folio); @@ -3209,6 +3199,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, folio_ref_unfreeze(folio, 1); /* Fall through */ } + mpol_cond_put(mpol);

hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, folio); /* If allocation is not consuming a reservation, also store the

This will allow preparation steps to be shared

Signed-off-by: Ackerley Tng ackerleytng@google.com --- include/linux/mm.h | 1 + mm/truncate.c | 26 ++++++++++++++++---------- 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/include/linux/mm.h b/include/linux/mm.h index c4b238a20b76..ffb4788295b4 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3442,6 +3442,7 @@ extern unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info); extern void truncate_inode_pages(struct address_space *, loff_t); extern void truncate_inode_pages_range(struct address_space *, loff_t lstart, loff_t lend); +extern void truncate_inode_pages_final_prepare(struct address_space *); extern void truncate_inode_pages_final(struct address_space *);

/* generic vm_area_ops exported for stackable file systems */ diff --git a/mm/truncate.c b/mm/truncate.c index 4d61fbdd4b2f..28cca86424f8 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -424,16 +424,7 @@ void truncate_inode_pages(struct address_space *mapping, loff_t lstart) } EXPORT_SYMBOL(truncate_inode_pages);

-/** - * truncate_inode_pages_final - truncate *all* pages before inode dies - * @mapping: mapping to truncate - * - * Called under (and serialized by) inode->i_rwsem. - * - * Filesystems have to use this in the .evict_inode path to inform the - * VM that this is the final truncate and the inode is going away. - */ -void truncate_inode_pages_final(struct address_space *mapping) +void truncate_inode_pages_final_prepare(struct address_space *mapping) { /* * Page reclaim can not participate in regular inode lifetime @@ -454,6 +445,21 @@ void truncate_inode_pages_final(struct address_space *mapping) xa_lock_irq(&mapping->i_pages); xa_unlock_irq(&mapping->i_pages); } +} +EXPORT_SYMBOL(truncate_inode_pages_final_prepare); + +/** + * truncate_inode_pages_final - truncate *all* pages before inode dies + * @mapping: mapping to truncate + * + * Called under (and serialized by) inode->i_rwsem. + * + * Filesystems have to use this in the .evict_inode path to inform the + * VM that this is the final truncate and the inode is going away. + */ +void truncate_inode_pages_final(struct address_space *mapping) +{ + truncate_inode_pages_final_prepare(mapping);

truncate_inode_pages(mapping, 0); }

__hugetlb_acct_memory() today does more than just memory accounting. when there's insufficient HugeTLB pages, __hugetlb_acct_memory() will attempt to get surplus pages.

This change adds a flag to disable getting surplus pages if there are insufficient HugeTLB pages.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- fs/hugetlbfs/inode.c | 2 +- include/linux/hugetlb.h | 2 +- mm/hugetlb.c | 43 ++++++++++++++++++++++++++++++----------- 3 files changed, 34 insertions(+), 13 deletions(-)

diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 9f6cff356796..300a6ef300c1 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -1488,7 +1488,7 @@ hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc) if (ctx->max_hpages != -1 || ctx->min_hpages != -1) { sbinfo->spool = hugepage_new_subpool(ctx->hstate, ctx->max_hpages, - ctx->min_hpages); + ctx->min_hpages, true); if (!sbinfo->spool) goto out_free; } diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 907cfbbd9e24..9ef1adbd3207 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -116,7 +116,7 @@ extern int hugetlb_max_hstate __read_mostly; for ((h) = hstates; (h) < &hstates[hugetlb_max_hstate]; (h)++)

struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, - long min_hpages); + long min_hpages, bool use_surplus); void hugepage_put_subpool(struct hugepage_subpool *spool);

long hugepage_subpool_get_pages(struct hugepage_subpool *spool, long delta); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 808915108126..efdb5772b367 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -92,6 +92,7 @@ static int num_fault_mutexes; struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;

/* Forward declaration */ +static int __hugetlb_acct_memory(struct hstate *h, long delta, bool use_surplus); static int hugetlb_acct_memory(struct hstate *h, long delta); static void hugetlb_vma_lock_free(struct vm_area_struct *vma); static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma); @@ -129,7 +130,7 @@ static inline void unlock_or_release_subpool(struct hugepage_subpool *spool, }

struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, - long min_hpages) + long min_hpages, bool use_surplus) { struct hugepage_subpool *spool;

@@ -143,7 +144,8 @@ struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages, spool->hstate = h; spool->min_hpages = min_hpages;

- if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) { + if (min_hpages != -1 && + __hugetlb_acct_memory(h, min_hpages, use_surplus)) { kfree(spool); return NULL; } @@ -2592,6 +2594,21 @@ static nodemask_t *policy_mbind_nodemask(gfp_t gfp) return NULL; }

+static int hugetlb_hstate_reserve_pages(struct hstate *h, + long num_pages_to_reserve) + __must_hold(&hugetlb_lock) +{ + long needed; + + needed = (h->resv_huge_pages + num_pages_to_reserve) - h->free_huge_pages; + if (needed <= 0) { + h->resv_huge_pages += num_pages_to_reserve; + return 0; + } + + return needed; +} + /* * Increase the hugetlb pool such that it can accommodate a reservation * of size 'delta'. @@ -2608,13 +2625,7 @@ static int gather_surplus_pages(struct hstate *h, long delta) int node; nodemask_t *mbind_nodemask = policy_mbind_nodemask(htlb_alloc_mask(h));

- lockdep_assert_held(&hugetlb_lock); - needed = (h->resv_huge_pages + delta) - h->free_huge_pages; - if (needed <= 0) { - h->resv_huge_pages += delta; - return 0; - } - + needed = delta; allocated = 0;

ret = -ENOMEM; @@ -5104,7 +5115,7 @@ unsigned long hugetlb_total_pages(void) return nr_total_pages; }

-static int hugetlb_acct_memory(struct hstate *h, long delta) +static int __hugetlb_acct_memory(struct hstate *h, long delta, bool use_surplus) { int ret = -ENOMEM;

@@ -5136,7 +5147,12 @@ static int hugetlb_acct_memory(struct hstate *h, long delta) * above. */ if (delta > 0) { - if (gather_surplus_pages(h, delta) < 0) + long required_surplus = hugetlb_hstate_reserve_pages(h, delta); + + if (!use_surplus && required_surplus > 0) + goto out; + + if (gather_surplus_pages(h, required_surplus) < 0) goto out;

if (delta > allowed_mems_nr(h)) { @@ -5154,6 +5170,11 @@ static int hugetlb_acct_memory(struct hstate *h, long delta) return ret; }

+static int hugetlb_acct_memory(struct hstate *h, long delta) +{ + return __hugetlb_acct_memory(h, delta, true); +} + static void hugetlb_vm_op_open(struct vm_area_struct *vma) { struct resv_map *resv = vma_resv_map(vma);

This will used by guest_memfd in a later patch.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- fs/hugetlbfs/inode.c | 33 +++++---------------------------- include/linux/hugetlb.h | 3 +++ mm/hugetlb.c | 21 +++++++++++++++++++++ 3 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 300a6ef300c1..f76001418672 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -720,29 +720,6 @@ static void hugetlb_vmtruncate(struct inode *inode, loff_t offset) remove_inode_hugepages(inode, offset, LLONG_MAX); }

-static void hugetlbfs_zero_partial_page(struct hstate *h, - struct address_space *mapping, - loff_t start, - loff_t end) -{ - pgoff_t idx = start >> huge_page_shift(h); - struct folio *folio; - - folio = filemap_lock_hugetlb_folio(h, mapping, idx); - if (IS_ERR(folio)) - return; - - start = start & ~huge_page_mask(h); - end = end & ~huge_page_mask(h); - if (!end) - end = huge_page_size(h); - - folio_zero_segment(folio, (size_t)start, (size_t)end); - - folio_unlock(folio); - folio_put(folio); -} - static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) { struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode); @@ -768,9 +745,10 @@ static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) i_mmap_lock_write(mapping);

/* If range starts before first full page, zero partial page. */ - if (offset < hole_start) - hugetlbfs_zero_partial_page(h, mapping, - offset, min(offset + len, hole_start)); + if (offset < hole_start) { + hugetlb_zero_partial_page(h, mapping, offset, + min(offset + len, hole_start)); + }

/* Unmap users of full pages in the hole. */ if (hole_end > hole_start) { @@ -782,8 +760,7 @@ static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)

/* If range extends beyond last full page, zero partial page. */ if ((offset + len) > hole_end && (offset + len) > hole_start) - hugetlbfs_zero_partial_page(h, mapping, - hole_end, offset + len); + hugetlb_zero_partial_page(h, mapping, hole_end, offset + len);

i_mmap_unlock_write(mapping);

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 4d47bf94c211..752062044b0b 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -124,6 +124,9 @@ long hugepage_subpool_put_pages(struct hugepage_subpool *spool, long delta);

int hugetlb_acct_memory(struct hstate *h, long delta);

+void hugetlb_zero_partial_page(struct hstate *h, struct address_space *mapping, + loff_t start, loff_t end); + void hugetlb_dup_vma_private(struct vm_area_struct *vma); void clear_vma_resv_huge_pages(struct vm_area_struct *vma); int move_hugetlb_page_tables(struct vm_area_struct *vma, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 5a37b03e1361..372d8294fb2f 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1989,6 +1989,27 @@ void free_huge_folio(struct folio *folio) } }

+void hugetlb_zero_partial_page(struct hstate *h, struct address_space *mapping, + loff_t start, loff_t end) +{ + pgoff_t idx = start >> huge_page_shift(h); + struct folio *folio; + + folio = filemap_lock_hugetlb_folio(h, mapping, idx); + if (IS_ERR(folio)) + return; + + start = start & ~huge_page_mask(h); + end = end & ~huge_page_mask(h); + if (!end) + end = huge_page_size(h); + + folio_zero_segment(folio, (size_t)start, (size_t)end); + + folio_unlock(folio); + folio_put(folio); +} + /* * Must be called with the hugetlb lock held */

First stage of hugetlb support: add initialization and cleanup routines.

After guest_mem was massaged to use guest_mem inodes instead of anonymous inodes in an earlier patch, the .evict_inode handler can now be overridden to do hugetlb metadata cleanup.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- include/uapi/linux/kvm.h | 26 ++++++ virt/kvm/guest_memfd.c | 177 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 197 insertions(+), 6 deletions(-)

diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 637efc055145..77de7c4432f6 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -13,6 +13,7 @@ #include <linux/compiler.h> #include <linux/ioctl.h> #include <asm/kvm.h> +#include <asm-generic/hugetlb_encode.h>

#define KVM_API_VERSION 12

@@ -1558,6 +1559,31 @@ struct kvm_memory_attributes {

#define KVM_CREATE_GUEST_MEMFD _IOWR(KVMIO, 0xd4, struct kvm_create_guest_memfd)

+#define KVM_GUEST_MEMFD_HUGETLB (1ULL << 1) + +/* + * Huge page size encoding when KVM_GUEST_MEMFD_HUGETLB is specified, and a huge + * page size other than the default is desired. See hugetlb_encode.h. All + * known huge page size encodings are provided here. It is the responsibility + * of the application to know which sizes are supported on the running system. + * See mmap(2) man page for details. + */ +#define KVM_GUEST_MEMFD_HUGE_SHIFT HUGETLB_FLAG_ENCODE_SHIFT +#define KVM_GUEST_MEMFD_HUGE_MASK HUGETLB_FLAG_ENCODE_MASK + +#define KVM_GUEST_MEMFD_HUGE_64KB HUGETLB_FLAG_ENCODE_64KB +#define KVM_GUEST_MEMFD_HUGE_512KB HUGETLB_FLAG_ENCODE_512KB +#define KVM_GUEST_MEMFD_HUGE_1MB HUGETLB_FLAG_ENCODE_1MB +#define KVM_GUEST_MEMFD_HUGE_2MB HUGETLB_FLAG_ENCODE_2MB +#define KVM_GUEST_MEMFD_HUGE_8MB HUGETLB_FLAG_ENCODE_8MB +#define KVM_GUEST_MEMFD_HUGE_16MB HUGETLB_FLAG_ENCODE_16MB +#define KVM_GUEST_MEMFD_HUGE_32MB HUGETLB_FLAG_ENCODE_32MB +#define KVM_GUEST_MEMFD_HUGE_256MB HUGETLB_FLAG_ENCODE_256MB +#define KVM_GUEST_MEMFD_HUGE_512MB HUGETLB_FLAG_ENCODE_512MB +#define KVM_GUEST_MEMFD_HUGE_1GB HUGETLB_FLAG_ENCODE_1GB +#define KVM_GUEST_MEMFD_HUGE_2GB HUGETLB_FLAG_ENCODE_2GB +#define KVM_GUEST_MEMFD_HUGE_16GB HUGETLB_FLAG_ENCODE_16GB + struct kvm_create_guest_memfd { __u64 size; __u64 flags; diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 5d7fd1f708a6..31e1115273e1 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -3,6 +3,7 @@ #include <linux/mount.h> #include <linux/backing-dev.h> #include <linux/falloc.h> +#include <linux/hugetlb.h> #include <linux/kvm_host.h> #include <linux/pseudo_fs.h> #include <linux/pagemap.h> @@ -18,6 +19,16 @@ struct kvm_gmem { struct list_head entry; };

+struct kvm_gmem_hugetlb { + struct hstate *h; + struct hugepage_subpool *spool; +}; + +static struct kvm_gmem_hugetlb *kvm_gmem_hgmem(struct inode *inode) +{ + return inode->i_mapping->i_private_data; +} + /** * folio_file_pfn - like folio_file_page, but return a pfn. * @folio: The folio which contains this index. @@ -154,6 +165,82 @@ static void kvm_gmem_invalidate_end(struct kvm_gmem *gmem, pgoff_t start, } }

+static inline void kvm_gmem_hugetlb_filemap_remove_folio(struct folio *folio) +{ + folio_lock(folio); + + folio_clear_dirty(folio); + folio_clear_uptodate(folio); + filemap_remove_folio(folio); + + folio_unlock(folio); +} + +/** + * Removes folios in range [@lstart, @lend) from page cache/filemap (@mapping), + * returning the number of pages freed. + */ +static int kvm_gmem_hugetlb_filemap_remove_folios(struct address_space *mapping, + struct hstate *h, + loff_t lstart, loff_t lend) +{ + const pgoff_t end = lend >> PAGE_SHIFT; + pgoff_t next = lstart >> PAGE_SHIFT; + struct folio_batch fbatch; + int num_freed = 0; + + folio_batch_init(&fbatch); + while (filemap_get_folios(mapping, &next, end - 1, &fbatch)) { + int i; + for (i = 0; i < folio_batch_count(&fbatch); ++i) { + struct folio *folio; + pgoff_t hindex; + u32 hash; + + folio = fbatch.folios[i]; + hindex = folio->index >> huge_page_order(h); + hash = hugetlb_fault_mutex_hash(mapping, hindex); + + mutex_lock(&hugetlb_fault_mutex_table[hash]); + kvm_gmem_hugetlb_filemap_remove_folio(folio); + mutex_unlock(&hugetlb_fault_mutex_table[hash]); + + num_freed++; + } + folio_batch_release(&fbatch); + cond_resched(); + } + + return num_freed; +} + +/** + * Removes folios in range [@lstart, @lend) from page cache of inode, updates + * inode metadata and hugetlb reservations. + */ +static void kvm_gmem_hugetlb_truncate_folios_range(struct inode *inode, + loff_t lstart, loff_t lend) +{ + struct kvm_gmem_hugetlb *hgmem; + struct hstate *h; + int gbl_reserve; + int num_freed; + + hgmem = kvm_gmem_hgmem(inode); + h = hgmem->h; + + num_freed = kvm_gmem_hugetlb_filemap_remove_folios(inode->i_mapping, + h, lstart, lend); + + gbl_reserve = hugepage_subpool_put_pages(hgmem->spool, num_freed); + hugetlb_acct_memory(h, -gbl_reserve); + + spin_lock(&inode->i_lock); + inode->i_blocks -= blocks_per_huge_page(h) * num_freed; + spin_unlock(&inode->i_lock); +} + + static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) { struct list_head *gmem_list = &inode->i_mapping->i_private_list; @@ -307,8 +394,33 @@ static inline struct file *kvm_gmem_get_file(struct kvm_memory_slot *slot) return get_file_active(&slot->gmem.file); }

+static void kvm_gmem_hugetlb_teardown(struct inode *inode) +{ + struct kvm_gmem_hugetlb *hgmem; + + truncate_inode_pages_final_prepare(inode->i_mapping); + kvm_gmem_hugetlb_truncate_folios_range(inode, 0, LLONG_MAX); + + hgmem = kvm_gmem_hgmem(inode); + hugepage_put_subpool(hgmem->spool); + kfree(hgmem); +} + +static void kvm_gmem_evict_inode(struct inode *inode) +{ + u64 flags = (u64)inode->i_private; + + if (flags & KVM_GUEST_MEMFD_HUGETLB) + kvm_gmem_hugetlb_teardown(inode); + else + truncate_inode_pages_final(inode->i_mapping); + + clear_inode(inode); +} + static const struct super_operations kvm_gmem_super_operations = { .statfs = simple_statfs, + .evict_inode = kvm_gmem_evict_inode, };

static int kvm_gmem_init_fs_context(struct fs_context *fc) @@ -431,6 +543,42 @@ static const struct inode_operations kvm_gmem_iops = { .setattr = kvm_gmem_setattr, };

+static int kvm_gmem_hugetlb_setup(struct inode *inode, loff_t size, u64 flags) +{ + struct kvm_gmem_hugetlb *hgmem; + struct hugepage_subpool *spool; + int page_size_log; + struct hstate *h; + long hpages; + + page_size_log = (flags >> KVM_GUEST_MEMFD_HUGE_SHIFT) & KVM_GUEST_MEMFD_HUGE_MASK; + h = hstate_sizelog(page_size_log); + + /* Round up to accommodate size requests that don't align with huge pages */ + hpages = round_up(size, huge_page_size(h)) >> huge_page_shift(h); + + spool = hugepage_new_subpool(h, hpages, hpages, false); + if (!spool) + goto err; + + hgmem = kzalloc(sizeof(*hgmem), GFP_KERNEL); + if (!hgmem) + goto err_subpool; + + inode->i_blkbits = huge_page_shift(h); + + hgmem->h = h; + hgmem->spool = spool; + inode->i_mapping->i_private_data = hgmem; + + return 0; + +err_subpool: + kfree(spool); +err: + return -ENOMEM; +} + static struct inode *kvm_gmem_inode_make_secure_inode(const char *name, loff_t size, u64 flags) { @@ -443,9 +591,13 @@ static struct inode *kvm_gmem_inode_make_secure_inode(const char *name, return inode;

err = security_inode_init_security_anon(inode, &qname, NULL); - if (err) { - iput(inode); - return ERR_PTR(err); + if (err) + goto out; + + if (flags & KVM_GUEST_MEMFD_HUGETLB) { + err = kvm_gmem_hugetlb_setup(inode, size, flags); + if (err) + goto out; }

inode->i_private = (void *)(unsigned long)flags; @@ -459,6 +611,11 @@ static struct inode *kvm_gmem_inode_make_secure_inode(const char *name, WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping));

return inode; + +out: + iput(inode); + + return ERR_PTR(err); }

static struct file *kvm_gmem_inode_create_getfile(void *priv, loff_t size, @@ -526,14 +683,22 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) return err; }

+#define KVM_GUEST_MEMFD_ALL_FLAGS KVM_GUEST_MEMFD_HUGETLB + int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args) { loff_t size = args->size; u64 flags = args->flags; - u64 valid_flags = 0;

- if (flags & ~valid_flags) - return -EINVAL; + if (flags & KVM_GUEST_MEMFD_HUGETLB) { + /* Allow huge page size encoding in flags */ + if (flags & ~(KVM_GUEST_MEMFD_ALL_FLAGS | + (KVM_GUEST_MEMFD_HUGE_MASK << KVM_GUEST_MEMFD_HUGE_SHIFT))) + return -EINVAL; + } else { + if (flags & ~KVM_GUEST_MEMFD_ALL_FLAGS) + return -EINVAL; + }

if (size <= 0 || !PAGE_ALIGNED(size)) return -EINVAL;

If HugeTLB is requested at guest_memfd creation time, HugeTLB pages will be used to back guest_memfd.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- virt/kvm/guest_memfd.c | 252 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 239 insertions(+), 13 deletions(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 31e1115273e1..2e6f12e2bac8 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -8,6 +8,8 @@ #include <linux/pseudo_fs.h> #include <linux/pagemap.h> #include <linux/anon_inodes.h> +#include <linux/memcontrol.h> +#include <linux/mempolicy.h>

#include "kvm_mm.h"

@@ -29,6 +31,13 @@ static struct kvm_gmem_hugetlb *kvm_gmem_hgmem(struct inode *inode) return inode->i_mapping->i_private_data; }

+static bool is_kvm_gmem_hugetlb(struct inode *inode) +{ + u64 flags = (u64)inode->i_private; + + return flags & KVM_GUEST_MEMFD_HUGETLB; +} + /** * folio_file_pfn - like folio_file_page, but return a pfn. * @folio: The folio which contains this index. @@ -58,6 +67,9 @@ static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slo return 0; }

+/** + * Use the uptodate flag to indicate that the folio is prepared for KVM's usage. + */ static inline void kvm_gmem_mark_prepared(struct folio *folio) { folio_mark_uptodate(folio); @@ -72,13 +84,18 @@ static inline void kvm_gmem_mark_prepared(struct folio *folio) static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, struct folio *folio) { - unsigned long nr_pages, i; pgoff_t index; int r;

- nr_pages = folio_nr_pages(folio); - for (i = 0; i < nr_pages; i++) - clear_highpage(folio_page(folio, i)); + if (folio_test_hugetlb(folio)) { + folio_zero_user(folio, folio->index << PAGE_SHIFT); + } else { + unsigned long nr_pages, i; + + nr_pages = folio_nr_pages(folio); + for (i = 0; i < nr_pages; i++) + clear_highpage(folio_page(folio, i)); + }

/* * Preparing huge folios should always be safe, since it should @@ -103,6 +120,174 @@ static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, return r; }

+static int kvm_gmem_get_mpol_node_nodemask(gfp_t gfp_mask, + struct mempolicy **mpol, + nodemask_t **nodemask) +{ + /* + * TODO: mempolicy would probably have to be stored on the inode, use + * task policy for now. + */ + *mpol = get_task_policy(current); + + /* TODO: ignore interleaving (set ilx to 0) for now. */ + return policy_node_nodemask(*mpol, gfp_mask, 0, nodemask); +} + +static struct folio *kvm_gmem_hugetlb_alloc_folio(struct hstate *h, + struct hugepage_subpool *spool) +{ + bool memcg_charge_was_prepared; + struct mem_cgroup *memcg; + struct mempolicy *mpol; + nodemask_t *nodemask; + struct folio *folio; + gfp_t gfp_mask; + int ret; + int nid; + + gfp_mask = htlb_alloc_mask(h); + + memcg = get_mem_cgroup_from_current(); + ret = mem_cgroup_hugetlb_try_charge(memcg, + gfp_mask | __GFP_RETRY_MAYFAIL, + pages_per_huge_page(h)); + if (ret == -ENOMEM) + goto err; + + memcg_charge_was_prepared = ret != -EOPNOTSUPP; + + /* Pages are only to be taken from guest_memfd subpool and nowhere else. */ + if (hugepage_subpool_get_pages(spool, 1)) + goto err_cancel_charge; + + nid = kvm_gmem_get_mpol_node_nodemask(htlb_alloc_mask(h), &mpol, + &nodemask); + /* + * charge_cgroup_reservation is false because we didn't make any cgroup + * reservations when creating the guest_memfd subpool. + * + * use_hstate_resv is true because we reserved from global hstate when + * creating the guest_memfd subpool. + */ + folio = hugetlb_alloc_folio(h, mpol, nid, nodemask, false, true); + mpol_cond_put(mpol); + + if (!folio) + goto err_put_pages; + + hugetlb_set_folio_subpool(folio, spool); + + if (memcg_charge_was_prepared) + mem_cgroup_commit_charge(folio, memcg); + +out: + mem_cgroup_put(memcg); + + return folio; + +err_put_pages: + hugepage_subpool_put_pages(spool, 1); + +err_cancel_charge: + if (memcg_charge_was_prepared) + mem_cgroup_cancel_charge(memcg, pages_per_huge_page(h)); + +err: + folio = ERR_PTR(-ENOMEM); + goto out; +} + +static int kvm_gmem_hugetlb_filemap_add_folio(struct address_space *mapping, + struct folio *folio, pgoff_t index, + gfp_t gfp) +{ + int ret; + + __folio_set_locked(folio); + ret = __filemap_add_folio(mapping, folio, index, gfp, NULL); + if (unlikely(ret)) { + __folio_clear_locked(folio); + return ret; + } + + /* + * In hugetlb_add_to_page_cache(), there is a call to + * folio_clear_hugetlb_restore_reserve(). This is handled when the pages + * are removed from the page cache in unmap_hugepage_range() -> + * __unmap_hugepage_range() by conditionally calling + * folio_set_hugetlb_restore_reserve(). In kvm_gmem_hugetlb's usage of + * hugetlb, there are no VMAs involved, and pages are never taken from + * the surplus, so when pages are freed, the hstate reserve must be + * restored. Hence, this function makes no call to + * folio_clear_hugetlb_restore_reserve(). + */ + + /* mark folio dirty so that it will not be removed from cache/inode */ + folio_mark_dirty(folio); + + return 0; +} + +static struct folio *kvm_gmem_hugetlb_alloc_and_cache_folio(struct inode *inode, + pgoff_t index) +{ + struct kvm_gmem_hugetlb *hgmem; + struct folio *folio; + int ret; + + hgmem = kvm_gmem_hgmem(inode); + folio = kvm_gmem_hugetlb_alloc_folio(hgmem->h, hgmem->spool); + if (IS_ERR(folio)) + return folio; + + /* TODO: Fix index here to be aligned to huge page size. */ + ret = kvm_gmem_hugetlb_filemap_add_folio( + inode->i_mapping, folio, index, htlb_alloc_mask(hgmem->h)); + if (ret) { + folio_put(folio); + return ERR_PTR(ret); + } + + spin_lock(&inode->i_lock); + inode->i_blocks += blocks_per_huge_page(hgmem->h); + spin_unlock(&inode->i_lock); + + return folio; +} + +static struct folio *kvm_gmem_get_hugetlb_folio(struct inode *inode, + pgoff_t index) +{ + struct address_space *mapping; + struct folio *folio; + struct hstate *h; + pgoff_t hindex; + u32 hash; + + h = kvm_gmem_hgmem(inode)->h; + hindex = index >> huge_page_order(h); + mapping = inode->i_mapping; + + /* To lock, we calculate the hash using the hindex and not index. */ + hash = hugetlb_fault_mutex_hash(mapping, hindex); + mutex_lock(&hugetlb_fault_mutex_table[hash]); + + /* + * The filemap is indexed with index and not hindex. Taking lock on + * folio to align with kvm_gmem_get_regular_folio() + */ + folio = filemap_lock_folio(mapping, index); + if (!IS_ERR(folio)) + goto out; + + folio = kvm_gmem_hugetlb_alloc_and_cache_folio(inode, index); +out: + mutex_unlock(&hugetlb_fault_mutex_table[hash]); + + return folio; +} + /* * Returns a locked folio on success. The caller is responsible for * setting the up-to-date flag before the memory is mapped into the guest. @@ -114,8 +299,10 @@ static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, */ static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index) { - /* TODO: Support huge pages. */ - return filemap_grab_folio(inode->i_mapping, index); + if (is_kvm_gmem_hugetlb(inode)) + return kvm_gmem_get_hugetlb_folio(inode, index); + else + return filemap_grab_folio(inode->i_mapping, index); }

static void kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start, @@ -240,6 +427,35 @@ static void kvm_gmem_hugetlb_truncate_folios_range(struct inode *inode, spin_unlock(&inode->i_lock); }

+static void kvm_gmem_hugetlb_truncate_range(struct inode *inode, loff_t lstart, + loff_t lend) +{ + loff_t full_hpage_start; + loff_t full_hpage_end; + unsigned long hsize; + struct hstate *h; + + h = kvm_gmem_hgmem(inode)->h; + hsize = huge_page_size(h); + + full_hpage_start = round_up(lstart, hsize); + full_hpage_end = round_down(lend, hsize); + + if (lstart < full_hpage_start) { + hugetlb_zero_partial_page(h, inode->i_mapping, lstart, + full_hpage_start); + } + + if (full_hpage_end > full_hpage_start) { + kvm_gmem_hugetlb_truncate_folios_range(inode, full_hpage_start, + full_hpage_end); + } + + if (lend > full_hpage_end) { + hugetlb_zero_partial_page(h, inode->i_mapping, full_hpage_end, + lend); + } +}

static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) { @@ -257,7 +473,12 @@ static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len) list_for_each_entry(gmem, gmem_list, entry) kvm_gmem_invalidate_begin(gmem, start, end);

- truncate_inode_pages_range(inode->i_mapping, offset, offset + len - 1); + if (is_kvm_gmem_hugetlb(inode)) { + kvm_gmem_hugetlb_truncate_range(inode, offset, offset + len); + } else { + truncate_inode_pages_range(inode->i_mapping, offset, + offset + len - 1); + }

list_for_each_entry(gmem, gmem_list, entry) kvm_gmem_invalidate_end(gmem, start, end); @@ -279,8 +500,15 @@ static long kvm_gmem_allocate(struct inode *inode, loff_t offset, loff_t len)

filemap_invalidate_lock_shared(mapping);

- start = offset >> PAGE_SHIFT; - end = (offset + len) >> PAGE_SHIFT; + if (is_kvm_gmem_hugetlb(inode)) { + unsigned long hsize = huge_page_size(kvm_gmem_hgmem(inode)->h); + + start = round_down(offset, hsize) >> PAGE_SHIFT; + end = round_down(offset + len, hsize) >> PAGE_SHIFT; + } else { + start = offset >> PAGE_SHIFT; + end = (offset + len) >> PAGE_SHIFT; + }

r = 0; for (index = start; index < end; ) { @@ -408,9 +636,7 @@ static void kvm_gmem_hugetlb_teardown(struct inode *inode)

static void kvm_gmem_evict_inode(struct inode *inode) { - u64 flags = (u64)inode->i_private; - - if (flags & KVM_GUEST_MEMFD_HUGETLB) + if (is_kvm_gmem_hugetlb(inode)) kvm_gmem_hugetlb_teardown(inode); else truncate_inode_pages_final(inode->i_mapping); @@ -827,7 +1053,7 @@ __kvm_gmem_get_pfn(struct file *file, struct kvm_memory_slot *slot,

*pfn = folio_file_pfn(folio, index); if (max_order) - *max_order = 0; + *max_order = folio_order(folio);

*is_prepared = folio_test_uptodate(folio); return folio;

Add tests for 2MB and 1GB page sizes, and update the invalid flags test for the new KVM_GUEST_MEMFD_HUGETLB flag.

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- .../testing/selftests/kvm/guest_memfd_test.c | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-)

diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c index ba0c8e996035..3618ce06663e 100644 --- a/tools/testing/selftests/kvm/guest_memfd_test.c +++ b/tools/testing/selftests/kvm/guest_memfd_test.c @@ -13,6 +13,7 @@

#include <linux/bitmap.h> #include <linux/falloc.h> +#include <linux/kvm.h> #include <sys/mman.h> #include <sys/types.h> #include <sys/stat.h> @@ -122,6 +123,7 @@ static void test_invalid_punch_hole(int fd, size_t page_size, size_t total_size)

static void test_create_guest_memfd_invalid(struct kvm_vm *vm) { + uint64_t valid_flags = KVM_GUEST_MEMFD_HUGETLB; size_t page_size = getpagesize(); uint64_t flag; size_t size; @@ -135,6 +137,9 @@ static void test_create_guest_memfd_invalid(struct kvm_vm *vm) }

for (flag = 0; flag; flag <<= 1) { + if (flag & valid_flags) + continue; + fd = __vm_create_guest_memfd(vm, page_size, flag); TEST_ASSERT(fd == -1 && errno == EINVAL, "guest_memfd() with flag '0x%lx' should fail with EINVAL", @@ -170,24 +175,16 @@ static void test_create_guest_memfd_multiple(struct kvm_vm *vm) close(fd1); }

-int main(int argc, char *argv[]) +static void test_guest_memfd(struct kvm_vm *vm, uint32_t flags, size_t page_size) { - size_t page_size; size_t total_size; int fd; - struct kvm_vm *vm;

TEST_REQUIRE(kvm_has_cap(KVM_CAP_GUEST_MEMFD));

- page_size = getpagesize(); total_size = page_size * 4;

- vm = vm_create_barebones(); - - test_create_guest_memfd_invalid(vm); - test_create_guest_memfd_multiple(vm); - - fd = vm_create_guest_memfd(vm, total_size, 0); + fd = vm_create_guest_memfd(vm, total_size, flags);

test_file_read_write(fd); test_mmap(fd, page_size); @@ -197,3 +194,31 @@ int main(int argc, char *argv[])

close(fd); } + +int main(int argc, char *argv[]) +{ + struct kvm_vm *vm; + + TEST_REQUIRE(kvm_has_cap(KVM_CAP_GUEST_MEMFD)); + + vm = vm_create_barebones(); + + test_create_guest_memfd_invalid(vm); + test_create_guest_memfd_multiple(vm); + + printf("Test guest_memfd with 4K pages\n"); + test_guest_memfd(vm, 0, getpagesize()); + printf("\tPASSED\n"); + + printf("Test guest_memfd with 2M pages\n"); + test_guest_memfd(vm, KVM_GUEST_MEMFD_HUGETLB | KVM_GUEST_MEMFD_HUGE_2MB, + 2UL << 20); + printf("\tPASSED\n"); + + printf("Test guest_memfd with 1G pages\n"); + test_guest_memfd(vm, KVM_GUEST_MEMFD_HUGETLB | KVM_GUEST_MEMFD_HUGE_1GB, + 1UL << 30); + printf("\tPASSED\n"); + + return 0; +}

Update private_mem_conversions_test for various private memory backing source types.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- .../kvm/x86_64/private_mem_conversions_test.c | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c index 82a8d88b5338..71f480c19f92 100644 --- a/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c +++ b/tools/testing/selftests/kvm/x86_64/private_mem_conversions_test.c @@ -366,14 +366,20 @@ static void *__test_mem_conversions(void *__vcpu) } }

-static void test_mem_conversions(enum vm_mem_backing_src_type src_type, uint32_t nr_vcpus, - uint32_t nr_memslots) +static void +test_mem_conversions(enum vm_mem_backing_src_type src_type, + enum vm_private_mem_backing_src_type private_mem_src_type, + uint32_t nr_vcpus, + uint32_t nr_memslots) { /* * Allocate enough memory so that each vCPU's chunk of memory can be * naturally aligned with respect to the size of the backing store. */ - const size_t alignment = max_t(size_t, SZ_2M, get_backing_src_pagesz(src_type)); + const size_t alignment = max_t(size_t, SZ_2M, + max_t(size_t, + get_private_mem_backing_src_pagesz(private_mem_src_type), + get_backing_src_pagesz(src_type))); const size_t per_cpu_size = align_up(PER_CPU_DATA_SIZE, alignment); const size_t memfd_size = per_cpu_size * nr_vcpus; const size_t slot_size = memfd_size / nr_memslots; @@ -394,7 +400,9 @@ static void test_mem_conversions(enum vm_mem_backing_src_type src_type, uint32_t

vm_enable_cap(vm, KVM_CAP_EXIT_HYPERCALL, (1 << KVM_HC_MAP_GPA_RANGE));

- memfd = vm_create_guest_memfd(vm, memfd_size, 0); + memfd = vm_create_guest_memfd( + vm, memfd_size, + vm_private_mem_backing_src_alias(private_mem_src_type)->flag);

for (i = 0; i < nr_memslots; i++) vm_mem_add(vm, src_type, BASE_DATA_GPA + slot_size * i, @@ -440,10 +448,12 @@ static void test_mem_conversions(enum vm_mem_backing_src_type src_type, uint32_t static void usage(const char *cmd) { puts(""); - printf("usage: %s [-h] [-m nr_memslots] [-s mem_type] [-n nr_vcpus]\n", cmd); + printf("usage: %s [-h] [-m nr_memslots] [-s mem_type] [-p private_mem_type] [-n nr_vcpus]\n", cmd); puts(""); backing_src_help("-s"); puts(""); + private_mem_backing_src_help("-p"); + puts(""); puts(" -n: specify the number of vcpus (default: 1)"); puts(""); puts(" -m: specify the number of memslots (default: 1)"); @@ -453,17 +463,21 @@ static void usage(const char *cmd) int main(int argc, char *argv[]) { enum vm_mem_backing_src_type src_type = DEFAULT_VM_MEM_SRC; + enum vm_private_mem_backing_src_type private_mem_src_type = DEFAULT_VM_PRIVATE_MEM_SRC; uint32_t nr_memslots = 1; uint32_t nr_vcpus = 1; int opt;

TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM));

- while ((opt = getopt(argc, argv, "hm:s:n:")) != -1) { + while ((opt = getopt(argc, argv, "hm:s:p:n:")) != -1) { switch (opt) { case 's': src_type = parse_backing_src_type(optarg); break; + case 'p': + private_mem_src_type = parse_private_mem_backing_src_type(optarg); + break; case 'n': nr_vcpus = atoi_positive("nr_vcpus", optarg); break; @@ -477,7 +491,7 @@ int main(int argc, char *argv[]) } }

- test_mem_conversions(src_type, nr_vcpus, nr_memslots); + test_mem_conversions(src_type, private_mem_src_type, nr_vcpus, nr_memslots);

return 0; }

Using HugeTLB as the huge page allocator for guest_memfd allows reuse of HugeTLB's reporting mechanism.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- tools/testing/selftests/kvm/Makefile | 1 + .../kvm/guest_memfd_hugetlb_reporting_test.c | 222 ++++++++++++++++++ 2 files changed, 223 insertions(+) create mode 100644 tools/testing/selftests/kvm/guest_memfd_hugetlb_reporting_test.c

diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 48d32c5aa3eb..b3b7e83f39fc 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -134,6 +134,7 @@ TEST_GEN_PROGS_x86_64 += demand_paging_test TEST_GEN_PROGS_x86_64 += dirty_log_test TEST_GEN_PROGS_x86_64 += dirty_log_perf_test TEST_GEN_PROGS_x86_64 += guest_memfd_test +TEST_GEN_PROGS_x86_64 += guest_memfd_hugetlb_reporting_test TEST_GEN_PROGS_x86_64 += guest_print_test TEST_GEN_PROGS_x86_64 += hardware_disable_test TEST_GEN_PROGS_x86_64 += kvm_create_max_vcpus diff --git a/tools/testing/selftests/kvm/guest_memfd_hugetlb_reporting_test.c b/tools/testing/selftests/kvm/guest_memfd_hugetlb_reporting_test.c new file mode 100644 index 000000000000..cb9fdf0d4ec8 --- /dev/null +++ b/tools/testing/selftests/kvm/guest_memfd_hugetlb_reporting_test.c @@ -0,0 +1,222 @@ +#include <fcntl.h> +#include <linux/falloc.h> +#include <linux/kvm.h> +#include <linux/limits.h> +#include <linux/memfd.h> +#include <string.h> +#include <sys/mman.h> + +#include "kvm_util.h" +#include "test_util.h" +#include "processor.h" + +static int read_int(const char *file_name) +{ + FILE *fp; + int num; + + fp = fopen(file_name, "r"); + TEST_ASSERT(fp != NULL, "Error opening file %s!\n", file_name); + + TEST_ASSERT_EQ(fscanf(fp, "%d", &num), 1); + + fclose(fp); + + return num; +} + +enum hugetlb_statistic { + FREE_HUGEPAGES, + NR_HUGEPAGES, + NR_OVERCOMMIT_HUGEPAGES, + RESV_HUGEPAGES, + SURPLUS_HUGEPAGES, + NR_TESTED_HUGETLB_STATISTICS, +}; + +static const char *hugetlb_statistics[NR_TESTED_HUGETLB_STATISTICS] = { + [FREE_HUGEPAGES] = "free_hugepages", + [NR_HUGEPAGES] = "nr_hugepages", + [NR_OVERCOMMIT_HUGEPAGES] = "nr_overcommit_hugepages", + [RESV_HUGEPAGES] = "resv_hugepages", + [SURPLUS_HUGEPAGES] = "surplus_hugepages", +}; + +enum test_page_size { + TEST_SZ_2M, + TEST_SZ_1G, + NR_TEST_SIZES, +}; + +struct test_param { + size_t page_size; + int memfd_create_flags; + int guest_memfd_flags; + char *path_suffix; +}; + +const struct test_param *test_params(enum test_page_size size) +{ + static const struct test_param params[] = { + [TEST_SZ_2M] = { + .page_size = PG_SIZE_2M, + .memfd_create_flags = MFD_HUGETLB | MFD_HUGE_2MB, + .guest_memfd_flags = KVM_GUEST_MEMFD_HUGETLB | KVM_GUEST_MEMFD_HUGE_2MB, + .path_suffix = "2048kB", + }, + [TEST_SZ_1G] = { + .page_size = PG_SIZE_1G, + .memfd_create_flags = MFD_HUGETLB | MFD_HUGE_1GB, + .guest_memfd_flags = KVM_GUEST_MEMFD_HUGETLB | KVM_GUEST_MEMFD_HUGE_1GB, + .path_suffix = "1048576kB", + }, + }; + + return &params[size]; +} + +static int read_statistic(enum test_page_size size, enum hugetlb_statistic statistic) +{ + char path[PATH_MAX] = "/sys/kernel/mm/hugepages/hugepages-"; + + strcat(path, test_params(size)->path_suffix); + strcat(path, "/"); + strcat(path, hugetlb_statistics[statistic]); + + return read_int(path); +} + +static int baseline[NR_TEST_SIZES][NR_TESTED_HUGETLB_STATISTICS]; + +static void establish_baseline(void) +{ + int i, j; + + for (i = 0; i < NR_TEST_SIZES; ++i) + for (j = 0; j < NR_TESTED_HUGETLB_STATISTICS; ++j) + baseline[i][j] = read_statistic(i, j); +} + +static void assert_stats_at_baseline(void) +{ + TEST_ASSERT_EQ(read_statistic(TEST_SZ_2M, FREE_HUGEPAGES), + baseline[TEST_SZ_2M][FREE_HUGEPAGES]); + TEST_ASSERT_EQ(read_statistic(TEST_SZ_2M, NR_HUGEPAGES), + baseline[TEST_SZ_2M][NR_HUGEPAGES]); + TEST_ASSERT_EQ(read_statistic(TEST_SZ_2M, NR_OVERCOMMIT_HUGEPAGES), + baseline[TEST_SZ_2M][NR_OVERCOMMIT_HUGEPAGES]); + TEST_ASSERT_EQ(read_statistic(TEST_SZ_2M, RESV_HUGEPAGES), + baseline[TEST_SZ_2M][RESV_HUGEPAGES]); + TEST_ASSERT_EQ(read_statistic(TEST_SZ_2M, SURPLUS_HUGEPAGES), + baseline[TEST_SZ_2M][SURPLUS_HUGEPAGES]); + + TEST_ASSERT_EQ(read_statistic(TEST_SZ_1G, FREE_HUGEPAGES), + baseline[TEST_SZ_1G][FREE_HUGEPAGES]); + TEST_ASSERT_EQ(read_statistic(TEST_SZ_1G, NR_HUGEPAGES), + baseline[TEST_SZ_1G][NR_HUGEPAGES]); + TEST_ASSERT_EQ(read_statistic(TEST_SZ_1G, NR_OVERCOMMIT_HUGEPAGES), + baseline[TEST_SZ_1G][NR_OVERCOMMIT_HUGEPAGES]); + TEST_ASSERT_EQ(read_statistic(TEST_SZ_1G, RESV_HUGEPAGES), + baseline[TEST_SZ_1G][RESV_HUGEPAGES]); + TEST_ASSERT_EQ(read_statistic(TEST_SZ_1G, SURPLUS_HUGEPAGES), + baseline[TEST_SZ_1G][SURPLUS_HUGEPAGES]); +} + +static void assert_stats(enum test_page_size size, int num_reserved, int num_faulted) +{ + TEST_ASSERT_EQ(read_statistic(size, FREE_HUGEPAGES), + baseline[size][FREE_HUGEPAGES] - num_faulted); + TEST_ASSERT_EQ(read_statistic(size, NR_HUGEPAGES), + baseline[size][NR_HUGEPAGES]); + TEST_ASSERT_EQ(read_statistic(size, NR_OVERCOMMIT_HUGEPAGES), + baseline[size][NR_OVERCOMMIT_HUGEPAGES]); + TEST_ASSERT_EQ(read_statistic(size, RESV_HUGEPAGES), + baseline[size][RESV_HUGEPAGES] + num_reserved - num_faulted); + TEST_ASSERT_EQ(read_statistic(size, SURPLUS_HUGEPAGES), + baseline[size][SURPLUS_HUGEPAGES]); +} + +/* Use hugetlb behavior as a baseline. guest_memfd should have comparable behavior. */ +static void test_hugetlb_behavior(enum test_page_size test_size) +{ + const struct test_param *param; + char *mem; + int memfd; + + param = test_params(test_size); + + assert_stats_at_baseline(); + + memfd = memfd_create("guest_memfd_hugetlb_reporting_test", + param->memfd_create_flags); + + mem = mmap(NULL, param->page_size, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_HUGETLB, memfd, 0); + TEST_ASSERT(mem != MAP_FAILED, "Couldn't mmap()"); + + assert_stats(test_size, 1, 0); + + *mem = 'A'; + + assert_stats(test_size, 1, 1); + + munmap(mem, param->page_size); + + assert_stats(test_size, 1, 1); + + madvise(mem, param->page_size, MADV_DONTNEED); + + assert_stats(test_size, 1, 1); + + madvise(mem, param->page_size, MADV_REMOVE); + + assert_stats(test_size, 1, 1); + + close(memfd); + + assert_stats_at_baseline(); +} + +static void test_guest_memfd_behavior(enum test_page_size test_size) +{ + const struct test_param *param; + struct kvm_vm *vm; + int guest_memfd; + + param = test_params(test_size); + + assert_stats_at_baseline(); + + vm = vm_create_barebones_type(KVM_X86_SW_PROTECTED_VM); + + guest_memfd = vm_create_guest_memfd(vm, param->page_size, + param->guest_memfd_flags); + + assert_stats(test_size, 1, 0); + + fallocate(guest_memfd, FALLOC_FL_KEEP_SIZE, 0, param->page_size); + + assert_stats(test_size, 1, 1); + + fallocate(guest_memfd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, + param->page_size); + + assert_stats(test_size, 1, 0); + + close(guest_memfd); + + assert_stats_at_baseline(); + + kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + establish_baseline(); + + test_hugetlb_behavior(TEST_SZ_2M); + test_hugetlb_behavior(TEST_SZ_1G); + + test_guest_memfd_behavior(TEST_SZ_2M); + test_guest_memfd_behavior(TEST_SZ_1G); +}

These functions will be used by guest_memfd to split/reconstruct HugeTLB pages.

Co-developed-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com Co-developed-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Vishal Annapurve vannapurve@google.com --- include/linux/hugetlb.h | 15 +++++++++++++++ mm/hugetlb.c | 8 ++------ 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 7ba4ed9e0001..ac9d4ada52bd 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -298,6 +298,21 @@ static inline void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct } #endif

+#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE +bool prep_compound_gigantic_folio(struct folio *folio, unsigned int order); +void destroy_compound_gigantic_folio(struct folio *folio, unsigned int order); +#else +bool prep_compound_gigantic_folio(struct folio *folio, unsigned int order) +{ + return false; +} + +static inline void destroy_compound_gigantic_folio(struct folio *folio, + unsigned int order) +{ +} +#endif + #else /* !CONFIG_HUGETLB_PAGE */

static inline void hugetlb_dup_vma_private(struct vm_area_struct *vma) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 372d8294fb2f..8f2b7b411b60 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -1533,8 +1533,7 @@ static void destroy_compound_hugetlb_folio_for_demote(struct folio *folio, }

#ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE -static void destroy_compound_gigantic_folio(struct folio *folio, - unsigned int order) +void destroy_compound_gigantic_folio(struct folio *folio, unsigned int order) { __destroy_compound_gigantic_folio(folio, order, false); } @@ -1609,8 +1608,6 @@ static struct folio *alloc_gigantic_folio(struct hstate *h, gfp_t gfp_mask, } static inline void free_gigantic_folio(struct folio *folio, unsigned int order) { } -static inline void destroy_compound_gigantic_folio(struct folio *folio, - unsigned int order) { } #endif

/* @@ -2120,8 +2117,7 @@ static bool __prep_compound_gigantic_folio(struct folio *folio, return false; }

-static bool prep_compound_gigantic_folio(struct folio *folio, - unsigned int order) +bool prep_compound_gigantic_folio(struct folio *folio, unsigned int order) { return __prep_compound_gigantic_folio(folio, order, false); }

From: Vishal Annapurve vannapurve@google.com

In this patch, newly allocated HugeTLB pages are split to 4K regular pages before providing them to the requester (fallocate() or KVM).

The pages are then reconstructed/merged to HugeTLB pages before the HugeTLB pages are returned to HugeTLB.

This is an intermediate step to build page splitting/merging functionality before allowing guest_memfd files to be mmap()ed.

Co-developed-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com Co-developed-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Vishal Annapurve vannapurve@google.com

--- virt/kvm/guest_memfd.c | 299 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 281 insertions(+), 18 deletions(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index eacbfdb950d1..8151df2c03e5 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -229,31 +229,206 @@ static int kvm_gmem_hugetlb_filemap_add_folio(struct address_space *mapping, return 0; }

+struct kvm_gmem_split_stash { + struct { + unsigned long _flags_2; + unsigned long _head_2; + + void *_hugetlb_subpool; + void *_hugetlb_cgroup; + void *_hugetlb_cgroup_rsvd; + void *_hugetlb_hwpoison; + }; + void *hugetlb_private; +}; + +static int kvm_gmem_hugetlb_stash_metadata(struct folio *folio) +{ + struct kvm_gmem_split_stash *stash; + + stash = kmalloc(sizeof(*stash), GFP_KERNEL); + if (!stash) + return -ENOMEM; + + stash->_flags_2 = folio->_flags_2; + stash->_head_2 = folio->_head_2; + stash->_hugetlb_subpool = folio->_hugetlb_subpool; + stash->_hugetlb_cgroup = folio->_hugetlb_cgroup; + stash->_hugetlb_cgroup_rsvd = folio->_hugetlb_cgroup_rsvd; + stash->_hugetlb_hwpoison = folio->_hugetlb_hwpoison; + stash->hugetlb_private = folio_get_private(folio); + + folio_change_private(folio, (void *)stash); + + return 0; +} + +static int kvm_gmem_hugetlb_unstash_metadata(struct folio *folio) +{ + struct kvm_gmem_split_stash *stash; + + stash = folio_get_private(folio); + + if (!stash) + return -EINVAL; + + folio->_flags_2 = stash->_flags_2; + folio->_head_2 = stash->_head_2; + folio->_hugetlb_subpool = stash->_hugetlb_subpool; + folio->_hugetlb_cgroup = stash->_hugetlb_cgroup; + folio->_hugetlb_cgroup_rsvd = stash->_hugetlb_cgroup_rsvd; + folio->_hugetlb_hwpoison = stash->_hugetlb_hwpoison; + folio_change_private(folio, stash->hugetlb_private); + + kfree(stash); + + return 0; +} + +/** + * Reconstruct a HugeTLB folio from a contiguous block of folios where the first + * of the contiguous folios is @folio. + * + * The size of the contiguous block is of huge_page_size(@h). All the folios in + * the block are checked to have a refcount of 1 before reconstruction. After + * reconstruction, the reconstructed folio has a refcount of 1. + * + * Return 0 on success and negative error otherwise. + */ +static int kvm_gmem_hugetlb_reconstruct_folio(struct hstate *h, struct folio *folio) +{ + int ret; + + WARN_ON((folio->index & (huge_page_order(h) - 1)) != 0); + + ret = kvm_gmem_hugetlb_unstash_metadata(folio); + if (ret) + return ret; + + if (!prep_compound_gigantic_folio(folio, huge_page_order(h))) { + kvm_gmem_hugetlb_stash_metadata(folio); + return -ENOMEM; + } + + __folio_set_hugetlb(folio); + + folio_set_count(folio, 1); + + hugetlb_vmemmap_optimize_folio(h, folio); + + return 0; +} + +/* Basically folio_set_order(folio, 1) without the checks. */ +static inline void kvm_gmem_folio_set_order(struct folio *folio, unsigned int order) +{ + folio->_flags_1 = (folio->_flags_1 & ~0xffUL) | order; +#ifdef CONFIG_64BIT + folio->_folio_nr_pages = 1U << order; +#endif +} + +/** + * Split a HugeTLB @folio of size huge_page_size(@h). + * + * After splitting, each split folio has a refcount of 1. There are no checks on + * refcounts before splitting. + * + * Return 0 on success and negative error otherwise. + */ +static int kvm_gmem_hugetlb_split_folio(struct hstate *h, struct folio *folio) +{ + int ret; + + ret = hugetlb_vmemmap_restore_folio(h, folio); + if (ret) + return ret; + + ret = kvm_gmem_hugetlb_stash_metadata(folio); + if (ret) { + hugetlb_vmemmap_optimize_folio(h, folio); + return ret; + } + + kvm_gmem_folio_set_order(folio, 0); + + destroy_compound_gigantic_folio(folio, huge_page_order(h)); + __folio_clear_hugetlb(folio); + + /* + * Remove the first folio from h->hugepage_activelist since it is no + * longer a HugeTLB page. The other split pages should not be on any + * lists. + */ + hugetlb_folio_list_del(folio); + + return 0; +} + static struct folio *kvm_gmem_hugetlb_alloc_and_cache_folio(struct inode *inode, pgoff_t index) { + struct folio *allocated_hugetlb_folio; + pgoff_t hugetlb_first_subpage_index; + struct page *hugetlb_first_subpage; struct kvm_gmem_hugetlb *hgmem; - struct folio *folio; + struct page *requested_page; int ret; + int i;

hgmem = kvm_gmem_hgmem(inode); - folio = kvm_gmem_hugetlb_alloc_folio(hgmem->h, hgmem->spool); - if (IS_ERR(folio)) - return folio; + allocated_hugetlb_folio = kvm_gmem_hugetlb_alloc_folio(hgmem->h, hgmem->spool); + if (IS_ERR(allocated_hugetlb_folio)) + return allocated_hugetlb_folio; + + requested_page = folio_file_page(allocated_hugetlb_folio, index); + hugetlb_first_subpage = folio_file_page(allocated_hugetlb_folio, 0); + hugetlb_first_subpage_index = index & (huge_page_mask(hgmem->h) >> PAGE_SHIFT);

- /* TODO: Fix index here to be aligned to huge page size. */ - ret = kvm_gmem_hugetlb_filemap_add_folio( - inode->i_mapping, folio, index, htlb_alloc_mask(hgmem->h)); + ret = kvm_gmem_hugetlb_split_folio(hgmem->h, allocated_hugetlb_folio); if (ret) { - folio_put(folio); + folio_put(allocated_hugetlb_folio); return ERR_PTR(ret); }

+ for (i = 0; i < pages_per_huge_page(hgmem->h); ++i) { + struct folio *folio = page_folio(nth_page(hugetlb_first_subpage, i)); + + ret = kvm_gmem_hugetlb_filemap_add_folio(inode->i_mapping, + folio, + hugetlb_first_subpage_index + i, + htlb_alloc_mask(hgmem->h)); + if (ret) { + /* TODO: handle cleanup properly. */ + pr_err("Handle cleanup properly index=%lx, ret=%d\n", + hugetlb_first_subpage_index + i, ret); + dump_page(nth_page(hugetlb_first_subpage, i), "check"); + return ERR_PTR(ret); + } + + /* + * Skip unlocking for the requested index since + * kvm_gmem_get_folio() returns a locked folio. + * + * Do folio_put() to drop the refcount that came with the folio, + * from splitting the folio. Splitting the folio has a refcount + * to be in line with hugetlb_alloc_folio(), which returns a + * folio with refcount 1. + * + * Skip folio_put() for requested index since + * kvm_gmem_get_folio() returns a folio with refcount 1. + */ + if (hugetlb_first_subpage_index + i != index) { + folio_unlock(folio); + folio_put(folio); + } + } + spin_lock(&inode->i_lock); inode->i_blocks += blocks_per_huge_page(hgmem->h); spin_unlock(&inode->i_lock);

- return folio; + return page_folio(requested_page); }

static struct folio *kvm_gmem_get_hugetlb_folio(struct inode *inode, @@ -365,7 +540,9 @@ static inline void kvm_gmem_hugetlb_filemap_remove_folio(struct folio *folio)

/** * Removes folios in range [@lstart, @lend) from page cache/filemap (@mapping), - * returning the number of pages freed. + * returning the number of HugeTLB pages freed. + * + * @lend - @lstart must be a multiple of the HugeTLB page size. */ static int kvm_gmem_hugetlb_filemap_remove_folios(struct address_space *mapping, struct hstate *h, @@ -373,37 +550,69 @@ static int kvm_gmem_hugetlb_filemap_remove_folios(struct address_space *mapping, { const pgoff_t end = lend >> PAGE_SHIFT; pgoff_t next = lstart >> PAGE_SHIFT; + LIST_HEAD(folios_to_reconstruct); struct folio_batch fbatch; + struct folio *folio, *tmp; int num_freed = 0; + int i;

+ /* + * TODO: Iterate over huge_page_size(h) blocks to avoid taking and + * releasing hugetlb_fault_mutex_table[hash] lock so often. When + * truncating, lstart and lend should be clipped to the size of this + * guest_memfd file, otherwise there would be too many iterations. + */ folio_batch_init(&fbatch); while (filemap_get_folios(mapping, &next, end - 1, &fbatch)) { - int i; for (i = 0; i < folio_batch_count(&fbatch); ++i) { struct folio *folio; pgoff_t hindex; u32 hash;

folio = fbatch.folios[i]; + hindex = folio->index >> huge_page_order(h); hash = hugetlb_fault_mutex_hash(mapping, hindex); - mutex_lock(&hugetlb_fault_mutex_table[hash]); + + /* + * Collect first pages of HugeTLB folios for + * reconstruction later. + */ + if ((folio->index & ~(huge_page_mask(h) >> PAGE_SHIFT)) == 0) + list_add(&folio->lru, &folios_to_reconstruct); + + /* + * Before removing from filemap, take a reference so + * sub-folios don't get freed. Don't free the sub-folios + * until after reconstruction. + */ + folio_get(folio); + kvm_gmem_hugetlb_filemap_remove_folio(folio); - mutex_unlock(&hugetlb_fault_mutex_table[hash]);

- num_freed++; + mutex_unlock(&hugetlb_fault_mutex_table[hash]); } folio_batch_release(&fbatch); cond_resched(); }

+ list_for_each_entry_safe(folio, tmp, &folios_to_reconstruct, lru) { + kvm_gmem_hugetlb_reconstruct_folio(h, folio); + hugetlb_folio_list_move(folio, &h->hugepage_activelist); + + folio_put(folio); + num_freed++; + } + return num_freed; }

/** * Removes folios in range [@lstart, @lend) from page cache of inode, updates * inode metadata and hugetlb reservations. + * + * @lend - @lstart must be a multiple of the HugeTLB page size. */ static void kvm_gmem_hugetlb_truncate_folios_range(struct inode *inode, loff_t lstart, loff_t lend) @@ -427,6 +636,56 @@ static void kvm_gmem_hugetlb_truncate_folios_range(struct inode *inode, spin_unlock(&inode->i_lock); }

+/** + * Zeroes offsets [@start, @end) in a folio from @mapping. + * + * [@start, @end) must be within the same folio. + */ +static void kvm_gmem_zero_partial_page( + struct address_space *mapping, loff_t start, loff_t end) +{ + struct folio *folio; + pgoff_t idx = start >> PAGE_SHIFT; + + folio = filemap_lock_folio(mapping, idx); + if (IS_ERR(folio)) + return; + + start = offset_in_folio(folio, start); + end = offset_in_folio(folio, end); + if (!end) + end = folio_size(folio); + + folio_zero_segment(folio, (size_t)start, (size_t)end); + folio_unlock(folio); + folio_put(folio); +} + +/** + * Zeroes all pages in range [@start, @end) in @mapping. + * + * hugetlb_zero_partial_page() would work if this had been a full page, but is + * not suitable since the pages have been split. + * + * truncate_inode_pages_range() isn't the right function because it removes + * pages from the page cache; this function only zeroes the pages. + */ +static void kvm_gmem_hugetlb_zero_split_pages(struct address_space *mapping, + loff_t start, loff_t end) +{ + loff_t aligned_start; + loff_t index; + + aligned_start = round_up(start, PAGE_SIZE); + + kvm_gmem_zero_partial_page(mapping, start, min(aligned_start, end)); + + for (index = aligned_start; index < end; index += PAGE_SIZE) { + kvm_gmem_zero_partial_page(mapping, index, + min((loff_t)(index + PAGE_SIZE), end)); + } +} + static void kvm_gmem_hugetlb_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) { @@ -442,8 +701,8 @@ static void kvm_gmem_hugetlb_truncate_range(struct inode *inode, loff_t lstart, full_hpage_end = round_down(lend, hsize);

if (lstart < full_hpage_start) { - hugetlb_zero_partial_page(h, inode->i_mapping, lstart, - full_hpage_start); + kvm_gmem_hugetlb_zero_split_pages(inode->i_mapping, lstart, + full_hpage_start); }

if (full_hpage_end > full_hpage_start) { @@ -452,8 +711,8 @@ static void kvm_gmem_hugetlb_truncate_range(struct inode *inode, loff_t lstart, }

if (lend > full_hpage_end) { - hugetlb_zero_partial_page(h, inode->i_mapping, full_hpage_end, - lend); + kvm_gmem_hugetlb_zero_split_pages(inode->i_mapping, full_hpage_end, + lend); } }

@@ -1060,6 +1319,10 @@ __kvm_gmem_get_pfn(struct file *file, struct kvm_memory_slot *slot,

if (folio_test_hwpoison(folio)) { folio_unlock(folio); + /* + * TODO: this folio may be part of a HugeTLB folio. Perhaps + * reconstruct and then free page? + */ folio_put(folio); return ERR_PTR(-EHWPOISON); }

Memory of a KVM_X86_SW_PROTECTED_VM defaults to faultable to align with the default in kvm->mem_attr_array.

For this RFC, determine default faultability when associating a range with a memslot.

Another option is to determine default faultability at guest_memfd creation time. guest_memfd is created for a specific VM, hence we can set default faultability based on the VM type.

In future, if different struct kvms are bound to the same guest_memfd inode, all the struct kvms must be of the same vm_type.

TODO: Perhaps faultability should be based on kvm->mem_attr_array?

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- virt/kvm/guest_memfd.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index fc2483e35876..1d4dfe0660ad 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -1256,6 +1256,23 @@ static struct file *kvm_gmem_inode_create_getfile(void *priv, loff_t size, return file; }

+static void kvm_gmem_set_default_faultability_by_vm_type(struct inode *inode, + u8 vm_type, + loff_t start, loff_t end) +{ + bool faultable; + + switch (vm_type) { + case KVM_X86_SW_PROTECTED_VM: + faultable = true; + break; + default: + faultable = false; + } + + WARN_ON(kvm_gmem_set_faultable(inode, start, end, faultable)); +} + static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) { struct kvm_gmem *gmem; @@ -1378,6 +1395,11 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, slot->gmem.pgoff = start;

xa_store_range(&gmem->bindings, start, end - 1, slot, GFP_KERNEL); + + kvm_gmem_set_default_faultability_by_vm_type(file_inode(file), + kvm->arch.vm_type, + start, end); + filemap_invalidate_unlock(inode->i_mapping);

/*

Since guest_memfd now supports mmap(), folios have to be prepared before they are faulted into userspace.

When memory attributes are switched between shared and private, the up-to-date flags will be cleared.

Use the folio's up-to-date flag to indicate being ready for the guest usage and can be used to mark whether the folio is ready for shared OR private use.

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- virt/kvm/guest_memfd.c | 131 ++++++++++++++++++++++++++++++++++++++++- virt/kvm/kvm_main.c | 2 + virt/kvm/kvm_mm.h | 7 +++ 3 files changed, 139 insertions(+), 1 deletion(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 110c4bbb004b..fb292e542381 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -129,13 +129,29 @@ static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slo }

/** - * Use the uptodate flag to indicate that the folio is prepared for KVM's usage. + * Use folio's up-to-date flag to indicate that this folio is prepared for usage + * by the guest. + * + * This flag can be used whether the folio is prepared for PRIVATE or SHARED + * usage. */ static inline void kvm_gmem_mark_prepared(struct folio *folio) { folio_mark_uptodate(folio); }

+/** + * Use folio's up-to-date flag to indicate that this folio is not yet prepared for + * usage by the guest. + * + * This flag can be used whether the folio is prepared for PRIVATE or SHARED + * usage. + */ +static inline void kvm_gmem_clear_prepared(struct folio *folio) +{ + folio_clear_uptodate(folio); +} + /* * Process @folio, which contains @gfn, so that the guest can use it. * The folio must be locked and the gfn must be contained in @slot. @@ -148,6 +164,12 @@ static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, pgoff_t index; int r;

+ /* + * Defensively zero folio to avoid leaking kernel memory in + * uninitialized pages. This is important since pages can now be mapped + * into userspace, where hardware (e.g. TDX) won't be clearing those + * pages. + */ if (folio_test_hugetlb(folio)) { folio_zero_user(folio, folio->index << PAGE_SHIFT); } else { @@ -1017,6 +1039,7 @@ static vm_fault_t kvm_gmem_fault(struct vm_fault *vmf) { struct inode *inode; struct folio *folio; + bool is_prepared;

inode = file_inode(vmf->vma->vm_file); if (!kvm_gmem_is_faultable(inode, vmf->pgoff)) @@ -1026,6 +1049,31 @@ static vm_fault_t kvm_gmem_fault(struct vm_fault *vmf) if (!folio) return VM_FAULT_SIGBUS;

+ is_prepared = folio_test_uptodate(folio); + if (!is_prepared) { + unsigned long nr_pages; + unsigned long i; + + if (folio_test_hugetlb(folio)) { + folio_zero_user(folio, folio->index << PAGE_SHIFT); + } else { + /* + * Defensively zero folio to avoid leaking kernel memory in + * uninitialized pages. This is important since pages can now be + * mapped into userspace, where hardware (e.g. TDX) won't be + * clearing those pages. + * + * Will probably need a version of kvm_gmem_prepare_folio() to + * prepare the page for SHARED use. + */ + nr_pages = folio_nr_pages(folio); + for (i = 0; i < nr_pages; i++) + clear_highpage(folio_page(folio, i)); + } + + kvm_gmem_mark_prepared(folio); + } + vmf->page = folio_file_page(folio, vmf->pgoff); return VM_FAULT_LOCKED; } @@ -1593,6 +1641,87 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long } EXPORT_SYMBOL_GPL(kvm_gmem_populate);

+static void kvm_gmem_clear_prepared_range(struct inode *inode, pgoff_t start, + pgoff_t end) +{ + pgoff_t index; + + filemap_invalidate_lock_shared(inode->i_mapping); + + /* TODO: replace iteration with filemap_get_folios() for efficiency. */ + for (index = start; index < end;) { + struct folio *folio; + + /* Don't use kvm_gmem_get_folio to avoid allocating */ + folio = filemap_lock_folio(inode->i_mapping, index); + if (IS_ERR(folio)) { + ++index; + continue; + } + + kvm_gmem_clear_prepared(folio); + + index = folio_next_index(folio); + folio_unlock(folio); + folio_put(folio); + } + + filemap_invalidate_unlock_shared(inode->i_mapping); +} + +/** + * Clear the prepared flag for all folios in gfn range [@start, @end) in memslot + * @slot. + */ +static void kvm_gmem_clear_prepared_slot(struct kvm_memory_slot *slot, gfn_t start, + gfn_t end) +{ + pgoff_t start_offset; + pgoff_t end_offset; + struct file *file; + + file = kvm_gmem_get_file(slot); + if (!file) + return; + + start_offset = start - slot->base_gfn + slot->gmem.pgoff; + end_offset = end - slot->base_gfn + slot->gmem.pgoff; + + kvm_gmem_clear_prepared_range(file_inode(file), start_offset, end_offset); + + fput(file); +} + +/** + * Clear the prepared flag for all folios for any slot in gfn range + * [@start, @end) in @kvm. + */ +void kvm_gmem_clear_prepared_vm(struct kvm *kvm, gfn_t start, gfn_t end) +{ + int i; + + lockdep_assert_held(&kvm->slots_lock); + + for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { + struct kvm_memslot_iter iter; + struct kvm_memslots *slots; + + slots = __kvm_memslots(kvm, i); + kvm_for_each_memslot_in_gfn_range(&iter, slots, start, end) { + struct kvm_memory_slot *slot; + gfn_t gfn_start; + gfn_t gfn_end; + + slot = iter.slot; + gfn_start = max(start, slot->base_gfn); + gfn_end = min(end, slot->base_gfn + slot->npages); + + if (iter.slot->flags & KVM_MEM_GUEST_MEMFD) + kvm_gmem_clear_prepared_slot(iter.slot, gfn_start, gfn_end); + } + } +} + /** * Returns true if pages in range [@start, @end) in inode @inode have no * userspace mappings. diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 1a7bbcc31b7e..255d27df7f5c 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2565,6 +2565,8 @@ static int kvm_vm_set_mem_attributes(struct kvm *kvm, gfn_t start, gfn_t end, KVM_BUG_ON(r, kvm); }

+ kvm_gmem_clear_prepared_vm(kvm, start, end); + kvm_handle_gfn_range(kvm, &post_set_range);

out_unlock: diff --git a/virt/kvm/kvm_mm.h b/virt/kvm/kvm_mm.h index d8ff2b380d0e..25fd0d9f66cc 100644 --- a/virt/kvm/kvm_mm.h +++ b/virt/kvm/kvm_mm.h @@ -43,6 +43,7 @@ int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot, void kvm_gmem_unbind(struct kvm_memory_slot *slot); int kvm_gmem_should_set_attributes(struct kvm *kvm, gfn_t start, gfn_t end, unsigned long attrs); +void kvm_gmem_clear_prepared_vm(struct kvm *kvm, gfn_t start, gfn_t end); #else static inline void kvm_gmem_init(struct module *module) { @@ -68,6 +69,12 @@ static inline int kvm_gmem_should_set_attributes(struct kvm *kvm, gfn_t start, return 0; }

+static inline void kvm_gmem_clear_prepared_slots(struct kvm *kvm, + gfn_t start, gfn_t end) +{ + WARN_ON_ONCE(1); +} + #endif /* CONFIG_KVM_PRIVATE_MEM */

#endif /* __KVM_MM_H__ */

No functional change intended.

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- tools/testing/selftests/kvm/include/kvm_util.h | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 63c2aaae51f3..d336cd0c8f19 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -374,8 +374,8 @@ static inline void vm_enable_cap(struct kvm_vm *vm, uint32_t cap, uint64_t arg0) vm_ioctl(vm, KVM_ENABLE_CAP, &enable_cap); }

-static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa, - uint64_t size, uint64_t attributes) +static inline int __vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa, + uint64_t size, uint64_t attributes) { struct kvm_memory_attributes attr = { .attributes = attributes, @@ -391,7 +391,15 @@ static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa, TEST_ASSERT(!attributes || attributes == KVM_MEMORY_ATTRIBUTE_PRIVATE, "Update me to support multiple attributes!");

- vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); + return __vm_ioctl(vm, KVM_SET_MEMORY_ATTRIBUTES, &attr); +} + +static inline void vm_set_memory_attributes(struct kvm_vm *vm, uint64_t gpa, + uint64_t size, uint64_t attributes) +{ + int ret = __vm_set_memory_attributes(vm, gpa, size, attributes); + + __TEST_ASSERT_VM_VCPU_IOCTL(!ret, "KVM_SET_MEMORY_ATTRIBUTES", ret, vm); }

Minimal test for guest_memfd to test that when memory is marked shared in a VM, the host can read and write to it via an mmap()ed address, and the guest can also read and write to it.

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- tools/testing/selftests/kvm/Makefile | 1 + .../selftests/kvm/guest_memfd_sharing_test.c | 160 ++++++++++++++++++ 2 files changed, 161 insertions(+) create mode 100644 tools/testing/selftests/kvm/guest_memfd_sharing_test.c

diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index b3b7e83f39fc..3c1f35456bfc 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -135,6 +135,7 @@ TEST_GEN_PROGS_x86_64 += dirty_log_test TEST_GEN_PROGS_x86_64 += dirty_log_perf_test TEST_GEN_PROGS_x86_64 += guest_memfd_test TEST_GEN_PROGS_x86_64 += guest_memfd_hugetlb_reporting_test +TEST_GEN_PROGS_x86_64 += guest_memfd_sharing_test TEST_GEN_PROGS_x86_64 += guest_print_test TEST_GEN_PROGS_x86_64 += hardware_disable_test TEST_GEN_PROGS_x86_64 += kvm_create_max_vcpus diff --git a/tools/testing/selftests/kvm/guest_memfd_sharing_test.c b/tools/testing/selftests/kvm/guest_memfd_sharing_test.c new file mode 100644 index 000000000000..fef5a73e5053 --- /dev/null +++ b/tools/testing/selftests/kvm/guest_memfd_sharing_test.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Minimal test for guest_memfd to test that when memory is marked shared in a + * VM, the host can read and write to it via an mmap()ed address, and the guest + * can also read and write to it. + * + * Copyright (c) 2024, Google LLC. + */ +#include <string.h> +#include <sys/mman.h> +#include <unistd.h> + +#include "test_util.h" +#include "kvm_util.h" +#include "ucall_common.h" + +#define GUEST_MEMFD_SHARING_TEST_SLOT 10 +#define GUEST_MEMFD_SHARING_TEST_GPA 0x50000000ULL +#define GUEST_MEMFD_SHARING_TEST_GVA 0x90000000ULL +#define GUEST_MEMFD_SHARING_TEST_OFFSET 0 +#define GUEST_MEMFD_SHARING_TEST_GUEST_TO_HOST_VALUE 0x11 +#define GUEST_MEMFD_SHARING_TEST_HOST_TO_GUEST_VALUE 0x22 + +static void guest_code(int page_size) +{ + char *mem; + int i; + + mem = (char *)GUEST_MEMFD_SHARING_TEST_GVA; + + for (i = 0; i < page_size; ++i) { + GUEST_ASSERT_EQ(mem[i], GUEST_MEMFD_SHARING_TEST_HOST_TO_GUEST_VALUE); + } + + memset(mem, GUEST_MEMFD_SHARING_TEST_GUEST_TO_HOST_VALUE, page_size); + + GUEST_DONE(); +} + +int run_test(struct kvm_vcpu *vcpu, void *hva, int page_size) +{ + struct ucall uc; + uint64_t uc_cmd; + + memset(hva, GUEST_MEMFD_SHARING_TEST_HOST_TO_GUEST_VALUE, page_size); + vcpu_args_set(vcpu, 1, page_size); + + /* Reset vCPU to guest_code every time run_test is called. */ + vcpu_arch_set_entry_point(vcpu, guest_code); + + vcpu_run(vcpu); + uc_cmd = get_ucall(vcpu, &uc); + + if (uc_cmd == UCALL_ABORT) { + REPORT_GUEST_ASSERT(uc); + return 1; + } else if (uc_cmd == UCALL_DONE) { + char *mem; + int i; + + mem = hva; + for (i = 0; i < page_size; ++i) + TEST_ASSERT_EQ(mem[i], GUEST_MEMFD_SHARING_TEST_GUEST_TO_HOST_VALUE); + + return 0; + } else { + TEST_FAIL("Unknown ucall 0x%lx.", uc.cmd); + return 1; + } +} + +void *add_memslot(struct kvm_vm *vm, int guest_memfd, size_t page_size, + bool back_shared_memory_with_guest_memfd) +{ + void *mem; + + if (back_shared_memory_with_guest_memfd) { + mem = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, + guest_memfd, GUEST_MEMFD_SHARING_TEST_OFFSET); + } else { + mem = mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + } + TEST_ASSERT(mem != MAP_FAILED, "mmap should return valid address"); + + /* + * Setting up this memslot with a KVM_X86_SW_PROTECTED_VM marks all + * offsets in the file as shared. + */ + vm_set_user_memory_region2(vm, GUEST_MEMFD_SHARING_TEST_SLOT, + KVM_MEM_GUEST_MEMFD, + GUEST_MEMFD_SHARING_TEST_GPA, page_size, mem, + guest_memfd, GUEST_MEMFD_SHARING_TEST_OFFSET); + + return mem; +} + +void test_sharing(bool back_shared_memory_with_guest_memfd) +{ + const struct vm_shape shape = { + .mode = VM_MODE_DEFAULT, + .type = KVM_X86_SW_PROTECTED_VM, + }; + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + size_t page_size; + int guest_memfd; + void *mem; + + TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); + + vm = vm_create_shape_with_one_vcpu(shape, &vcpu, &guest_code); + + page_size = getpagesize(); + + guest_memfd = vm_create_guest_memfd(vm, page_size, 0); + + mem = add_memslot(vm, guest_memfd, page_size, back_shared_memory_with_guest_memfd); + + virt_map(vm, GUEST_MEMFD_SHARING_TEST_GVA, GUEST_MEMFD_SHARING_TEST_GPA, 1); + + run_test(vcpu, mem, page_size); + + /* Toggle private flag of memory attributes and run the test again. */ + if (back_shared_memory_with_guest_memfd) { + /* + * Use MADV_REMOVE to release the backing guest_memfd memory + * back to the system before it is used again. Test that this is + * only necessary when guest_memfd is used to back shared + * memory. + */ + madvise(mem, page_size, MADV_REMOVE); + } + vm_mem_set_private(vm, GUEST_MEMFD_SHARING_TEST_GPA, page_size); + vm_mem_set_shared(vm, GUEST_MEMFD_SHARING_TEST_GPA, page_size); + + run_test(vcpu, mem, page_size); + + kvm_vm_free(vm); + munmap(mem, page_size); + close(guest_memfd); +} + +int main(int argc, char *argv[]) +{ + /* + * Confidence check that when guest_memfd is associated with a memslot + * but only anonymous memory is used to back shared memory, sharing + * memory between guest and host works as expected. + */ + test_sharing(false); + + /* + * Memory sharing should work as expected when shared memory is backed + * with guest_memfd. + */ + test_sharing(true); + + return 0; +}

CONFIG_GUP_TEST provides userspace with an ioctl to invoke pin_user_pages(), and this test uses the ioctl to pin pages, to check that memory attributes cannot be set to private if shared pages are pinned.

Signed-off-by: Ackerley Tng ackerleytng@google.com --- tools/testing/selftests/kvm/Makefile | 1 + .../selftests/kvm/guest_memfd_pin_test.c | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 tools/testing/selftests/kvm/guest_memfd_pin_test.c

diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 3c1f35456bfc..c5a1c8c7125a 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -136,6 +136,7 @@ TEST_GEN_PROGS_x86_64 += dirty_log_perf_test TEST_GEN_PROGS_x86_64 += guest_memfd_test TEST_GEN_PROGS_x86_64 += guest_memfd_hugetlb_reporting_test TEST_GEN_PROGS_x86_64 += guest_memfd_sharing_test +TEST_GEN_PROGS_x86_64 += guest_memfd_pin_test TEST_GEN_PROGS_x86_64 += guest_print_test TEST_GEN_PROGS_x86_64 += hardware_disable_test TEST_GEN_PROGS_x86_64 += kvm_create_max_vcpus diff --git a/tools/testing/selftests/kvm/guest_memfd_pin_test.c b/tools/testing/selftests/kvm/guest_memfd_pin_test.c new file mode 100644 index 000000000000..b45fb8024970 --- /dev/null +++ b/tools/testing/selftests/kvm/guest_memfd_pin_test.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Test that pinned pages block KVM from setting memory attributes to PRIVATE. + * + * Copyright (c) 2024, Google LLC. + */ +#include <string.h> +#include <sys/mman.h> +#include <unistd.h> + +#include "test_util.h" +#include "kvm_util.h" +#include "../../../../mm/gup_test.h" + +#define GUEST_MEMFD_PIN_TEST_SLOT 10 +#define GUEST_MEMFD_PIN_TEST_GPA 0x50000000ULL +#define GUEST_MEMFD_PIN_TEST_OFFSET 0 + +static int gup_test_fd; + +void pin_pages(void *vaddr, uint64_t size) +{ + const struct pin_longterm_test args = { + .addr = (uint64_t)vaddr, + .size = size, + .flags = PIN_LONGTERM_TEST_FLAG_USE_WRITE, + }; + + TEST_ASSERT_EQ(ioctl(gup_test_fd, PIN_LONGTERM_TEST_START, &args), 0); +} + +void unpin_pages(void) +{ + TEST_ASSERT_EQ(ioctl(gup_test_fd, PIN_LONGTERM_TEST_STOP), 0); +} + +void run_test(void) +{ + struct kvm_vm *vm; + size_t page_size; + void *mem; + int fd; + + vm = vm_create_barebones_type(KVM_X86_SW_PROTECTED_VM); + + page_size = getpagesize(); + fd = vm_create_guest_memfd(vm, page_size, 0); + + mem = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + GUEST_MEMFD_PIN_TEST_OFFSET); + TEST_ASSERT(mem != MAP_FAILED, "mmap should return valid address"); + + /* + * Setting up this memslot with a KVM_X86_SW_PROTECTED_VM marks all + * offsets in the file as shared. + */ + vm_set_user_memory_region2(vm, GUEST_MEMFD_PIN_TEST_SLOT, + KVM_MEM_GUEST_MEMFD, + GUEST_MEMFD_PIN_TEST_GPA, page_size, mem, fd, + GUEST_MEMFD_PIN_TEST_OFFSET); + + /* Before pinning pages, toggling memory attributes should be fine. */ + vm_mem_set_private(vm, GUEST_MEMFD_PIN_TEST_GPA, page_size); + vm_mem_set_shared(vm, GUEST_MEMFD_PIN_TEST_GPA, page_size); + + pin_pages(mem, page_size); + + /* + * Pinning also faults pages in, so remove these pages from userspace + * page tables to properly test that pinning blocks setting memory + * attributes to private. + */ + TEST_ASSERT_EQ(madvise(mem, page_size, MADV_DONTNEED), 0); + + /* Should fail since the page is still faulted in. */ + TEST_ASSERT_EQ(__vm_set_memory_attributes(vm, GUEST_MEMFD_PIN_TEST_GPA, + page_size, + KVM_MEMORY_ATTRIBUTE_PRIVATE), + -1); + TEST_ASSERT_EQ(errno, EINVAL); + + unpin_pages(); + + /* With the pages unpinned, kvm can set this page to private. */ + vm_mem_set_private(vm, GUEST_MEMFD_PIN_TEST_GPA, page_size); + + kvm_vm_free(vm); + close(fd); +} + +int main(int argc, char *argv[]) +{ + gup_test_fd = open("/sys/kernel/debug/gup_test", O_RDWR); + /* + * This test depends on CONFIG_GUP_TEST to provide a kernel module that + * exposes pin_user_pages() to userspace. + */ + TEST_REQUIRE(gup_test_fd != -1); + TEST_REQUIRE(kvm_check_cap(KVM_CAP_VM_TYPES) & BIT(KVM_X86_SW_PROTECTED_VM)); + + run_test(); + + return 0; +}

A contiguous GPA range may not be contiguous in HVA.

This helper performs madvise, given a GPA range, by madvising in blocks according to memslot configuration.

Signed-off-by: Ackerley Tng ackerleytng@google.com

--- tools/include/linux/kernel.h | 4 +-- .../testing/selftests/kvm/include/kvm_util.h | 2 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 30 +++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/tools/include/linux/kernel.h b/tools/include/linux/kernel.h index 07cfad817d53..5454cd3272ed 100644 --- a/tools/include/linux/kernel.h +++ b/tools/include/linux/kernel.h @@ -54,8 +54,8 @@ _min1 < _min2 ? _min1 : _min2; }) #endif

-#define max_t(type, x, y) max((type)x, (type)y) -#define min_t(type, x, y) min((type)x, (type)y) +#define max_t(type, x, y) max((type)(x), (type)(y)) +#define min_t(type, x, y) min((type)(x), (type)(y)) #define clamp(val, lo, hi) min((typeof(val))max(val, lo), hi)

#ifndef BUG_ON diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 1576e7e4aefe..58b516c23574 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -433,6 +433,8 @@ static inline void vm_mem_set_shared(struct kvm_vm *vm, uint64_t gpa, void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t gpa, uint64_t size, bool punch_hole);

+void vm_guest_mem_madvise(struct kvm_vm *vm, vm_paddr_t gpa_start, uint64_t size, + int advice); static inline void vm_guest_mem_punch_hole(struct kvm_vm *vm, uint64_t gpa, uint64_t size) { diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 9bdd03a5da90..21ea6616124c 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1416,6 +1416,36 @@ void vm_guest_mem_fallocate(struct kvm_vm *vm, uint64_t base, uint64_t size, } }

+void vm_guest_mem_madvise(struct kvm_vm *vm, vm_paddr_t gpa_start, uint64_t size, + int advice) +{ + size_t madvise_len; + vm_paddr_t gpa_end; + vm_paddr_t gpa; + + gpa_end = gpa_start + size; + for (gpa = gpa_start; gpa < gpa_end; gpa += madvise_len) { + struct userspace_mem_region *region; + void *hva_start; + uint64_t memslot_end; + int ret; + + region = userspace_mem_region_find(vm, gpa, gpa); + TEST_ASSERT(region, "Memory region not found for GPA 0x%lx", gpa); + + hva_start = addr_gpa2hva(vm, gpa); + memslot_end = region->region.userspace_addr + + region->region.memory_size; + madvise_len = min_t(size_t, memslot_end - (uint64_t)hva_start, + gpa_end - gpa); + + ret = madvise(hva_start, madvise_len, advice); + TEST_ASSERT(!ret, "madvise(addr=%p, len=%lx, advice=%x) failed\n", + hva_start, madvise_len, advice); + } +} + + /* Returns the size of a vCPU's kvm_run structure. */ static int vcpu_mmap_sz(void) {

From: Vishal Annapurve vannapurve@google.com

The faultability of a page is used to determine whether to split or reconstruct a page.

If there is any page in a folio that is faultable, split the folio. If all pages in a folio are not faultable, reconstruct the folio.

On truncation, always reconstruct and free regardless of faultability (as long as a HugeTLB page's worth of pages is truncated).

Co-developed-by: Vishal Annapurve vannapurve@google.com Signed-off-by: Vishal Annapurve vannapurve@google.com Co-developed-by: Ackerley Tng ackerleytng@google.com Signed-off-by: Ackerley Tng ackerleytng@google.com

--- virt/kvm/guest_memfd.c | 678 +++++++++++++++++++++++++++-------------- 1 file changed, 456 insertions(+), 222 deletions(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index fb292e542381..0afc111099c0 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -99,6 +99,23 @@ static bool kvm_gmem_is_faultable(struct inode *inode, pgoff_t index) return xa_to_value(xa_load(faultability, index)) == KVM_GMEM_FAULTABILITY_VALUE; }

+/** + * Return true if any of the @nr_pages beginning at @index is allowed to be + * faulted in. + */ +static bool kvm_gmem_is_any_faultable(struct inode *inode, pgoff_t index, + int nr_pages) +{ + pgoff_t i; + + for (i = index; i < index + nr_pages; ++i) { + if (kvm_gmem_is_faultable(inode, i)) + return true; + } + + return false; +} + /** * folio_file_pfn - like folio_file_page, but return a pfn. * @folio: The folio which contains this index. @@ -312,6 +329,40 @@ static int kvm_gmem_hugetlb_filemap_add_folio(struct address_space *mapping, return 0; }

+static inline void kvm_gmem_hugetlb_filemap_remove_folio(struct folio *folio) +{ + folio_lock(folio); + + folio_clear_dirty(folio); + folio_clear_uptodate(folio); + filemap_remove_folio(folio); + + folio_unlock(folio); +} + +/* + * Locks a block of nr_pages (1 << huge_page_order(h)) pages within @mapping + * beginning at @index. Take either this or filemap_invalidate_lock() whenever + * the filemap is accessed. + */ +static u32 hugetlb_fault_mutex_lock(struct address_space *mapping, pgoff_t index) +{ + pgoff_t hindex; + u32 hash; + + hindex = index >> huge_page_order(kvm_gmem_hgmem(mapping->host)->h); + hash = hugetlb_fault_mutex_hash(mapping, hindex); + + mutex_lock(&hugetlb_fault_mutex_table[hash]); + + return hash; +} + +static void hugetlb_fault_mutex_unlock(u32 hash) +{ + mutex_unlock(&hugetlb_fault_mutex_table[hash]); +} + struct kvm_gmem_split_stash { struct { unsigned long _flags_2; @@ -394,15 +445,136 @@ static int kvm_gmem_hugetlb_reconstruct_folio(struct hstate *h, struct folio *fo }

__folio_set_hugetlb(folio); - - folio_set_count(folio, 1); + hugetlb_folio_list_add(folio, &h->hugepage_activelist);

hugetlb_vmemmap_optimize_folio(h, folio);

+ folio_set_count(folio, 1); + return 0; }

-/* Basically folio_set_order(folio, 1) without the checks. */ +/** + * Reconstruct a HugeTLB folio out of folio_nr_pages(@first_folio) pages. Will + * clean up subfolios from filemap and add back the reconstructed folio. Folios + * to be reconstructed must not be locked, and reconstructed folio will not be + * locked. Return 0 on success or negative error otherwise. + * + * hugetlb_fault_mutex_lock() has to be held when calling this function. + * + * Expects that before this call, the filemap's refcounts are the only refcounts + * for the folios in the filemap. After this function returns, the filemap's + * refcount will be the only refcount on the reconstructed folio. + */ +static int kvm_gmem_reconstruct_folio_in_filemap(struct hstate *h, + struct folio *first_folio) +{ + struct address_space *mapping; + struct folio_batch fbatch; + unsigned long end; + pgoff_t index; + pgoff_t next; + int ret; + int i; + + if (folio_order(first_folio) == huge_page_order(h)) + return 0; + + index = first_folio->index; + mapping = first_folio->mapping; + + next = index; + end = index + (1UL << huge_page_order(h)); + folio_batch_init(&fbatch); + while (filemap_get_folios(mapping, &next, end - 1, &fbatch)) { + for (i = 0; i < folio_batch_count(&fbatch); ++i) { + struct folio *folio; + + folio = fbatch.folios[i]; + + /* + * Before removing from filemap, take a reference so + * sub-folios don't get freed when removing from + * filemap. + */ + folio_get(folio); + + kvm_gmem_hugetlb_filemap_remove_folio(folio); + } + folio_batch_release(&fbatch); + } + + ret = kvm_gmem_hugetlb_reconstruct_folio(h, first_folio); + if (ret) { + /* TODO: handle cleanup properly. */ + WARN_ON(ret); + return ret; + } + + kvm_gmem_hugetlb_filemap_add_folio(mapping, first_folio, index, + htlb_alloc_mask(h)); + + folio_unlock(first_folio); + folio_put(first_folio); + + return ret; +} + +/** + * Reconstruct any HugeTLB folios in range [@start, @end), if all the subfolios + * are not faultable. Return 0 on success or negative error otherwise. + * + * Will skip any folios that are already reconstructed. + */ +static int kvm_gmem_try_reconstruct_folios_range(struct inode *inode, + pgoff_t start, pgoff_t end) +{ + unsigned int nr_pages; + pgoff_t aligned_start; + pgoff_t aligned_end; + struct hstate *h; + pgoff_t index; + int ret; + + if (!is_kvm_gmem_hugetlb(inode)) + return 0; + + h = kvm_gmem_hgmem(inode)->h; + nr_pages = 1UL << huge_page_order(h); + + aligned_start = round_up(start, nr_pages); + aligned_end = round_down(end, nr_pages); + + ret = 0; + for (index = aligned_start; !ret && index < aligned_end; index += nr_pages) { + struct folio *folio; + u32 hash; + + hash = hugetlb_fault_mutex_lock(inode->i_mapping, index); + + folio = filemap_get_folio(inode->i_mapping, index); + if (!IS_ERR(folio)) { + /* + * Drop refcount because reconstruction expects an equal number + * of refcounts for all subfolios - just keep the refcount taken + * by the filemap. + */ + folio_put(folio); + + /* Merge only when the entire block of nr_pages is not faultable. */ + if (!kvm_gmem_is_any_faultable(inode, index, nr_pages)) { + ret = kvm_gmem_reconstruct_folio_in_filemap(h, folio); + WARN_ON(ret); + } + } + + hugetlb_fault_mutex_unlock(hash); + } + + return ret; +} + +/* Basically folio_set_order() without the checks. */ static inline void kvm_gmem_folio_set_order(struct folio *folio, unsigned int order) { folio->_flags_1 = (folio->_flags_1 & ~0xffUL) | order; @@ -414,8 +586,8 @@ static inline void kvm_gmem_folio_set_order(struct folio *folio, unsigned int or /** * Split a HugeTLB @folio of size huge_page_size(@h). * - * After splitting, each split folio has a refcount of 1. There are no checks on - * refcounts before splitting. + * Folio must have refcount of 1 when this function is called. After splitting, + * each split folio has a refcount of 1. * * Return 0 on success and negative error otherwise. */ @@ -423,14 +595,18 @@ static int kvm_gmem_hugetlb_split_folio(struct hstate *h, struct folio *folio) { int ret;

+ VM_WARN_ON_ONCE_FOLIO(folio_ref_count(folio) != 1, folio); + + folio_set_count(folio, 0); + ret = hugetlb_vmemmap_restore_folio(h, folio); if (ret) - return ret; + goto out;

ret = kvm_gmem_hugetlb_stash_metadata(folio); if (ret) { hugetlb_vmemmap_optimize_folio(h, folio); - return ret; + goto out; }

kvm_gmem_folio_set_order(folio, 0); @@ -439,109 +615,183 @@ static int kvm_gmem_hugetlb_split_folio(struct hstate *h, struct folio *folio) __folio_clear_hugetlb(folio);

/* - * Remove the first folio from h->hugepage_activelist since it is no + * Remove the original folio from h->hugepage_activelist since it is no * longer a HugeTLB page. The other split pages should not be on any * lists. */ hugetlb_folio_list_del(folio);

- return 0; + ret = 0; +out: + folio_set_count(folio, 1); + return ret; }

-static struct folio *kvm_gmem_hugetlb_alloc_and_cache_folio(struct inode *inode, - pgoff_t index) +/** + * Split a HugeTLB folio into folio_nr_pages(@folio) pages. Will clean up folio + * from filemap and add back the split folios. @folio must not be locked, and + * all split folios will not be locked. Return 0 on success or negative error + * otherwise. + * + * hugetlb_fault_mutex_lock() has to be held when calling this function. + * + * Expects that before this call, the filemap's refcounts are the only refcounts + * for the folio. After this function returns, the filemap's refcounts will be + * the only refcounts on the split folios. + */ +static int kvm_gmem_split_folio_in_filemap(struct hstate *h, struct folio *folio) { - struct folio *allocated_hugetlb_folio; - pgoff_t hugetlb_first_subpage_index; - struct page *hugetlb_first_subpage; - struct kvm_gmem_hugetlb *hgmem; - struct page *requested_page; + struct address_space *mapping; + struct page *first_subpage; + pgoff_t index; int ret; int i;

- hgmem = kvm_gmem_hgmem(inode); - allocated_hugetlb_folio = kvm_gmem_hugetlb_alloc_folio(hgmem->h, hgmem->spool); - if (IS_ERR(allocated_hugetlb_folio)) - return allocated_hugetlb_folio; + if (folio_order(folio) == 0) + return 0;

- requested_page = folio_file_page(allocated_hugetlb_folio, index); - hugetlb_first_subpage = folio_file_page(allocated_hugetlb_folio, 0); - hugetlb_first_subpage_index = index & (huge_page_mask(hgmem->h) >> PAGE_SHIFT); + index = folio->index; + mapping = folio->mapping;

- ret = kvm_gmem_hugetlb_split_folio(hgmem->h, allocated_hugetlb_folio); + first_subpage = folio_page(folio, 0); + + /* + * Take reference so that folio will not be released when removed from + * filemap. + */ + folio_get(folio); + + kvm_gmem_hugetlb_filemap_remove_folio(folio); + + ret = kvm_gmem_hugetlb_split_folio(h, folio); if (ret) { - folio_put(allocated_hugetlb_folio); - return ERR_PTR(ret); + WARN_ON(ret); + kvm_gmem_hugetlb_filemap_add_folio(mapping, folio, index, + htlb_alloc_mask(h)); + folio_put(folio); + return ret; }

- for (i = 0; i < pages_per_huge_page(hgmem->h); ++i) { - struct folio *folio = page_folio(nth_page(hugetlb_first_subpage, i)); + for (i = 0; i < pages_per_huge_page(h); ++i) { + struct folio *folio = page_folio(nth_page(first_subpage, i));

- ret = kvm_gmem_hugetlb_filemap_add_folio(inode->i_mapping, - folio, - hugetlb_first_subpage_index + i, - htlb_alloc_mask(hgmem->h)); + ret = kvm_gmem_hugetlb_filemap_add_folio(mapping, folio, + index + i, + htlb_alloc_mask(h)); if (ret) { /* TODO: handle cleanup properly. */ - pr_err("Handle cleanup properly index=%lx, ret=%d\n", - hugetlb_first_subpage_index + i, ret); - dump_page(nth_page(hugetlb_first_subpage, i), "check"); - return ERR_PTR(ret); + WARN_ON(ret); + return ret; }

+ folio_unlock(folio); + /* - * Skip unlocking for the requested index since - * kvm_gmem_get_folio() returns a locked folio. - * - * Do folio_put() to drop the refcount that came with the folio, - * from splitting the folio. Splitting the folio has a refcount - * to be in line with hugetlb_alloc_folio(), which returns a - * folio with refcount 1. - * - * Skip folio_put() for requested index since - * kvm_gmem_get_folio() returns a folio with refcount 1. + * Drop reference so that the only remaining reference is the + * one held by the filemap. */ - if (hugetlb_first_subpage_index + i != index) { - folio_unlock(folio); - folio_put(folio); - } + folio_put(folio); }

+ return ret; +} + +/* + * Allocates and then caches a folio in the filemap. Returns a folio with + * refcount of 2: 1 after allocation, and 1 taken by the filemap. + */ +static struct folio *kvm_gmem_hugetlb_alloc_and_cache_folio(struct inode *inode, + pgoff_t index) +{ + struct kvm_gmem_hugetlb *hgmem; + pgoff_t aligned_index; + struct folio *folio; + int nr_pages; + int ret; + + hgmem = kvm_gmem_hgmem(inode); + folio = kvm_gmem_hugetlb_alloc_folio(hgmem->h, hgmem->spool); + if (IS_ERR(folio)) + return folio; + + nr_pages = 1UL << huge_page_order(hgmem->h); + aligned_index = round_down(index, nr_pages); + + ret = kvm_gmem_hugetlb_filemap_add_folio(inode->i_mapping, folio, + aligned_index, + htlb_alloc_mask(hgmem->h)); + WARN_ON(ret); + spin_lock(&inode->i_lock); inode->i_blocks += blocks_per_huge_page(hgmem->h); spin_unlock(&inode->i_lock);

- return page_folio(requested_page); + return folio; +} + +/** + * Split @folio if any of the subfolios are faultable. Returns the split + * (locked, refcount=2) folio at @index. + * + * Expects a locked folio with 1 refcount in addition to filemap's refcounts. + * + * After splitting, the subfolios in the filemap will be unlocked and have + * refcount 1 (other than the returned folio, which will be locked and have + * refcount 2). + */ +static struct folio *kvm_gmem_maybe_split_folio(struct folio *folio, pgoff_t index) +{ + pgoff_t aligned_index; + struct inode *inode; + struct hstate *h; + int nr_pages; + int ret; + + inode = folio->mapping->host; + h = kvm_gmem_hgmem(inode)->h; + nr_pages = 1UL << huge_page_order(h); + aligned_index = round_down(index, nr_pages); + + if (!kvm_gmem_is_any_faultable(inode, aligned_index, nr_pages)) + return folio; + + /* Drop lock and refcount in preparation for splitting. */ + folio_unlock(folio); + folio_put(folio); + + ret = kvm_gmem_split_folio_in_filemap(h, folio); + if (ret) { + kvm_gmem_hugetlb_filemap_remove_folio(folio); + return ERR_PTR(ret); + } + + /* + * At this point, the filemap has the only reference on the folio. Take + * lock and refcount on folio to align with kvm_gmem_get_folio(). + */ + return filemap_lock_folio(inode->i_mapping, index); }

static struct folio *kvm_gmem_get_hugetlb_folio(struct inode *inode, pgoff_t index) { - struct address_space *mapping; struct folio *folio; - struct hstate *h; - pgoff_t hindex; u32 hash;

- h = kvm_gmem_hgmem(inode)->h; - hindex = index >> huge_page_order(h); - mapping = inode->i_mapping; - - /* To lock, we calculate the hash using the hindex and not index. */ - hash = hugetlb_fault_mutex_hash(mapping, hindex); - mutex_lock(&hugetlb_fault_mutex_table[hash]); + hash = hugetlb_fault_mutex_lock(inode->i_mapping, index);

/* - * The filemap is indexed with index and not hindex. Taking lock on - * folio to align with kvm_gmem_get_regular_folio() + * The filemap is indexed with index and not hindex. Take lock on folio + * to align with kvm_gmem_get_regular_folio() */ - folio = filemap_lock_folio(mapping, index); + folio = filemap_lock_folio(inode->i_mapping, index); + if (IS_ERR(folio)) + folio = kvm_gmem_hugetlb_alloc_and_cache_folio(inode, index); + if (!IS_ERR(folio)) - goto out; + folio = kvm_gmem_maybe_split_folio(folio, index);

- folio = kvm_gmem_hugetlb_alloc_and_cache_folio(inode, index); -out: - mutex_unlock(&hugetlb_fault_mutex_table[hash]); + hugetlb_fault_mutex_unlock(hash);

return folio; } @@ -610,17 +860,6 @@ static void kvm_gmem_invalidate_end(struct kvm_gmem *gmem, pgoff_t start, } }

-static inline void kvm_gmem_hugetlb_filemap_remove_folio(struct folio *folio) -{ - folio_lock(folio); - - folio_clear_dirty(folio); - folio_clear_uptodate(folio); - filemap_remove_folio(folio); - - folio_unlock(folio); -} - /** * Removes folios in range [@lstart, @lend) from page cache/filemap (@mapping), * returning the number of HugeTLB pages freed. @@ -631,61 +870,30 @@ static int kvm_gmem_hugetlb_filemap_remove_folios(struct address_space *mapping, struct hstate *h, loff_t lstart, loff_t lend) { - const pgoff_t end = lend >> PAGE_SHIFT; - pgoff_t next = lstart >> PAGE_SHIFT; - LIST_HEAD(folios_to_reconstruct); - struct folio_batch fbatch; - struct folio *folio, *tmp; - int num_freed = 0; - int i; - - /* - * TODO: Iterate over huge_page_size(h) blocks to avoid taking and - * releasing hugetlb_fault_mutex_table[hash] lock so often. When - * truncating, lstart and lend should be clipped to the size of this - * guest_memfd file, otherwise there would be too many iterations. - */ - folio_batch_init(&fbatch); - while (filemap_get_folios(mapping, &next, end - 1, &fbatch)) { - for (i = 0; i < folio_batch_count(&fbatch); ++i) { - struct folio *folio; - pgoff_t hindex; - u32 hash; - - folio = fbatch.folios[i]; + loff_t offset; + int num_freed;

- hindex = folio->index >> huge_page_order(h); - hash = hugetlb_fault_mutex_hash(mapping, hindex); - mutex_lock(&hugetlb_fault_mutex_table[hash]); + num_freed = 0; + for (offset = lstart; offset < lend; offset += huge_page_size(h)) { + struct folio *folio; + pgoff_t index; + u32 hash;

- /* - * Collect first pages of HugeTLB folios for - * reconstruction later. - */ - if ((folio->index & ~(huge_page_mask(h) >> PAGE_SHIFT)) == 0) - list_add(&folio->lru, &folios_to_reconstruct); + index = offset >> PAGE_SHIFT; + hash = hugetlb_fault_mutex_lock(mapping, index);

- /* - * Before removing from filemap, take a reference so - * sub-folios don't get freed. Don't free the sub-folios - * until after reconstruction. - */ - folio_get(folio); + folio = filemap_get_folio(mapping, index); + if (!IS_ERR(folio)) { + /* Drop refcount so that filemap holds only reference. */ + folio_put(folio);

+ kvm_gmem_reconstruct_folio_in_filemap(h, folio); kvm_gmem_hugetlb_filemap_remove_folio(folio);

- mutex_unlock(&hugetlb_fault_mutex_table[hash]); + num_freed++; } - folio_batch_release(&fbatch); - cond_resched(); - } - - list_for_each_entry_safe(folio, tmp, &folios_to_reconstruct, lru) { - kvm_gmem_hugetlb_reconstruct_folio(h, folio); - hugetlb_folio_list_move(folio, &h->hugepage_activelist);

- folio_put(folio); - num_freed++; + hugetlb_fault_mutex_unlock(hash); }

return num_freed; @@ -705,6 +913,10 @@ static void kvm_gmem_hugetlb_truncate_folios_range(struct inode *inode, int gbl_reserve; int num_freed;

+ /* No point truncating more than inode size. */ + lstart = min(lstart, inode->i_size); + lend = min(lend, inode->i_size); + hgmem = kvm_gmem_hgmem(inode); h = hgmem->h;

@@ -1042,13 +1254,27 @@ static vm_fault_t kvm_gmem_fault(struct vm_fault *vmf) bool is_prepared;

inode = file_inode(vmf->vma->vm_file); - if (!kvm_gmem_is_faultable(inode, vmf->pgoff)) + + /* + * Use filemap_invalidate_lock_shared() to make sure + * kvm_gmem_get_folio() doesn't race with faultability updates. + */ + filemap_invalidate_lock_shared(inode->i_mapping); + + if (!kvm_gmem_is_faultable(inode, vmf->pgoff)) { + filemap_invalidate_unlock_shared(inode->i_mapping); return VM_FAULT_SIGBUS; + }

folio = kvm_gmem_get_folio(inode, vmf->pgoff); + + filemap_invalidate_unlock_shared(inode->i_mapping); + if (!folio) return VM_FAULT_SIGBUS;

+ WARN(folio_test_hugetlb(folio), "should not be faulting in hugetlb folio=%p\n", folio); + is_prepared = folio_test_uptodate(folio); if (!is_prepared) { unsigned long nr_pages; @@ -1731,8 +1957,6 @@ static bool kvm_gmem_no_mappings_range(struct inode *inode, pgoff_t start, pgoff pgoff_t index; bool checked_indices_unmapped;

- filemap_invalidate_lock_shared(inode->i_mapping); - /* TODO: replace iteration with filemap_get_folios() for efficiency. */ checked_indices_unmapped = true; for (index = start; checked_indices_unmapped && index < end;) { @@ -1754,98 +1978,130 @@ static bool kvm_gmem_no_mappings_range(struct inode *inode, pgoff_t start, pgoff folio_put(folio); }

- filemap_invalidate_unlock_shared(inode->i_mapping); return checked_indices_unmapped; }

/** - * Returns true if pages in range [@start, @end) in memslot @slot have no - * userspace mappings. + * Split any HugeTLB folios in range [@start, @end), if any of the offsets in + * the folio are faultable. Return 0 on success or negative error otherwise. + * + * Will skip any folios that are already split. */ -static bool kvm_gmem_no_mappings_slot(struct kvm_memory_slot *slot, - gfn_t start, gfn_t end) +static int kvm_gmem_try_split_folios_range(struct inode *inode, + pgoff_t start, pgoff_t end) { - pgoff_t offset_start; - pgoff_t offset_end; - struct file *file; - bool ret; - - offset_start = start - slot->base_gfn + slot->gmem.pgoff; - offset_end = end - slot->base_gfn + slot->gmem.pgoff; - - file = kvm_gmem_get_file(slot); - if (!file) - return false; - - ret = kvm_gmem_no_mappings_range(file_inode(file), offset_start, offset_end); + unsigned int nr_pages; + pgoff_t aligned_start; + pgoff_t aligned_end; + struct hstate *h; + pgoff_t index; + int ret;

- fput(file); + if (!is_kvm_gmem_hugetlb(inode)) + return 0;

- return ret; -} + h = kvm_gmem_hgmem(inode)->h; + nr_pages = 1UL << huge_page_order(h);

-/** - * Returns true if pages in range [@start, @end) have no host userspace mappings. - */ -static bool kvm_gmem_no_mappings(struct kvm *kvm, gfn_t start, gfn_t end) -{ - int i; + aligned_start = round_down(start, nr_pages); + aligned_end = round_up(end, nr_pages);

- lockdep_assert_held(&kvm->slots_lock); + ret = 0; + for (index = aligned_start; !ret && index < aligned_end; index += nr_pages) { + struct folio *folio; + u32 hash;

- for (i = 0; i < kvm_arch_nr_memslot_as_ids(kvm); i++) { - struct kvm_memslot_iter iter; - struct kvm_memslots *slots; + hash = hugetlb_fault_mutex_lock(inode->i_mapping, index);

- slots = __kvm_memslots(kvm, i); - kvm_for_each_memslot_in_gfn_range(&iter, slots, start, end) { - struct kvm_memory_slot *slot; - gfn_t gfn_start; - gfn_t gfn_end; - - slot = iter.slot; - gfn_start = max(start, slot->base_gfn); - gfn_end = min(end, slot->base_gfn + slot->npages); + folio = filemap_get_folio(inode->i_mapping, index); + if (!IS_ERR(folio)) { + /* + * Drop refcount so that the only references held are refcounts + * from the filemap. + */ + folio_put(folio);

- if (iter.slot->flags & KVM_MEM_GUEST_MEMFD && - !kvm_gmem_no_mappings_slot(iter.slot, gfn_start, gfn_end)) - return false; + if (kvm_gmem_is_any_faultable(inode, index, nr_pages)) { + ret = kvm_gmem_split_folio_in_filemap(h, folio); + if (ret) { + /* TODO cleanup properly. */ + WARN_ON(ret); + } + } } + + hugetlb_fault_mutex_unlock(hash); }

- return true; + return ret; }

/** - * Set faultability of given range of gfns [@start, @end) in memslot @slot to - * @faultable. + * Returns 0 if guest_memfd permits setting range [@start, @end) with + * faultability @faultable within memslot @slot, or negative error otherwise. + * + * If a request was made to set the memory to PRIVATE (not faultable), the pages + * in the range must not be pinned or mapped for the request to be permitted. + * + * Because this may allow pages to be faulted in to userspace when requested to + * set attributes to shared, this must only be called after the pages have been + * invalidated from guest page tables. */ -static void kvm_gmem_set_faultable_slot(struct kvm_memory_slot *slot, gfn_t start, - gfn_t end, bool faultable) +static int kvm_gmem_try_set_faultable_slot(struct kvm_memory_slot *slot, + gfn_t start, gfn_t end, + bool faultable) { pgoff_t start_offset; + struct inode *inode; pgoff_t end_offset; struct file *file; + int ret;

file = kvm_gmem_get_file(slot); if (!file) - return; + return 0;

start_offset = start - slot->base_gfn + slot->gmem.pgoff; end_offset = end - slot->base_gfn + slot->gmem.pgoff;

- WARN_ON(kvm_gmem_set_faultable(file_inode(file), start_offset, end_offset, - faultable)); + inode = file_inode(file); + + /* + * Use filemap_invalidate_lock_shared() to make sure + * splitting/reconstruction doesn't race with faultability updates. + */ + filemap_invalidate_lock(inode->i_mapping); + + kvm_gmem_set_faultable(inode, start_offset, end_offset, faultable); + + if (faultable) { + ret = kvm_gmem_try_split_folios_range(inode, start_offset, + end_offset); + } else { + if (kvm_gmem_no_mappings_range(inode, start_offset, end_offset)) { + ret = kvm_gmem_try_reconstruct_folios_range(inode, + start_offset, + end_offset); + } else { + ret = -EINVAL; + } + } + + filemap_invalidate_unlock(inode->i_mapping);

fput(file); + + return ret; }

/** - * Set faultability of given range of gfns [@start, @end) in memslot @slot to - * @faultable. + * Returns 0 if guest_memfd permits setting range [@start, @end) with + * faultability @faultable within VM @kvm, or negative error otherwise. + * + * See kvm_gmem_try_set_faultable_slot() for details. */ -static void kvm_gmem_set_faultable_vm(struct kvm *kvm, gfn_t start, gfn_t end, - bool faultable) +static int kvm_gmem_try_set_faultable_vm(struct kvm *kvm, gfn_t start, gfn_t end, + bool faultable) { int i;

@@ -1866,43 +2122,15 @@ static void kvm_gmem_set_faultable_vm(struct kvm *kvm, gfn_t start, gfn_t end, gfn_end = min(end, slot->base_gfn + slot->npages);

if (iter.slot->flags & KVM_MEM_GUEST_MEMFD) { - kvm_gmem_set_faultable_slot(slot, gfn_start, - gfn_end, faultable); + int ret; + + ret = kvm_gmem_try_set_faultable_slot(slot, gfn_start, + gfn_end, faultable); + if (ret) + return ret; } } } -} - -/** - * Returns true if guest_memfd permits setting range [@start, @end) to PRIVATE. - * - * If memory is faulted in to host userspace and a request was made to set the - * memory to PRIVATE, the faulted in pages must not be pinned for the request to - * be permitted. - */ -static int kvm_gmem_should_set_attributes_private(struct kvm *kvm, gfn_t start, - gfn_t end) -{ - kvm_gmem_set_faultable_vm(kvm, start, end, false); - - if (kvm_gmem_no_mappings(kvm, start, end)) - return 0; - - kvm_gmem_set_faultable_vm(kvm, start, end, true); - return -EINVAL; -} - -/** - * Returns true if guest_memfd permits setting range [@start, @end) to SHARED. - * - * Because this allows pages to be faulted in to userspace, this must only be - * called after the pages have been invalidated from guest page tables. - */ -static int kvm_gmem_should_set_attributes_shared(struct kvm *kvm, gfn_t start, - gfn_t end) -{ - /* Always okay to set shared, hence set range faultable here. */ - kvm_gmem_set_faultable_vm(kvm, start, end, true);

return 0; } @@ -1922,10 +2150,16 @@ static int kvm_gmem_should_set_attributes_shared(struct kvm *kvm, gfn_t start, int kvm_gmem_should_set_attributes(struct kvm *kvm, gfn_t start, gfn_t end, unsigned long attrs) { - if (attrs & KVM_MEMORY_ATTRIBUTE_PRIVATE) - return kvm_gmem_should_set_attributes_private(kvm, start, end); - else - return kvm_gmem_should_set_attributes_shared(kvm, start, end); + bool faultable; + int ret; + + faultable = !(attrs & KVM_MEMORY_ATTRIBUTE_PRIVATE); + + ret = kvm_gmem_try_set_faultable_vm(kvm, start, end, faultable); + if (ret) + WARN_ON(kvm_gmem_try_set_faultable_vm(kvm, start, end, !faultable)); + + return ret; }

#endif