KVM guest_memfd is implementing its own inodes to store metadata for backing memory using a custom filesystem. This requires the ability to allocate an anonymous inode with security context using anon_inode_make_secure_inode().

As guest_memfd currently resides in the KVM module, we need to export this symbol for use outside the core kernel. In the future, guest_memfd might be moved to core-mm, at which point the symbols no longer would have to be exported. When/if that happens is still unclear.

Signed-off-by: Shivank Garg shivankg@amd.com ---

The handling of the S_PRIVATE flag for these inodes was discussed extensively ([1], [2]) My understanding [3], is that because KVM guest_memfd and secretmem results in user-visible file descriptors, its inodes should not bypass LSM security checks. Therefore, anon_inode_make_secure_inode() (as implemented in this patch) correctly clears the S_PRIVATE flag set by alloc_anon_inode() to ensure proper security policy enforcement.

[1] https://lore.kernel.org/all/b9e5fa41-62fd-4b3d-bb2d-24ae9d3c33da@redhat.com [2] https://lore.kernel.org/all/cover.1748890962.git.ackerleytng@google.com [3] https://lore.kernel.org/all/647ab7a4-790f-4858-acf2-0f6bae5b7f99@amd.com

fs/anon_inodes.c | 20 +++++++++++++++++--- include/linux/fs.h | 2 ++ 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c index e51e7d88980a..441fff40b55a 100644 --- a/fs/anon_inodes.c +++ b/fs/anon_inodes.c @@ -98,14 +98,26 @@ static struct file_system_type anon_inode_fs_type = { .kill_sb = kill_anon_super, };

-static struct inode *anon_inode_make_secure_inode( +/** + * anon_inode_make_secure_inode - allocate an anonymous inode with security context + * @sb: [in] Superblock to allocate from + * @name: [in] Name of the class of the newfile (e.g., "secretmem") + * @context_inode: + * [in] Optional parent inode for security inheritance + * + * The function ensures proper security initialization through the LSM hook + * security_inode_init_security_anon(). + * + * Return: Pointer to new inode on success, ERR_PTR on failure. + */ +struct inode *anon_inode_make_secure_inode(struct super_block *sb, const char *name, const struct inode *context_inode) { struct inode *inode; int error;

- inode = alloc_anon_inode(anon_inode_mnt->mnt_sb); + inode = alloc_anon_inode(sb); if (IS_ERR(inode)) return inode; inode->i_flags &= ~S_PRIVATE; @@ -118,6 +130,7 @@ static struct inode *anon_inode_make_secure_inode( } return inode; } +EXPORT_SYMBOL_GPL(anon_inode_make_secure_inode);

static struct file *__anon_inode_getfile(const char *name, const struct file_operations *fops, @@ -132,7 +145,8 @@ static struct file *__anon_inode_getfile(const char *name, return ERR_PTR(-ENOENT);

if (make_inode) { - inode = anon_inode_make_secure_inode(name, context_inode); + inode = anon_inode_make_secure_inode(anon_inode_mnt->mnt_sb, + name, context_inode); if (IS_ERR(inode)) { file = ERR_CAST(inode); goto err; diff --git a/include/linux/fs.h b/include/linux/fs.h index 96c7925a6551..7ba45be0d7a0 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3604,6 +3604,8 @@ extern int simple_write_begin(struct file *file, struct address_space *mapping, extern const struct address_space_operations ram_aops; extern int always_delete_dentry(const struct dentry *); extern struct inode *alloc_anon_inode(struct super_block *); +extern struct inode *anon_inode_make_secure_inode(struct super_block *sb, + const char *name, const struct inode *context_inode); extern int simple_nosetlease(struct file *, int, struct file_lease **, void **); extern const struct dentry_operations simple_dentry_operations;

From: Shivansh Dhiman shivansh.dhiman@amd.com

Add NUMA mempolicy support to the filemap allocation path by introducing new APIs that take a mempolicy argument: - filemap_grab_folio_mpol() - filemap_alloc_folio_mpol() - __filemap_get_folio_mpol()

These APIs allow callers to specify a NUMA policy during page cache allocations, enabling fine-grained control over memory placement. This is particularly needed by KVM when using guest-memfd memory backends, where the guest memory needs to be allocated according to the NUMA policy specified by VMM.

The existing non-mempolicy APIs remain unchanged and continue to use the default allocation behavior.

Signed-off-by: Shivansh Dhiman shivansh.dhiman@amd.com Co-developed-by: Shivank Garg shivankg@amd.com Signed-off-by: Shivank Garg shivankg@amd.com --- include/linux/pagemap.h | 41 +++++++++++++++++++++++++++++++++++++++++ mm/filemap.c | 27 +++++++++++++++++++++++---- 2 files changed, 64 insertions(+), 4 deletions(-)

diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index e63fbfbd5b0f..6558c672740d 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -647,15 +647,24 @@ static inline void *detach_page_private(struct page *page)

#ifdef CONFIG_NUMA struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order); +struct folio *filemap_alloc_folio_mpol_noprof(gfp_t gfp, unsigned int order, + struct mempolicy *mpol, pgoff_t ilx); #else static inline struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order) { return folio_alloc_noprof(gfp, order); } +static inline struct folio *filemap_alloc_folio_mpol_noprof(gfp_t gfp, + unsigned int order, struct mempolicy *mpol, pgoff_t ilx) +{ + return filemap_alloc_folio_noprof(gfp, order); +} #endif

#define filemap_alloc_folio(...) \ alloc_hooks(filemap_alloc_folio_noprof(__VA_ARGS__)) +#define filemap_alloc_folio_mpol(...) \ + alloc_hooks(filemap_alloc_folio_mpol_noprof(__VA_ARGS__))

static inline struct page *__page_cache_alloc(gfp_t gfp) { @@ -747,6 +756,8 @@ static inline fgf_t fgf_set_order(size_t size) void *filemap_get_entry(struct address_space *mapping, pgoff_t index); struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, fgf_t fgp_flags, gfp_t gfp); +struct folio *__filemap_get_folio_mpol(struct address_space *mapping, + pgoff_t index, fgf_t fgp_flags, gfp_t gfp, struct mempolicy *mpol, pgoff_t ilx); struct page *pagecache_get_page(struct address_space *mapping, pgoff_t index, fgf_t fgp_flags, gfp_t gfp);

@@ -805,6 +816,36 @@ static inline struct folio *filemap_grab_folio(struct address_space *mapping, mapping_gfp_mask(mapping)); }

+/** + * filemap_grab_folio_mpol - grab a folio from the page cache. + * @mapping: The address space to search. + * @index: The page index. + * @mpol: The mempolicy to apply when allocating a new folio. + * @ilx: The interleave index, for use only with MPOL_INTERLEAVE or + * MPOL_WEIGHTED_INTERLEAVE. + * + * Same as filemap_grab_folio(), except that it allocates the folio using + * given memory policy. + * + * Return: A found or created folio. ERR_PTR(-ENOMEM) if no folio is found + * and failed to create a folio. + */ +#ifdef CONFIG_NUMA +static inline struct folio *filemap_grab_folio_mpol(struct address_space *mapping, + pgoff_t index, struct mempolicy *mpol, pgoff_t ilx) +{ + return __filemap_get_folio_mpol(mapping, index, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, + mapping_gfp_mask(mapping), mpol, ilx); +} +#else +static inline struct folio *filemap_grab_folio_mpol(struct address_space *mapping, + pgoff_t index, struct mempolicy *mpol, pgoff_t ilx) +{ + return filemap_grab_folio(mapping, index); +} +#endif /* CONFIG_NUMA */ + /** * find_get_page - find and get a page reference * @mapping: the address_space to search diff --git a/mm/filemap.c b/mm/filemap.c index bada249b9fb7..c7e913b91636 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1007,6 +1007,15 @@ struct folio *filemap_alloc_folio_noprof(gfp_t gfp, unsigned int order) return folio_alloc_noprof(gfp, order); } EXPORT_SYMBOL(filemap_alloc_folio_noprof); + +struct folio *filemap_alloc_folio_mpol_noprof(gfp_t gfp, unsigned int order, + struct mempolicy *mpol, pgoff_t ilx) +{ + if (mpol) + return folio_alloc_mpol_noprof(gfp, order, mpol, + ilx, numa_node_id()); + return filemap_alloc_folio_noprof(gfp, order); +} #endif

/* @@ -1891,11 +1900,14 @@ void *filemap_get_entry(struct address_space *mapping, pgoff_t index) }

/** - * __filemap_get_folio - Find and get a reference to a folio. + * __filemap_get_folio_mpol - Find and get a reference to a folio. * @mapping: The address_space to search. * @index: The page index. * @fgp_flags: %FGP flags modify how the folio is returned. * @gfp: Memory allocation flags to use if %FGP_CREAT is specified. + * @mpol: The mempolicy to apply when allocating a new folio. + * @ilx: The interleave index, for use only with MPOL_INTERLEAVE or + * MPOL_WEIGHTED_INTERLEAVE. * * Looks up the page cache entry at @mapping & @index. * @@ -1906,8 +1918,8 @@ void *filemap_get_entry(struct address_space *mapping, pgoff_t index) * * Return: The found folio or an ERR_PTR() otherwise. */ -struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, - fgf_t fgp_flags, gfp_t gfp) +struct folio *__filemap_get_folio_mpol(struct address_space *mapping, pgoff_t index, + fgf_t fgp_flags, gfp_t gfp, struct mempolicy *mpol, pgoff_t ilx) { struct folio *folio;

@@ -1977,7 +1989,7 @@ struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, err = -ENOMEM; if (order > min_order) alloc_gfp |= __GFP_NORETRY | __GFP_NOWARN; - folio = filemap_alloc_folio(alloc_gfp, order); + folio = filemap_alloc_folio_mpol(alloc_gfp, order, mpol, ilx); if (!folio) continue;

@@ -2024,6 +2036,13 @@ struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, folio_clear_dropbehind(folio); return folio; } +EXPORT_SYMBOL_GPL(__filemap_get_folio_mpol); + +struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, + fgf_t fgp_flags, gfp_t gfp) +{ + return __filemap_get_folio_mpol(mapping, index, fgp_flags, gfp, NULL, 0); +} EXPORT_SYMBOL(__filemap_get_folio);

static inline struct folio *find_get_entry(struct xa_state *xas, pgoff_t max,

KVM guest_memfd wants to implement support for NUMA policies just like shmem already does using the shared policy infrastructure. As guest_memfd currently resides in KVM module code, we have to export the relevant symbols.

In the future, guest_memfd might be moved to core-mm, at which point the symbols no longer would have to be exported. When/if that happens is still unclear.

Acked-by: David Hildenbrand david@redhat.com Acked-by: Vlastimil Babka vbabka@suse.cz Signed-off-by: Shivank Garg shivankg@amd.com --- mm/mempolicy.c | 6 ++++++ 1 file changed, 6 insertions(+)

diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 3b1dfd08338b..d98243cdf090 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -354,6 +354,7 @@ struct mempolicy *get_task_policy(struct task_struct *p)

return &default_policy; } +EXPORT_SYMBOL_GPL(get_task_policy);

static const struct mempolicy_operations { int (*create)(struct mempolicy *pol, const nodemask_t *nodes); @@ -487,6 +488,7 @@ void __mpol_put(struct mempolicy *pol) return; kmem_cache_free(policy_cache, pol); } +EXPORT_SYMBOL_GPL(__mpol_put);

static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes) { @@ -2888,6 +2890,7 @@ struct mempolicy *mpol_shared_policy_lookup(struct shared_policy *sp, read_unlock(&sp->lock); return pol; } +EXPORT_SYMBOL_GPL(mpol_shared_policy_lookup);

static void sp_free(struct sp_node *n) { @@ -3173,6 +3176,7 @@ void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol) mpol_put(mpol); /* drop our incoming ref on sb mpol */ } } +EXPORT_SYMBOL_GPL(mpol_shared_policy_init);

int mpol_set_shared_policy(struct shared_policy *sp, struct vm_area_struct *vma, struct mempolicy *pol) @@ -3191,6 +3195,7 @@ int mpol_set_shared_policy(struct shared_policy *sp, sp_free(new); return err; } +EXPORT_SYMBOL_GPL(mpol_set_shared_policy);

/* Free a backing policy store on inode delete. */ void mpol_free_shared_policy(struct shared_policy *sp) @@ -3209,6 +3214,7 @@ void mpol_free_shared_policy(struct shared_policy *sp) } write_unlock(&sp->lock); } +EXPORT_SYMBOL_GPL(mpol_free_shared_policy);

#ifdef CONFIG_NUMA_BALANCING static int __initdata numabalancing_override;

Add dedicated inode structure (kvm_gmem_inode_info) and slab-allocated inode cache for guest memory backing, similar to how shmem handles inodes.

This adds the necessary allocation/destruction functions and prepares for upcoming guest_memfd NUMA policy support changes.

Signed-off-by: Shivank Garg shivankg@amd.com --- virt/kvm/guest_memfd.c | 51 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 159df462d193..5a1ce6f5e287 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -17,6 +17,15 @@ struct kvm_gmem { struct list_head entry; };

+struct kvm_gmem_inode_info { + struct inode vfs_inode; +}; + +static inline struct kvm_gmem_inode_info *KVM_GMEM_I(struct inode *inode) +{ + return container_of(inode, struct kvm_gmem_inode_info, vfs_inode); +} + /** * folio_file_pfn - like folio_file_page, but return a pfn. * @folio: The folio which contains this index. @@ -392,8 +401,33 @@ static struct file_operations kvm_gmem_fops = { .fallocate = kvm_gmem_fallocate, };

+static struct kmem_cache *kvm_gmem_inode_cachep; + +static struct inode *kvm_gmem_alloc_inode(struct super_block *sb) +{ + struct kvm_gmem_inode_info *info; + + info = alloc_inode_sb(sb, kvm_gmem_inode_cachep, GFP_KERNEL); + if (!info) + return NULL; + + return &info->vfs_inode; +} + +static void kvm_gmem_destroy_inode(struct inode *inode) +{ +} + +static void kvm_gmem_free_inode(struct inode *inode) +{ + kmem_cache_free(kvm_gmem_inode_cachep, KVM_GMEM_I(inode)); +} + static const struct super_operations kvm_gmem_super_operations = { .statfs = simple_statfs, + .alloc_inode = kvm_gmem_alloc_inode, + .destroy_inode = kvm_gmem_destroy_inode, + .free_inode = kvm_gmem_free_inode, };

static int kvm_gmem_init_fs_context(struct fs_context *fc) @@ -426,10 +460,26 @@ static int kvm_gmem_init_mount(void) return 0; }

+static void kvm_gmem_init_inode(void *foo) +{ + struct kvm_gmem_inode_info *info = foo; + + inode_init_once(&info->vfs_inode); +} + +static void kvm_gmem_init_inodecache(void) +{ + kvm_gmem_inode_cachep = kmem_cache_create("kvm_gmem_inode_cache", + sizeof(struct kvm_gmem_inode_info), + 0, SLAB_ACCOUNT, + kvm_gmem_init_inode); +} + int kvm_gmem_init(struct module *module) { kvm_gmem_fops.owner = module;

+ kvm_gmem_init_inodecache(); return kvm_gmem_init_mount(); }

@@ -437,6 +487,7 @@ void kvm_gmem_exit(void) { kern_unmount(kvm_gmem_mnt); kvm_gmem_mnt = NULL; + kmem_cache_destroy(kvm_gmem_inode_cachep); }

static int kvm_gmem_migrate_folio(struct address_space *mapping,

Previously, guest-memfd allocations followed local NUMA node id in absence of process mempolicy, resulting in arbitrary memory allocation. Moreover, mbind() couldn't be used by the VMM as guest memory wasn't mapped into userspace when allocation occurred.

Enable NUMA policy support by implementing vm_ops for guest-memfd mmap operation. This allows the VMM to map the memory and use mbind() to set the desired NUMA policy. The policy is stored in the inode structure via kvm_gmem_inode_info, as memory policy is a property of the memory (struct inode) itself. The policy is then retrieved via mpol_shared_policy_lookup() and passed to filemap_grab_folio_mpol() to ensure that allocations follow the specified memory policy.

This enables the VMM to control guest memory NUMA placement by calling mbind() on the mapped memory regions, providing fine-grained control over guest memory allocation across NUMA nodes.

The policy change only affect future allocations and does not migrate existing memory. This matches mbind(2)'s default behavior which affects only new allocations unless overridden with MPOL_MF_MOVE/MPOL_MF_MOVE_ALL flags, which are not supported for guest_memfd as it is unmovable.

Suggested-by: David Hildenbrand david@redhat.com Signed-off-by: Shivank Garg shivankg@amd.com --- virt/kvm/guest_memfd.c | 69 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 2 deletions(-)

diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 5a1ce6f5e287..2bd5ff3abd87 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -4,6 +4,7 @@ #include <linux/falloc.h> #include <linux/fs.h> #include <linux/kvm_host.h> +#include <linux/mempolicy.h> #include <linux/pseudo_fs.h> #include <linux/pagemap.h>

@@ -18,6 +19,7 @@ struct kvm_gmem { };

struct kvm_gmem_inode_info { + struct shared_policy policy; struct inode vfs_inode; };

@@ -26,6 +28,9 @@ static inline struct kvm_gmem_inode_info *KVM_GMEM_I(struct inode *inode) return container_of(inode, struct kvm_gmem_inode_info, vfs_inode); }

+static struct mempolicy *kvm_gmem_get_pgoff_policy(struct kvm_gmem_inode_info *info, + pgoff_t index); + /** * folio_file_pfn - like folio_file_page, but return a pfn. * @folio: The folio which contains this index. @@ -112,7 +117,24 @@ static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index) { /* TODO: Support huge pages. */ - return filemap_grab_folio(inode->i_mapping, index); + struct mempolicy *policy; + struct folio *folio; + + /* + * Fast-path: See if folio is already present in mapping to avoid + * policy_lookup. + */ + folio = __filemap_get_folio(inode->i_mapping, index, + FGP_LOCK | FGP_ACCESSED, 0); + if (!IS_ERR(folio)) + return folio; + + policy = kvm_gmem_get_pgoff_policy(KVM_GMEM_I(inode), index); + folio = filemap_grab_folio_mpol(inode->i_mapping, index, policy, + NO_INTERLEAVE_INDEX); + mpol_cond_put(policy); + + return folio; }

static void kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start, @@ -375,12 +397,52 @@ static vm_fault_t kvm_gmem_fault_shared(struct vm_fault *vmf) return ret; }

+#ifdef CONFIG_NUMA +static int kvm_gmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) +{ + struct inode *inode = file_inode(vma->vm_file); + + return mpol_set_shared_policy(&KVM_GMEM_I(inode)->policy, vma, mpol); +} + +static struct mempolicy *kvm_gmem_get_policy(struct vm_area_struct *vma, + unsigned long addr, pgoff_t *pgoff) +{ + struct inode *inode = file_inode(vma->vm_file); + + *pgoff = vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT); + return mpol_shared_policy_lookup(&KVM_GMEM_I(inode)->policy, *pgoff); +} + +static struct mempolicy *kvm_gmem_get_pgoff_policy(struct kvm_gmem_inode_info *info, + pgoff_t index) +{ + struct mempolicy *mpol; + + mpol = mpol_shared_policy_lookup(&info->policy, index); + return mpol ? mpol : get_task_policy(current); +} +#else +static struct mempolicy *kvm_gmem_get_pgoff_policy(struct kvm_gmem_inode_info *info, + pgoff_t index) +{ + return NULL; +} +#endif /* CONFIG_NUMA */ + static const struct vm_operations_struct kvm_gmem_vm_ops = { - .fault = kvm_gmem_fault_shared, + .fault = kvm_gmem_fault_shared, +#ifdef CONFIG_NUMA + .get_policy = kvm_gmem_get_policy, + .set_policy = kvm_gmem_set_policy, +#endif };

static int kvm_gmem_mmap(struct file *file, struct vm_area_struct *vma) { + struct kvm_gmem *gmem = file->private_data; + struct kvm *kvm = gmem->kvm; + if (!kvm_gmem_supports_shared(file_inode(file))) return -ENODEV;

@@ -411,11 +473,14 @@ static struct inode *kvm_gmem_alloc_inode(struct super_block *sb) if (!info) return NULL;

+ mpol_shared_policy_init(&info->policy, NULL); + return &info->vfs_inode; }

static void kvm_gmem_destroy_inode(struct inode *inode) { + mpol_free_shared_policy(&KVM_GMEM_I(inode)->policy); }

static void kvm_gmem_free_inode(struct inode *inode)