* Mike Rapoport rppt@kernel.org [251125 13:39]:
From: "Mike Rapoport (Microsoft)" rppt@kernel.org
When userspace resolves a page fault in a shmem VMA with UFFDIO_CONTINUE it needs to get a folio that already exists in the pagecache backing that VMA.
Instead of using shmem_get_folio() for that, add a get_folio() method to 'struct vm_operations_struct' that will return a folio if it exists in the VMA's pagecache at given pgoff.
Implement get_folio() method for shmem and slightly refactor userfaultfd's mfill_atomic() and mfill_atomic_pte_continue() to support this new API.
Signed-off-by: Mike Rapoport (Microsoft) rppt@kernel.org
Reviewed-by: Liam R. Howlett Liam.Howlett@oracle.com
include/linux/mm.h | 9 ++++++++ mm/shmem.c | 18 ++++++++++++++++ mm/userfaultfd.c | 52 +++++++++++++++++++++++++++++----------------- 3 files changed, 60 insertions(+), 19 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h index 7c79b3369b82..c8647707d75b 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -690,6 +690,15 @@ struct vm_operations_struct { struct page *(*find_normal_page)(struct vm_area_struct *vma, unsigned long addr); #endif /* CONFIG_FIND_NORMAL_PAGE */ +#ifdef CONFIG_USERFAULTFD
- /*
* Called by userfault to resolve UFFDIO_CONTINUE request.* Should return the folio found at pgoff in the VMA's pagecache if it* exists or ERR_PTR otherwise.* The returned folio is locked and with reference held.*/- struct folio *(*get_folio)(struct inode *inode, pgoff_t pgoff);
+#endif }; #ifdef CONFIG_NUMA_BALANCING diff --git a/mm/shmem.c b/mm/shmem.c index 58701d14dd96..e16c7c8c3e1e 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -3263,6 +3263,18 @@ int shmem_mfill_atomic_pte(pmd_t *dst_pmd, shmem_inode_unacct_blocks(inode, 1); return ret; }
+static struct folio *shmem_get_folio_noalloc(struct inode *inode, pgoff_t pgoff) +{
- struct folio *folio;
- int err;
- err = shmem_get_folio(inode, pgoff, 0, &folio, SGP_NOALLOC);
- if (err)
return ERR_PTR(err);- return folio;
+} #endif /* CONFIG_USERFAULTFD */ #ifdef CONFIG_TMPFS @@ -5295,6 +5307,9 @@ static const struct vm_operations_struct shmem_vm_ops = { .set_policy = shmem_set_policy, .get_policy = shmem_get_policy, #endif +#ifdef CONFIG_USERFAULTFD
- .get_folio = shmem_get_folio_noalloc,
+#endif }; static const struct vm_operations_struct shmem_anon_vm_ops = { @@ -5304,6 +5319,9 @@ static const struct vm_operations_struct shmem_anon_vm_ops = { .set_policy = shmem_set_policy, .get_policy = shmem_get_policy, #endif +#ifdef CONFIG_USERFAULTFD
- .get_folio = shmem_get_folio_noalloc,
+#endif }; int shmem_init_fs_context(struct fs_context *fc) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 8dc964389b0d..9f0f879b603a 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -388,15 +388,12 @@ static int mfill_atomic_pte_continue(pmd_t *dst_pmd, struct page *page; int ret;
- ret = shmem_get_folio(inode, pgoff, 0, &folio, SGP_NOALLOC);
- folio = dst_vma->vm_ops->get_folio(inode, pgoff); /* Our caller expects us to return -EFAULT if we failed to find folio */
- if (ret == -ENOENT)
ret = -EFAULT;- if (ret)
goto out;- if (!folio) {
ret = -EFAULT;goto out;
- if (IS_ERR_OR_NULL(folio)) {
if (PTR_ERR(folio) == -ENOENT || !folio)return -EFAULT;return PTR_ERR(folio);page = folio_file_page(folio, pgoff); @@ -411,13 +408,12 @@ static int mfill_atomic_pte_continue(pmd_t *dst_pmd, goto out_release; folio_unlock(folio);
- ret = 0;
-out:
- return ret;
- return 0;
out_release: folio_unlock(folio); folio_put(folio);
- goto out;
- return ret;
I really like this part.
} /* Handles UFFDIO_POISON for all non-hugetlb VMAs. */ @@ -694,6 +690,15 @@ static __always_inline ssize_t mfill_atomic_pte(pmd_t *dst_pmd, return err; } +static __always_inline bool vma_can_mfill_atomic(struct vm_area_struct *vma,
uffd_flags_t flags)+{
- if (uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE))
return vma->vm_ops && vma->vm_ops->get_folio;- return vma_is_anonymous(vma) || vma_is_shmem(vma);
+}
static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, unsigned long dst_start, unsigned long src_start, @@ -766,10 +771,7 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, return mfill_atomic_hugetlb(ctx, dst_vma, dst_start, src_start, len, flags);
- if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma))
goto out_unlock;- if (!vma_is_shmem(dst_vma) &&
uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE))
- if (!vma_can_mfill_atomic(dst_vma, flags)) goto out_unlock;
while (src_addr < src_start + len) { @@ -1985,9 +1987,21 @@ bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags, if (vma->vm_flags & VM_DROPPABLE) return false;
- if ((vm_flags & VM_UFFD_MINOR) &&
(!is_vm_hugetlb_page(vma) && !vma_is_shmem(vma)))return false;
- if (vm_flags & VM_UFFD_MINOR) {
/** If only MINOR mode is requested and we can request an* existing folio from VMA's page cache, allow it*/if (vm_flags == VM_UFFD_MINOR && vma->vm_ops &&vma->vm_ops->get_folio)return true;/** Only hugetlb and shmem can support MINOR mode in combination* with other modes*/if (!is_vm_hugetlb_page(vma) && !vma_is_shmem(vma))return false;- }
/* * If wp async enabled, and WP is the only mode enabled, allow any -- 2.50.1