These are on performance paths so we protect them using the CONFIG_IOMMUFD_TEST to not hit during normal operation.
These are useful when running the test suite and syzkaller to find data structure inconsistencies early.
Signed-off-by: Jason Gunthorpe jgg@nvidia.com --- drivers/iommu/iommufd/device.c | 5 ++++ drivers/iommu/iommufd/io_pagetable.c | 22 +++++++++++++++ drivers/iommu/iommufd/io_pagetable.h | 3 +++ drivers/iommu/iommufd/pages.c | 40 ++++++++++++++++++++++++++-- 4 files changed, 68 insertions(+), 2 deletions(-)
diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c index 8ca5a38b2a79b8..536a34d099968d 100644 --- a/drivers/iommu/iommufd/device.c +++ b/drivers/iommu/iommufd/device.c @@ -589,6 +589,11 @@ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, bool first = true; int rc;
+ /* Driver didn't specify needs_pin_pages in its ops */ + if (IS_ENABLED(CONFIG_IOMMUFD_TEST) && + WARN_ON(access->iova_alignment != PAGE_SIZE)) + return -EINVAL; + if (!length) return -EINVAL; if (check_add_overflow(iova, length - 1, &last_iova)) diff --git a/drivers/iommu/iommufd/io_pagetable.c b/drivers/iommu/iommufd/io_pagetable.c index a7d16cd14ea200..432592fc026f4e 100644 --- a/drivers/iommu/iommufd/io_pagetable.c +++ b/drivers/iommu/iommufd/io_pagetable.c @@ -261,6 +261,11 @@ static int iopt_alloc_area_pages(struct io_pagetable *iopt, (uintptr_t)elm->pages->uptr + elm->start_byte, length); if (rc) goto out_unlock; + if (IS_ENABLED(CONFIG_IOMMUFD_TEST) && + WARN_ON(iopt_check_iova(iopt, *dst_iova, length))) { + rc = -EINVAL; + goto out_unlock; + } } else { rc = iopt_check_iova(iopt, *dst_iova, length); if (rc) @@ -287,6 +292,8 @@ static int iopt_alloc_area_pages(struct io_pagetable *iopt,
static void iopt_abort_area(struct iopt_area *area) { + if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) + WARN_ON(area->pages); if (area->iopt) { down_write(&area->iopt->iova_rwsem); interval_tree_remove(&area->node, &area->iopt->area_itree); @@ -652,6 +659,9 @@ void iopt_destroy_table(struct io_pagetable *iopt) { struct interval_tree_node *node;
+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) + iopt_remove_reserved_iova(iopt, NULL); + while ((node = interval_tree_iter_first(&iopt->allowed_itree, 0, ULONG_MAX))) { interval_tree_remove(node, &iopt->allowed_itree); @@ -698,6 +708,8 @@ static void iopt_unfill_domain(struct io_pagetable *iopt, continue;
mutex_lock(&pages->mutex); + if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) + WARN_ON(!area->storage_domain); if (area->storage_domain == domain) area->storage_domain = storage_domain; mutex_unlock(&pages->mutex); @@ -802,6 +814,16 @@ static int iopt_check_iova_alignment(struct io_pagetable *iopt, (iopt_area_length(area) & align_mask) || (area->page_offset & align_mask)) return -EADDRINUSE; + + if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) { + struct iommufd_access *access; + unsigned long index; + + xa_for_each(&iopt->access_list, index, access) + if (WARN_ON(access->iova_alignment > + new_iova_alignment)) + return -EADDRINUSE; + } return 0; }
diff --git a/drivers/iommu/iommufd/io_pagetable.h b/drivers/iommu/iommufd/io_pagetable.h index 9a1c8a5ae3c883..3b85fa344f6be3 100644 --- a/drivers/iommu/iommufd/io_pagetable.h +++ b/drivers/iommu/iommufd/io_pagetable.h @@ -101,6 +101,9 @@ static inline size_t iopt_area_length(struct iopt_area *area) static inline unsigned long iopt_area_start_byte(struct iopt_area *area, unsigned long iova) { + if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) + WARN_ON(iova < iopt_area_iova(area) || + iova > iopt_area_last_iova(area)); return (iova - iopt_area_iova(area)) + area->page_offset + iopt_area_index(area) * PAGE_SIZE; } diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c index 2ddcb0d4f71e04..c535be1bedb9ba 100644 --- a/drivers/iommu/iommufd/pages.c +++ b/drivers/iommu/iommufd/pages.c @@ -101,12 +101,20 @@ static void *temp_kmalloc(size_t *size, void *backup, size_t backup_len)
static void iopt_pages_add_npinned(struct iopt_pages *pages, size_t npages) { - pages->npinned += npages; + int rc; + + rc = check_add_overflow(pages->npinned, npages, &pages->npinned); + if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) + WARN_ON(rc || pages->npinned > pages->npages); }
static void iopt_pages_sub_npinned(struct iopt_pages *pages, size_t npages) { - pages->npinned -= npages; + int rc; + + rc = check_sub_overflow(pages->npinned, npages, &pages->npinned); + if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) + WARN_ON(rc || pages->npinned > pages->npages); }
static void iopt_pages_err_unpin(struct iopt_pages *pages, @@ -128,6 +136,9 @@ static void iopt_pages_err_unpin(struct iopt_pages *pages, static unsigned long iopt_area_index_to_iova(struct iopt_area *area, unsigned long index) { + if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) + WARN_ON(index < iopt_area_index(area) || + index > iopt_area_last_index(area)); index -= iopt_area_index(area); if (index == 0) return iopt_area_iova(area); @@ -137,6 +148,9 @@ static unsigned long iopt_area_index_to_iova(struct iopt_area *area, static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area, unsigned long index) { + if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) + WARN_ON(index < iopt_area_index(area) || + index > iopt_area_last_index(area)); if (index == iopt_area_last_index(area)) return iopt_area_last_iova(area); return iopt_area_iova(area) - area->page_offset + @@ -240,6 +254,8 @@ static int __batch_init(struct pfn_batch *batch, size_t max_pages, void *backup, batch->pfns = temp_kmalloc(&size, backup, backup_len); if (!batch->pfns) return -ENOMEM; + if (IS_ENABLED(CONFIG_IOMMUFD_TEST) && WARN_ON(size < elmsz)) + return -EINVAL; batch->array_size = size / elmsz; batch->npfns = (u32 *)(batch->pfns + batch->array_size); batch_clear(batch); @@ -367,6 +383,10 @@ static int batch_iommu_map_small(struct iommu_domain *domain, unsigned long start_iova = iova; int rc;
+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) + WARN_ON(paddr % PAGE_SIZE || iova % PAGE_SIZE || + size % PAGE_SIZE); + while (size) { rc = iommu_map(domain, iova, paddr, PAGE_SIZE, prot); if (rc) @@ -652,6 +672,10 @@ static int pfn_reader_user_pin(struct pfn_reader_user *user, uintptr_t uptr; long rc;
+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST) && + WARN_ON(last_index < start_index)) + return -EINVAL; + if (!user->upages) { /* All undone in pfn_reader_destroy() */ user->upages_len = @@ -890,6 +914,10 @@ static int pfn_reader_fill_span(struct pfn_reader *pfns) struct iopt_area *area; int rc;
+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST) && + WARN_ON(span->last_used < start_index)) + return -EINVAL; + if (span->is_used == 1) { batch_from_xarray(&pfns->batch, &pfns->pages->pinned_pfns, start_index, span->last_used); @@ -942,6 +970,10 @@ static int pfn_reader_next(struct pfn_reader *pfns) while (pfns->batch_end_index != pfns->last_index + 1) { unsigned int npfns = pfns->batch.total_pfns;
+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST) && + WARN_ON(interval_tree_double_span_iter_done(&pfns->span))) + return -EINVAL; + rc = pfn_reader_fill_span(pfns); if (rc) return rc; @@ -1025,6 +1057,10 @@ static int pfn_reader_first(struct pfn_reader *pfns, struct iopt_pages *pages, { int rc;
+ if (IS_ENABLED(CONFIG_IOMMUFD_TEST) && + WARN_ON(last_index < start_index)) + return -EINVAL; + rc = pfn_reader_init(pfns, pages, start_index, last_index); if (rc) return rc;