Enable IOVA (un)mapping at a specific IOVA address, independent of allocating/freeing IOVA area, introducing the following dma_(un)map_page_*at*() functions:

dma_map_page_at() dma_unmap_page_at()

The above create a mapping between pre-allocated iova and a page, and remov just a mapping, leaving iova itself allocated. At mapping, it also checks if IOVA is already reserved or not.

There are the version with the prefix "arm_iommu_", and they are exactly same as the above.

Signed-off-by: Hiroshi DOYU hdoyu@nvidia.com --- arch/arm/include/asm/dma-iommu.h | 29 +++++++- arch/arm/include/asm/dma-mapping.h | 1 + arch/arm/mm/dma-mapping.c | 158 +++++++++++++++++++++++++++--------- 3 files changed, 150 insertions(+), 38 deletions(-)

diff --git a/arch/arm/include/asm/dma-iommu.h b/arch/arm/include/asm/dma-iommu.h index 2595928..99eba3d 100644 --- a/arch/arm/include/asm/dma-iommu.h +++ b/arch/arm/include/asm/dma-iommu.h @@ -30,9 +30,36 @@ void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping); int arm_iommu_attach_device(struct device *dev, struct dma_iommu_mapping *mapping);

-dma_addr_t arm_iommu_alloc_iova(struct device *dev, size_t size); +dma_addr_t arm_iommu_alloc_iova_at(struct device *dev, dma_addr_t addr, + size_t size); + +static inline dma_addr_t arm_iommu_alloc_iova(struct device *dev, size_t size) +{ + return arm_iommu_alloc_iova_at(dev, DMA_ANON_ADDR, size); +}

void arm_iommu_free_iova(struct device *dev, dma_addr_t addr, size_t size);

+dma_addr_t arm_iommu_map_page_at(struct device *dev, struct page *page, + dma_addr_t addr, unsigned long offset, size_t size, + enum dma_data_direction dir, struct dma_attrs *attrs); + +static inline dma_addr_t dma_map_page_at(struct device *d, struct page *p, + dma_addr_t a, size_t o, size_t s, + enum dma_data_direction r) +{ + return arm_iommu_map_page_at(d, p, a, o, s, r, 0); +} + +void arm_iommu_unmap_page_at(struct device *dev, dma_addr_t handle, + size_t size, enum dma_data_direction dir, + struct dma_attrs *attrs); + +static inline void dma_unmap_page_at(struct device *d, dma_addr_t a, size_t s, + enum dma_data_direction r) +{ + return arm_iommu_unmap_page_at(d, a, s, r, 0); +} + #endif /* __KERNEL__ */ #endif diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index bbef15d..b73eb73 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -12,6 +12,7 @@ #include <asm/memory.h>

#define DMA_ERROR_CODE (~0) +#define DMA_ANON_ADDR (~0) extern struct dma_map_ops arm_dma_ops;

static inline struct dma_map_ops *get_dma_ops(struct device *dev) diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index bca1715..b98e668 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1013,48 +1013,65 @@ fs_initcall(dma_debug_do_init);

/* IOMMU */

-static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping, - size_t size) +static dma_addr_t __alloc_iova_at(struct dma_iommu_mapping *mapping, + dma_addr_t iova, size_t size) { unsigned int order = get_order(size); unsigned int align = 0; - unsigned int count, start; + unsigned int count, start, orig = 0; unsigned long flags; + bool anon = (iova == DMA_ANON_ADDR) ? true : false;

count = ((PAGE_ALIGN(size) >> PAGE_SHIFT) + (1 << mapping->order) - 1) >> mapping->order;

- if (order > mapping->order) + if (anon && (order > mapping->order)) align = (1 << (order - mapping->order)) - 1;

spin_lock_irqsave(&mapping->lock, flags); - start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, 0, - count, align); - if (start > mapping->bits) { - spin_unlock_irqrestore(&mapping->lock, flags); - return DMA_ERROR_CODE; - } + if (!anon) + orig = (iova - mapping->base) >> (mapping->order + PAGE_SHIFT); + + start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, + orig, count, align); + if (start > mapping->bits) + goto not_found; + + if (!anon && (orig != start)) + goto not_found;

bitmap_set(mapping->bitmap, start, count); spin_unlock_irqrestore(&mapping->lock, flags);

return mapping->base + (start << (mapping->order + PAGE_SHIFT)); + +not_found: + spin_unlock_irqrestore(&mapping->lock, flags); + return DMA_ERROR_CODE; +} + +static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping, + size_t size) +{ + return __alloc_iova_at(mapping, DMA_ANON_ADDR, size); }

/** - * arm_iommu_alloc_iova + * arm_iommu_alloc_iova_at * @dev: valid struct device pointer + * @iova: iova address being requested. Set DMA_ANON_ADDR for arbitral * @size: size of buffer to allocate * * Allocate IOVA address range */ -dma_addr_t arm_iommu_alloc_iova(struct device *dev, size_t size) +dma_addr_t arm_iommu_alloc_iova_at(struct device *dev, dma_addr_t iova, + size_t size) { struct dma_iommu_mapping *mapping = dev->archdata.mapping;

- return __alloc_iova(mapping, size); + return __alloc_iova_at(mapping, iova, size); } -EXPORT_SYMBOL_GPL(arm_iommu_alloc_iova); +EXPORT_SYMBOL_GPL(arm_iommu_alloc_iova_at);

static inline void __free_iova(struct dma_iommu_mapping *mapping, dma_addr_t addr, size_t size) @@ -1507,6 +1524,41 @@ void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg, __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); }

+static dma_addr_t __arm_iommu_map_page_at(struct device *dev, struct page *page, + dma_addr_t req, unsigned long offset, size_t size, + enum dma_data_direction dir, struct dma_attrs *attrs) +{ + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + dma_addr_t dma_addr; + int ret, len = PAGE_ALIGN(size + offset); + + if (!arch_is_coherent()) + __dma_page_cpu_to_dev(page, offset, size, dir); + + dma_addr = __alloc_iova_at(mapping, req, len); + if (dma_addr == DMA_ERROR_CODE) { + if (req == DMA_ANON_ADDR) + return DMA_ERROR_CODE; + /* + * Verified that iova(req) is reserved in advance if + * @req is specified. + */ + dma_addr = req; + } + + if (req != DMA_ANON_ADDR) + BUG_ON(dma_addr != req); + + ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, 0); + if (ret < 0) + goto fail; + + return dma_addr + offset; +fail: + if (req == DMA_ANON_ADDR) + __free_iova(mapping, dma_addr, len); + return DMA_ERROR_CODE; +}

/** * arm_iommu_map_page @@ -1522,25 +1574,47 @@ static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { - struct dma_iommu_mapping *mapping = dev->archdata.mapping; - dma_addr_t dma_addr; - int ret, len = PAGE_ALIGN(size + offset); + return __arm_iommu_map_page_at(dev, page, DMA_ANON_ADDR, + offset, size, dir, attrs); +}

- if (!arch_is_coherent()) - __dma_page_cpu_to_dev(page, offset, size, dir); +/** + * arm_iommu_map_page_at + * @dev: valid struct device pointer + * @page: page that buffer resides in + * @req: iova address being requested. Set DMA_ANON_ADDR for arbitral + * @offset: offset into page for start of buffer + * @size: size of buffer to map + * @dir: DMA transfer direction + * + * The version with a specified iova address of arm_iommu_map_page(). + */ +dma_addr_t arm_iommu_map_page_at(struct device *dev, struct page *page, + dma_addr_t req, unsigned long offset, size_t size, + enum dma_data_direction dir, struct dma_attrs *attrs) +{ + return __arm_iommu_map_page_at(dev, page, req, offset, size, dir, + attrs); +} +EXPORT_SYMBOL_GPL(arm_iommu_map_page_at);

- dma_addr = __alloc_iova(mapping, len); - if (dma_addr == DMA_ERROR_CODE) - return dma_addr; +static inline int __arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, + size_t size, enum dma_data_direction dir, + struct dma_attrs *attrs) +{ + dma_addr_t iova = handle & PAGE_MASK; + struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); + int offset = handle & ~PAGE_MASK; + int len = PAGE_ALIGN(size + offset);

- ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, 0); - if (ret < 0) - goto fail; + if (!iova) + return -EINVAL;

- return dma_addr + offset; -fail: - __free_iova(mapping, dma_addr, len); - return DMA_ERROR_CODE; + if (!arch_is_coherent()) + __dma_page_dev_to_cpu(page, offset, size, dir); + + iommu_unmap(mapping->domain, iova, len); + return 0; }

/** @@ -1558,20 +1632,30 @@ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, { struct dma_iommu_mapping *mapping = dev->archdata.mapping; dma_addr_t iova = handle & PAGE_MASK; - struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); - int offset = handle & ~PAGE_MASK; int len = PAGE_ALIGN(size + offset);

- if (!iova) + if (__arm_iommu_unmap_page(dev, handle, size, dir, attrs)) return; - - if (!arch_is_coherent()) - __dma_page_dev_to_cpu(page, offset, size, dir); - - iommu_unmap(mapping->domain, iova, len); __free_iova(mapping, iova, len); }

+/** + * arm_iommu_unmap_page_at + * @dev: valid struct device pointer + * @handle: DMA address of buffer + * @size: size of buffer (same as passed to dma_map_page) + * @dir: DMA transfer direction (same as passed to dma_map_page) + * + * The version without freeing iova of arm_iommu_unmap_page(). + */ +void arm_iommu_unmap_page_at(struct device *dev, dma_addr_t handle, + size_t size, enum dma_data_direction dir, + struct dma_attrs *attrs) +{ + __arm_iommu_unmap_page(dev, handle, size, dir, attrs); +} +EXPORT_SYMBOL_GPL(arm_iommu_unmap_page_at); + static void arm_iommu_sync_single_for_cpu(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir) {