Add initial proof of concept implementation of DMA-mapping API for devices that have IOMMU support. Right now only dma_alloc_coherent, dma_free_coherent and dma_mmap_coherent functions are supported.
Signed-off-by: Marek Szyprowski m.szyprowski@samsung.com Signed-off-by: Kyungmin Park kyungmin.park@samsung.com --- arch/arm/Kconfig | 1 + arch/arm/include/asm/device.h | 2 + arch/arm/include/asm/dma-iommu.h | 30 ++++ arch/arm/mm/dma-mapping.c | 326 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 359 insertions(+), 0 deletions(-) create mode 100644 arch/arm/include/asm/dma-iommu.h
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 377a7a5..61900f1 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -8,6 +8,8 @@ config ARM select RTC_LIB select SYS_SUPPORTS_APM_EMULATION select GENERIC_ATOMIC64 if (CPU_V6 || !CPU_32v6K || !AEABI) + select GENERIC_ALLOCATOR + select HAVE_DMA_ATTRS select HAVE_OPROFILE if (HAVE_PERF_EVENTS) select HAVE_ARCH_KGDB select HAVE_KPROBES if (!XIP_KERNEL && !THUMB2_KERNEL) diff --git a/arch/arm/include/asm/device.h b/arch/arm/include/asm/device.h index 005791a..d3ec1e9 100644 --- a/arch/arm/include/asm/device.h +++ b/arch/arm/include/asm/device.h @@ -11,6 +11,8 @@ struct dev_archdata { #ifdef CONFIG_DMABOUNCE struct dmabounce_device_info *dmabounce; #endif + void *iommu_priv; + struct dma_iommu_mapping *mapping; };
struct pdev_archdata { diff --git a/arch/arm/include/asm/dma-iommu.h b/arch/arm/include/asm/dma-iommu.h new file mode 100644 index 0000000..c246ff3 --- /dev/null +++ b/arch/arm/include/asm/dma-iommu.h @@ -0,0 +1,30 @@ +#ifndef ASMARM_DMA_IOMMU_H +#define ASMARM_DMA_IOMMU_H + +#ifdef __KERNEL__ + +#include <linux/mm_types.h> +#include <linux/scatterlist.h> +#include <linux/dma-debug.h> +#include <linux/kmemcheck.h> + +#include <asm/memory.h> + +struct dma_iommu_mapping { + /* iommu specific data */ + struct iommu_domain *domain; + + /* address space data */ + struct gen_pool *pool; + + dma_addr_t base; + size_t size; + + atomic_t ref_count; + struct mutex lock; +}; + +int __init arm_iommu_assign_device(struct device *dev, dma_addr_t base, dma_addr_t size); + +#endif /* __KERNEL__ */ +#endif diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index f8c6972..b6397c1 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -19,6 +19,7 @@ #include <linux/dma-mapping.h> #include <linux/highmem.h> #include <linux/slab.h> +#include <linux/genalloc.h>
#include <asm/memory.h> #include <asm/highmem.h> @@ -26,6 +27,9 @@ #include <asm/tlbflush.h> #include <asm/sizes.h>
+#include <linux/iommu.h> +#include <asm/dma-iommu.h> + #ifdef __arch_page_to_dma #error Please update to __arch_pfn_to_dma #endif @@ -1040,3 +1044,325 @@ static int __init dma_debug_do_init(void) return 0; } fs_initcall(dma_debug_do_init); + + +/* IOMMU */ + +/* + * Allocate a DMA buffer for 'dev' of size 'size' using the + * specified gfp mask. Note that 'size' must be page aligned. + */ +static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp_t gfp) +{ + struct page **pages; + int count = size >> PAGE_SHIFT; + void *ptr; + int i; + + pages = kzalloc(count * sizeof(struct page*), gfp); + if (!pages) + return NULL; + + printk("IOMMU: page table allocated\n"); + + for (i=0; i<count; i++) { + pages[i] = alloc_page(gfp); //alloc_pages(gfp, 0); + + if (!pages[i]) + goto error; + + /* + * Ensure that the allocated pages are zeroed, and that any data + * lurking in the kernel direct-mapped region is invalidated. + */ + ptr = page_address(pages[i]); + memset(ptr, 0, PAGE_SIZE); + dmac_flush_range(ptr, ptr + PAGE_SIZE); + outer_flush_range(__pa(ptr), __pa(ptr) + PAGE_SIZE); + } + printk("IOMMU: pages allocated\n"); + return pages; +error: + printk("IOMMU: error allocating pages\n"); + while (--i) + if (pages[i]) + __free_pages(pages[i], 0); + kfree(pages); + return NULL; +} + +static int __iommu_free_buffer(struct device *dev, struct page **pages, size_t size) +{ + int count = size >> PAGE_SHIFT; + int i; + for (i=0; i< count; i++) + if (pages[i]) + __free_pages(pages[i], 0); + kfree(pages); + return 0; +} + +static void * +__iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot) +{ + struct arm_vmregion *c; + size_t align; + size_t count = size >> PAGE_SHIFT; + int bit; + + if (!consistent_pte[0]) { + printk(KERN_ERR "%s: not initialised\n", __func__); + dump_stack(); + return NULL; + } + + /* + * Align the virtual region allocation - maximum alignment is + * a section size, minimum is a page size. This helps reduce + * fragmentation of the DMA space, and also prevents allocations + * smaller than a section from crossing a section boundary. + */ + bit = fls(size - 1); + if (bit > SECTION_SHIFT) + bit = SECTION_SHIFT; + align = 1 << bit; + + /* + * Allocate a virtual address in the consistent mapping region. + */ + c = arm_vmregion_alloc(&consistent_head, align, size, + gfp & ~(__GFP_DMA | __GFP_HIGHMEM)); + if (c) { + pte_t *pte; + int idx = CONSISTENT_PTE_INDEX(c->vm_start); + int i = 0; + u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1); + + pte = consistent_pte[idx] + off; + c->priv = pages; + + do { + BUG_ON(!pte_none(*pte)); + + set_pte_ext(pte, mk_pte(pages[i], prot), 0); + pte++; + off++; + i++; + if (off >= PTRS_PER_PTE) { + off = 0; + pte = consistent_pte[++idx]; + } + } while (i < count); + + dsb(); + + return (void *)c->vm_start; + } + return NULL; +} + +static dma_addr_t __iommu_create_mapping(struct device *dev, struct page **pages, size_t size) +{ + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + unsigned int count = size >> PAGE_SHIFT; + dma_addr_t dma_addr, iova; + int i, ret = 0; + + printk("IOMMU: mapping %p\n", mapping); + + iova = gen_pool_alloc(mapping->pool, size); + + printk("IOMMU: gen_alloc res %x\n", iova); + + if (iova == 0) + goto fail; + + dma_addr = iova; + + for (i=0; i<count; i++) { + unsigned int phys = page_to_phys(pages[i]); + ret = iommu_map(mapping->domain, iova, phys, 0, 0); + if (ret < 0) + goto fail; + iova += PAGE_SIZE; + } + + return dma_addr; +fail: + return 0; +} + +static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size) +{ + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + unsigned int count = size >> PAGE_SHIFT; + int i; + + gen_pool_free(mapping->pool, iova, size); + + for (i=0; i<count; i++) { + iommu_unmap(mapping->domain, iova, 0); + iova += PAGE_SIZE; + } + return 0; +} + +int arm_iommu_init(struct device *dev); + +static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, + dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) +{ + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + struct page **pages; + void *addr = NULL; + pgprot_t prot; + + if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs)) + prot = pgprot_writecombine(pgprot_kernel); + else + prot = pgprot_dmacoherent(pgprot_kernel); + + arm_iommu_init(dev); + + mutex_lock(&mapping->lock); + + *handle = ~0; + size = PAGE_ALIGN(size); + + printk("IOMMU: requested size %d\n", size); + + pages = __iommu_alloc_buffer(dev, size, gfp); + if (!pages) + return NULL; + + printk("IOMMU: allocated pages: %p\n", pages); + + *handle = __iommu_create_mapping(dev, pages, size); + + printk("IOMMU: created iova: %08x\n", *handle); + + if (!*handle) + goto err_buffer; + + addr = __iommu_alloc_remap(pages, size, gfp, prot); + if (!addr) + goto err_iommu; + + printk("IOMMU: allocated iova %08x, virt %p\n", *handle, addr); + +err_iommu: +err_buffer: + mutex_unlock(&mapping->lock); + return addr; +} + +static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + struct dma_attrs *attrs) +{ + unsigned long user_size; + struct arm_vmregion *c; + + if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs)) + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + else + vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot); + + printk("IOMMU: mmap virt %p, dma %08x, size %d\n", cpu_addr, dma_addr, size); + + user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + + c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr); + if (c) { + struct page **pages = c->priv; + + unsigned long uaddr = vma->vm_start; + unsigned long usize = vma->vm_end - vma->vm_start; + int i = 0; + + do { + int ret; + + ret = vm_insert_page(vma, uaddr, pages[i++]); + if (ret) { + printk(KERN_ERR "Remapping memory, error: %d\n", ret); + return ret; + } + + uaddr += PAGE_SIZE; + usize -= PAGE_SIZE; + } while (usize > 0); + } + return 0; +} + +/* + * free a page as defined by the above mapping. + * Must not be called with IRQs disabled. + */ +void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle, struct dma_attrs *attrs) +{ + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + struct arm_vmregion *c; + size = PAGE_ALIGN(size); + + mutex_lock(&mapping->lock); + c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr); + if (c) { + struct page **pages = c->priv; + __dma_free_remap(cpu_addr, size); + __iommu_remove_mapping(dev, handle, size); + __iommu_free_buffer(dev, pages, size); + } + mutex_unlock(&mapping->lock); +} + +struct arm_dma_map_ops iommu_ops = { + .alloc_attrs = arm_iommu_alloc_attrs, + .free_attrs = arm_iommu_free_attrs, + .mmap_attrs = arm_iommu_mmap_attrs, +}; +EXPORT_SYMBOL_GPL(iommu_ops); + +int arm_iommu_init(struct device *dev) +{ + struct dma_iommu_mapping *mapping = dev->archdata.mapping; + + if (mapping->pool) + return 0; + + mutex_init(&mapping->lock); + + mapping->pool = gen_pool_create(16, -1); + if (!mapping->pool) + return -ENOMEM; + + if (gen_pool_add(mapping->pool, mapping->base, mapping->size, -1) != 0) + return -ENOMEM; + + mapping->domain = iommu_domain_alloc(); + if (!mapping->domain) + return -ENOMEM; + + if (iommu_attach_device(mapping->domain, dev) != 0) + return -ENOMEM; + + return 0; +} + +int __init arm_iommu_assign_device(struct device *dev, dma_addr_t base, dma_addr_t size) +{ + struct dma_iommu_mapping *mapping; + mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL); + if (!mapping) + return -ENOMEM; + mapping->base = base; + mapping->size = size; + + dev->archdata.mapping = mapping; + set_dma_ops(dev, &iommu_ops); + printk(KERN_INFO "Assigned IOMMU device to %s\n", dev_name(dev)); + + return 0; +}