From: KAMEZAWA Hiroyuki kamezawa.hiroyu@jp.fujitsu.com

Memory hotplug is a logic for making pages unused in the specified range of pfn. So, some of core logics can be used for other purpose as allocating a very large contigous memory block.

This patch moves some functions from mm/memory_hotplug.c to mm/page_isolation.c. This helps adding a function for large-alloc in page_isolation.c with memory-unplug technique.

Signed-off-by: KAMEZAWA Hiroyuki kamezawa.hiroyu@jp.fujitsu.com [m.nazarewicz: reworded commit message] Signed-off-by: Michal Nazarewicz m.nazarewicz@samsung.com Signed-off-by: Kyungmin Park kyungmin.park@samsung.com [m.szyprowski: rebased and updated to Linux v3.0-rc1] Signed-off-by: Marek Szyprowski m.szyprowski@samsung.com CC: Michal Nazarewicz mina86@mina86.com Acked-by: Arnd Bergmann arnd@arndb.de --- include/linux/page-isolation.h | 7 +++ mm/memory_hotplug.c | 111 -------------------------------------- mm/page_isolation.c | 114 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 111 deletions(-)

diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h index 051c1b1..58cdbac 100644 --- a/include/linux/page-isolation.h +++ b/include/linux/page-isolation.h @@ -33,5 +33,12 @@ test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn); extern int set_migratetype_isolate(struct page *page); extern void unset_migratetype_isolate(struct page *page);

+/* + * For migration. + */ + +int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn); +unsigned long scan_lru_pages(unsigned long start, unsigned long end); +int do_migrate_range(unsigned long start_pfn, unsigned long end_pfn);

#endif diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 6e7d8b2..3419dd6 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -707,117 +707,6 @@ int is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages) }

/* - * Confirm all pages in a range [start, end) is belongs to the same zone. - */ -static int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn) -{ - unsigned long pfn; - struct zone *zone = NULL; - struct page *page; - int i; - for (pfn = start_pfn; - pfn < end_pfn; - pfn += MAX_ORDER_NR_PAGES) { - i = 0; - /* This is just a CONFIG_HOLES_IN_ZONE check.*/ - while ((i < MAX_ORDER_NR_PAGES) && !pfn_valid_within(pfn + i)) - i++; - if (i == MAX_ORDER_NR_PAGES) - continue; - page = pfn_to_page(pfn + i); - if (zone && page_zone(page) != zone) - return 0; - zone = page_zone(page); - } - return 1; -} - -/* - * Scanning pfn is much easier than scanning lru list. - * Scan pfn from start to end and Find LRU page. - */ -static unsigned long scan_lru_pages(unsigned long start, unsigned long end) -{ - unsigned long pfn; - struct page *page; - for (pfn = start; pfn < end; pfn++) { - if (pfn_valid(pfn)) { - page = pfn_to_page(pfn); - if (PageLRU(page)) - return pfn; - } - } - return 0; -} - -static struct page * -hotremove_migrate_alloc(struct page *page, unsigned long private, int **x) -{ - /* This should be improooooved!! */ - return alloc_page(GFP_HIGHUSER_MOVABLE); -} - -#define NR_OFFLINE_AT_ONCE_PAGES (256) -static int -do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) -{ - unsigned long pfn; - struct page *page; - int move_pages = NR_OFFLINE_AT_ONCE_PAGES; - int not_managed = 0; - int ret = 0; - LIST_HEAD(source); - - for (pfn = start_pfn; pfn < end_pfn && move_pages > 0; pfn++) { - if (!pfn_valid(pfn)) - continue; - page = pfn_to_page(pfn); - if (!get_page_unless_zero(page)) - continue; - /* - * We can skip free pages. And we can only deal with pages on - * LRU. - */ - ret = isolate_lru_page(page); - if (!ret) { /* Success */ - put_page(page); - list_add_tail(&page->lru, &source); - move_pages--; - inc_zone_page_state(page, NR_ISOLATED_ANON + - page_is_file_cache(page)); - - } else { -#ifdef CONFIG_DEBUG_VM - printk(KERN_ALERT "removing pfn %lx from LRU failed\n", - pfn); - dump_page(page); -#endif - put_page(page); - /* Because we don't have big zone->lock. we should - check this again here. */ - if (page_count(page)) { - not_managed++; - ret = -EBUSY; - break; - } - } - } - if (!list_empty(&source)) { - if (not_managed) { - putback_lru_pages(&source); - goto out; - } - /* this function returns # of failed pages */ - ret = migrate_pages(&source, hotremove_migrate_alloc, 0, - true, true); - if (ret) - putback_lru_pages(&source); - } -out: - return ret; -} - -/* * remove from free_area[] and mark all as Reserved. */ static int diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 4ae42bb..270a026 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -5,6 +5,9 @@ #include <linux/mm.h> #include <linux/page-isolation.h> #include <linux/pageblock-flags.h> +#include <linux/memcontrol.h> +#include <linux/migrate.h> +#include <linux/mm_inline.h> #include "internal.h"

static inline struct page * @@ -139,3 +142,114 @@ int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn) spin_unlock_irqrestore(&zone->lock, flags); return ret ? 0 : -EBUSY; } + + +/* + * Confirm all pages in a range [start, end) is belongs to the same zone. + */ +int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn) +{ + unsigned long pfn; + struct zone *zone = NULL; + struct page *page; + int i; + for (pfn = start_pfn; + pfn < end_pfn; + pfn += MAX_ORDER_NR_PAGES) { + i = 0; + /* This is just a CONFIG_HOLES_IN_ZONE check.*/ + while ((i < MAX_ORDER_NR_PAGES) && !pfn_valid_within(pfn + i)) + i++; + if (i == MAX_ORDER_NR_PAGES) + continue; + page = pfn_to_page(pfn + i); + if (zone && page_zone(page) != zone) + return 0; + zone = page_zone(page); + } + return 1; +} + +/* + * Scanning pfn is much easier than scanning lru list. + * Scan pfn from start to end and Find LRU page. + */ +unsigned long scan_lru_pages(unsigned long start, unsigned long end) +{ + unsigned long pfn; + struct page *page; + for (pfn = start; pfn < end; pfn++) { + if (pfn_valid(pfn)) { + page = pfn_to_page(pfn); + if (PageLRU(page)) + return pfn; + } + } + return 0; +} + +struct page * +hotremove_migrate_alloc(struct page *page, unsigned long private, int **x) +{ + /* This should be improooooved!! */ + return alloc_page(GFP_HIGHUSER_MOVABLE); +} + +#define NR_OFFLINE_AT_ONCE_PAGES (256) +int do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) +{ + unsigned long pfn; + struct page *page; + int move_pages = NR_OFFLINE_AT_ONCE_PAGES; + int not_managed = 0; + int ret = 0; + LIST_HEAD(source); + + for (pfn = start_pfn; pfn < end_pfn && move_pages > 0; pfn++) { + if (!pfn_valid(pfn)) + continue; + page = pfn_to_page(pfn); + if (!get_page_unless_zero(page)) + continue; + /* + * We can skip free pages. And we can only deal with pages on + * LRU. + */ + ret = isolate_lru_page(page); + if (!ret) { /* Success */ + put_page(page); + list_add_tail(&page->lru, &source); + move_pages--; + inc_zone_page_state(page, NR_ISOLATED_ANON + + page_is_file_cache(page)); + + } else { +#ifdef CONFIG_DEBUG_VM + printk(KERN_ALERT "removing pfn %lx from LRU failed\n", + pfn); + dump_page(page); +#endif + put_page(page); + /* Because we don't have big zone->lock. we should + check this again here. */ + if (page_count(page)) { + not_managed++; + ret = -EBUSY; + break; + } + } + } + if (!list_empty(&source)) { + if (not_managed) { + putback_lru_pages(&source); + goto out; + } + /* this function returns # of failed pages */ + ret = migrate_pages(&source, hotremove_migrate_alloc, 0, + true, true); + if (ret) + putback_lru_pages(&source); + } +out: + return ret; +}

From: Michal Nazarewicz m.nazarewicz@samsung.com

This commit adds the alloc_contig_range() function which tries to allecate given range of pages. It tries to migrate all already allocated pages that fall in the range thus freeing them. Once all pages in the range are freed they are removed from the buddy system thus allocated for the caller to use.

Signed-off-by: Michal Nazarewicz m.nazarewicz@samsung.com Signed-off-by: Kyungmin Park kyungmin.park@samsung.com [m.szyprowski: renamed some variables for easier code reading] Signed-off-by: Marek Szyprowski m.szyprowski@samsung.com CC: Michal Nazarewicz mina86@mina86.com Acked-by: Arnd Bergmann arnd@arndb.de --- include/linux/page-isolation.h | 2 + mm/page_alloc.c | 144 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 0 deletions(-)

diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h index f1417ed..c5d1a7c 100644 --- a/include/linux/page-isolation.h +++ b/include/linux/page-isolation.h @@ -34,6 +34,8 @@ extern int set_migratetype_isolate(struct page *page); extern void unset_migratetype_isolate(struct page *page); extern unsigned long alloc_contig_freed_pages(unsigned long start, unsigned long end, gfp_t flag); +extern int alloc_contig_range(unsigned long start, unsigned long end, + gfp_t flags); extern void free_contig_pages(struct page *page, int nr_pages);

/* diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ad6ae3f..35423c2 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5706,6 +5706,150 @@ unsigned long alloc_contig_freed_pages(unsigned long start, unsigned long end, return pfn; }

+static unsigned long pfn_to_maxpage(unsigned long pfn) +{ + return pfn & ~(MAX_ORDER_NR_PAGES - 1); +} + +static unsigned long pfn_to_maxpage_up(unsigned long pfn) +{ + return ALIGN(pfn, MAX_ORDER_NR_PAGES); +} + +#define MIGRATION_RETRY 5 +static int __alloc_contig_migrate_range(unsigned long start, unsigned long end) +{ + int migration_failed = 0, ret; + unsigned long pfn = start; + + /* + * Some code "borrowed" from KAMEZAWA Hiroyuki's + * __alloc_contig_pages(). + */ + + for (;;) { + pfn = scan_lru_pages(pfn, end); + if (!pfn || pfn >= end) + break; + + ret = do_migrate_range(pfn, end); + if (!ret) { + migration_failed = 0; + } else if (ret != -EBUSY + || ++migration_failed >= MIGRATION_RETRY) { + return ret; + } else { + /* There are unstable pages.on pagevec. */ + lru_add_drain_all(); + /* + * there may be pages on pcplist before + * we mark the range as ISOLATED. + */ + drain_all_pages(); + } + cond_resched(); + } + + if (!migration_failed) { + /* drop all pages in pagevec and pcp list */ + lru_add_drain_all(); + drain_all_pages(); + } + + /* Make sure all pages are isolated */ + if (WARN_ON(test_pages_isolated(start, end))) + return -EBUSY; + + return 0; +} + +/** + * alloc_contig_range() -- tries to allocate given range of pages + * @start: start PFN to allocate + * @end: one-past-the-last PFN to allocate + * @flags: flags passed to alloc_contig_freed_pages(). + * + * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES + * aligned, hovewer it's callers responsibility to guarantee that we + * are the only thread that changes migrate type of pageblocks the + * pages fall in. + * + * Returns zero on success or negative error code. On success all + * pages which PFN is in (start, end) are allocated for the caller and + * need to be freed with free_contig_pages(). + */ +int alloc_contig_range(unsigned long start, unsigned long end, + gfp_t flags) +{ + unsigned long outer_start, outer_end; + int ret; + + /* + * What we do here is we mark all pageblocks in range as + * MIGRATE_ISOLATE. Because of the way page allocator work, we + * align the range to MAX_ORDER pages so that page allocator + * won't try to merge buddies from different pageblocks and + * change MIGRATE_ISOLATE to some other migration type. + * + * Once the pageblocks are marked as MIGRATE_ISOLATE, we + * migrate the pages from an unaligned range (ie. pages that + * we are interested in). This will put all the pages in + * range back to page allocator as MIGRATE_ISOLATE. + * + * When this is done, we take the pages in range from page + * allocator removing them from the buddy system. This way + * page allocator will never consider using them. + * + * This lets us mark the pageblocks back as + * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the + * MAX_ORDER aligned range but not in the unaligned, original + * range are put back to page allocator so that buddy can use + * them. + */ + + ret = start_isolate_page_range(pfn_to_maxpage(start), + pfn_to_maxpage_up(end)); + if (ret) + goto done; + + ret = __alloc_contig_migrate_range(start, end); + if (ret) + goto done; + + /* + * Pages from [start, end) are within a MAX_ORDER_NR_PAGES + * aligned blocks that are marked as MIGRATE_ISOLATE. What's + * more, all pages in [start, end) are free in page allocator. + * What we are going to do is to allocate all pages from + * [start, end) (that is remove them from page allocater). + * + * The only problem is that pages at the beginning and at the + * end of interesting range may be not aligned with pages that + * page allocator holds, ie. they can be part of higher order + * pages. Because of this, we reserve the bigger range and + * once this is done free the pages we are not interested in. + */ + + ret = 0; + while (!PageBuddy(pfn_to_page(start & (~0UL << ret)))) + if (WARN_ON(++ret >= MAX_ORDER)) + return -EINVAL; + + outer_start = start & (~0UL << ret); + outer_end = alloc_contig_freed_pages(outer_start, end, flags); + + /* Free head and tail (if any) */ + if (start != outer_start) + free_contig_pages(pfn_to_page(outer_start), start - outer_start); + if (end != outer_end) + free_contig_pages(pfn_to_page(end), outer_end - end); + + ret = 0; +done: + undo_isolate_page_range(pfn_to_maxpage(start), pfn_to_maxpage_up(end)); + return ret; +} + void free_contig_pages(struct page *page, int nr_pages) { for (; nr_pages; --nr_pages, ++page)

From: Michal Nazarewicz m.nazarewicz@samsung.com

This commit changes various functions that change pages and pageblocks migrate type between MIGRATE_ISOLATE and MIGRATE_MOVABLE in such a way as to allow to work with MIGRATE_CMA migrate type.

Signed-off-by: Michal Nazarewicz m.nazarewicz@samsung.com Signed-off-by: Kyungmin Park kyungmin.park@samsung.com Signed-off-by: Marek Szyprowski m.szyprowski@samsung.com CC: Michal Nazarewicz mina86@mina86.com Acked-by: Arnd Bergmann arnd@arndb.de --- include/linux/page-isolation.h | 40 +++++++++++++++++++++++++++------------- mm/page_alloc.c | 19 ++++++++++++------- mm/page_isolation.c | 15 ++++++++------- 3 files changed, 47 insertions(+), 27 deletions(-)

diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h index 856d9cf..b2a81fd 100644 --- a/include/linux/page-isolation.h +++ b/include/linux/page-isolation.h @@ -3,39 +3,53 @@

/* * Changes migrate type in [start_pfn, end_pfn) to be MIGRATE_ISOLATE. - * If specified range includes migrate types other than MOVABLE, + * If specified range includes migrate types other than MOVABLE or CMA, * this will fail with -EBUSY. * * For isolating all pages in the range finally, the caller have to * free all pages in the range. test_page_isolated() can be used for * test it. */ -extern int -start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn); +int __start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, + unsigned migratetype); + +static inline int +start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn) +{ + return __start_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); +} + +int __undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, + unsigned migratetype);

/* * Changes MIGRATE_ISOLATE to MIGRATE_MOVABLE. * target range is [start_pfn, end_pfn) */ -extern int -undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn); +static inline int +undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn) +{ + return __undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); +}

/* - * test all pages in [start_pfn, end_pfn)are isolated or not. + * Test all pages in [start_pfn, end_pfn) are isolated or not. */ -extern int -test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn); +int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn);

/* - * Internal funcs.Changes pageblock's migrate type. - * Please use make_pagetype_isolated()/make_pagetype_movable(). + * Internal functions. Changes pageblock's migrate type. */ -extern int set_migratetype_isolate(struct page *page); -extern void unset_migratetype_isolate(struct page *page); +int set_migratetype_isolate(struct page *page); +void __unset_migratetype_isolate(struct page *page, unsigned migratetype); +static inline void unset_migratetype_isolate(struct page *page) +{ + __unset_migratetype_isolate(page, MIGRATE_MOVABLE); +} extern unsigned long alloc_contig_freed_pages(unsigned long start, unsigned long end, gfp_t flag); extern int alloc_contig_range(unsigned long start, unsigned long end, - gfp_t flags); + gfp_t flags, unsigned migratetype); extern void free_contig_pages(struct page *page, int nr_pages);

/* diff --git a/mm/page_alloc.c b/mm/page_alloc.c index aecf32a..e3d756a 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -5702,7 +5702,7 @@ out: return ret; }

-void unset_migratetype_isolate(struct page *page) +void __unset_migratetype_isolate(struct page *page, unsigned migratetype) { struct zone *zone; unsigned long flags; @@ -5710,8 +5710,8 @@ void unset_migratetype_isolate(struct page *page) spin_lock_irqsave(&zone->lock, flags); if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE) goto out; - set_pageblock_migratetype(page, MIGRATE_MOVABLE); - move_freepages_block(zone, page, MIGRATE_MOVABLE); + set_pageblock_migratetype(page, migratetype); + move_freepages_block(zone, page, migratetype); out: spin_unlock_irqrestore(&zone->lock, flags); } @@ -5816,6 +5816,10 @@ static int __alloc_contig_migrate_range(unsigned long start, unsigned long end) * @start: start PFN to allocate * @end: one-past-the-last PFN to allocate * @flags: flags passed to alloc_contig_freed_pages(). + * @migratetype: migratetype of the underlaying pageblocks (either + * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks + * in range must have the same migratetype and it must + * be either of the two. * * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES * aligned, hovewer it's callers responsibility to guarantee that we @@ -5827,7 +5831,7 @@ static int __alloc_contig_migrate_range(unsigned long start, unsigned long end) * need to be freed with free_contig_pages(). */ int alloc_contig_range(unsigned long start, unsigned long end, - gfp_t flags) + gfp_t flags, unsigned migratetype) { unsigned long outer_start, outer_end; int ret; @@ -5855,8 +5859,8 @@ int alloc_contig_range(unsigned long start, unsigned long end, * them. */

- ret = start_isolate_page_range(pfn_to_maxpage(start), - pfn_to_maxpage_up(end)); + ret = __start_isolate_page_range(pfn_to_maxpage(start), + pfn_to_maxpage_up(end), migratetype); if (ret) goto done;

@@ -5894,7 +5898,8 @@ int alloc_contig_range(unsigned long start, unsigned long end,

ret = 0; done: - undo_isolate_page_range(pfn_to_maxpage(start), pfn_to_maxpage_up(end)); + __undo_isolate_page_range(pfn_to_maxpage(start), pfn_to_maxpage_up(end), + migratetype); return ret; }

diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 270a026..e232b25 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -23,10 +23,11 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages) }

/* - * start_isolate_page_range() -- make page-allocation-type of range of pages + * __start_isolate_page_range() -- make page-allocation-type of range of pages * to be MIGRATE_ISOLATE. * @start_pfn: The lower PFN of the range to be isolated. * @end_pfn: The upper PFN of the range to be isolated. + * @migratetype: migrate type to set in error recovery. * * Making page-allocation-type to be MIGRATE_ISOLATE means free pages in * the range will never be allocated. Any free pages and pages freed in the @@ -35,8 +36,8 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages) * start_pfn/end_pfn must be aligned to pageblock_order. * Returns 0 on success and -EBUSY if any part of range cannot be isolated. */ -int -start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn) +int __start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, + unsigned migratetype) { unsigned long pfn; unsigned long undo_pfn; @@ -59,7 +60,7 @@ undo: for (pfn = start_pfn; pfn < undo_pfn; pfn += pageblock_nr_pages) - unset_migratetype_isolate(pfn_to_page(pfn)); + __unset_migratetype_isolate(pfn_to_page(pfn), migratetype);

return -EBUSY; } @@ -67,8 +68,8 @@ undo: /* * Make isolated pages available again. */ -int -undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn) +int __undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, + unsigned migratetype) { unsigned long pfn; struct page *page; @@ -80,7 +81,7 @@ undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn) page = __first_valid_page(pfn, pageblock_nr_pages); if (!page || get_pageblock_migratetype(page) != MIGRATE_ISOLATE) continue; - unset_migratetype_isolate(page); + __unset_migratetype_isolate(page, migratetype); } return 0; }

From: Russell King rmk+kernel@arm.linux.org.uk

Steal memory from the kernel to provide coherent DMA memory to drivers. This avoids the problem with multiple mappings with differing attributes on later CPUs.

Signed-off-by: Russell King rmk+kernel@arm.linux.org.uk [m.szyprowski: rebased onto 3.1-rc1] Signed-off-by: Marek Szyprowski m.szyprowski@samsung.com --- arch/arm/include/asm/dma-mapping.h | 3 +- arch/arm/include/asm/mach/map.h | 2 + arch/arm/include/asm/memory.h | 7 + arch/arm/mm/dma-mapping.c | 313 +++++++++++++++++++----------------- arch/arm/mm/init.c | 1 + arch/arm/mm/mm.h | 2 + arch/arm/mm/mmu.c | 24 +++ 7 files changed, 202 insertions(+), 150 deletions(-)

diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index 7a21d0b..2f50659 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -199,8 +199,7 @@ int dma_mmap_coherent(struct device *, struct vm_area_struct *, extern void *dma_alloc_writecombine(struct device *, size_t, dma_addr_t *, gfp_t);

-#define dma_free_writecombine(dev,size,cpu_addr,handle) \ - dma_free_coherent(dev,size,cpu_addr,handle) +extern void dma_free_writecombine(struct device *, size_t, void *, dma_addr_t);

int dma_mmap_writecombine(struct device *, struct vm_area_struct *, void *, dma_addr_t, size_t); diff --git a/arch/arm/include/asm/mach/map.h b/arch/arm/include/asm/mach/map.h index d2fedb5..3845215 100644 --- a/arch/arm/include/asm/mach/map.h +++ b/arch/arm/include/asm/mach/map.h @@ -29,6 +29,8 @@ struct map_desc { #define MT_MEMORY_NONCACHED 11 #define MT_MEMORY_DTCM 12 #define MT_MEMORY_ITCM 13 +#define MT_DMA_COHERENT 14 +#define MT_WC_COHERENT 15

#ifdef CONFIG_MMU extern void iotable_init(struct map_desc *, int); diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h index b8de516..334b288 100644 --- a/arch/arm/include/asm/memory.h +++ b/arch/arm/include/asm/memory.h @@ -88,6 +88,13 @@ #define CONSISTENT_END (0xffe00000UL) #define CONSISTENT_BASE (CONSISTENT_END - CONSISTENT_DMA_SIZE)

+#ifndef CONSISTENT_WC_SIZE +#define CONSISTENT_WC_SIZE SZ_2M +#endif + +#define CONSISTENT_WC_END CONSISTENT_BASE +#define CONSISTENT_WC_BASE (CONSISTENT_WC_END - CONSISTENT_WC_SIZE) + #else /* CONFIG_MMU */

/* diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 0a0a1e7..b643262 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -18,12 +18,14 @@ #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/highmem.h> +#include <linux/memblock.h>

#include <asm/memory.h> #include <asm/highmem.h> #include <asm/cacheflush.h> #include <asm/tlbflush.h> #include <asm/sizes.h> +#include <asm/mach/map.h>

#include "mm.h"

@@ -117,93 +119,128 @@ static void __dma_free_buffer(struct page *page, size_t size) }

#ifdef CONFIG_MMU -/* Sanity check size */ -#if (CONSISTENT_DMA_SIZE % SZ_2M) -#error "CONSISTENT_DMA_SIZE must be multiple of 2MiB" +/* Sanity check sizes */ +#if CONSISTENT_DMA_SIZE % SECTION_SIZE +#error "CONSISTENT_DMA_SIZE must be a multiple of the section size" +#endif +#if CONSISTENT_WC_SIZE % SECTION_SIZE +#error "CONSISTENT_WC_SIZE must be a multiple of the section size" +#endif +#if ((CONSISTENT_DMA_SIZE + CONSISTENT_WC_SIZE) % SZ_2M) +#error "Sum of CONSISTENT_DMA_SIZE and CONSISTENT_WC_SIZE must be multiple of 2MiB" #endif

-#define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT) -#define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT) -#define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT) +#include "vmregion.h"

-/* - * These are the page tables (2MB each) covering uncached, DMA consistent allocations - */ -static pte_t *consistent_pte[NUM_CONSISTENT_PTES]; +struct dma_coherent_area { + struct arm_vmregion_head vm; + unsigned long pfn; + unsigned int type; + const char *name; +};

-#include "vmregion.h" +static struct dma_coherent_area coherent_wc_head = { + .vm = { + .vm_start = CONSISTENT_WC_BASE, + .vm_end = CONSISTENT_WC_END, + }, + .type = MT_WC_COHERENT, + .name = "WC ", +};

-static struct arm_vmregion_head consistent_head = { - .vm_lock = __SPIN_LOCK_UNLOCKED(&consistent_head.vm_lock), - .vm_list = LIST_HEAD_INIT(consistent_head.vm_list), - .vm_start = CONSISTENT_BASE, - .vm_end = CONSISTENT_END, +static struct dma_coherent_area coherent_dma_head = { + .vm = { + .vm_start = CONSISTENT_BASE, + .vm_end = CONSISTENT_END, + }, + .type = MT_DMA_COHERENT, + .name = "DMA coherent ", };

-#ifdef CONFIG_HUGETLB_PAGE -#error ARM Coherent DMA allocator does not (yet) support huge TLB -#endif +static struct dma_coherent_area *coherent_areas[2] __initdata = + { &coherent_wc_head, &coherent_dma_head };

-/* - * Initialise the consistent memory allocation. - */ -static int __init consistent_init(void) +static struct dma_coherent_area *coherent_map[2]; +#define coherent_wc_area coherent_map[0] +#define coherent_dma_area coherent_map[1] + +void dma_coherent_reserve(void) { - int ret = 0; - pgd_t *pgd; - pud_t *pud; - pmd_t *pmd; - pte_t *pte; - int i = 0; - u32 base = CONSISTENT_BASE; + phys_addr_t base, max_addr; + unsigned long size; + int can_share, i;

- do { - pgd = pgd_offset(&init_mm, base); + if (arch_is_coherent()) + return;

- pud = pud_alloc(&init_mm, pgd, base); - if (!pud) { - printk(KERN_ERR "%s: no pud tables\n", __func__); - ret = -ENOMEM; - break; - } +#ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE + /* ARMv6: only when DMA_MEM_BUFFERABLE is enabled */ + can_share = cpu_architecture() >= CPU_ARCH_ARMv6; +#else + /* ARMv7+: WC and DMA areas have the same properties, so can share */ + can_share = cpu_architecture() >= CPU_ARCH_ARMv7; +#endif + if (can_share) { + coherent_wc_head.name = "DMA coherent/WC "; + coherent_wc_head.vm.vm_end = coherent_dma_head.vm.vm_end; + coherent_dma_head.vm.vm_start = coherent_dma_head.vm.vm_end; + coherent_dma_area = coherent_wc_area = &coherent_wc_head; + } else { + memcpy(coherent_map, coherent_areas, sizeof(coherent_map)); + }

- pmd = pmd_alloc(&init_mm, pud, base); - if (!pmd) { - printk(KERN_ERR "%s: no pmd tables\n", __func__); - ret = -ENOMEM; - break; - } - WARN_ON(!pmd_none(*pmd)); +#ifdef CONFIG_ARM_DMA_ZONE_SIZE + max_addr = PHYS_OFFSET + ARM_DMA_ZONE_SIZE - 1; +#else + max_addr = MEMBLOCK_ALLOC_ANYWHERE; +#endif + for (i = 0; i < ARRAY_SIZE(coherent_areas); i++) { + struct dma_coherent_area *area = coherent_areas[i];

- pte = pte_alloc_kernel(pmd, base); - if (!pte) { - printk(KERN_ERR "%s: no pte tables\n", __func__); - ret = -ENOMEM; - break; - } + size = area->vm.vm_end - area->vm.vm_start; + if (!size) + continue;

- consistent_pte[i++] = pte; - base += (1 << PGDIR_SHIFT); - } while (base < CONSISTENT_END); + spin_lock_init(&area->vm.vm_lock); + INIT_LIST_HEAD(&area->vm.vm_list);

- return ret; + base = memblock_alloc_base(size, SZ_1M, max_addr); + memblock_free(base, size); + memblock_remove(base, size); + + area->pfn = __phys_to_pfn(base); + + pr_info("DMA: %luMiB %smemory allocated at 0x%08llx phys\n", + size / 1048576, area->name, (unsigned long long)base); + } }

-core_initcall(consistent_init); +void __init dma_coherent_mapping(void) +{ + struct map_desc map[ARRAY_SIZE(coherent_areas)]; + int nr;

-static void * -__dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot) + for (nr = 0; nr < ARRAY_SIZE(map); nr++) { + struct dma_coherent_area *area = coherent_areas[nr]; + + map[nr].pfn = area->pfn; + map[nr].virtual = area->vm.vm_start; + map[nr].length = area->vm.vm_end - area->vm.vm_start; + map[nr].type = area->type; + if (map[nr].length == 0) + break; + } + + iotable_init(map, nr); +} + +static void *dma_alloc_area(size_t size, unsigned long *pfn, gfp_t gfp, + struct dma_coherent_area *area) { struct arm_vmregion *c; size_t align; int bit;

- if (!consistent_pte[0]) { - printk(KERN_ERR "%s: not initialised\n", __func__); - dump_stack(); - return NULL; - } - /* * Align the virtual region allocation - maximum alignment is * a section size, minimum is a page size. This helps reduce @@ -218,45 +255,21 @@ __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot) /* * Allocate a virtual address in the consistent mapping region. */ - c = arm_vmregion_alloc(&consistent_head, align, size, + c = arm_vmregion_alloc(&area->vm, align, size, gfp & ~(__GFP_DMA | __GFP_HIGHMEM)); - if (c) { - pte_t *pte; - int idx = CONSISTENT_PTE_INDEX(c->vm_start); - u32 off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1); - - pte = consistent_pte[idx] + off; - c->vm_pages = page; - - do { - BUG_ON(!pte_none(*pte)); - - set_pte_ext(pte, mk_pte(page, prot), 0); - page++; - pte++; - off++; - if (off >= PTRS_PER_PTE) { - off = 0; - pte = consistent_pte[++idx]; - } - } while (size -= PAGE_SIZE); - - dsb(); + if (!c) + return NULL;

- return (void *)c->vm_start; - } - return NULL; + memset((void *)c->vm_start, 0, size); + *pfn = area->pfn + ((c->vm_start - area->vm.vm_start) >> PAGE_SHIFT); + return (void *)c->vm_start; }

-static void __dma_free_remap(void *cpu_addr, size_t size) +static void dma_free_area(void *cpu_addr, size_t size, struct dma_coherent_area *area) { struct arm_vmregion *c; - unsigned long addr; - pte_t *ptep; - int idx; - u32 off;

- c = arm_vmregion_find_remove(&consistent_head, (unsigned long)cpu_addr); + c = arm_vmregion_find_remove(&area->vm, (unsigned long)cpu_addr); if (!c) { printk(KERN_ERR "%s: trying to free invalid coherent area: %p\n", __func__, cpu_addr); @@ -271,61 +284,62 @@ static void __dma_free_remap(void *cpu_addr, size_t size) size = c->vm_end - c->vm_start; }

- idx = CONSISTENT_PTE_INDEX(c->vm_start); - off = CONSISTENT_OFFSET(c->vm_start) & (PTRS_PER_PTE-1); - ptep = consistent_pte[idx] + off; - addr = c->vm_start; - do { - pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep); - - ptep++; - addr += PAGE_SIZE; - off++; - if (off >= PTRS_PER_PTE) { - off = 0; - ptep = consistent_pte[++idx]; - } + arm_vmregion_free(&area->vm, c); +}

- if (pte_none(pte) || !pte_present(pte)) - printk(KERN_CRIT "%s: bad page in kernel page table\n", - __func__); - } while (size -= PAGE_SIZE); +#define nommu() (0)

- flush_tlb_kernel_range(c->vm_start, c->vm_end); +#else /* !CONFIG_MMU */

- arm_vmregion_free(&consistent_head, c); -} +#define dma_alloc_area(size, pfn, gfp, area) ({ *(pfn) = 0; NULL }) +#define dma_free_area(addr, size, area) do { } while (0)

-#else /* !CONFIG_MMU */ +#define nommu() (1) +#define coherent_wc_area NULL +#define coherent_dma_area NULL

-#define __dma_alloc_remap(page, size, gfp, prot) page_address(page) -#define __dma_free_remap(addr, size) do { } while (0) +void dma_coherent_reserve(void) +{ +}

#endif /* CONFIG_MMU */

static void * __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, - pgprot_t prot) + struct dma_coherent_area *area) { - struct page *page; - void *addr; + unsigned long pfn; + void *ret;

*handle = ~0; size = PAGE_ALIGN(size);

- page = __dma_alloc_buffer(dev, size, gfp); - if (!page) - return NULL; + if (arch_is_coherent() || nommu()) { + struct page *page = __dma_alloc_buffer(dev, size, gfp); + if (!page) + return NULL; + pfn = page_to_pfn(page); + ret = page_address(page); + } else { + ret = dma_alloc_area(size, &pfn, gfp, area); + }

- if (!arch_is_coherent()) - addr = __dma_alloc_remap(page, size, gfp, prot); - else - addr = page_address(page); + if (ret) + *handle = pfn_to_dma(dev, pfn);

- if (addr) - *handle = pfn_to_dma(dev, page_to_pfn(page)); + return ret; +} + +static void __dma_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle, struct dma_coherent_area *area) +{ + size = PAGE_ALIGN(size);

- return addr; + if (arch_is_coherent() || nommu()) { + __dma_free_buffer(pfn_to_page(dma_to_pfn(dev, handle)), size); + } else { + dma_free_area(cpu_addr, size, area); + } }

/* @@ -340,8 +354,7 @@ dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gf if (dma_alloc_from_coherent(dev, size, handle, &memory)) return memory;

- return __dma_alloc(dev, size, handle, gfp, - pgprot_dmacoherent(pgprot_kernel)); + return __dma_alloc(dev, size, handle, gfp, coherent_dma_area); } EXPORT_SYMBOL(dma_alloc_coherent);

@@ -352,13 +365,13 @@ EXPORT_SYMBOL(dma_alloc_coherent); void * dma_alloc_writecombine(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp) { - return __dma_alloc(dev, size, handle, gfp, - pgprot_writecombine(pgprot_kernel)); + return __dma_alloc(dev, size, handle, gfp, coherent_wc_area); } EXPORT_SYMBOL(dma_alloc_writecombine);

static int dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size) + void *cpu_addr, dma_addr_t dma_addr, size_t size, + struct dma_coherent_area *area) { int ret = -ENXIO; #ifdef CONFIG_MMU @@ -367,7 +380,7 @@ static int dma_mmap(struct device *dev, struct vm_area_struct *vma,

user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;

- c = arm_vmregion_find(&consistent_head, (unsigned long)cpu_addr); + c = arm_vmregion_find(&area->vm, (unsigned long)cpu_addr); if (c) { unsigned long off = vma->vm_pgoff;

@@ -390,7 +403,7 @@ int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size) { vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot); - return dma_mmap(dev, vma, cpu_addr, dma_addr, size); + return dma_mmap(dev, vma, cpu_addr, dma_addr, size, coherent_dma_area); } EXPORT_SYMBOL(dma_mmap_coherent);

@@ -398,7 +411,7 @@ int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size) { vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); - return dma_mmap(dev, vma, cpu_addr, dma_addr, size); + return dma_mmap(dev, vma, cpu_addr, dma_addr, size, coherent_wc_area); } EXPORT_SYMBOL(dma_mmap_writecombine);

@@ -413,14 +426,18 @@ void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr if (dma_release_from_coherent(dev, get_order(size), cpu_addr)) return;

- size = PAGE_ALIGN(size); + __dma_free(dev, size, cpu_addr, handle, coherent_dma_area); +} +EXPORT_SYMBOL(dma_free_coherent);

- if (!arch_is_coherent()) - __dma_free_remap(cpu_addr, size); +void dma_free_writecombine(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle) +{ + WARN_ON(irqs_disabled());

- __dma_free_buffer(pfn_to_page(dma_to_pfn(dev, handle)), size); + __dma_free(dev, size, cpu_addr, handle, coherent_wc_area); } -EXPORT_SYMBOL(dma_free_coherent); +EXPORT_SYMBOL(dma_free_writecombine);

/* * Make an area consistent for devices. diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 2fee782..77076a6 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -365,6 +365,7 @@ void __init arm_memblock_init(struct meminfo *mi, struct machine_desc *mdesc)

arm_mm_memblock_reserve(); arm_dt_memblock_reserve(); + dma_coherent_reserve();

/* reserve any platform specific memblock areas */ if (mdesc->reserve) diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h index 0105667..3abaa2c 100644 --- a/arch/arm/mm/mm.h +++ b/arch/arm/mm/mm.h @@ -31,3 +31,5 @@ extern u32 arm_dma_limit;

void __init bootmem_init(void); void arm_mm_memblock_reserve(void); +void dma_coherent_reserve(void); +void dma_coherent_mapping(void); diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 594d677..027f118 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -273,6 +273,16 @@ static struct mem_type mem_types[] = { .prot_l1 = PMD_TYPE_TABLE, .domain = DOMAIN_KERNEL, }, + [MT_DMA_COHERENT] = { + .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | + PMD_SECT_S, + .domain = DOMAIN_IO, + }, + [MT_WC_COHERENT] = { + .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | + PMD_SECT_S, + .domain = DOMAIN_IO, + }, };

const struct mem_type *get_mem_type(unsigned int type) @@ -353,6 +363,7 @@ static void __init build_mem_type_table(void) mem_types[MT_DEVICE_NONSHARED].prot_sect |= PMD_SECT_XN; mem_types[MT_DEVICE_CACHED].prot_sect |= PMD_SECT_XN; mem_types[MT_DEVICE_WC].prot_sect |= PMD_SECT_XN; + mem_types[MT_DMA_COHERENT].prot_sect |= PMD_SECT_XN; } if (cpu_arch >= CPU_ARCH_ARMv7 && (cr & CR_TRE)) { /* @@ -457,13 +468,24 @@ static void __init build_mem_type_table(void) /* Non-cacheable Normal is XCB = 001 */ mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_BUFFERED; + mem_types[MT_WC_COHERENT].prot_sect |= + PMD_SECT_BUFFERED; + mem_types[MT_DMA_COHERENT].prot_sect |= + PMD_SECT_BUFFERED; } else { /* For both ARMv6 and non-TEX-remapping ARMv7 */ mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_TEX(1); + mem_types[MT_WC_COHERENT].prot_sect |= + PMD_SECT_TEX(1); +#ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE + mem_types[MT_DMA_COHERENT].prot_sect |= + PMD_SECT_TEX(1); +#endif } } else { mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE; + mem_types[MT_WC_COHERENT].prot_sect |= PMD_SECT_BUFFERED; }

for (i = 0; i < 16; i++) { @@ -976,6 +998,8 @@ static void __init devicemaps_init(struct machine_desc *mdesc) create_mapping(&map); }

+ dma_coherent_mapping(); + /* * Ask the machine support to map in the statically mapped devices. */

This patch adds support for CMA to dma-mapping subsystem for ARM architecture. By default a global CMA area is used, but specific devices are allowed to have their private memory areas if required (they can be created with dma_declare_contiguous() function during board initialization).

Contiguous memory areas reserved for DMA are remapped with 2-level page tables on boot. Once a buffer is requested, a low memory kernel mapping is updated to to match requested memory access type.

GFP_ATOMIC allocations are performed from special memory area which is exclusive from system memory to avoid remapping page attributes what might be not allowed in atomic context on some systems. If CMA has been disabled then all DMA allocations are performed from this area.

Signed-off-by: Marek Szyprowski m.szyprowski@samsung.com Signed-off-by: Kyungmin Park kyungmin.park@samsung.com --- arch/arm/Kconfig | 1 + arch/arm/include/asm/device.h | 3 + arch/arm/include/asm/dma-contiguous.h | 33 +++++++ arch/arm/include/asm/mach/map.h | 5 +- arch/arm/mm/dma-mapping.c | 169 +++++++++++++++++++++++++-------- arch/arm/mm/init.c | 5 +- arch/arm/mm/mm.h | 3 + arch/arm/mm/mmu.c | 29 ++++-- 8 files changed, 196 insertions(+), 52 deletions(-) create mode 100644 arch/arm/include/asm/dma-contiguous.h

diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 2c71a8f..20fa729 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -3,6 +3,7 @@ config ARM default y select HAVE_AOUT select HAVE_DMA_API_DEBUG + select HAVE_DMA_CONTIGUOUS select HAVE_IDE select HAVE_MEMBLOCK select RTC_LIB diff --git a/arch/arm/include/asm/device.h b/arch/arm/include/asm/device.h index 9f390ce..942913e 100644 --- a/arch/arm/include/asm/device.h +++ b/arch/arm/include/asm/device.h @@ -10,6 +10,9 @@ struct dev_archdata { #ifdef CONFIG_DMABOUNCE struct dmabounce_device_info *dmabounce; #endif +#ifdef CONFIG_CMA + struct cma *cma_area; +#endif };

struct pdev_archdata { diff --git a/arch/arm/include/asm/dma-contiguous.h b/arch/arm/include/asm/dma-contiguous.h new file mode 100644 index 0000000..99bf7c8 --- /dev/null +++ b/arch/arm/include/asm/dma-contiguous.h @@ -0,0 +1,33 @@ +#ifndef ASMARM_DMA_CONTIGUOUS_H +#define ASMARM_DMA_CONTIGUOUS_H + +#ifdef __KERNEL__ + +#include <linux/device.h> +#include <linux/dma-contiguous.h> + +#ifdef CONFIG_CMA + +#define MAX_CMA_AREAS (8) + +void dma_contiguous_early_fixup(phys_addr_t base, unsigned long size); + +static inline struct cma *get_dev_cma_area(struct device *dev) +{ + if (dev->archdata.cma_area) + return dev->archdata.cma_area; + return dma_contiguous_default_area; +} + +static inline void set_dev_cma_area(struct device *dev, struct cma *cma) +{ + dev->archdata.cma_area = cma; +} + +#else + +#define MAX_CMA_AREAS (0) + +#endif +#endif +#endif diff --git a/arch/arm/include/asm/mach/map.h b/arch/arm/include/asm/mach/map.h index 3845215..5982a83 100644 --- a/arch/arm/include/asm/mach/map.h +++ b/arch/arm/include/asm/mach/map.h @@ -29,8 +29,9 @@ struct map_desc { #define MT_MEMORY_NONCACHED 11 #define MT_MEMORY_DTCM 12 #define MT_MEMORY_ITCM 13 -#define MT_DMA_COHERENT 14 -#define MT_WC_COHERENT 15 +#define MT_MEMORY_DMA_READY 14 +#define MT_DMA_COHERENT 15 +#define MT_WC_COHERENT 16

#ifdef CONFIG_MMU extern void iotable_init(struct map_desc *, int); diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index b643262..63175d1 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -17,6 +17,7 @@ #include <linux/init.h> #include <linux/device.h> #include <linux/dma-mapping.h> +#include <linux/dma-contiguous.h> #include <linux/highmem.h> #include <linux/memblock.h>

@@ -26,6 +27,7 @@ #include <asm/tlbflush.h> #include <asm/sizes.h> #include <asm/mach/map.h> +#include <asm/dma-contiguous.h>

#include "mm.h"

@@ -56,6 +58,24 @@ static u64 get_coherent_dma_mask(struct device *dev) return mask; }

+static struct page *__dma_alloc_system_pages(size_t count, gfp_t gfp, + unsigned long order) +{ + struct page *page, *p, *e; + + page = alloc_pages(gfp, order); + if (!page) + return NULL; + + /* + * Now split the huge page and free the excess pages + */ + split_page(page, order); + for (p = page + count, e = page + (1 << order); p < e; p++) + __free_page(p); + return page; +} + /* * Allocate a DMA buffer for 'dev' of size 'size' using the * specified gfp mask. Note that 'size' must be page aligned. @@ -63,7 +83,8 @@ static u64 get_coherent_dma_mask(struct device *dev) static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp) { unsigned long order = get_order(size); - struct page *page, *p, *e; + size_t count = size >> PAGE_SHIFT; + struct page *page; void *ptr; u64 mask = get_coherent_dma_mask(dev);

@@ -82,16 +103,16 @@ static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gf if (mask < 0xffffffffULL) gfp |= GFP_DMA;

- page = alloc_pages(gfp, order); - if (!page) - return NULL; - /* - * Now split the huge page and free the excess pages + * Allocate contiguous memory */ - split_page(page, order); - for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++) - __free_page(p); + if (cma_available()) + page = dma_alloc_from_contiguous(dev, count, order); + else + page = __dma_alloc_system_pages(count, gfp, order); + + if (!page) + return NULL;

/* * Ensure that the allocated pages are zeroed, and that any data @@ -108,7 +129,7 @@ static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gf /* * Free a DMA buffer. 'size' must be page aligned. */ -static void __dma_free_buffer(struct page *page, size_t size) +static void __dma_free_system_buffer(struct page *page, size_t size) { struct page *e = page + (size >> PAGE_SHIFT);

@@ -136,6 +157,7 @@ struct dma_coherent_area { struct arm_vmregion_head vm; unsigned long pfn; unsigned int type; + pgprot_t prot; const char *name; };

@@ -232,6 +254,55 @@ void __init dma_coherent_mapping(void) }

iotable_init(map, nr); + coherent_dma_area->prot = pgprot_dmacoherent(pgprot_kernel); + coherent_wc_area->prot = pgprot_writecombine(pgprot_kernel); +} + +struct dma_contiguous_early_reserve { + phys_addr_t base; + unsigned long size; +}; + +static struct dma_contiguous_early_reserve +dma_mmu_remap[MAX_CMA_AREAS] __initdata; + +static int dma_mmu_remap_num __initdata; + +void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) +{ + dma_mmu_remap[dma_mmu_remap_num].base = base; + dma_mmu_remap[dma_mmu_remap_num].size = size; + dma_mmu_remap_num++; +} + +void __init dma_contiguous_remap(void) +{ + int i; + for (i = 0; i < dma_mmu_remap_num; i++) { + phys_addr_t start = dma_mmu_remap[i].base; + phys_addr_t end = start + dma_mmu_remap[i].size; + struct map_desc map; + unsigned long addr; + + if (end > arm_lowmem_limit) + end = arm_lowmem_limit; + if (start >= end) + return; + + map.pfn = __phys_to_pfn(start); + map.virtual = __phys_to_virt(start); + map.length = end - start; + map.type = MT_MEMORY_DMA_READY; + + /* + * Clear previous low-memory mapping + */ + for (addr = __phys_to_virt(start); addr < __phys_to_virt(end); + addr += PGDIR_SIZE) + pmd_clear(pmd_off_k(addr)); + + iotable_init(&map, 1); + } }

static void *dma_alloc_area(size_t size, unsigned long *pfn, gfp_t gfp, @@ -289,10 +360,34 @@ static void dma_free_area(void *cpu_addr, size_t size, struct dma_coherent_area

#define nommu() (0)

+static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr, + void *data) +{ + struct page *page = virt_to_page(addr); + pgprot_t prot = *(pgprot_t *)data; + + set_pte_ext(pte, mk_pte(page, prot), 0); + return 0; +} + +static void dma_remap_area(struct page *page, size_t size, pgprot_t prot) +{ + unsigned long start = (unsigned long) page_address(page); + unsigned end = start + size; + + if (arch_is_coherent()) + return; + + apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot); + dsb(); + flush_tlb_kernel_range(start, end); +} + #else /* !CONFIG_MMU */

#define dma_alloc_area(size, pfn, gfp, area) ({ *(pfn) = 0; NULL }) #define dma_free_area(addr, size, area) do { } while (0) +#define dma_remap_area(page, size, prot) do { } while (0)

#define nommu() (1) #define coherent_wc_area NULL @@ -308,19 +403,27 @@ static void * __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, struct dma_coherent_area *area) { - unsigned long pfn; - void *ret; + unsigned long pfn = 0; + void *ret = NULL;

*handle = ~0; size = PAGE_ALIGN(size);

- if (arch_is_coherent() || nommu()) { + if (arch_is_coherent() || nommu() || + (cma_available() && !(gfp & GFP_ATOMIC))) { + /* + * Allocate from system or CMA pages + */ struct page *page = __dma_alloc_buffer(dev, size, gfp); if (!page) return NULL; + dma_remap_area(page, size, area->prot); pfn = page_to_pfn(page); ret = page_address(page); } else { + /* + * Allocate from reserved DMA coherent/wc area + */ ret = dma_alloc_area(size, &pfn, gfp, area); }

@@ -333,12 +436,19 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, static void __dma_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle, struct dma_coherent_area *area) { + struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); size = PAGE_ALIGN(size);

if (arch_is_coherent() || nommu()) { - __dma_free_buffer(pfn_to_page(dma_to_pfn(dev, handle)), size); - } else { + WARN_ON(irqs_disabled()); + __dma_free_system_buffer(page, size); + } else if ((unsigned long)cpu_addr >= area->vm.vm_start && + (unsigned long)cpu_addr < area->vm.vm_end) { dma_free_area(cpu_addr, size, area); + } else { + WARN_ON(irqs_disabled()); + dma_remap_area(page, size, pgprot_kernel); + dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); } }

@@ -375,27 +485,12 @@ static int dma_mmap(struct device *dev, struct vm_area_struct *vma, { int ret = -ENXIO; #ifdef CONFIG_MMU - unsigned long user_size, kern_size; - struct arm_vmregion *c; - - user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; - - c = arm_vmregion_find(&area->vm, (unsigned long)cpu_addr); - if (c) { - unsigned long off = vma->vm_pgoff; - - kern_size = (c->vm_end - c->vm_start) >> PAGE_SHIFT; - - if (off < kern_size && - user_size <= (kern_size - off)) { - ret = remap_pfn_range(vma, vma->vm_start, - page_to_pfn(c->vm_pages) + off, - user_size << PAGE_SHIFT, - vma->vm_page_prot); - } - } + unsigned long pfn = dma_to_pfn(dev, dma_addr); + ret = remap_pfn_range(vma, vma->vm_start, + pfn + vma->vm_pgoff, + vma->vm_end - vma->vm_start, + vma->vm_page_prot); #endif /* CONFIG_MMU */ - return ret; }

@@ -421,8 +516,6 @@ EXPORT_SYMBOL(dma_mmap_writecombine); */ void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle) { - WARN_ON(irqs_disabled()); - if (dma_release_from_coherent(dev, get_order(size), cpu_addr)) return;

@@ -433,8 +526,6 @@ EXPORT_SYMBOL(dma_free_coherent); void dma_free_writecombine(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle) { - WARN_ON(irqs_disabled()); - __dma_free(dev, size, cpu_addr, handle, coherent_wc_area); } EXPORT_SYMBOL(dma_free_writecombine); diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 77076a6..0f2dbb8 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -20,6 +20,7 @@ #include <linux/gfp.h> #include <linux/memblock.h> #include <linux/sort.h> +#include <linux/dma-contiguous.h>

#include <asm/mach-types.h> #include <asm/prom.h> @@ -365,12 +366,14 @@ void __init arm_memblock_init(struct meminfo *mi, struct machine_desc *mdesc)

arm_mm_memblock_reserve(); arm_dt_memblock_reserve(); - dma_coherent_reserve();

/* reserve any platform specific memblock areas */ if (mdesc->reserve) mdesc->reserve();

+ dma_coherent_reserve(); + dma_contiguous_reserve(); + memblock_analyze(); memblock_dump_all(); } diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h index 3abaa2c..46101be 100644 --- a/arch/arm/mm/mm.h +++ b/arch/arm/mm/mm.h @@ -29,7 +29,10 @@ extern u32 arm_dma_limit; #define arm_dma_limit ((u32)~0) #endif

+extern phys_addr_t arm_lowmem_limit; + void __init bootmem_init(void); void arm_mm_memblock_reserve(void); void dma_coherent_reserve(void); void dma_coherent_mapping(void); +void dma_contiguous_remap(void); diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 027f118..9dc18d4 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -273,6 +273,11 @@ static struct mem_type mem_types[] = { .prot_l1 = PMD_TYPE_TABLE, .domain = DOMAIN_KERNEL, }, + [MT_MEMORY_DMA_READY] = { + .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY, + .prot_l1 = PMD_TYPE_TABLE, + .domain = DOMAIN_KERNEL, + }, [MT_DMA_COHERENT] = { .prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_S, @@ -425,6 +430,7 @@ static void __init build_mem_type_table(void) if (arch_is_coherent() && cpu_is_xsc3()) { mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED; + mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED; mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S; mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED; } @@ -454,6 +460,7 @@ static void __init build_mem_type_table(void) mem_types[MT_DEVICE_CACHED].prot_pte |= L_PTE_SHARED; mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; mem_types[MT_MEMORY].prot_pte |= L_PTE_SHARED; + mem_types[MT_MEMORY_DMA_READY].prot_pte |= L_PTE_SHARED; mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_S; mem_types[MT_MEMORY_NONCACHED].prot_pte |= L_PTE_SHARED; } @@ -504,6 +511,7 @@ static void __init build_mem_type_table(void) mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask; mem_types[MT_MEMORY].prot_sect |= ecc_mask | cp->pmd; mem_types[MT_MEMORY].prot_pte |= kern_pgprot; + mem_types[MT_MEMORY_DMA_READY].prot_pte |= kern_pgprot; mem_types[MT_MEMORY_NONCACHED].prot_sect |= ecc_mask; mem_types[MT_ROM].prot_sect |= cp->pmd;

@@ -583,7 +591,7 @@ static void __init alloc_init_section(pud_t *pud, unsigned long addr, * L1 entries, whereas PGDs refer to a group of L1 entries making * up one logical pointer to an L2 table. */ - if (((addr | end | phys) & ~SECTION_MASK) == 0) { + if (type->prot_sect && ((addr | end | phys) & ~SECTION_MASK) == 0) { pmd_t *p = pmd;

if (addr & SECTION_SIZE) @@ -779,7 +787,7 @@ static int __init early_vmalloc(char *arg) } early_param("vmalloc", early_vmalloc);

-static phys_addr_t lowmem_limit __initdata = 0; +phys_addr_t arm_lowmem_limit __initdata = 0;

void __init sanity_check_meminfo(void) { @@ -848,8 +856,8 @@ void __init sanity_check_meminfo(void) bank->size = newsize; } #endif - if (!bank->highmem && bank->start + bank->size > lowmem_limit) - lowmem_limit = bank->start + bank->size; + if (!bank->highmem && bank->start + bank->size > arm_lowmem_limit) + arm_lowmem_limit = bank->start + bank->size;

j++; } @@ -874,7 +882,7 @@ void __init sanity_check_meminfo(void) } #endif meminfo.nr_banks = j; - memblock_set_current_limit(lowmem_limit); + memblock_set_current_limit(arm_lowmem_limit); }

static inline void prepare_page_table(void) @@ -899,8 +907,8 @@ static inline void prepare_page_table(void) * Find the end of the first block of lowmem. */ end = memblock.memory.regions[0].base + memblock.memory.regions[0].size; - if (end >= lowmem_limit) - end = lowmem_limit; + if (end >= arm_lowmem_limit) + end = arm_lowmem_limit;

/* * Clear out all the kernel space mappings, except for the first @@ -1034,8 +1042,8 @@ static void __init map_lowmem(void) phys_addr_t end = start + reg->size; struct map_desc map;

- if (end > lowmem_limit) - end = lowmem_limit; + if (end > arm_lowmem_limit) + end = arm_lowmem_limit; if (start >= end) break;

@@ -1056,11 +1064,12 @@ void __init paging_init(struct machine_desc *mdesc) { void *zero_page;

- memblock_set_current_limit(lowmem_limit); + memblock_set_current_limit(arm_lowmem_limit);

build_mem_type_table(); prepare_page_table(); map_lowmem(); + dma_contiguous_remap(); devicemaps_init(mdesc); kmap_init();

This patch is an example how device private CMA area can be activated. It creates one CMA region and assigns it to the first s5p-fimc device on Samsung Goni S5PC110 board.

Signed-off-by: Marek Szyprowski m.szyprowski@samsung.com Signed-off-by: Kyungmin Park kyungmin.park@samsung.com --- arch/arm/mach-s5pv210/Kconfig | 1 + arch/arm/mach-s5pv210/mach-goni.c | 4 ++++ 2 files changed, 5 insertions(+), 0 deletions(-)

diff --git a/arch/arm/mach-s5pv210/Kconfig b/arch/arm/mach-s5pv210/Kconfig index 69dd87c..697a5be 100644 --- a/arch/arm/mach-s5pv210/Kconfig +++ b/arch/arm/mach-s5pv210/Kconfig @@ -64,6 +64,7 @@ menu "S5PC110 Machines" config MACH_AQUILA bool "Aquila" select CPU_S5PV210 + select CMA select S3C_DEV_FB select S5P_DEV_FIMC0 select S5P_DEV_FIMC1 diff --git a/arch/arm/mach-s5pv210/mach-goni.c b/arch/arm/mach-s5pv210/mach-goni.c index 85c2d51..665c4ae 100644 --- a/arch/arm/mach-s5pv210/mach-goni.c +++ b/arch/arm/mach-s5pv210/mach-goni.c @@ -26,6 +26,7 @@ #include <linux/input.h> #include <linux/gpio.h> #include <linux/interrupt.h> +#include <linux/dma-contiguous.h>

#include <asm/mach/arch.h> #include <asm/mach/map.h> @@ -848,6 +849,9 @@ static void __init goni_map_io(void) static void __init goni_reserve(void) { s5p_mfc_reserve_mem(0x43000000, 8 << 20, 0x51000000, 8 << 20); + + /* Create private 16MiB contiguous memory area for s5p-fimc.0 device */ + dma_declare_contiguous(&s5p_device_fimc0.dev, 16*SZ_1M, 0); }

static void __init goni_machine_init(void)