The system_cc_shared heap currently allocates its backing pages using the normal system-heap order policy and changes each resulting compound page to shared state. That is unsafe when an architecture requires state changes in units larger than PAGE_SIZE: an order-0 tail is neither sufficiently aligned nor large enough to transition independently.
Use the common CoCo shared-page allocator for every backing allocation of the shared heap. Preserve the existing preferred-order search and its GFP policy: each candidate order is passed to the common allocator as a byte request, and that allocator rounds it up when the architecture shared granule is larger.
Add __GFP_COMP for shared allocations because an order-0 candidate can be rounded into a high-order allocation. The system heap uses compound_order() and page_size() for accounting and release, so the returned allocation must retain compound-page semantics.
Calculate a rounded internal backing length but retain the original length in dma_buf::size. The preceding scatterlist-length change ensures that the rounded tail is not included in DMA mappings or other operations. This permits a 4 KiB request on a 64 KiB shared-granule system without exposing the extra 60 KiB to an importer.
Cc: Sumit Semwal sumit.semwal@linaro.org Cc: Benjamin Gaignard benjamin.gaignard@collabora.com Cc: Brian Starkey Brian.Starkey@arm.com Cc: John Stultz jstultz@google.com Cc: "T.J. Mercier" tjmercier@google.com Cc: "Christian König" christian.koenig@amd.com Signed-off-by: Aneesh Kumar K.V (Arm) aneesh.kumar@kernel.org --- drivers/dma-buf/heaps/system_heap.c | 121 ++++++++++++---------------- 1 file changed, 53 insertions(+), 68 deletions(-)
diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c index b5b8cdf65f23..5970b76416ae 100644 --- a/drivers/dma-buf/heaps/system_heap.c +++ b/drivers/dma-buf/heaps/system_heap.c @@ -11,14 +11,13 @@ */
#include <linux/cc_platform.h> +#include <linux/cc_shared.h> #include <linux/dma-buf.h> #include <linux/dma-mapping.h> #include <linux/dma-heap.h> #include <linux/err.h> #include <linux/highmem.h> -#include <linux/mem_encrypt.h> #include <linux/mm.h> -#include <linux/set_memory.h> #include <linux/module.h> #include <linux/pgtable.h> #include <linux/scatterlist.h> @@ -65,34 +64,6 @@ static gfp_t order_flags[] = {HIGH_ORDER_GFP, HIGH_ORDER_GFP, LOW_ORDER_GFP}; static const unsigned int orders[] = {8, 4, 0}; #define NUM_ORDERS ARRAY_SIZE(orders)
-static int system_heap_set_page_decrypted(struct page *page) -{ - unsigned long addr = (unsigned long)page_address(page); - unsigned int nr_pages = 1 << compound_order(page); - int ret; - - ret = set_memory_decrypted(addr, nr_pages); - if (ret) - pr_warn_ratelimited("dma-buf system heap: failed to decrypt page at %p\n", - page_address(page)); - - return ret; -} - -static int system_heap_set_page_encrypted(struct page *page) -{ - unsigned long addr = (unsigned long)page_address(page); - unsigned int nr_pages = 1 << compound_order(page); - int ret; - - ret = set_memory_encrypted(addr, nr_pages); - if (ret) - pr_warn_ratelimited("dma-buf system heap: failed to re-encrypt page at %p, leaking memory\n", - page_address(page)); - - return ret; -} - static int dup_sg_table(struct sg_table *from, struct sg_table *to) { struct scatterlist *sg, *new_sg; @@ -337,6 +308,20 @@ static void system_heap_vunmap(struct dma_buf *dmabuf, struct iosys_map *map) iosys_map_clear(map); }
+static void system_heap_free_page(struct page *page, bool cc_shared) +{ + struct cc_shared_pages mem; + + if (!cc_shared) { + __free_pages(page, compound_order(page)); + return; + } + + mem.page = page; + mem.shared_size = page_size(page); + free_cc_shared_pages(&mem); +} + static void system_heap_dma_buf_release(struct dma_buf *dmabuf) { struct system_heap_buffer *buffer = dmabuf->priv; @@ -345,19 +330,8 @@ static void system_heap_dma_buf_release(struct dma_buf *dmabuf) int i;
table = &buffer->sg_table; - for_each_sgtable_sg(table, sg, i) { - struct page *page = sg_page(sg); - - /* - * Intentionally leak pages that cannot be re-encrypted - * to prevent shared memory from being reused. - */ - if (cc_shared_buffer(buffer) && - system_heap_set_page_encrypted(page)) - continue; - - __free_pages(page, compound_order(page)); - } + for_each_sgtable_sg(table, sg, i) + system_heap_free_page(sg_page(sg), cc_shared_buffer(buffer)); sg_free_table(table); kfree(buffer); } @@ -375,22 +349,39 @@ static const struct dma_buf_ops system_heap_buf_ops = { .release = system_heap_dma_buf_release, };
+static struct page *system_heap_alloc_order(unsigned int order, + gfp_t flags, bool cc_shared) +{ + struct cc_shared_pages mem; + + if (!cc_shared) + return alloc_pages(flags, order); + + /* The shared granule can raise the actual allocation order. */ + flags |= __GFP_COMP; + if (alloc_cc_shared_pages(flags, PAGE_SIZE << order, &mem)) + return NULL; + + return mem.page; +} + static struct page *alloc_largest_available(unsigned long size, - unsigned int max_order) + unsigned int max_order, bool cc_shared) { struct page *page; - int i; gfp_t flags; + int i;
for (i = 0; i < NUM_ORDERS; i++) { if (size < (PAGE_SIZE << orders[i])) continue; if (max_order < orders[i]) continue; + flags = order_flags[i]; if (mem_accounting) flags |= __GFP_ACCOUNT; - page = alloc_pages(flags, orders[i]); + page = system_heap_alloc_order(orders[i], flags, cc_shared); if (!page) continue; return page; @@ -405,6 +396,7 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, { struct system_heap_buffer *buffer; DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + struct cc_shared_layout layout; unsigned long size_remaining = len; unsigned long sg_remaining = len; unsigned int max_order = orders[0]; @@ -417,6 +409,16 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, struct page *page, *tmp_page; int i, ret = -ENOMEM;
+ if (cc_shared) { + int err; + + err = cc_shared_calc_layout(len, &layout); + if (err) + return ERR_PTR(err); + + size_remaining = layout.shared_size; + } + buffer = kzalloc_obj(*buffer); if (!buffer) return ERR_PTR(-ENOMEM); @@ -439,7 +441,8 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, goto free_buffer; }
- page = alloc_largest_available(size_remaining, max_order); + page = alloc_largest_available(size_remaining, max_order, + cc_shared); if (!page) goto free_buffer;
@@ -464,14 +467,6 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, list_del(&page->lru); }
- if (cc_shared_buffer(buffer)) { - for_each_sgtable_sg(table, sg, i) { - ret = system_heap_set_page_decrypted(sg_page(sg)); - if (ret) - goto free_pages; - } - } - /* create the dmabuf */ exp_info.exp_name = dma_heap_get_name(heap); exp_info.ops = &system_heap_buf_ops; @@ -486,22 +481,12 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, return dmabuf;
free_pages: - for_each_sgtable_sg(table, sg, i) { - struct page *p = sg_page(sg); - - /* - * Intentionally leak pages that cannot be re-encrypted - * to prevent shared memory from being reused. - */ - if (cc_shared_buffer(buffer) && - system_heap_set_page_encrypted(p)) - continue; - __free_pages(p, compound_order(p)); - } + for_each_sgtable_sg(table, sg, i) + system_heap_free_page(sg_page(sg), cc_shared); sg_free_table(table); free_buffer: list_for_each_entry_safe(page, tmp_page, &pages, lru) - __free_pages(page, compound_order(page)); + system_heap_free_page(page, cc_shared); kfree(buffer);
return ERR_PTR(ret);