Use the common CoCo shared-memory geometry helpers for allocations backed directly by CMA or the page allocator. Round the backing allocation to a whole shared granule and pass the resolved shared-allocation attribute through the DMA contiguous allocator so it can enforce the mandatory alignment. Transition the complete range through the common shared/private helpers.
Recompute the layout before freeing ordinary direct allocations so the transition back to private memory and dma_free_contiguous() cover exactly the range acquired by the allocation path. If restoring private state fails, retain the existing fail-safe behavior and leak the pages rather than returning potentially shared memory to the allocator.
This also applies the same rules to dma_direct_alloc_pages(), covering callers which require a struct page result rather than a CPU virtual address.
Cc: Marek Szyprowski m.szyprowski@samsung.com Cc: Robin Murphy robin.murphy@arm.com Signed-off-by: Aneesh Kumar K.V (Arm) aneesh.kumar@kernel.org --- kernel/dma/direct.c | 51 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 12 deletions(-)
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 7e71f96b6ffd..675ed698a3ad 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -11,10 +11,10 @@ #include <linux/scatterlist.h> #include <linux/pfn.h> #include <linux/vmalloc.h> -#include <linux/set_memory.h> #include <linux/slab.h> #include <linux/pci-p2pdma.h> #include <linux/cc_platform.h> +#include <linux/cc_shared.h>
#include "direct.h"
@@ -85,7 +85,7 @@ static int dma_set_decrypted(struct device *dev, void *vaddr, size_t size) { int ret;
- ret = set_memory_decrypted((unsigned long)vaddr, PFN_UP(size)); + ret = cc_make_shared(vaddr, size); if (ret) pr_warn_ratelimited("leaking DMA memory that can't be decrypted\n"); return ret; @@ -95,7 +95,7 @@ static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size) { int ret;
- ret = set_memory_encrypted((unsigned long)vaddr, PFN_UP(size)); + ret = cc_make_private(vaddr, size); if (ret) pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n"); return ret; @@ -115,7 +115,7 @@ static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size, }
static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, - gfp_t gfp, bool allow_highmem) + gfp_t gfp, bool allow_highmem, unsigned long attrs) { int node = dev_to_node(dev); struct page *page; @@ -125,7 +125,7 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, WARN_ON_ONCE(!PAGE_ALIGNED(size));
gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit); - page = dma_alloc_contiguous(dev, size, gfp, 0); + page = dma_alloc_contiguous(dev, size, gfp, attrs); if (page) { if (dma_coherent_ok(dev, page_to_phys(page), size) && (allow_highmem || !PageHighMem(page))) @@ -202,7 +202,7 @@ static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size, { struct page *page;
- page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true); + page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true, 0); if (!page) return NULL;
@@ -223,6 +223,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, bool remap = false, set_uncached = false; bool mark_mem_decrypt = false; bool allow_highmem = true; + struct cc_shared_layout layout; struct page *page; void *cpu_addr;
@@ -302,8 +303,15 @@ void *dma_direct_alloc(struct device *dev, size_t size, return NULL; }
+ if (mark_mem_decrypt) { + if (cc_shared_calc_layout(size, &layout)) + return NULL; + size = layout.shared_size; + } + /* we always manually zero the memory once we are done */ - page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, allow_highmem); + page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, + allow_highmem, attrs); if (!page) return NULL;
@@ -322,7 +330,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, void *lm_addr;
lm_addr = page_address(page); - if (set_memory_decrypted((unsigned long)lm_addr, PFN_UP(size))) + if (dma_set_decrypted(dev, lm_addr, size)) goto out_leak_pages; }
@@ -381,6 +389,7 @@ void dma_direct_free(struct device *dev, size_t size, phys_addr_t phys; bool mark_mem_encrypted = false; struct io_tlb_pool *swiotlb_pool; + struct cc_shared_layout layout; unsigned int page_order = get_order(size);
/* @@ -424,6 +433,12 @@ void dma_direct_free(struct device *dev, size_t size, /* Swiotlb doesn't need a page attribute update on free */ mark_mem_encrypted = false;
+ if (mark_mem_encrypted) { + if (WARN_ON_ONCE(cc_shared_calc_layout(size, &layout))) + return; + size = layout.shared_size; + } + if (is_vmalloc_addr(cpu_addr)) { vunmap(cpu_addr); } else { @@ -435,10 +450,8 @@ void dma_direct_free(struct device *dev, size_t size, void *lm_addr;
lm_addr = phys_to_virt(phys); - if (set_memory_encrypted((unsigned long)lm_addr, PFN_UP(size))) { - pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n"); + if (dma_set_encrypted(dev, lm_addr, size)) return; - } }
if (swiotlb_pool) @@ -451,6 +464,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size, dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) { unsigned long attrs = 0; + struct cc_shared_layout layout; struct page *page; void *cpu_addr; bool mark_mem_decrypt; @@ -474,7 +488,13 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size, goto setup_page; }
- page = __dma_direct_alloc_pages(dev, size, gfp, false); + if (mark_mem_decrypt) { + if (cc_shared_calc_layout(size, &layout)) + return NULL; + size = layout.shared_size; + } + + page = __dma_direct_alloc_pages(dev, size, gfp, false, attrs); if (!page) return NULL;
@@ -500,6 +520,7 @@ void dma_direct_free_pages(struct device *dev, size_t size, phys_addr_t phys; void *vaddr = page_address(page); struct io_tlb_pool *swiotlb_pool; + struct cc_shared_layout layout; /* * if the device had requested for an unencrypted buffer, * convert it to encrypted on free @@ -516,6 +537,12 @@ void dma_direct_free_pages(struct device *dev, size_t size, if (swiotlb_pool) mark_mem_encrypted = false;
+ if (mark_mem_encrypted) { + if (WARN_ON_ONCE(cc_shared_calc_layout(size, &layout))) + return; + size = layout.shared_size; + } + if (mark_mem_encrypted && dma_set_encrypted(dev, vaddr, size)) return;