Confidential-computing shared DMA allocations require CMA backing memory to be aligned to the architecture shared-memory granule. Passing explicit alignment orders through the DMA allocation stack exposes this derived constraint to callers and requires separate preferred and mandatory values.
The DMA core already records whether an allocation needs shared backing in __DMA_ATTR_ALLOC_CC_SHARED. Pass allocation attributes to the DMA-contiguous entry points and derive the mandatory alignment there before calling CMA. Preserve the existing alignment clamping for ordinary allocations, while rejecting shared allocations whose granule exceeds CONFIG_CMA_ALIGNMENT so callers can fall back to the page allocator.
Cc: Russell King linux@armlinux.org.uk Cc: Robin Murphy robin.murphy@arm.com Cc: "Joerg Roedel (AMD)" joro@8bytes.org Cc: Will Deacon will@kernel.org Cc: Marek Szyprowski m.szyprowski@samsung.com Cc: Andrew Morton akpm@linux-foundation.org Cc: Baoquan He baoquan.he@linux.dev Cc: Mike Rapoport rppt@kernel.org Cc: Pasha Tatashin pasha.tatashin@soleen.com Cc: Pratyush Yadav pratyush@kernel.org Signed-off-by: Aneesh Kumar K.V (Arm) aneesh.kumar@kernel.org --- arch/arm/mm/dma-mapping.c | 5 +++-- drivers/iommu/dma-iommu.c | 2 +- include/linux/dma-map-ops.h | 9 ++++---- kernel/dma/contiguous.c | 41 +++++++++++++++++++++++++++---------- kernel/dma/direct.c | 2 +- kernel/dma/ops_helpers.c | 2 +- kernel/dma/pool.c | 2 +- kernel/kexec_file.c | 3 ++- 8 files changed, 44 insertions(+), 22 deletions(-)
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 7761099dde9e..9714fcd51941 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -398,7 +398,8 @@ static void *__alloc_from_contiguous(struct device *dev, size_t size, struct page *page; void *ptr = NULL;
- page = dma_alloc_from_contiguous(dev, count, order, gfp & __GFP_NOWARN); + page = dma_alloc_from_contiguous(dev, count, order, 0, + gfp & __GFP_NOWARN); if (!page) return NULL;
@@ -866,7 +867,7 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, unsigned long order = get_order(size); struct page *page;
- page = dma_alloc_from_contiguous(dev, count, order, + page = dma_alloc_from_contiguous(dev, count, order, 0, gfp & __GFP_NOWARN); if (!page) goto error; diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 58c624513cd4..59baf2687612 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1627,7 +1627,7 @@ static void *iommu_dma_alloc_pages(struct device *dev, size_t size, struct page *page = NULL; void *cpu_addr;
- page = dma_alloc_contiguous(dev, alloc_size, gfp); + page = dma_alloc_contiguous(dev, alloc_size, gfp, 0); if (!page) page = alloc_pages_node(node, gfp, get_order(alloc_size)); if (!page) diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 8fae2b7deb20..f3490dd3f8ac 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -99,10 +99,11 @@ int __init dma_contiguous_reserve_area(phys_addr_t size, phys_addr_t base, phys_addr_t limit, struct cma **res_cma, bool fixed);
struct page *dma_alloc_from_contiguous(struct device *dev, size_t count, - unsigned int order, bool no_warn); + unsigned int order, unsigned long attrs, bool no_warn); bool dma_release_from_contiguous(struct device *dev, struct page *pages, int count); -struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp); +struct page *dma_alloc_contiguous(struct device *dev, size_t size, + gfp_t gfp, unsigned long attrs); void dma_free_contiguous(struct device *dev, struct page *page, size_t size);
void dma_contiguous_early_fixup(phys_addr_t base, unsigned long size); @@ -125,7 +126,7 @@ static inline int dma_contiguous_reserve_area(phys_addr_t size, return -ENOSYS; } static inline struct page *dma_alloc_from_contiguous(struct device *dev, - size_t count, unsigned int order, bool no_warn) + size_t count, unsigned int order, unsigned long attrs, bool no_warn) { return NULL; } @@ -136,7 +137,7 @@ static inline bool dma_release_from_contiguous(struct device *dev, } /* Use fallback alloc() and free() when CONFIG_DMA_CMA=n */ static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size, - gfp_t gfp) + gfp_t gfp, unsigned long attrs) { return NULL; } diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index 66093460584e..1a29565c9185 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -39,6 +39,7 @@
#include <asm/page.h>
+#include <linux/cc_shared.h> #include <linux/memblock.h> #include <linux/err.h> #include <linux/sizes.h> @@ -357,19 +358,29 @@ int __init dma_contiguous_reserve_area(phys_addr_t size, phys_addr_t base, * dma_alloc_from_contiguous() - allocate pages from contiguous area * @dev: Pointer to device for which the allocation is performed. * @count: Requested number of pages. - * @align: Requested alignment of pages (in PAGE_SIZE order). + * @align: Preferred alignment of pages (in PAGE_SIZE order). + * @attrs: DMA allocation attributes. * @no_warn: Avoid printing message about failed allocation. * * This function allocates memory buffer for specified device. It uses * device specific contiguous memory area if available or the default * global one. Requires architecture specific dev_get_cma_area() helper * function. + * + * The preferred alignment is capped at CONFIG_CMA_ALIGNMENT. The internal + * shared-allocation attribute requires at least the architecture shared + * granule alignment and fails if that exceeds the CMA alignment limit. */ struct page *dma_alloc_from_contiguous(struct device *dev, size_t count, - unsigned int align, bool no_warn) + unsigned int align, unsigned long attrs, bool no_warn) { - if (align > CONFIG_CMA_ALIGNMENT) - align = CONFIG_CMA_ALIGNMENT; + unsigned int required_align = 0; + + if (attrs & __DMA_ATTR_ALLOC_CC_SHARED) + required_align = get_order(cc_shared_granule_size()); + if (required_align > CONFIG_CMA_ALIGNMENT) + return NULL; + align = min(max(align, required_align), CONFIG_CMA_ALIGNMENT);
return cma_alloc(dev_get_cma_area(dev), count, align, no_warn); } @@ -390,10 +401,9 @@ bool dma_release_from_contiguous(struct device *dev, struct page *pages, return cma_release(dev_get_cma_area(dev), pages, count); }
-static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp) +static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp, + unsigned int align) { - unsigned int align = min(get_order(size), CONFIG_CMA_ALIGNMENT); - return cma_alloc(cma, size >> PAGE_SHIFT, align, gfp & __GFP_NOWARN); }
@@ -402,6 +412,7 @@ static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp) * @dev: Pointer to device for which the allocation is performed. * @size: Requested allocation size. * @gfp: Allocation flags. + * @attrs: DMA allocation attributes. * * tries to use device specific contiguous memory area if available, or it * tries to use per-numa cma, if the allocation fails, it will fallback to @@ -412,8 +423,11 @@ static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp) * there is no need to waste CMA pages for that kind; it also helps reduce * fragmentations. */ -struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) +struct page *dma_alloc_contiguous(struct device *dev, size_t size, + gfp_t gfp, unsigned long attrs) { + unsigned int required_align = 0; + unsigned int align = get_order(size); #ifdef CONFIG_DMA_NUMA_CMA int nid = dev_to_node(dev); #endif @@ -421,8 +435,13 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) /* CMA can be used only in the context which permits sleeping */ if (!gfpflags_allow_blocking(gfp)) return NULL; + if (attrs & __DMA_ATTR_ALLOC_CC_SHARED) + required_align = get_order(cc_shared_granule_size()); + if (required_align > CONFIG_CMA_ALIGNMENT) + return NULL; + align = min(max(align, required_align), CONFIG_CMA_ALIGNMENT); if (dev->cma_area) - return cma_alloc_aligned(dev->cma_area, size, gfp); + return cma_alloc_aligned(dev->cma_area, size, gfp, align); if (size <= PAGE_SIZE) return NULL;
@@ -431,7 +450,7 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) struct cma *cma = dma_contiguous_numa_area[nid]; struct page *page; if (cma) { - page = cma_alloc_aligned(cma, size, gfp); + page = cma_alloc_aligned(cma, size, gfp, align); if (page) return page; } @@ -440,7 +459,7 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) if (!dma_contiguous_default_area) return NULL;
- return cma_alloc_aligned(dma_contiguous_default_area, size, gfp); + return cma_alloc_aligned(dma_contiguous_default_area, size, gfp, align); }
/** diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index 356d4e09e1c8..cb14419f8a09 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -124,7 +124,7 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, WARN_ON_ONCE(!PAGE_ALIGNED(size));
gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit); - page = dma_alloc_contiguous(dev, size, gfp); + page = dma_alloc_contiguous(dev, size, gfp, 0); if (page) { if (dma_coherent_ok(dev, page_to_phys(page), size) && (allow_highmem || !PageHighMem(page))) diff --git a/kernel/dma/ops_helpers.c b/kernel/dma/ops_helpers.c index 6b5f9208d31c..8320cc0fada5 100644 --- a/kernel/dma/ops_helpers.c +++ b/kernel/dma/ops_helpers.c @@ -66,7 +66,7 @@ struct page *dma_common_alloc_pages(struct device *dev, size_t size, struct page *page; phys_addr_t phys;
- page = dma_alloc_contiguous(dev, size, gfp); + page = dma_alloc_contiguous(dev, size, gfp, 0); if (!page) page = alloc_pages_node(dev_to_node(dev), gfp, get_order(size)); if (!page) diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c index 00f422a1e896..70b7f64b17ab 100644 --- a/kernel/dma/pool.c +++ b/kernel/dma/pool.c @@ -99,7 +99,7 @@ static int atomic_pool_expand(struct dma_gen_pool *dma_pool, size_t pool_size, pool_size = 1 << (PAGE_SHIFT + order); if (cma_in_zone(gfp)) page = dma_alloc_from_contiguous(NULL, 1 << order, - order, false); + order, 0, false); if (!page) page = alloc_pages(gfp | __GFP_NOWARN, order); } while (!page && order-- > 0); diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 59fb9d71e9d8..2a337ce7264e 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -682,7 +682,8 @@ static int kexec_alloc_contig(struct kexec_buf *kbuf) if (kbuf->image->type == KEXEC_TYPE_CRASH) return -EPERM;
- p = dma_alloc_from_contiguous(NULL, nr_pages, get_order(kbuf->buf_align), true); + p = dma_alloc_from_contiguous(NULL, nr_pages, + get_order(kbuf->buf_align), 0, true); if (!p) return -ENOMEM;