Confidential-computing guests may require memory shared with the host to be aligned and transitioned in units larger than PAGE_SIZE. Several DMA users need struct page-backed allocations satisfying these requirements. Provide a common allocator instead of requiring each user to open-code this sequence.

Add alloc_cc_shared_pages() and its node-aware variant. The allocator rounds the requested size to the architecture's shared granule, allocates suitably aligned contiguous pages and transitions the complete range to shared state. It also preserves the caller's GFP policy.

A private-to-shared transition may alter memory contents. Mask __GFP_ZERO from the underlying allocation and when requested, clear the complete transitioned range after cc_make_shared() succeeds.

Return the page and transitioned size so free_cc_shared_pages() can restore the complete range to private state before freeing it. If private state cannot be established, retain the allocation instead of returning a possibly shared page to the buddy allocator.

Also provide the shared-granule geometry and byte-oriented transition helpers used by the allocator and by callers managing their own backing memory.

Signed-off-by: Aneesh Kumar K.V (Arm) aneesh.kumar@kernel.org --- include/linux/cc_shared.h | 39 +++++++++ mm/Makefile | 1 + mm/cc_shared.c | 172 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 212 insertions(+) create mode 100644 include/linux/cc_shared.h create mode 100644 mm/cc_shared.c

diff --git a/include/linux/cc_shared.h b/include/linux/cc_shared.h new file mode 100644 index 000000000000..5f8db7c468c5 --- /dev/null +++ b/include/linux/cc_shared.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_CC_SHARED_H +#define _LINUX_CC_SHARED_H + +#include <linux/gfp_types.h> +#include <linux/types.h> + +struct page; + +struct cc_shared_pages { + struct page *page; + size_t shared_size; +}; + +struct cc_shared_layout { + size_t requested_size; + size_t shared_size; + size_t alignment; +}; + +/* + * Architectures may override this to return the granule used for transitions + * between private and shared memory. The value must be a power of two and no + * smaller than PAGE_SIZE. + */ +size_t arch_cc_shared_granule_size(void); + +size_t cc_shared_granule_size(void); +int cc_shared_calc_layout(size_t requested, struct cc_shared_layout *layout); +bool cc_shared_range_valid(phys_addr_t base, size_t size); +int cc_make_shared(void *addr, size_t size); +int cc_make_private(void *addr, size_t size); +int alloc_cc_shared_pages_node(int nid, gfp_t gfp, + size_t requested, struct cc_shared_pages *mem); +int alloc_cc_shared_pages(gfp_t gfp, + size_t requested, struct cc_shared_pages *mem); +void free_cc_shared_pages(struct cc_shared_pages *mem); + +#endif /* _LINUX_CC_SHARED_H */ diff --git a/mm/Makefile b/mm/Makefile index e7245cb88c66..6e6544428422 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -56,6 +56,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ compaction.o show_mem.o \ interval_tree.o list_lru.o workingset.o \ debug.o gup.o mmap_lock.o vma_init.o $(mmu-y) +obj-y += cc_shared.o

# Give 'page_alloc' its own module-parameter namespace page-alloc-y := page_alloc.o diff --git a/mm/cc_shared.c b/mm/cc_shared.c new file mode 100644 index 000000000000..3e33681218f1 --- /dev/null +++ b/mm/cc_shared.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026 ARM Ltd. + */ +#include <linux/align.h> +#include <linux/cc_shared.h> +#include <linux/errno.h> +#include <linux/export.h> +#include <linux/gfp.h> +#include <linux/log2.h> +#include <linux/mm.h> +#include <linux/mem_encrypt.h> +#include <linux/numa.h> +#include <linux/overflow.h> +#include <linux/set_memory.h> + +size_t __weak arch_cc_shared_granule_size(void) +{ + return PAGE_SIZE; +} + +size_t cc_shared_granule_size(void) +{ + size_t granule = arch_cc_shared_granule_size(); + + if (WARN_ON_ONCE(granule < PAGE_SIZE || !is_power_of_2(granule))) + return PAGE_SIZE; + + return granule; +} +EXPORT_SYMBOL_GPL(cc_shared_granule_size); + +int cc_shared_calc_layout(size_t requested, struct cc_shared_layout *layout) +{ + size_t granule, rounded; + + if (!requested || !layout) + return -EINVAL; + + granule = cc_shared_granule_size(); + if (check_add_overflow(requested, granule - 1, &rounded)) + return -EOVERFLOW; + + rounded = ALIGN_DOWN(rounded, granule); + layout->requested_size = requested; + layout->shared_size = rounded; + layout->alignment = granule; + + return 0; +} +EXPORT_SYMBOL_GPL(cc_shared_calc_layout); + +bool cc_shared_range_valid(phys_addr_t base, size_t size) +{ + size_t granule = cc_shared_granule_size(); + + if (!size) + return false; + + return IS_ALIGNED(base, granule) && IS_ALIGNED(size, granule); +} +EXPORT_SYMBOL_GPL(cc_shared_range_valid); + +static int cc_validate_transition(void *addr, size_t size) +{ + phys_addr_t phys; + + if (!addr || !size || !PAGE_ALIGNED(addr) || + !virt_addr_valid(addr)) + return -EINVAL; + + phys = page_to_phys(virt_to_page(addr)); + if (!cc_shared_range_valid(phys, size)) + return -EINVAL; + + return 0; +} + +int cc_make_shared(void *addr, size_t size) +{ + int ret = cc_validate_transition(addr, size); + + if (ret) + return ret; + + return set_memory_decrypted((unsigned long)addr, size >> PAGE_SHIFT); +} + +int cc_make_private(void *addr, size_t size) +{ + int ret = cc_validate_transition(addr, size); + + if (ret) + return ret; + + return set_memory_encrypted((unsigned long)addr, size >> PAGE_SHIFT); +} + +int alloc_cc_shared_pages_node(int nid, gfp_t gfp, + size_t requested, struct cc_shared_pages *mem) +{ + struct cc_shared_layout layout; + struct page *page; + unsigned int order; + bool zero = gfp & __GFP_ZERO; + int ret; + + if (!mem) + return -EINVAL; + + ret = cc_shared_calc_layout(requested, &layout); + if (ret) + return ret; + + order = get_order(layout.shared_size); + if (order > MAX_PAGE_ORDER) + return -EINVAL; + + /* + * State transitions require a linear-map address and may modify memory. + * Allocate from low memory and defer requested zeroing until afterwards. + */ + gfp &= ~(__GFP_HIGHMEM | __GFP_ZERO); + if (nid == NUMA_NO_NODE) + page = alloc_pages(gfp, order); + else + page = alloc_pages_node(nid, gfp, order); + if (!page) + return -ENOMEM; + + ret = cc_make_shared(page_address(page), layout.shared_size); + if (ret) { + if (!cc_make_private(page_address(page), layout.shared_size)) + __free_pages(page, order); + else + pr_warn_ratelimited("leaking %zu bytes with uncertain shared state\n", + layout.shared_size); + return ret; + } + + if (zero) + memset(page_address(page), 0, layout.shared_size); + + mem->page = page; + mem->shared_size = layout.shared_size; + return 0; +} +EXPORT_SYMBOL_GPL(alloc_cc_shared_pages_node); + +int alloc_cc_shared_pages(gfp_t gfp, + size_t requested, struct cc_shared_pages *mem) +{ + return alloc_cc_shared_pages_node(NUMA_NO_NODE, gfp, requested, mem); +} +EXPORT_SYMBOL_GPL(alloc_cc_shared_pages); + +void free_cc_shared_pages(struct cc_shared_pages *mem) +{ + if (!mem || !mem->page) + return; + + if (cc_make_private(page_address(mem->page), mem->shared_size)) { + pr_warn_ratelimited("leaking %zu bytes that cannot be made private\n", + mem->shared_size); + return; + } + + __free_pages(mem->page, get_order(mem->shared_size)); + mem->page = NULL; + mem->shared_size = 0; +} +EXPORT_SYMBOL_GPL(free_cc_shared_pages);

alloc_pages_node() resolves NUMA_NO_NODE to numa_mem_id() internally. In preparation for switching ITS allocations to the CoCo shared memory allocator, resolve the default node explicitly in its_alloc_pages().

The CoCo shared memory allocator interprets NUMA_NO_NODE as a request to apply the current task's memory policy. Passing numa_mem_id() instead preserves the existing nearest-memory-node placement across the allocator switch. This patch does not change the current allocation behavior.

Signed-off-by: Aneesh Kumar K.V (Arm) aneesh.kumar@kernel.org --- drivers/irqchip/irq-gic-v3-its.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index e9807af23537..ac5507acb1a0 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -32,6 +32,7 @@ #include <linux/set_memory.h> #include <linux/slab.h> #include <linux/syscore_ops.h> +#include <linux/topology.h>

#include <linux/irqchip.h> #include <linux/irqchip/arm-gic-v3.h> @@ -236,7 +237,7 @@ static struct page *its_alloc_pages_node(int node, gfp_t gfp,

static struct page *its_alloc_pages(gfp_t gfp, unsigned int order) { - return its_alloc_pages_node(NUMA_NO_NODE, gfp, order); + return its_alloc_pages_node(numa_mem_id(), gfp, order); }

static void its_free_pages(void *addr, unsigned int order)

Confidential-computing shared allocations need CMA backing allocations to be aligned to the architecture private/shared state-change granule size. Passing a DMA attribute into CMA would make a generic physical allocator interpret DMA and confidential-computing policy that has already been resolved by its caller.

Add an explicit minimum alignment order to dma_alloc_contiguous(). Preserve the existing size-derived alignment for ordinary callers by passing zero, and reject a request that exceeds CONFIG_CMA_ALIGNMENT. For supported requests, use the larger of the size-derived and requested orders, capped by the configured CMA alignment as before.

Also distinguish preferred and required alignment for dma_alloc_from_contiguous(). Existing callers pass zero as the required alignment and retain the current clamping behavior. Callers that require a minimum alignment can request it explicitly and receive NULL when CMA cannot satisfy it.

Signed-off-by: Aneesh Kumar K.V (Arm) aneesh.kumar@kernel.org --- arch/arm/mm/dma-mapping.c | 5 +++-- drivers/iommu/dma-iommu.c | 2 +- include/linux/dma-map-ops.h | 10 ++++++---- kernel/dma/contiguous.c | 33 +++++++++++++++++++++++---------- kernel/dma/direct.c | 2 +- kernel/dma/ops_helpers.c | 2 +- kernel/dma/pool.c | 2 +- kernel/kexec_file.c | 3 ++- 8 files changed, 38 insertions(+), 21 deletions(-)

diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 7761099dde9e..9714fcd51941 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -398,7 +398,8 @@ static void *__alloc_from_contiguous(struct device *dev, size_t size, struct page *page; void *ptr = NULL;

- page = dma_alloc_from_contiguous(dev, count, order, gfp & __GFP_NOWARN); + page = dma_alloc_from_contiguous(dev, count, order, 0, + gfp & __GFP_NOWARN); if (!page) return NULL;

@@ -866,7 +867,7 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, unsigned long order = get_order(size); struct page *page;

- page = dma_alloc_from_contiguous(dev, count, order, + page = dma_alloc_from_contiguous(dev, count, order, 0, gfp & __GFP_NOWARN); if (!page) goto error; diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 58c624513cd4..59baf2687612 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1627,7 +1627,7 @@ static void *iommu_dma_alloc_pages(struct device *dev, size_t size, struct page *page = NULL; void *cpu_addr;

- page = dma_alloc_contiguous(dev, alloc_size, gfp); + page = dma_alloc_contiguous(dev, alloc_size, gfp, 0); if (!page) page = alloc_pages_node(node, gfp, get_order(alloc_size)); if (!page) diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 8fae2b7deb20..5fccda7e5c69 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -99,10 +99,11 @@ int __init dma_contiguous_reserve_area(phys_addr_t size, phys_addr_t base, phys_addr_t limit, struct cma **res_cma, bool fixed);

struct page *dma_alloc_from_contiguous(struct device *dev, size_t count, - unsigned int order, bool no_warn); + unsigned int order, unsigned int required_order, bool no_warn); bool dma_release_from_contiguous(struct device *dev, struct page *pages, int count); -struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp); +struct page *dma_alloc_contiguous(struct device *dev, size_t size, + gfp_t gfp, unsigned int align_order); void dma_free_contiguous(struct device *dev, struct page *page, size_t size);

void dma_contiguous_early_fixup(phys_addr_t base, unsigned long size); @@ -125,7 +126,8 @@ static inline int dma_contiguous_reserve_area(phys_addr_t size, return -ENOSYS; } static inline struct page *dma_alloc_from_contiguous(struct device *dev, - size_t count, unsigned int order, bool no_warn) + size_t count, unsigned int order, unsigned int required_order, + bool no_warn) { return NULL; } @@ -136,7 +138,7 @@ static inline bool dma_release_from_contiguous(struct device *dev, } /* Use fallback alloc() and free() when CONFIG_DMA_CMA=n */ static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size, - gfp_t gfp) + gfp_t gfp, unsigned int align_order) { return NULL; } diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index 66093460584e..a3eb3299b817 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -357,19 +357,25 @@ int __init dma_contiguous_reserve_area(phys_addr_t size, phys_addr_t base, * dma_alloc_from_contiguous() - allocate pages from contiguous area * @dev: Pointer to device for which the allocation is performed. * @count: Requested number of pages. - * @align: Requested alignment of pages (in PAGE_SIZE order). + * @align: Preferred alignment of pages (in PAGE_SIZE order). + * @required_align: Minimum required alignment (in PAGE_SIZE order). * @no_warn: Avoid printing message about failed allocation. * * This function allocates memory buffer for specified device. It uses * device specific contiguous memory area if available or the default * global one. Requires architecture specific dev_get_cma_area() helper * function. + * + * The preferred alignment is capped at CONFIG_CMA_ALIGNMENT. Return NULL + * if the required alignment exceeds this limit. A required alignment of + * zero preserves the preferred-alignment clamping behavior. */ struct page *dma_alloc_from_contiguous(struct device *dev, size_t count, - unsigned int align, bool no_warn) + unsigned int align, unsigned int required_align, bool no_warn) { - if (align > CONFIG_CMA_ALIGNMENT) - align = CONFIG_CMA_ALIGNMENT; + if (required_align > CONFIG_CMA_ALIGNMENT) + return NULL; + align = min(max(align, required_align), CONFIG_CMA_ALIGNMENT);

return cma_alloc(dev_get_cma_area(dev), count, align, no_warn); } @@ -390,9 +396,14 @@ bool dma_release_from_contiguous(struct device *dev, struct page *pages, return cma_release(dev_get_cma_area(dev), pages, count); }

-static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp) +static struct page *cma_alloc_aligned(struct cma *cma, size_t size, + gfp_t gfp, unsigned int align_order) { - unsigned int align = min(get_order(size), CONFIG_CMA_ALIGNMENT); + unsigned int align; + + if (align_order > CONFIG_CMA_ALIGNMENT) + return NULL; + align = min(max(get_order(size), align_order), CONFIG_CMA_ALIGNMENT);

return cma_alloc(cma, size >> PAGE_SHIFT, align, gfp & __GFP_NOWARN); } @@ -402,6 +413,7 @@ static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp) * @dev: Pointer to device for which the allocation is performed. * @size: Requested allocation size. * @gfp: Allocation flags. + * @align_order: Minimum alignment as a power-of-two page order. * * tries to use device specific contiguous memory area if available, or it * tries to use per-numa cma, if the allocation fails, it will fallback to @@ -412,7 +424,8 @@ static struct page *cma_alloc_aligned(struct cma *cma, size_t size, gfp_t gfp) * there is no need to waste CMA pages for that kind; it also helps reduce * fragmentations. */ -struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) +struct page *dma_alloc_contiguous(struct device *dev, size_t size, + gfp_t gfp, unsigned int align_order) { #ifdef CONFIG_DMA_NUMA_CMA int nid = dev_to_node(dev); @@ -422,7 +435,7 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) if (!gfpflags_allow_blocking(gfp)) return NULL; if (dev->cma_area) - return cma_alloc_aligned(dev->cma_area, size, gfp); + return cma_alloc_aligned(dev->cma_area, size, gfp, align_order); if (size <= PAGE_SIZE) return NULL;

@@ -431,7 +444,7 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) struct cma *cma = dma_contiguous_numa_area[nid]; struct page *page; if (cma) { - page = cma_alloc_aligned(cma, size, gfp); + page = cma_alloc_aligned(cma, size, gfp, align_order); if (page) return page; } @@ -440,7 +453,7 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp) if (!dma_contiguous_default_area) return NULL;

- return cma_alloc_aligned(dma_contiguous_default_area, size, gfp); + return cma_alloc_aligned(dma_contiguous_default_area, size, gfp, align_order); }

/** diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index da665ca22d5c..d968a0c81e73 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -124,7 +124,7 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, WARN_ON_ONCE(!PAGE_ALIGNED(size));

gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit); - page = dma_alloc_contiguous(dev, size, gfp); + page = dma_alloc_contiguous(dev, size, gfp, 0); if (page) { if (dma_coherent_ok(dev, page_to_phys(page), size) && (allow_highmem || !PageHighMem(page))) diff --git a/kernel/dma/ops_helpers.c b/kernel/dma/ops_helpers.c index 6b5f9208d31c..8320cc0fada5 100644 --- a/kernel/dma/ops_helpers.c +++ b/kernel/dma/ops_helpers.c @@ -66,7 +66,7 @@ struct page *dma_common_alloc_pages(struct device *dev, size_t size, struct page *page; phys_addr_t phys;

- page = dma_alloc_contiguous(dev, size, gfp); + page = dma_alloc_contiguous(dev, size, gfp, 0); if (!page) page = alloc_pages_node(dev_to_node(dev), gfp, get_order(size)); if (!page) diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c index 00f422a1e896..70b7f64b17ab 100644 --- a/kernel/dma/pool.c +++ b/kernel/dma/pool.c @@ -99,7 +99,7 @@ static int atomic_pool_expand(struct dma_gen_pool *dma_pool, size_t pool_size, pool_size = 1 << (PAGE_SHIFT + order); if (cma_in_zone(gfp)) page = dma_alloc_from_contiguous(NULL, 1 << order, - order, false); + order, 0, false); if (!page) page = alloc_pages(gfp | __GFP_NOWARN, order); } while (!page && order-- > 0); diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 59fb9d71e9d8..2a337ce7264e 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -682,7 +682,8 @@ static int kexec_alloc_contig(struct kexec_buf *kbuf) if (kbuf->image->type == KEXEC_TYPE_CRASH) return -EPERM;

- p = dma_alloc_from_contiguous(NULL, nr_pages, get_order(kbuf->buf_align), true); + p = dma_alloc_from_contiguous(NULL, nr_pages, + get_order(kbuf->buf_align), 0, true); if (!p) return -ENOMEM;

Use the common CoCo shared-memory geometry helpers for allocations backed directly by CMA or the page allocator. Round the backing allocation to a whole shared granule, pass the required alignment order through the DMA contiguous allocator, and transition the complete range through the common shared/private helpers.

Recompute the layout before freeing ordinary direct allocations so the transition back to private memory and dma_free_contiguous() cover exactly the range acquired by the allocation path. If restoring private state fails, retain the existing fail-safe behavior and leak the pages rather than returning potentially shared memory to the allocator.

This also applies the same rules to dma_direct_alloc_pages(), covering callers which require a struct page result rather than a CPU virtual address.

Signed-off-by: Aneesh Kumar K.V (Arm) aneesh.kumar@kernel.org --- kernel/dma/direct.c | 55 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 12 deletions(-)

diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index d968a0c81e73..d293198384c3 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -11,10 +11,10 @@ #include <linux/scatterlist.h> #include <linux/pfn.h> #include <linux/vmalloc.h> -#include <linux/set_memory.h> #include <linux/slab.h> #include <linux/pci-p2pdma.h> #include <linux/cc_platform.h> +#include <linux/cc_shared.h>

#include "direct.h"

@@ -85,7 +85,7 @@ static int dma_set_decrypted(struct device *dev, void *vaddr, size_t size) { int ret;

- ret = set_memory_decrypted((unsigned long)vaddr, PFN_UP(size)); + ret = cc_make_shared(vaddr, size); if (ret) pr_warn_ratelimited("leaking DMA memory that can't be decrypted\n"); return ret; @@ -95,7 +95,7 @@ static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size) { int ret;

- ret = set_memory_encrypted((unsigned long)vaddr, PFN_UP(size)); + ret = cc_make_private(vaddr, size); if (ret) pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n"); return ret; @@ -115,7 +115,7 @@ static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size, }

static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, - gfp_t gfp, bool allow_highmem) + gfp_t gfp, bool allow_highmem, unsigned int align_order) { int node = dev_to_node(dev); struct page *page; @@ -124,7 +124,7 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, WARN_ON_ONCE(!PAGE_ALIGNED(size));

gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit); - page = dma_alloc_contiguous(dev, size, gfp, 0); + page = dma_alloc_contiguous(dev, size, gfp, align_order); if (page) { if (dma_coherent_ok(dev, page_to_phys(page), size) && (allow_highmem || !PageHighMem(page))) @@ -184,7 +184,7 @@ static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size, { struct page *page;

- page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true); + page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true, 0); if (!page) return NULL;

@@ -205,6 +205,8 @@ void *dma_direct_alloc(struct device *dev, size_t size, bool remap = false, set_uncached = false; bool mark_mem_decrypt = false; bool allow_highmem = true; + struct cc_shared_layout layout; + unsigned int align_order = 0; struct page *page; void *cpu_addr;

@@ -285,8 +287,16 @@ void *dma_direct_alloc(struct device *dev, size_t size, return NULL; }

+ if (mark_mem_decrypt) { + if (cc_shared_calc_layout(size, &layout)) + return NULL; + size = layout.shared_size; + align_order = get_order(layout.alignment); + } + /* we always manually zero the memory once we are done */ - page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, allow_highmem); + page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, + allow_highmem, align_order); if (!page) return NULL;

@@ -305,7 +315,7 @@ void *dma_direct_alloc(struct device *dev, size_t size, void *lm_addr;

lm_addr = page_address(page); - if (set_memory_decrypted((unsigned long)lm_addr, PFN_UP(size))) + if (dma_set_decrypted(dev, lm_addr, size)) goto out_leak_pages; }

@@ -362,6 +372,7 @@ void dma_direct_free(struct device *dev, size_t size, phys_addr_t phys; bool mark_mem_encrypted = false; struct io_tlb_pool *swiotlb_pool; + struct cc_shared_layout layout; unsigned int page_order = get_order(size);

/* @@ -406,6 +417,12 @@ void dma_direct_free(struct device *dev, size_t size, /* Swiotlb doesn't need a page attribute update on free */ mark_mem_encrypted = false;

+ if (mark_mem_encrypted) { + if (WARN_ON_ONCE(cc_shared_calc_layout(size, &layout))) + return; + size = layout.shared_size; + } + if (is_vmalloc_addr(cpu_addr)) { vunmap(cpu_addr); } else { @@ -417,10 +434,8 @@ void dma_direct_free(struct device *dev, size_t size, void *lm_addr;

lm_addr = phys_to_virt(phys); - if (set_memory_encrypted((unsigned long)lm_addr, PFN_UP(size))) { - pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n"); + if (dma_set_encrypted(dev, lm_addr, size)) return; - } }

if (swiotlb_pool) @@ -433,6 +448,8 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size, dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) { unsigned long attrs = 0; + struct cc_shared_layout layout; + unsigned int align_order = 0; struct page *page; void *cpu_addr;

@@ -452,7 +469,14 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size, goto setup_page; }

- page = __dma_direct_alloc_pages(dev, size, gfp, false); + if (attrs & __DMA_ATTR_ALLOC_CC_SHARED) { + if (cc_shared_calc_layout(size, &layout)) + return NULL; + size = layout.shared_size; + align_order = get_order(layout.alignment); + } + + page = __dma_direct_alloc_pages(dev, size, gfp, false, align_order); if (!page) return NULL;

@@ -476,6 +500,7 @@ void dma_direct_free_pages(struct device *dev, size_t size, phys_addr_t phys; void *vaddr = page_address(page); struct io_tlb_pool *swiotlb_pool; + struct cc_shared_layout layout; /* * if the device had requested for an unencrypted buffer, * convert it to encrypted on free @@ -492,6 +517,12 @@ void dma_direct_free_pages(struct device *dev, size_t size, if (swiotlb_pool) mark_mem_encrypted = false;

+ if (mark_mem_encrypted) { + if (WARN_ON_ONCE(cc_shared_calc_layout(size, &layout))) + return; + size = layout.shared_size; + } + if (mark_mem_encrypted && dma_set_encrypted(dev, vaddr, size)) return;

A restricted DMA pool is a firmware-described ownership boundary. Unlike a pool allocated by SWIOTLB itself, its base cannot be moved and its size cannot be rounded without claiming memory outside the reserved region.

Confidential-computing guests may require private/shared state changes to cover units larger than PAGE_SIZE. Passing a misaligned restricted region to set_memory_decrypted() can therefore fail.

Validate both the physical base and the complete reserved size with the common CoCo shared-range helper before allocating SWIOTLB metadata or performing a state transition. Perform the transition through cc_make_shared().

Signed-off-by: Aneesh Kumar K.V (Arm) aneesh.kumar@kernel.org --- kernel/dma/swiotlb.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index 9928d75efc2d..cb67105b8812 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -2010,6 +2010,14 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem, return -EINVAL; }

+ if (cc_platform_has(CC_ATTR_MEM_ENCRYPT) && + !cc_shared_range_valid(rmem->base, rmem->size)) { + dev_err(dev, + "Restricted DMA pool must be aligned to %#zx bytes for memory encryption\n", + cc_shared_granule_size()); + return -EINVAL; + } + /* * Since multiple devices can share the same pool, the private data, * io_tlb_mem struct, will be initialized by the first device attached @@ -2043,8 +2051,7 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem, int ret;

mem->cc_shared = true; - ret = set_memory_decrypted((unsigned long)phys_to_virt(rmem->base), - rmem->size >> PAGE_SHIFT); + ret = cc_make_shared(phys_to_virt(rmem->base), rmem->size); if (ret) { dev_err(dev, "Failed to decrypt restricted DMA pool\n"); kfree(pool->areas);

The system_cc_shared heap currently allocates its backing pages using the normal system-heap order policy and changes each resulting compound page to shared state. That is unsafe when an architecture requires state changes in units larger than PAGE_SIZE: an order-0 tail is neither sufficiently aligned nor large enough to transition independently.

Use the common CoCo shared-page allocator for every backing allocation of the shared heap. Preserve the existing preferred-order search and its GFP policy: each candidate order is passed to the common allocator as a byte request, and that allocator rounds it up when the architecture shared granule is larger.

Add __GFP_COMP for shared allocations because an order-0 candidate can be rounded into a high-order allocation. The system heap uses compound_order() and page_size() for accounting and release, so the returned allocation must retain compound-page semantics.

Calculate a rounded internal backing length but retain the original length in dma_buf::size. The preceding scatterlist-length change ensures that the rounded tail is not included in DMA mappings or other operations. This permits a 4 KiB request on a 64 KiB shared-granule system without exposing the extra 60 KiB to an importer.

Signed-off-by: Aneesh Kumar K.V (Arm) aneesh.kumar@kernel.org --- drivers/dma-buf/heaps/system_heap.c | 119 ++++++++++++---------------- 1 file changed, 51 insertions(+), 68 deletions(-)

diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c index b5b8cdf65f23..35ac029dc41b 100644 --- a/drivers/dma-buf/heaps/system_heap.c +++ b/drivers/dma-buf/heaps/system_heap.c @@ -11,14 +11,13 @@ */

#include <linux/cc_platform.h> +#include <linux/cc_shared.h> #include <linux/dma-buf.h> #include <linux/dma-mapping.h> #include <linux/dma-heap.h> #include <linux/err.h> #include <linux/highmem.h> -#include <linux/mem_encrypt.h> #include <linux/mm.h> -#include <linux/set_memory.h> #include <linux/module.h> #include <linux/pgtable.h> #include <linux/scatterlist.h> @@ -65,34 +64,6 @@ static gfp_t order_flags[] = {HIGH_ORDER_GFP, HIGH_ORDER_GFP, LOW_ORDER_GFP}; static const unsigned int orders[] = {8, 4, 0}; #define NUM_ORDERS ARRAY_SIZE(orders)

-static int system_heap_set_page_decrypted(struct page *page) -{ - unsigned long addr = (unsigned long)page_address(page); - unsigned int nr_pages = 1 << compound_order(page); - int ret; - - ret = set_memory_decrypted(addr, nr_pages); - if (ret) - pr_warn_ratelimited("dma-buf system heap: failed to decrypt page at %p\n", - page_address(page)); - - return ret; -} - -static int system_heap_set_page_encrypted(struct page *page) -{ - unsigned long addr = (unsigned long)page_address(page); - unsigned int nr_pages = 1 << compound_order(page); - int ret; - - ret = set_memory_encrypted(addr, nr_pages); - if (ret) - pr_warn_ratelimited("dma-buf system heap: failed to re-encrypt page at %p, leaking memory\n", - page_address(page)); - - return ret; -} - static int dup_sg_table(struct sg_table *from, struct sg_table *to) { struct scatterlist *sg, *new_sg; @@ -337,6 +308,20 @@ static void system_heap_vunmap(struct dma_buf *dmabuf, struct iosys_map *map) iosys_map_clear(map); }

+static void system_heap_free_page(struct page *page, bool cc_shared) +{ + struct cc_shared_pages mem; + + if (!cc_shared) { + __free_pages(page, compound_order(page)); + return; + } + + mem.page = page; + mem.shared_size = page_size(page); + free_cc_shared_pages(&mem); +} + static void system_heap_dma_buf_release(struct dma_buf *dmabuf) { struct system_heap_buffer *buffer = dmabuf->priv; @@ -345,19 +330,8 @@ static void system_heap_dma_buf_release(struct dma_buf *dmabuf) int i;

table = &buffer->sg_table; - for_each_sgtable_sg(table, sg, i) { - struct page *page = sg_page(sg); - - /* - * Intentionally leak pages that cannot be re-encrypted - * to prevent shared memory from being reused. - */ - if (cc_shared_buffer(buffer) && - system_heap_set_page_encrypted(page)) - continue; - - __free_pages(page, compound_order(page)); - } + for_each_sgtable_sg(table, sg, i) + system_heap_free_page(sg_page(sg), cc_shared_buffer(buffer)); sg_free_table(table); kfree(buffer); } @@ -375,22 +349,39 @@ static const struct dma_buf_ops system_heap_buf_ops = { .release = system_heap_dma_buf_release, };

+static struct page *system_heap_alloc_order(unsigned int order, + gfp_t flags, bool cc_shared) +{ + struct cc_shared_pages mem; + + if (!cc_shared) + return alloc_pages(flags, order); + + /* The shared granule can raise the actual allocation order. */ + flags |= __GFP_COMP; + if (alloc_cc_shared_pages(flags, PAGE_SIZE << order, &mem)) + return NULL; + + return mem.page; +} + static struct page *alloc_largest_available(unsigned long size, - unsigned int max_order) + unsigned int max_order, bool cc_shared) { struct page *page; - int i; gfp_t flags; + int i;

for (i = 0; i < NUM_ORDERS; i++) { if (size < (PAGE_SIZE << orders[i])) continue; if (max_order < orders[i]) continue; + flags = order_flags[i]; if (mem_accounting) flags |= __GFP_ACCOUNT; - page = alloc_pages(flags, orders[i]); + page = system_heap_alloc_order(orders[i], flags, cc_shared); if (!page) continue; return page; @@ -405,6 +396,7 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, { struct system_heap_buffer *buffer; DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + struct cc_shared_layout layout; unsigned long size_remaining = len; unsigned long sg_remaining = len; unsigned int max_order = orders[0]; @@ -417,6 +409,14 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, struct page *page, *tmp_page; int i, ret = -ENOMEM;

+ if (cc_shared) { + ret = cc_shared_calc_layout(len, &layout); + if (ret) + return ERR_PTR(ret); + + size_remaining = layout.shared_size; + } + buffer = kzalloc_obj(*buffer); if (!buffer) return ERR_PTR(-ENOMEM); @@ -439,7 +439,8 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, goto free_buffer; }

- page = alloc_largest_available(size_remaining, max_order); + page = alloc_largest_available(size_remaining, max_order, + cc_shared); if (!page) goto free_buffer;

@@ -464,14 +465,6 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, list_del(&page->lru); }

- if (cc_shared_buffer(buffer)) { - for_each_sgtable_sg(table, sg, i) { - ret = system_heap_set_page_decrypted(sg_page(sg)); - if (ret) - goto free_pages; - } - } - /* create the dmabuf */ exp_info.exp_name = dma_heap_get_name(heap); exp_info.ops = &system_heap_buf_ops; @@ -486,22 +479,12 @@ static struct dma_buf *system_heap_allocate(struct dma_heap *heap, return dmabuf;

free_pages: - for_each_sgtable_sg(table, sg, i) { - struct page *p = sg_page(sg); - - /* - * Intentionally leak pages that cannot be re-encrypted - * to prevent shared memory from being reused. - */ - if (cc_shared_buffer(buffer) && - system_heap_set_page_encrypted(p)) - continue; - __free_pages(p, compound_order(p)); - } + for_each_sgtable_sg(table, sg, i) + system_heap_free_page(sg_page(sg), cc_shared); sg_free_table(table); free_buffer: list_for_each_entry_safe(page, tmp_page, &pages, lru) - __free_pages(page, compound_order(page)); + system_heap_free_page(page, cc_shared); kfree(buffer);

return ERR_PTR(ret);