VPE L1 tables are allocated from the CPU-starting callback, where sleeping is not allowed. Remove the page allocation from this atomic path.
Use a blocking allocation for the boot CPU and record the resulting table order. Allocate a page of that order from the sleepable CPU hotplug prepare stage for each secondary CPU. Consume the preallocated page only when the CPU cannot inherit an existing redistributor or ITS table.
Reclaim unused preallocations after the CPU reaches the online state, and from the prepare-state teardown when CPU bring-up aborts.
Cc: Marc Zyngier maz@kernel.org Cc: Thomas Gleixner tglx@kernel.org Cc: Radu Rendec radu@rendec.net Signed-off-by: Aneesh Kumar K.V (Arm) aneesh.kumar@kernel.org --- drivers/irqchip/irq-gic-v3-its.c | 110 +++++++++++++++++++++++++++-- drivers/irqchip/irq-gic-v3.c | 4 +- include/linux/irqchip/arm-gic-v3.h | 3 +- 3 files changed, 108 insertions(+), 9 deletions(-)
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 9ad66253ab16..5628f497d389 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -197,6 +197,7 @@ static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count); static LIST_HEAD(its_nodes); static DEFINE_RAW_SPINLOCK(its_lock); static struct rdists *gic_rdists; +static unsigned int vpe_l1_prealloc_order = UINT_MAX; static struct irq_domain *its_parent;
static unsigned long its_list_map; @@ -2892,12 +2893,35 @@ static bool allocate_vpe_l2_table(int cpu, u32 id) return true; }
-static int allocate_vpe_l1_table(void) +static unsigned int vpe_l1_table_order(u64 val) +{ + unsigned int psz; + u64 npg; + + switch (FIELD_GET(GICR_VPROPBASER_4_1_PAGE_SIZE, val)) { + default: + case GIC_PAGE_SIZE_4K: + psz = SZ_4K; + break; + case GIC_PAGE_SIZE_16K: + psz = SZ_16K; + break; + case GIC_PAGE_SIZE_64K: + psz = SZ_64K; + break; + } + + npg = FIELD_GET(GICR_VPROPBASER_4_1_SIZE, val) + 1; + return get_order(npg * psz); +} + +static int allocate_vpe_l1_table(bool use_prealloc) { void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); u64 val, gpsz, npg, pa; unsigned int psz = SZ_64K; unsigned int np, epp, esz; + unsigned int order; struct page *page;
if (!gic_rdists->has_rvpeid) @@ -2990,7 +3014,16 @@ static int allocate_vpe_l1_table(void)
pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n", np, npg, psz, epp, esz); - page = its_alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE)); + order = get_order(np * PAGE_SIZE); + if (use_prealloc) { + if (WARN_ON_ONCE(order != vpe_l1_prealloc_order)) + return -EINVAL; + + page = gic_data_rdist()->vpe_l1_prealloc; + gic_data_rdist()->vpe_l1_prealloc = NULL; + } else { + page = its_alloc_pages(GFP_KERNEL | __GFP_ZERO, order); + } if (!page) return -ENOMEM;
@@ -3007,6 +3040,9 @@ static int allocate_vpe_l1_table(void) val |= GICR_VPROPBASER_4_1_VALID;
out: + if (!use_prealloc) + vpe_l1_prealloc_order = vpe_l1_table_order(val); + gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask);
@@ -3146,7 +3182,7 @@ static u64 its_clear_vpend_valid(void __iomem *vlpi_base, u64 clr, u64 set) return val; }
-static void its_cpu_init_lpis(void) +static void its_cpu_init_lpis(bool use_prealloc) { void __iomem *rbase = gic_data_rdist_rd_base(); struct page *pend_page; @@ -3259,7 +3295,7 @@ static void its_cpu_init_lpis(void) val = its_clear_vpend_valid(vlpi_base, 0, 0); }
- if (allocate_vpe_l1_table()) { + if (allocate_vpe_l1_table(use_prealloc)) { /* * If the allocation has failed, we're in massive trouble. * Disable direct injection, and pray that no VM was @@ -5436,7 +5472,7 @@ static int redist_disable_lpis(void) return 0; }
-int its_cpu_init(void) +int its_cpu_init(bool use_prealloc) { if (!list_empty(&its_nodes)) { int ret; @@ -5445,13 +5481,66 @@ int its_cpu_init(void) if (ret) return ret;
- its_cpu_init_lpis(); + its_cpu_init_lpis(use_prealloc); its_cpu_init_collections(); }
return 0; }
+static int its_vpe_l1_prepare(unsigned int cpu) +{ + struct page **prealloc; + + if (!gic_rdists->has_rvpeid || + vpe_l1_prealloc_order == UINT_MAX || + (gic_data_rdist_cpu(cpu)->flags & RD_LOCAL_LPI_ENABLED)) + return 0; + + prealloc = &gic_data_rdist_cpu(cpu)->vpe_l1_prealloc; + if (*prealloc) + return 0; + + *prealloc = its_alloc_pages_node(cpu_to_node(cpu), + GFP_KERNEL | __GFP_ZERO, + vpe_l1_prealloc_order); + return *prealloc ? 0 : -ENOMEM; +} + +static int its_vpe_l1_cleanup(unsigned int cpu) +{ + struct page *page; + + page = xchg(&gic_data_rdist_cpu(cpu)->vpe_l1_prealloc, NULL); + if (page) + its_free_pages(page_address(page), vpe_l1_prealloc_order); + + return 0; +} + +static int __init its_vpe_l1_cpuhp_init(void) +{ + int prepare_state, state; + + state = cpuhp_setup_state_nocalls(CPUHP_BP_PREPARE_DYN, + "irqchip/arm/gicv3-vpe:prepare", + its_vpe_l1_prepare, + its_vpe_l1_cleanup); + if (state < 0) + return state; + prepare_state = state; + + state = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "irqchip/arm/gicv3-vpe:online", + its_vpe_l1_cleanup, NULL); + if (state < 0) { + cpuhp_remove_state_nocalls(prepare_state); + return state; + } + + return 0; +} + static void rdist_memreserve_cpuhp_cleanup_workfn(struct work_struct *work) { cpuhp_remove_state_nocalls(gic_rdists->cpuhp_memreserve_state); @@ -5876,6 +5965,15 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, if (WARN_ON(!has_v4_1 && rdists->has_rvpeid)) rdists->has_rvpeid = false;
+ if (rdists->has_rvpeid) { + err = its_vpe_l1_cpuhp_init(); + if (err) { + rdists->has_rvpeid = false; + rdists->has_vlpis = false; + pr_err("ITS: Failed to prepare VPE tables, disabling GICv4 support\n"); + } + } + if (has_v4 & rdists->has_vlpis) { const struct irq_domain_ops *sgi_ops;
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 94873646d46f..361ab07d7d66 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1322,7 +1322,7 @@ static int gic_starting_cpu(unsigned int cpu) gic_cpu_init();
if (gic_dist_supports_lpis()) - its_cpu_init(); + its_cpu_init(true);
return 0; } @@ -2056,7 +2056,7 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base,
if (gic_dist_supports_lpis()) { its_init(handle, &gic_data.rdists, gic_data.domain, dist_prio_irq); - its_cpu_init(); + its_cpu_init(false); its_lpi_memreserve_init(); } else { if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index ea5fd2374ebe..a5565a745100 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -621,6 +621,7 @@ struct rdists { u64 flags; cpumask_t *vpe_table_mask; void *vpe_l1_base; + struct page *vpe_l1_prealloc; } __percpu *rdist; phys_addr_t prop_table_pa; void *prop_table_va; @@ -637,7 +638,7 @@ struct rdists { struct irq_domain; struct fwnode_handle; int __init its_lpi_memreserve_init(void); -int its_cpu_init(void); +int its_cpu_init(bool use_prealloc); int its_init(struct fwnode_handle *handle, struct rdists *rdists, struct irq_domain *domain, u8 irq_prio); int mbi_init(struct fwnode_handle *fwnode, struct irq_domain *parent);