From: "T.J. Mercier" tjmercier@google.com
Introduce a MEMCG_DMABUF stat counter and helpers for charging, cancelling, and uncharging dma-buf memory against a memory cgroup.
The charge is tracked on struct dma_buf via a new memcg pointer. The memcg reference is stored into (or cleared from) dmabuf->memcg by the helpers, removing the need for callers to manage those details. The release path in dma_buf_release() is reduced to a single mem_cgroup_uncharge_dmabuf() call.
Rather than charging at the page level via __GFP_ACCOUNT, which would require exporters to expose their backing pages for cross-cgroup transfer via new dma_buf_ops, charges are tracked externally on struct dma_buf through a single accounting path.
The next patch wires up the charging policy in the dma-heap ioctl.
Signed-off-by: T.J. Mercier tjmercier@google.com Signed-off-by: Albert Esteve aesteve@redhat.com --- Documentation/admin-guide/cgroup-v2.rst | 4 ++ drivers/dma-buf/dma-buf.c | 3 ++ include/linux/dma-buf.h | 4 ++ include/linux/memcontrol.h | 27 +++++++++++++ mm/memcontrol.c | 70 +++++++++++++++++++++++++++++++++ 5 files changed, 108 insertions(+)
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 86a2a0099178..a3265a4a953e 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1662,6 +1662,10 @@ The following nested keys are defined. Amount of memory used for storing in-kernel data structures.
+ dmabuf (npn) + Amount of memory used for exported DMA buffers allocated by the cgroup. + Stays with the allocating cgroup regardless of how the buffer is shared. + workingset_refault_anon Number of refaults of previously evicted anonymous pages.
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 4c9add51f9ef..1c6b9cee7ac2 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -14,6 +14,7 @@ #include <linux/fs.h> #include <linux/slab.h> #include <linux/dma-buf.h> +#include <linux/memcontrol.h> #include <linux/dma-fence.h> #include <linux/dma-fence-unwrap.h> #include <linux/anon_inodes.h> @@ -182,6 +183,8 @@ static void dma_buf_release(struct dentry *dentry)
dmabuf->ops->release(dmabuf);
+ mem_cgroup_uncharge_dmabuf(dmabuf); + if (dmabuf->resv == (struct dma_resv *)&dmabuf[1]) dma_resv_fini(dmabuf->resv);
diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h index d15b2b31d3c9..23ec0ff23fba 100644 --- a/include/linux/dma-buf.h +++ b/include/linux/dma-buf.h @@ -27,6 +27,7 @@ struct device; struct dma_buf; struct dma_buf_attachment; +struct mem_cgroup;
/** * struct dma_buf_ops - operations possible on struct dma_buf @@ -429,6 +430,9 @@ struct dma_buf {
__poll_t active; } cb_in, cb_out; + + /** @memcg: the cgroup to which this buffer is currently attributed */ + struct mem_cgroup *memcg; };
/** diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 7d1c0ce189a8..4d1e82cf0833 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -10,6 +10,9 @@
#ifndef _LINUX_MEMCONTROL_H #define _LINUX_MEMCONTROL_H + +struct dma_buf; + #include <linux/cgroup.h> #include <linux/vm_event_item.h> #include <linux/hardirq.h> @@ -40,6 +43,7 @@ enum memcg_stat_item { MEMCG_ZSWAP_B, MEMCG_ZSWAPPED, MEMCG_ZSWAP_INCOMP, + MEMCG_DMABUF, MEMCG_NR_STAT, };
@@ -667,6 +671,9 @@ int mem_cgroup_charge_hugetlb(struct folio* folio, gfp_t gfp); int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id, struct mm_struct *mm, gfp_t gfp);
+bool mem_cgroup_reserve_dmabuf(struct mem_cgroup *memcg, gfp_t gfp, + unsigned int nr_pages); + void __mem_cgroup_uncharge(struct folio *folio);
/** @@ -682,6 +689,10 @@ static inline void mem_cgroup_uncharge(struct folio *folio) __mem_cgroup_uncharge(folio); }
+void mem_cgroup_cancel_dmabuf(struct mem_cgroup *memcg, unsigned int nr_pages); + +void mem_cgroup_uncharge_dmabuf(struct dma_buf *dmabuf); + void __mem_cgroup_uncharge_folios(struct folio_batch *folios); static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios) { @@ -1167,10 +1178,26 @@ static inline int mem_cgroup_swapin_charge_folio(struct folio *folio, return 0; }
+static inline bool mem_cgroup_reserve_dmabuf(struct mem_cgroup *memcg, + gfp_t gfp, + unsigned int nr_pages) +{ + return true; +} + +static inline void mem_cgroup_cancel_dmabuf(struct mem_cgroup *memcg, + unsigned int nr_pages) +{ +} + static inline void mem_cgroup_uncharge(struct folio *folio) { }
+static inline void mem_cgroup_uncharge_dmabuf(struct dma_buf *dmabuf) +{ +} + static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios) { } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 856a7d07586c..e53dd7eebc3b 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -69,6 +69,7 @@ #include <net/ip.h> #include "slab.h" #include "memcontrol-v1.h" +#include <linux/dma-buf.h>
#include <linux/uaccess.h>
@@ -442,6 +443,7 @@ static const unsigned int memcg_stat_items[] = { MEMCG_ZSWAP_B, MEMCG_ZSWAPPED, MEMCG_ZSWAP_INCOMP, + MEMCG_DMABUF, };
#define NR_MEMCG_NODE_STAT_ITEMS ARRAY_SIZE(memcg_node_stat_items) @@ -1625,6 +1627,7 @@ static const struct memory_stat memory_stats[] = { #ifdef CONFIG_HUGETLB_PAGE { "hugetlb", NR_HUGETLB }, #endif + { "dmabuf", MEMCG_DMABUF },
/* The memory events */ { "workingset_refault_anon", WORKINGSET_REFAULT_ANON }, @@ -5658,6 +5661,73 @@ void mem_cgroup_flush_workqueue(void) flush_workqueue(memcg_wq); }
+/** + * mem_cgroup_reserve_dmabuf - charge a memcg for a dma-buf allocation + * @memcg: memcg to charge (may be %NULL) + * @gfp_mask: reclaim mode + * @nr_pages: number of pages to charge + * + * Charges @nr_pages against @memcg. Call this before allocating pages so + * the cgroup limit is enforced up front. On success the caller must + * finalize the charge with the MEMCG_DMABUF stat update, or call + * mem_cgroup_cancel_dmabuf() to undo it. + * + * Returns %true on success, %false if the charge would exceed the limit. + */ +bool mem_cgroup_reserve_dmabuf(struct mem_cgroup *memcg, gfp_t gfp_mask, + unsigned int nr_pages) +{ + if (!memcg || mem_cgroup_disabled()) + return true; + + if (try_charge(memcg, gfp_mask, nr_pages)) + return false; + + return true; +} +EXPORT_SYMBOL_GPL(mem_cgroup_reserve_dmabuf); + +/** + * mem_cgroup_cancel_dmabuf - undo a dma-buf charge + * @memcg: memcg to uncharge (may be %NULL) + * @nr_pages: same value passed to mem_cgroup_reserve_dmabuf() + * + * Returns @nr_pages of charge to @memcg. Use when the allocation fails + * after mem_cgroup_reserve_dmabuf(). + */ +void mem_cgroup_cancel_dmabuf(struct mem_cgroup *memcg, unsigned int nr_pages) +{ + if (!memcg) + return; + + if (!mem_cgroup_is_root(memcg)) + refill_stock(memcg, nr_pages); +} +EXPORT_SYMBOL_GPL(mem_cgroup_cancel_dmabuf); + +/** + * mem_cgroup_uncharge_dmabuf - uncharge a dma-buf from its memcg + * @dmabuf: buffer to uncharge + * + * Reverses a finalized dma-buf charge. Drops the memcg + * reference stored in @dmabuf->memcg and clears the pointer. + * Safe to call when @dmabuf->memcg is %NULL. + */ +void mem_cgroup_uncharge_dmabuf(struct dma_buf *dmabuf) +{ + unsigned int nr_pages; + struct mem_cgroup *memcg = dmabuf->memcg; + + if (!memcg) + return; + + nr_pages = PAGE_ALIGN(dmabuf->size) >> PAGE_SHIFT; + mod_memcg_state(memcg, MEMCG_DMABUF, -nr_pages); + mem_cgroup_cancel_dmabuf(memcg, nr_pages); + mem_cgroup_put(memcg); + dmabuf->memcg = NULL; +} + static int __init cgroup_memory(char *s) { char *token;