From: "T.J. Mercier" tjmercier@google.com
Introduce a MEMCG_DMABUF stat counter and helpers for charging, and uncharging dma-buf memory against a memory cgroup:
The charge is tracked on struct dma_buf via a new memcg pointer. The page count is derived internally from PAGE_ALIGN(dmabuf->size), and the memcg reference is stored into (or cleared from) dmabuf->memcg by the helpers themselves, removing the need for callers to manage those details. The release path in dma_buf_release() is reduced to a single mem_cgroup_uncharge_dmabuf() call.
This decouples accounting from the backing page allocator, avoiding the need for every exporter to pass __GFP_ACCOUNT and enabling cross-cgroup charge transfer, which is impractical with page-level accounting.
Remove __GFP_ACCOUNT from system-heap page allocations (added by commit 3c227be90659 ("dma-buf: system_heap: account for system heap allocation in memcg")) to avoid double counting once callers adopt this path.
The next patch wires up the charging policy in the dma-heap ioctl.
Signed-off-by: T.J. Mercier tjmercier@google.com Signed-off-by: Albert Esteve aesteve@redhat.com --- Documentation/admin-guide/cgroup-v2.rst | 4 +++ drivers/dma-buf/dma-buf.c | 3 ++ drivers/dma-buf/heaps/system_heap.c | 2 -- include/linux/dma-buf.h | 4 +++ include/linux/memcontrol.h | 20 ++++++++++++ mm/memcontrol.c | 54 +++++++++++++++++++++++++++++++++ 6 files changed, 85 insertions(+), 2 deletions(-)
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 86a2a0099178..a3265a4a953e 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1662,6 +1662,10 @@ The following nested keys are defined. Amount of memory used for storing in-kernel data structures.
+ dmabuf (npn) + Amount of memory used for exported DMA buffers allocated by the cgroup. + Stays with the allocating cgroup regardless of how the buffer is shared. + workingset_refault_anon Number of refaults of previously evicted anonymous pages.
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 4c9add51f9ef..f31f49ebd0af 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -14,6 +14,7 @@ #include <linux/fs.h> #include <linux/slab.h> #include <linux/dma-buf.h> +#include <linux/memcontrol.h> #include <linux/dma-fence.h> #include <linux/dma-fence-unwrap.h> #include <linux/anon_inodes.h> @@ -180,6 +181,8 @@ static void dma_buf_release(struct dentry *dentry) */ BUG_ON(dmabuf->cb_in.active || dmabuf->cb_out.active);
+ mem_cgroup_uncharge_dmabuf(dmabuf); + dmabuf->ops->release(dmabuf);
if (dmabuf->resv == (struct dma_resv *)&dmabuf[1]) diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c index c8959eadc71d..0a99ac0183a8 100644 --- a/drivers/dma-buf/heaps/system_heap.c +++ b/drivers/dma-buf/heaps/system_heap.c @@ -388,8 +388,6 @@ static struct page *alloc_largest_available(unsigned long size, if (max_order < orders[i]) continue; flags = order_flags[i]; - if (mem_accounting) - flags |= __GFP_ACCOUNT; page = alloc_pages(flags, orders[i]); if (!page) continue; diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h index d15b2b31d3c9..23ec0ff23fba 100644 --- a/include/linux/dma-buf.h +++ b/include/linux/dma-buf.h @@ -27,6 +27,7 @@ struct device; struct dma_buf; struct dma_buf_attachment; +struct mem_cgroup;
/** * struct dma_buf_ops - operations possible on struct dma_buf @@ -429,6 +430,9 @@ struct dma_buf {
__poll_t active; } cb_in, cb_out; + + /** @memcg: the cgroup to which this buffer is currently attributed */ + struct mem_cgroup *memcg; };
/** diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 7d1c0ce189a8..aa08fbb0c26c 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -10,6 +10,9 @@
#ifndef _LINUX_MEMCONTROL_H #define _LINUX_MEMCONTROL_H + +struct dma_buf; + #include <linux/cgroup.h> #include <linux/vm_event_item.h> #include <linux/hardirq.h> @@ -40,6 +43,7 @@ enum memcg_stat_item { MEMCG_ZSWAP_B, MEMCG_ZSWAPPED, MEMCG_ZSWAP_INCOMP, + MEMCG_DMABUF, MEMCG_NR_STAT, };
@@ -667,6 +671,9 @@ int mem_cgroup_charge_hugetlb(struct folio* folio, gfp_t gfp); int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id, struct mm_struct *mm, gfp_t gfp);
+bool mem_cgroup_charge_dmabuf(struct dma_buf *dmabuf, struct mem_cgroup *memcg, + gfp_t gfp_mask); + void __mem_cgroup_uncharge(struct folio *folio);
/** @@ -682,6 +689,8 @@ static inline void mem_cgroup_uncharge(struct folio *folio) __mem_cgroup_uncharge(folio); }
+void mem_cgroup_uncharge_dmabuf(struct dma_buf *dmabuf); + void __mem_cgroup_uncharge_folios(struct folio_batch *folios); static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios) { @@ -1167,10 +1176,21 @@ static inline int mem_cgroup_swapin_charge_folio(struct folio *folio, return 0; }
+static inline bool mem_cgroup_charge_dmabuf(struct dma_buf *dmabuf, + struct mem_cgroup *memcg, + gfp_t gfp_mask) +{ + return true; +} + static inline void mem_cgroup_uncharge(struct folio *folio) { }
+static inline void mem_cgroup_uncharge_dmabuf(struct dma_buf *dmabuf) +{ +} + static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios) { } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 856a7d07586c..e94222237d31 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -69,6 +69,7 @@ #include <net/ip.h> #include "slab.h" #include "memcontrol-v1.h" +#include <linux/dma-buf.h>
#include <linux/uaccess.h>
@@ -442,6 +443,7 @@ static const unsigned int memcg_stat_items[] = { MEMCG_ZSWAP_B, MEMCG_ZSWAPPED, MEMCG_ZSWAP_INCOMP, + MEMCG_DMABUF, };
#define NR_MEMCG_NODE_STAT_ITEMS ARRAY_SIZE(memcg_node_stat_items) @@ -1625,6 +1627,7 @@ static const struct memory_stat memory_stats[] = { #ifdef CONFIG_HUGETLB_PAGE { "hugetlb", NR_HUGETLB }, #endif + { "dmabuf", MEMCG_DMABUF },
/* The memory events */ { "workingset_refault_anon", WORKINGSET_REFAULT_ANON }, @@ -5658,6 +5661,57 @@ void mem_cgroup_flush_workqueue(void) flush_workqueue(memcg_wq); }
+/** + * mem_cgroup_charge_dmabuf - charge a dma-buf to a memcg + * @dmabuf: buffer to charge + * @memcg: memcg to charge; caller must hold a reference + * @gfp_mask: reclaim mode + * + * Charges the memory backing @dmabuf to @memcg and, on success, stores + * @memcg in @dmabuf->memcg (transferring the caller's reference). + * Returns %true on success, %false if the charge would exceed the limit. + * On failure the caller retains its reference to @memcg. + */ +bool mem_cgroup_charge_dmabuf(struct dma_buf *dmabuf, struct mem_cgroup *memcg, + gfp_t gfp_mask) +{ + unsigned int nr_pages = PAGE_ALIGN(dmabuf->size) >> PAGE_SHIFT; + + if (mem_cgroup_disabled()) + return true; + + if (try_charge(memcg, gfp_mask, nr_pages)) + return false; + + mod_memcg_state(memcg, MEMCG_DMABUF, nr_pages); + dmabuf->memcg = memcg; + return true; +} + +/** + * mem_cgroup_uncharge_dmabuf - uncharge a dma-buf from its memcg + * @dmabuf: buffer to uncharge + * + * Reverses a successful mem_cgroup_charge_dmabuf(). Drops the memcg + * reference stored in @dmabuf->memcg and clears the pointer. + * Safe to call when @dmabuf->memcg is %NULL. + */ +void mem_cgroup_uncharge_dmabuf(struct dma_buf *dmabuf) +{ + unsigned int nr_pages; + struct mem_cgroup *memcg = dmabuf->memcg; + + if (!memcg) + return; + + nr_pages = PAGE_ALIGN(dmabuf->size) >> PAGE_SHIFT; + mod_memcg_state(memcg, MEMCG_DMABUF, -nr_pages); + if (!mem_cgroup_is_root(memcg)) + refill_stock(memcg, nr_pages); + mem_cgroup_put(memcg); + dmabuf->memcg = NULL; +} + static int __init cgroup_memory(char *s) { char *token;