From: "T.J. Mercier" tjmercier@google.com
Introduce a MEMCG_DMABUF stat counter and helpers for charging, cancelling, and uncharging dma-buf memory against a memory cgroup.
The charge is tracked on struct dma_buf via a new objcg pointer. The object cgroup reference is stored into (or cleared from) dmabuf->objcg by the helpers, removing the need for callers to manage those details. The release path in dma_buf_release() is reduced to a single mem_cgroup_uncharge_dmabuf() call.
obj_cgroup_reserve_dmabuf() takes nr_pages as unsigned long and rejects values above INT_MAX. This differs from the rest of memcg API. Specifically, dma-buf sizes can be userspace-controlled and long-sized (size_t), while memcg charge and stat updates are int-sized. Charges that pass the reservation are therefore safe to handle as unsigned int in the export, transfer, and uncharge paths.
Rather than charging at the page level via __GFP_ACCOUNT, which would require exporters to expose their backing pages for cross-cgroup transfer via new dma_buf_ops, charges are tracked externally on struct dma_buf through a single accounting path.
The next patch wires up the charging policy in the dma-heap ioctl.
Signed-off-by: T.J. Mercier tjmercier@google.com Signed-off-by: Albert Esteve aesteve@redhat.com --- Documentation/admin-guide/cgroup-v2.rst | 4 ++ drivers/dma-buf/dma-buf.c | 3 + include/linux/dma-buf.h | 4 ++ include/linux/memcontrol.h | 32 +++++++++ mm/memcontrol.c | 112 ++++++++++++++++++++++++++++++++ 5 files changed, 155 insertions(+)
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 9e3c67b02ae2..f1f6808c5600 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1666,6 +1666,10 @@ The following nested keys are defined. Amount of memory used for storing in-kernel data structures.
+ dmabuf (npn) + Amount of memory used for exported DMA buffers allocated by the cgroup. + Stays with the allocating cgroup regardless of how the buffer is shared. + workingset_refault_anon Number of refaults of previously evicted anonymous pages.
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 36cdc342627e..06462faf762e 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -14,6 +14,7 @@ #include <linux/fs.h> #include <linux/slab.h> #include <linux/dma-buf.h> +#include <linux/memcontrol.h> #include <linux/dma-fence.h> #include <linux/dma-fence-unwrap.h> #include <linux/anon_inodes.h> @@ -182,6 +183,8 @@ static void dma_buf_release(struct dentry *dentry)
dmabuf->ops->release(dmabuf);
+ mem_cgroup_uncharge_dmabuf(dmabuf); + if (dmabuf->resv == (struct dma_resv *)&dmabuf[1]) dma_resv_fini(dmabuf->resv);
diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h index d15b2b31d3c9..2f56afd1f830 100644 --- a/include/linux/dma-buf.h +++ b/include/linux/dma-buf.h @@ -27,6 +27,7 @@ struct device; struct dma_buf; struct dma_buf_attachment; +struct obj_cgroup;
/** * struct dma_buf_ops - operations possible on struct dma_buf @@ -429,6 +430,9 @@ struct dma_buf {
__poll_t active; } cb_in, cb_out; + + /** @objcg: object cgroup to which this buffer is currently attributed */ + struct obj_cgroup *objcg; };
/** diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 69e1bcc67c58..46fbb418bf39 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -10,6 +10,9 @@
#ifndef _LINUX_MEMCONTROL_H #define _LINUX_MEMCONTROL_H + +struct dma_buf; + #include <linux/cgroup.h> #include <linux/vm_event_item.h> #include <linux/hardirq.h> @@ -41,6 +44,7 @@ enum memcg_stat_item { MEMCG_ZSWAP_B, MEMCG_ZSWAPPED, MEMCG_ZSWAP_INCOMP, + MEMCG_DMABUF, MEMCG_NR_STAT, };
@@ -688,6 +692,9 @@ int mem_cgroup_charge_hugetlb(struct folio *folio, struct mm_struct *mm, int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id, struct mm_struct *mm, gfp_t gfp);
+bool obj_cgroup_reserve_dmabuf(struct obj_cgroup *objcg, gfp_t gfp, + unsigned long nr_pages); + void __mem_cgroup_uncharge(struct folio *folio);
/** @@ -703,6 +710,11 @@ static inline void mem_cgroup_uncharge(struct folio *folio) __mem_cgroup_uncharge(folio); }
+void obj_cgroup_cancel_dmabuf(struct obj_cgroup *objcg, unsigned long nr_pages); + +void mem_cgroup_commit_dmabuf(struct dma_buf *dmabuf); +void mem_cgroup_uncharge_dmabuf(struct dma_buf *dmabuf); + void __mem_cgroup_uncharge_folios(struct folio_batch *folios); static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios) { @@ -1203,10 +1215,30 @@ static inline int mem_cgroup_swapin_charge_folio(struct folio *folio, return 0; }
+static inline bool obj_cgroup_reserve_dmabuf(struct obj_cgroup *objcg, + gfp_t gfp, + unsigned long nr_pages) +{ + return true; +} + +static inline void obj_cgroup_cancel_dmabuf(struct obj_cgroup *objcg, + unsigned long nr_pages) +{ +} + +static inline void mem_cgroup_commit_dmabuf(struct dma_buf *dmabuf) +{ +} + static inline void mem_cgroup_uncharge(struct folio *folio) { }
+static inline void mem_cgroup_uncharge_dmabuf(struct dma_buf *dmabuf) +{ +} + static inline void mem_cgroup_uncharge_folios(struct folio_batch *folios) { } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index a05d9a8baa1b..45e82cda7c44 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -70,6 +70,7 @@ #include <net/ip.h> #include "slab.h" #include "memcontrol-v1.h" +#include <linux/dma-buf.h>
#include <linux/uaccess.h>
@@ -459,6 +460,7 @@ static const unsigned int memcg_stat_items[] = { MEMCG_ZSWAP_B, MEMCG_ZSWAPPED, MEMCG_ZSWAP_INCOMP, + MEMCG_DMABUF, };
#define NR_MEMCG_NODE_STAT_ITEMS ARRAY_SIZE(memcg_node_stat_items) @@ -1644,6 +1646,7 @@ static const struct memory_stat memory_stats[] = { #ifdef CONFIG_HUGETLB_PAGE { "hugetlb", NR_HUGETLB }, #endif + { "dmabuf", MEMCG_DMABUF },
/* The memory events */ { "workingset_refault_anon", WORKINGSET_REFAULT_ANON }, @@ -5821,6 +5824,115 @@ void mem_cgroup_flush_workqueue(void) flush_workqueue(memcg_wq); }
+/** + * obj_cgroup_reserve_dmabuf - charge an object cgroup for a dma-buf allocation + * @objcg: object cgroup to charge (may be %NULL) + * @gfp_mask: reclaim mode + * @nr_pages: number of pages to charge + * + * Charges @nr_pages against @objcg. Call this before allocating pages so + * the cgroup limit is enforced up front. On success the caller must + * finalize the charge with the MEMCG_DMABUF stat update, or call + * obj_cgroup_cancel_dmabuf() to undo it. + * + * @nr_pages is derived from the buffer size, which may be controlled by + * userspace, so it is taken as unsigned long and rejected above what + * a single memcg charge can account. + * + * Returns %true on success, %false if @nr_pages is too large or the + * charge would exceed the limit. + */ +bool obj_cgroup_reserve_dmabuf(struct obj_cgroup *objcg, gfp_t gfp_mask, + unsigned long nr_pages) +{ + struct mem_cgroup *memcg; + int ret; + + if (!objcg || mem_cgroup_disabled()) + return true; + + /* a single memcg charge accounts at most INT_MAX pages */ + if (nr_pages > INT_MAX) + return false; + + memcg = get_mem_cgroup_from_objcg(objcg); + ret = try_charge(memcg, gfp_mask, nr_pages); + mem_cgroup_put(memcg); + return !ret; +} +EXPORT_SYMBOL_GPL(obj_cgroup_reserve_dmabuf); + +/** + * obj_cgroup_cancel_dmabuf - undo a dma-buf charge + * @objcg: object cgroup to uncharge (may be %NULL) + * @nr_pages: same value passed to obj_cgroup_reserve_dmabuf() + * + * Returns @nr_pages of charge to @objcg. Use when the allocation fails + * after obj_cgroup_reserve_dmabuf(). + */ +void obj_cgroup_cancel_dmabuf(struct obj_cgroup *objcg, unsigned long nr_pages) +{ + struct mem_cgroup *memcg; + + if (!objcg) + return; + + memcg = get_mem_cgroup_from_objcg(objcg); + if (!mem_cgroup_is_root(memcg)) + refill_stock(memcg, nr_pages); + mem_cgroup_put(memcg); +} +EXPORT_SYMBOL_GPL(obj_cgroup_cancel_dmabuf); + +/** + * mem_cgroup_commit_dmabuf - finalize a dma-buf charge + * @dmabuf: buffer whose charge is being finalized + * + * Adds @dmabuf's size to the %MEMCG_DMABUF stat of the cgroup its object + * cgroup currently resolves to. Call after obj_cgroup_reserve_dmabuf() + * succeeded and the buffer is about to be exported. Safe to call when + * @dmabuf->objcg is %NULL. + */ +void mem_cgroup_commit_dmabuf(struct dma_buf *dmabuf) +{ + struct mem_cgroup *memcg; + + if (!dmabuf->objcg) + return; + + memcg = get_mem_cgroup_from_objcg(dmabuf->objcg); + mod_memcg_state(memcg, MEMCG_DMABUF, + PAGE_ALIGN(dmabuf->size) >> PAGE_SHIFT); + mem_cgroup_put(memcg); +} + +/** + * mem_cgroup_uncharge_dmabuf - uncharge a dma-buf from its object cgroup + * @dmabuf: buffer to uncharge + * + * Reverses a finalized dma-buf charge. Drops the object cgroup + * reference stored in @dmabuf->objcg and clears the pointer. + * Safe to call when @dmabuf->objcg is %NULL. + */ +void mem_cgroup_uncharge_dmabuf(struct dma_buf *dmabuf) +{ + struct obj_cgroup *objcg = dmabuf->objcg; + struct mem_cgroup *memcg; + unsigned long nr_pages; + + if (!objcg) + return; + + memcg = get_mem_cgroup_from_objcg(objcg); + nr_pages = PAGE_ALIGN(dmabuf->size) >> PAGE_SHIFT; + mod_memcg_state(memcg, MEMCG_DMABUF, -nr_pages); + if (!mem_cgroup_is_root(memcg)) + refill_stock(memcg, nr_pages); + mem_cgroup_put(memcg); + obj_cgroup_put(objcg); + dmabuf->objcg = NULL; +} + static int __init cgroup_memory(char *s) { char *token;