dma-heap already supports charging a specific cgroup at allocation time via charge_fd in DMA_HEAP_IOCTL_ALLOC. However, dma-bufs can originate from any exporter (DRM, V4L2, etc.), and the allocator may not know the final owner at allocation time.
Add DMA_BUF_IOCTL_XFER_CHARGE to transfer a dma-buf's memcg charge to a different cgroup after allocation. The target is identified by charge_fd, which may be a cgroupfd or a pidfd.
The transfer charges the new cgroup before uncharging the old one, preserving the original charge on failure. An atomic cmpxchg on dmabuf->memcg prevents races with concurrent transfers.
Usage example: int cg_fd = open("/sys/fs/cgroup/clients/app1", O_RDONLY | O_DIRECTORY);
struct dma_buf_transfer_charge xfer = { .charge_fd = cg_fd, }; ioctl(dmabuf_fd, DMA_BUF_IOCTL_XFER_CHARGE, &xfer); /* charge now on app1's cgroup */ close(cg_fd);
Signed-off-by: Albert Esteve aesteve@redhat.com --- drivers/dma-buf/dma-buf.c | 77 ++++++++++++++++++++++++++++++++++++++++++++ include/linux/memcontrol.h | 9 ++++++ include/uapi/linux/dma-buf.h | 26 +++++++++++++++ mm/memcontrol.c | 40 +++++++++++++++++++++++ 4 files changed, 152 insertions(+)
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index f31f49ebd0af..7fa607d9bf65 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -540,6 +540,80 @@ static long dma_buf_import_sync_file(struct dma_buf *dmabuf, } #endif
+static long dma_buf_xfer_charge(struct dma_buf *dmabuf, + const void __user *user_data) +{ + struct cgroup *cgrp; + struct cgroup_subsys_state *css; + struct dma_buf_transfer_charge arg; + struct mem_cgroup *memcg; + struct task_struct *task; + const struct cred *tcred; + unsigned int pidfd_flags; + int ret = 0; + + if (copy_from_user(&arg, user_data, sizeof(arg))) + return -EFAULT; + + if (!IS_ENABLED(CONFIG_MEMCG)) + return 0; + + if (arg.__padding != 0) + return -EINVAL; + + cgrp = cgroup_get_from_fd(arg.charge_fd); + if (!IS_ERR(cgrp)) { + /* charge_fd is a cgroupfd */ + rcu_read_lock(); + css = cgroup_css(cgrp, &memory_cgrp_subsys); + if (!css || !css_tryget_online(css)) { + rcu_read_unlock(); + cgroup_put(cgrp); + return -ENOENT; + } + + memcg = mem_cgroup_from_css(css); + rcu_read_unlock(); + + ret = security_dma_buf_charge_cgroup(current_cred(), cgrp); + cgroup_put(cgrp); + if (ret) { + mem_cgroup_put(memcg); + return ret; + } + } else { + /* charge_fd is a pidfd */ + task = pidfd_get_task(arg.charge_fd, &pidfd_flags); + if (IS_ERR(task)) + return PTR_ERR(task); + + tcred = get_task_cred(task); + ret = security_dma_buf_charge_task(current_cred(), tcred); + put_cred(tcred); + if (ret) { + put_task_struct(task); + return ret; + } + + memcg = get_mem_cgroup_from_mm(task->mm); + put_task_struct(task); + } + + if (dmabuf->memcg == memcg) + goto skip_transfer; + + if (!mem_cgroup_transfer_dmabuf_charge(dmabuf, memcg)) { + ret = -ENOMEM; + goto skip_transfer; + } + + return 0; + +skip_transfer: + mem_cgroup_put(memcg); + return ret; +} + static long dma_buf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -590,6 +664,9 @@ static long dma_buf_ioctl(struct file *file, return dma_buf_import_sync_file(dmabuf, (const void __user *)arg); #endif
+ case DMA_BUF_IOCTL_XFER_CHARGE: + return dma_buf_xfer_charge(dmabuf, (const void __user *)arg); + default: return -ENOTTY; } diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index aa08fbb0c26c..3a6acfda1423 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -674,6 +674,9 @@ int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id, bool mem_cgroup_charge_dmabuf(struct dma_buf *dmabuf, struct mem_cgroup *memcg, gfp_t gfp_mask);
+bool mem_cgroup_transfer_dmabuf_charge(struct dma_buf *dmabuf, + struct mem_cgroup *new_memcg); + void __mem_cgroup_uncharge(struct folio *folio);
/** @@ -1183,6 +1186,12 @@ static inline bool mem_cgroup_charge_dmabuf(struct dma_buf *dmabuf, return true; }
+static inline bool mem_cgroup_transfer_dmabuf_charge(struct dma_buf *dmabuf, + struct mem_cgroup *new_memcg) +{ + return true; +} + static inline void mem_cgroup_uncharge(struct folio *folio) { } diff --git a/include/uapi/linux/dma-buf.h b/include/uapi/linux/dma-buf.h index e827c9d20c5d..4970a7b8e08e 100644 --- a/include/uapi/linux/dma-buf.h +++ b/include/uapi/linux/dma-buf.h @@ -168,6 +168,31 @@ struct dma_buf_import_sync_file { __s32 fd; };
+/** + * struct dma_buf_transfer_charge - Transfer a dma-buf's memcg charge + * + * Any holder of a dma-buf file descriptor can perform a + * DMA_BUF_IOCTL_XFER_CHARGE to attribute the buffer's memory + * cost to a specific cgroup, or move it from one cgroup to + * another. The ioctl charges the new cgroup before uncharging + * the old one atomically. On failure the original charge is + * preserved. If the buffer has no existing charge, a new + * charge is established. + */ +struct dma_buf_transfer_charge { + /** + * @charge_fd: File descriptor identifying the target cgroup. + * + * May be either a pidfd, in which case the charge moves to the + * memory cgroup of the referenced process, or a cgroupfd obtained + * by opening a cgroup v2 directory, in which case the charge moves + * directly to that cgroup's memory controller. + */ + __u32 charge_fd; + /** @__padding: Reserved, must be zero */ + __u32 __padding; +}; + #define DMA_BUF_BASE 'b' #define DMA_BUF_IOCTL_SYNC _IOW(DMA_BUF_BASE, 0, struct dma_buf_sync)
@@ -179,5 +204,6 @@ struct dma_buf_import_sync_file { #define DMA_BUF_SET_NAME_B _IOW(DMA_BUF_BASE, 1, __u64) #define DMA_BUF_IOCTL_EXPORT_SYNC_FILE _IOWR(DMA_BUF_BASE, 2, struct dma_buf_export_sync_file) #define DMA_BUF_IOCTL_IMPORT_SYNC_FILE _IOW(DMA_BUF_BASE, 3, struct dma_buf_import_sync_file) +#define DMA_BUF_IOCTL_XFER_CHARGE _IOW(DMA_BUF_BASE, 4, struct dma_buf_transfer_charge)
#endif diff --git a/mm/memcontrol.c b/mm/memcontrol.c index e94222237d31..d994a4b2d35e 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5688,6 +5688,46 @@ bool mem_cgroup_charge_dmabuf(struct dma_buf *dmabuf, struct mem_cgroup *memcg, return true; }
+/** + * mem_cgroup_transfer_dmabuf_charge - move a dma-buf charge + * @dmabuf: buffer to transfer charge + * @new_memcg: target memcg; caller must hold a reference + * + * Transfers the charge for @dmabuf to @new_memcg and, on + * success, stores @new_memcg in @dmabuf->memcg. + * Returns %true on success, %false if the charge against @new_memcg + * failed or a concurrent transfer raced on @dmabuf->memcg. + * On failure the caller retains its reference to @new_memcg. + */ +bool mem_cgroup_transfer_dmabuf_charge(struct dma_buf *dmabuf, + struct mem_cgroup *new_memcg) +{ + struct mem_cgroup *old_memcg = READ_ONCE(dmabuf->memcg); + unsigned int nr_pages = PAGE_ALIGN(dmabuf->size) >> PAGE_SHIFT; + + if (!mem_cgroup_disabled() && try_charge(new_memcg, GFP_KERNEL, nr_pages)) + return false; + + if (cmpxchg(&dmabuf->memcg, old_memcg, new_memcg) != old_memcg) { + if (!mem_cgroup_is_root(new_memcg)) + refill_stock(new_memcg, nr_pages); + return false; + } + + mod_memcg_state(new_memcg, MEMCG_DMABUF, nr_pages); + + if (old_memcg) { + /* Uncharge old cgroup */ + mod_memcg_state(old_memcg, MEMCG_DMABUF, -nr_pages); + if (!mem_cgroup_is_root(old_memcg)) + refill_stock(old_memcg, nr_pages); + /* unref from buffer - buffer keeps new ref to new_memcg */ + mem_cgroup_put(old_memcg); + } + + return true; +} + /** * mem_cgroup_uncharge_dmabuf - uncharge a dma-buf from its memcg * @dmabuf: buffer to uncharge