Allow not to charge memory used by bpf progs. This includes the memory used by bpf progs itself, auxxiliary data, statistics and bpf line info.
Signed-off-by: Yafang Shao laoar.shao@gmail.com --- kernel/bpf/core.c | 12 ++++++++++-- kernel/bpf/syscall.c | 6 ++++-- 2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 0f68b8203c18..7aa750e8bd7d 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -144,12 +144,17 @@ EXPORT_SYMBOL_GPL(bpf_prog_alloc);
int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) { + gfp_t gfp_flags = GFP_KERNEL | __GFP_NOWARN; + if (!prog->aux->nr_linfo || !prog->jit_requested) return 0;
+ if (!prog->aux->no_charge) + gfp_flags |= __GFP_ACCOUNT; + prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo, sizeof(*prog->aux->jited_linfo), - GFP_KERNEL_ACCOUNT | __GFP_NOWARN); + gfp_flags); if (!prog->aux->jited_linfo) return -ENOMEM;
@@ -224,7 +229,7 @@ void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, gfp_t gfp_extra_flags) { - gfp_t gfp_flags = GFP_KERNEL_ACCOUNT | __GFP_ZERO | gfp_extra_flags; + gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; struct bpf_prog *fp; u32 pages;
@@ -233,6 +238,9 @@ struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, if (pages <= fp_old->pages) return fp_old;
+ if (!fp_old->aux->no_charge) + gfp_flags |= __GFP_ACCOUNT; + fp = __vmalloc(size, gfp_flags); if (fp) { memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index fdfbb4d0d5e0..e386b549fafc 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2218,9 +2218,10 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr) enum bpf_prog_type type = attr->prog_type; struct bpf_prog *prog, *dst_prog = NULL; struct btf *attach_btf = NULL; - int err; + gfp_t gfp_flags = GFP_USER; char license[128]; bool is_gpl; + int err;
if (CHECK_ATTR(BPF_PROG_LOAD)) return -EINVAL; @@ -2305,7 +2306,8 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr) }
/* plain bpf_prog allocation */ - prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER | __GFP_ACCOUNT); + prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), + prog_flags_no_charge(gfp_flags, attr)); if (!prog) { if (dst_prog) bpf_prog_put(dst_prog);