This patch series introduces support for the PROBE_MEM32, bpf_addr_space_cast and PROBE_ATOMIC instructions in the powerpc BPF JIT, facilitating the implementation of BPF arena and arena atomics.
All selftests related to bpf_arena, bpf_arena_atomic(except load_acquire/store_release) enablement are passing:
# ./test_progs -t arena_list #5/1 arena_list/arena_list_1:OK #5/2 arena_list/arena_list_1000:OK #5 arena_list:OK Summary: 1/2 PASSED, 0 SKIPPED, 0 FAILED
# ./test_progs -t arena_htab #4/1 arena_htab/arena_htab_llvm:OK #4/2 arena_htab/arena_htab_asm:OK #4 arena_htab:OK Summary: 1/2 PASSED, 0 SKIPPED, 0 FAILED
# ./test_progs -t verifier_arena #464/1 verifier_arena/basic_alloc1:OK #464/2 verifier_arena/basic_alloc2:OK #464/3 verifier_arena/basic_alloc3:OK #464/4 verifier_arena/iter_maps1:OK #464/5 verifier_arena/iter_maps2:OK #464/6 verifier_arena/iter_maps3:OK #464 verifier_arena:OK #465/1 verifier_arena_large/big_alloc1:OK #465/2 verifier_arena_large/big_alloc2:OK #465 verifier_arena_large:OK Summary: 2/8 PASSED, 0 SKIPPED, 0 FAILED
# ./test_progs -t arena_atomics #3/1 arena_atomics/add:OK #3/2 arena_atomics/sub:OK #3/3 arena_atomics/and:OK #3/4 arena_atomics/or:OK #3/5 arena_atomics/xor:OK #3/6 arena_atomics/cmpxchg:OK #3/7 arena_atomics/xchg:OK #3/8 arena_atomics/uaf:OK #3/9 arena_atomics/load_acquire:SKIP #3/10 arena_atomics/store_release:SKIP #3 arena_atomics:OK (SKIP: 2/10) Summary: 1/8 PASSED, 2 SKIPPED, 0 FAILED
Changes since v2: * Dropped arena_spin_lock selftest fix patch from the patchset as it has to go via bpf-next while these changes will go via powerpc tree.
v2:https://lore.kernel.org/all/20250829165135.1273071-1-skb99@linux.ibm.com/
Changes since v1:
Addressed comments from Chris: * Squashed introduction of bpf_jit_emit_probe_mem_store() and its usage in one patch. * Defined and used PPC_RAW_RLDICL_DOT to avoid the CMPDI. * Removed conditional statement for fixup[0] = PPC_RAW_LI(dst_reg, 0); * Indicated this change is limited to powerpc64 in subject.
Addressed comments from Alexei: * Removed skel->rodata->nr_cpus = get_nprocs() and its usage to get currently online cpus(as it needs to be updated from userspace).
Addressed comments from Hari: * Updated the bpf jit stack layout and associated macros to accommodate new NVR.
v1:https://lore.kernel.org/all/20250805062747.3479221-1-skb99@linux.ibm.com/
Saket Kumar Bhaskar (4): powerpc64/bpf: Implement PROBE_MEM32 pseudo instructions powerpc64/bpf: Implement bpf_addr_space_cast instruction powerpc64/bpf: Introduce bpf_jit_emit_atomic_ops() to emit atomic instructions powerpc64/bpf: Implement PROBE_ATOMIC instructions
arch/powerpc/include/asm/ppc-opcode.h | 1 + arch/powerpc/net/bpf_jit.h | 6 +- arch/powerpc/net/bpf_jit_comp.c | 32 +- arch/powerpc/net/bpf_jit_comp32.c | 2 +- arch/powerpc/net/bpf_jit_comp64.c | 401 +++++++++++++++++++------- 5 files changed, 330 insertions(+), 112 deletions(-)
Add support for [LDX | STX | ST], PROBE_MEM32, [B | H | W | DW] instructions. They are similar to PROBE_MEM instructions with the following differences: - PROBE_MEM32 supports store. - PROBE_MEM32 relies on the verifier to clear upper 32-bit of the src/dst register - PROBE_MEM32 adds 64-bit kern_vm_start address (which is stored in _R26 in the prologue). Due to bpf_arena constructions such _R26 + reg + off16 access is guaranteed to be within arena virtual range, so no address check at run-time. - PROBE_MEM32 allows STX and ST. If they fault the store is a nop. When LDX faults the destination register is zeroed.
To support these on powerpc, we do tmp1 = _R26 + src/dst reg and then use tmp1 as the new src/dst register. This allows us to reuse most of the code for normal [LDX | STX | ST].
Additionally, bpf_jit_emit_probe_mem_store() is introduced to emit instructions for storing memory values depending on the size (byte, halfword, word, doubleword).
Stack layout is adjusted to introduce a new NVR (_R26) and to make BPF_PPC_STACKFRAME quadword aligned (local_tmp_var is increased by 8 bytes).
Reviewed-by: Hari Bathini hbathini@linux.ibm.com Tested-by: Venkat Rao Bagalkote venkat88@linux.ibm.com Signed-off-by: Saket Kumar Bhaskar skb99@linux.ibm.com --- arch/powerpc/net/bpf_jit.h | 5 +- arch/powerpc/net/bpf_jit_comp.c | 10 +- arch/powerpc/net/bpf_jit_comp32.c | 2 +- arch/powerpc/net/bpf_jit_comp64.c | 162 ++++++++++++++++++++++++++---- 4 files changed, 155 insertions(+), 24 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h index 4c26912c2e3c..2d095a873305 100644 --- a/arch/powerpc/net/bpf_jit.h +++ b/arch/powerpc/net/bpf_jit.h @@ -161,9 +161,10 @@ struct codegen_context { unsigned int seen; unsigned int idx; unsigned int stack_size; - int b2p[MAX_BPF_JIT_REG + 2]; + int b2p[MAX_BPF_JIT_REG + 3]; unsigned int exentry_idx; unsigned int alt_exit_addr; + u64 arena_vm_start; };
#define bpf_to_ppc(r) (ctx->b2p[r]) @@ -201,7 +202,7 @@ int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg,
int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass, struct codegen_context *ctx, int insn_idx, - int jmp_off, int dst_reg); + int jmp_off, int dst_reg, u32 code);
#endif
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index c0684733e9d6..7d070232159f 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -204,6 +204,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
/* Make sure that the stack is quadword aligned. */ cgctx.stack_size = round_up(fp->aux->stack_depth, 16); + cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena);
/* Scouting faux-generate pass 0 */ if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { @@ -326,7 +327,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) */ int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass, struct codegen_context *ctx, int insn_idx, int jmp_off, - int dst_reg) + int dst_reg, u32 code) { off_t offset; unsigned long pc; @@ -355,6 +356,9 @@ int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass (ctx->exentry_idx * BPF_FIXUP_LEN * 4);
fixup[0] = PPC_RAW_LI(dst_reg, 0); + if (BPF_CLASS(code) == BPF_ST || BPF_CLASS(code) == BPF_STX) + fixup[0] = PPC_RAW_NOP(); + if (IS_ENABLED(CONFIG_PPC32)) fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */
@@ -579,7 +583,7 @@ static void bpf_trampoline_setup_tail_call_cnt(u32 *image, struct codegen_contex { if (IS_ENABLED(CONFIG_PPC64)) { /* See bpf_jit_stack_tailcallcnt() */ - int tailcallcnt_offset = 6 * 8; + int tailcallcnt_offset = 7 * 8;
EMIT(PPC_RAW_LL(_R3, _R1, func_frame_offset - tailcallcnt_offset)); EMIT(PPC_RAW_STL(_R3, _R1, -tailcallcnt_offset)); @@ -594,7 +598,7 @@ static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_cont { if (IS_ENABLED(CONFIG_PPC64)) { /* See bpf_jit_stack_tailcallcnt() */ - int tailcallcnt_offset = 6 * 8; + int tailcallcnt_offset = 7 * 8;
EMIT(PPC_RAW_LL(_R3, _R1, -tailcallcnt_offset)); EMIT(PPC_RAW_STL(_R3, _R1, func_frame_offset - tailcallcnt_offset)); diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c index 0aace304dfe1..3087e744fb25 100644 --- a/arch/powerpc/net/bpf_jit_comp32.c +++ b/arch/powerpc/net/bpf_jit_comp32.c @@ -1087,7 +1087,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code }
ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, insn_idx, - jmp_off, dst_reg); + jmp_off, dst_reg, code); if (ret) return ret; } diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 025524378443..569619f1b31c 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -25,18 +25,18 @@ * with our redzone usage. * * [ prev sp ] <------------- - * [ nv gpr save area ] 5*8 | + * [ nv gpr save area ] 6*8 | * [ tail_call_cnt ] 8 | - * [ local_tmp_var ] 16 | + * [ local_tmp_var ] 24 | * fp (r31) --> [ ebpf stack space ] upto 512 | * [ frame header ] 32/112 | * sp (r1) ---> [ stack pointer ] -------------- */
/* for gpr non volatile registers BPG_REG_6 to 10 */ -#define BPF_PPC_STACK_SAVE (5*8) +#define BPF_PPC_STACK_SAVE (6*8) /* for bpf JIT code internal usage */ -#define BPF_PPC_STACK_LOCALS 24 +#define BPF_PPC_STACK_LOCALS 32 /* stack frame excluding BPF stack, ensure this is quadword aligned */ #define BPF_PPC_STACKFRAME (STACK_FRAME_MIN_SIZE + \ BPF_PPC_STACK_LOCALS + BPF_PPC_STACK_SAVE) @@ -44,6 +44,7 @@ /* BPF register usage */ #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) +#define ARENA_VM_START (MAX_BPF_JIT_REG + 2)
/* BPF to ppc register mappings */ void bpf_jit_init_reg_mapping(struct codegen_context *ctx) @@ -67,10 +68,12 @@ void bpf_jit_init_reg_mapping(struct codegen_context *ctx) ctx->b2p[BPF_REG_AX] = _R12; ctx->b2p[TMP_REG_1] = _R9; ctx->b2p[TMP_REG_2] = _R10; + /* non volatile register for kern_vm_start address */ + ctx->b2p[ARENA_VM_START] = _R26; }
-/* PPC NVR range -- update this if we ever use NVRs below r27 */ -#define BPF_PPC_NVR_MIN _R27 +/* PPC NVR range -- update this if we ever use NVRs below r26 */ +#define BPF_PPC_NVR_MIN _R26
static inline bool bpf_has_stack_frame(struct codegen_context *ctx) { @@ -89,9 +92,9 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx) * [ prev sp ] <------------- * [ ... ] | * sp (r1) ---> [ stack pointer ] -------------- - * [ nv gpr save area ] 5*8 + * [ nv gpr save area ] 6*8 * [ tail_call_cnt ] 8 - * [ local_tmp_var ] 16 + * [ local_tmp_var ] 24 * [ unused red zone ] 224 */ static int bpf_jit_stack_local(struct codegen_context *ctx) @@ -99,12 +102,12 @@ static int bpf_jit_stack_local(struct codegen_context *ctx) if (bpf_has_stack_frame(ctx)) return STACK_FRAME_MIN_SIZE + ctx->stack_size; else - return -(BPF_PPC_STACK_SAVE + 24); + return -(BPF_PPC_STACK_SAVE + 32); }
static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx) { - return bpf_jit_stack_local(ctx) + 16; + return bpf_jit_stack_local(ctx) + 24; }
static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) @@ -170,10 +173,17 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) if (bpf_is_seen_register(ctx, bpf_to_ppc(i))) EMIT(PPC_RAW_STD(bpf_to_ppc(i), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i))));
+ if (ctx->arena_vm_start) + EMIT(PPC_RAW_STD(bpf_to_ppc(ARENA_VM_START), _R1, + bpf_jit_stack_offsetof(ctx, bpf_to_ppc(ARENA_VM_START)))); + /* Setup frame pointer to point to the bpf stack area */ if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP))) EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1, STACK_FRAME_MIN_SIZE + ctx->stack_size)); + + if (ctx->arena_vm_start) + PPC_LI64(bpf_to_ppc(ARENA_VM_START), ctx->arena_vm_start); }
static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx) @@ -185,6 +195,10 @@ static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx if (bpf_is_seen_register(ctx, bpf_to_ppc(i))) EMIT(PPC_RAW_LD(bpf_to_ppc(i), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i))));
+ if (ctx->arena_vm_start) + EMIT(PPC_RAW_LD(bpf_to_ppc(ARENA_VM_START), _R1, + bpf_jit_stack_offsetof(ctx, bpf_to_ppc(ARENA_VM_START)))); + /* Tear down our stack frame */ if (bpf_has_stack_frame(ctx)) { EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME + ctx->stack_size)); @@ -396,11 +410,11 @@ void bpf_stf_barrier(void); asm ( " .global bpf_stf_barrier ;" " bpf_stf_barrier: ;" -" std 21,-64(1) ;" -" std 22,-56(1) ;" +" std 21,-80(1) ;" +" std 22,-72(1) ;" " sync ;" -" ld 21,-64(1) ;" -" ld 22,-56(1) ;" +" ld 21,-80(1) ;" +" ld 22,-72(1) ;" " ori 31,31,0 ;" " .rept 14 ;" " b 1f ;" @@ -409,6 +423,36 @@ asm ( " blr ;" );
+static int bpf_jit_emit_probe_mem_store(struct codegen_context *ctx, u32 src_reg, s16 off, + u32 code, u32 *image) +{ + u32 tmp1_reg = bpf_to_ppc(TMP_REG_1); + u32 tmp2_reg = bpf_to_ppc(TMP_REG_2); + + switch (BPF_SIZE(code)) { + case BPF_B: + EMIT(PPC_RAW_STB(src_reg, tmp1_reg, off)); + break; + case BPF_H: + EMIT(PPC_RAW_STH(src_reg, tmp1_reg, off)); + break; + case BPF_W: + EMIT(PPC_RAW_STW(src_reg, tmp1_reg, off)); + break; + case BPF_DW: + if (off % 4) { + EMIT(PPC_RAW_LI(tmp2_reg, off)); + EMIT(PPC_RAW_STDX(src_reg, tmp1_reg, tmp2_reg)); + } else { + EMIT(PPC_RAW_STD(src_reg, tmp1_reg, off)); + } + break; + default: + return -EINVAL; + } + return 0; +} + static int emit_atomic_ld_st(const struct bpf_insn insn, struct codegen_context *ctx, u32 *image) { u32 code = insn.code; @@ -960,6 +1004,50 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code } break;
+ case BPF_STX | BPF_PROBE_MEM32 | BPF_B: + case BPF_STX | BPF_PROBE_MEM32 | BPF_H: + case BPF_STX | BPF_PROBE_MEM32 | BPF_W: + case BPF_STX | BPF_PROBE_MEM32 | BPF_DW: + + EMIT(PPC_RAW_ADD(tmp1_reg, dst_reg, bpf_to_ppc(ARENA_VM_START))); + + ret = bpf_jit_emit_probe_mem_store(ctx, src_reg, off, code, image); + if (ret) + return ret; + + ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, + ctx->idx - 1, 4, -1, code); + if (ret) + return ret; + + break; + + case BPF_ST | BPF_PROBE_MEM32 | BPF_B: + case BPF_ST | BPF_PROBE_MEM32 | BPF_H: + case BPF_ST | BPF_PROBE_MEM32 | BPF_W: + case BPF_ST | BPF_PROBE_MEM32 | BPF_DW: + + EMIT(PPC_RAW_ADD(tmp1_reg, dst_reg, bpf_to_ppc(ARENA_VM_START))); + + if (BPF_SIZE(code) == BPF_W || BPF_SIZE(code) == BPF_DW) { + PPC_LI32(tmp2_reg, imm); + src_reg = tmp2_reg; + } else { + EMIT(PPC_RAW_LI(tmp2_reg, imm)); + src_reg = tmp2_reg; + } + + ret = bpf_jit_emit_probe_mem_store(ctx, src_reg, off, code, image); + if (ret) + return ret; + + ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, + ctx->idx - 1, 4, -1, code); + if (ret) + return ret; + + break; + /* * BPF_STX ATOMIC (atomic ops) */ @@ -1112,9 +1200,10 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code * Check if 'off' is word aligned for BPF_DW, because * we might generate two instructions. */ - if ((BPF_SIZE(code) == BPF_DW || - (BPF_SIZE(code) == BPF_B && BPF_MODE(code) == BPF_PROBE_MEMSX)) && - (off & 3)) + if ((BPF_SIZE(code) == BPF_DW && (off & 3)) || + (BPF_SIZE(code) == BPF_B && + BPF_MODE(code) == BPF_PROBE_MEMSX) || + (BPF_SIZE(code) == BPF_B && BPF_MODE(code) == BPF_MEMSX)) PPC_JMP((ctx->idx + 3) * 4); else PPC_JMP((ctx->idx + 2) * 4); @@ -1160,12 +1249,49 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
if (BPF_MODE(code) == BPF_PROBE_MEM) { ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, - ctx->idx - 1, 4, dst_reg); + ctx->idx - 1, 4, dst_reg, code); if (ret) return ret; } break;
+ /* dst = *(u64 *)(ul) (src + ARENA_VM_START + off) */ + case BPF_LDX | BPF_PROBE_MEM32 | BPF_B: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_H: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_W: + case BPF_LDX | BPF_PROBE_MEM32 | BPF_DW: + + EMIT(PPC_RAW_ADD(tmp1_reg, src_reg, bpf_to_ppc(ARENA_VM_START))); + + switch (size) { + case BPF_B: + EMIT(PPC_RAW_LBZ(dst_reg, tmp1_reg, off)); + break; + case BPF_H: + EMIT(PPC_RAW_LHZ(dst_reg, tmp1_reg, off)); + break; + case BPF_W: + EMIT(PPC_RAW_LWZ(dst_reg, tmp1_reg, off)); + break; + case BPF_DW: + if (off % 4) { + EMIT(PPC_RAW_LI(tmp2_reg, off)); + EMIT(PPC_RAW_LDX(dst_reg, tmp1_reg, tmp2_reg)); + } else { + EMIT(PPC_RAW_LD(dst_reg, tmp1_reg, off)); + } + break; + } + + if (size != BPF_DW && insn_is_zext(&insn[i + 1])) + addrs[++i] = ctx->idx * 4; + + ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, + ctx->idx - 1, 4, dst_reg, code); + if (ret) + return ret; + break; + /* * Doubleword load * 16 byte instruction that uses two 'struct bpf_insn'
LLVM generates bpf_addr_space_cast instruction while translating pointers between native (zero) address space and __attribute__((address_space(N))). The addr_space=0 is reserved as bpf_arena address space.
rY = addr_space_cast(rX, 0, 1) is processed by the verifier and converted to normal 32-bit move: wX = wY.
rY = addr_space_cast(rX, 1, 0) : used to convert a bpf arena pointer to a pointer in the userspace vma. This has to be converted by the JIT.
PPC_RAW_RLDICL_DOT, a variant of PPC_RAW_RLDICL is introduced to set condition register as well.
Reviewed-by: Hari Bathini hbathini@linux.ibm.com Tested-by: Venkat Rao Bagalkote venkat88@linux.ibm.com Signed-off-by: Saket Kumar Bhaskar skb99@linux.ibm.com --- arch/powerpc/include/asm/ppc-opcode.h | 1 + arch/powerpc/net/bpf_jit.h | 1 + arch/powerpc/net/bpf_jit_comp.c | 6 ++++++ arch/powerpc/net/bpf_jit_comp64.c | 10 ++++++++++ 4 files changed, 18 insertions(+)
diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index 8053b24afc39..55ca49d18319 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -571,6 +571,7 @@ (0x54000001 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH(i) | __PPC_MB(mb) | __PPC_ME(me)) #define PPC_RAW_RLWIMI(d, a, i, mb, me) (0x50000000 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH(i) | __PPC_MB(mb) | __PPC_ME(me)) #define PPC_RAW_RLDICL(d, a, i, mb) (0x78000000 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH64(i) | __PPC_MB64(mb)) +#define PPC_RAW_RLDICL_DOT(d, a, i, mb) (0x78000000 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH64(i) | __PPC_MB64(mb) | 0x1) #define PPC_RAW_RLDICR(d, a, i, me) (0x78000004 | ___PPC_RA(d) | ___PPC_RS(a) | __PPC_SH64(i) | __PPC_ME64(me))
/* slwi = rlwinm Rx, Ry, n, 0, 31-n */ diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h index 2d095a873305..748e30e8b5b4 100644 --- a/arch/powerpc/net/bpf_jit.h +++ b/arch/powerpc/net/bpf_jit.h @@ -165,6 +165,7 @@ struct codegen_context { unsigned int exentry_idx; unsigned int alt_exit_addr; u64 arena_vm_start; + u64 user_vm_start; };
#define bpf_to_ppc(r) (ctx->b2p[r]) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 7d070232159f..cfa84cab0a18 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -205,6 +205,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) /* Make sure that the stack is quadword aligned. */ cgctx.stack_size = round_up(fp->aux->stack_depth, 16); cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); + cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena);
/* Scouting faux-generate pass 0 */ if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { @@ -439,6 +440,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; }
+bool bpf_jit_supports_arena(void) +{ + return IS_ENABLED(CONFIG_PPC64); +} + bool bpf_jit_supports_far_kfunc_call(void) { return IS_ENABLED(CONFIG_PPC64); diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 569619f1b31c..76efb47f02a6 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -812,6 +812,16 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code */ case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */ case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */ + + if (insn_is_cast_user(&insn[i])) { + EMIT(PPC_RAW_RLDICL_DOT(tmp1_reg, src_reg, 0, 32)); + PPC_LI64(dst_reg, (ctx->user_vm_start & 0xffffffff00000000UL)); + PPC_BCC_SHORT(COND_EQ, (ctx->idx + 2) * 4); + EMIT(PPC_RAW_OR(tmp1_reg, dst_reg, tmp1_reg)); + EMIT(PPC_RAW_MR(dst_reg, tmp1_reg)); + break; + } + if (imm == 1) { /* special mov32 for zext */ EMIT(PPC_RAW_RLWINM(dst_reg, dst_reg, 0, 0, 31));
The existing code for emitting bpf atomic instruction sequences for atomic operations such as XCHG, CMPXCHG, ADD, AND, OR, and XOR has been refactored into a reusable function, bpf_jit_emit_ppc_atomic_op(). It also computes the jump offset and tracks the instruction index for jited LDARX/LWARX to be used in case it causes a fault.
Reviewed-by: Hari Bathini hbathini@linux.ibm.com Tested-by: Venkat Rao Bagalkote venkat88@linux.ibm.com Signed-off-by: Saket Kumar Bhaskar skb99@linux.ibm.com --- arch/powerpc/net/bpf_jit_comp64.c | 203 +++++++++++++++++------------- 1 file changed, 115 insertions(+), 88 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 76efb47f02a6..cb4d1e954961 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -423,6 +423,111 @@ asm ( " blr ;" );
+static int bpf_jit_emit_atomic_ops(u32 *image, struct codegen_context *ctx, + const struct bpf_insn *insn, u32 *jmp_off, + u32 *tmp_idx, u32 *addrp) +{ + u32 tmp1_reg = bpf_to_ppc(TMP_REG_1); + u32 tmp2_reg = bpf_to_ppc(TMP_REG_2); + u32 size = BPF_SIZE(insn->code); + u32 src_reg = bpf_to_ppc(insn->src_reg); + u32 dst_reg = bpf_to_ppc(insn->dst_reg); + s32 imm = insn->imm; + + u32 save_reg = tmp2_reg; + u32 ret_reg = src_reg; + u32 fixup_idx; + + /* Get offset into TMP_REG_1 */ + EMIT(PPC_RAW_LI(tmp1_reg, insn->off)); + /* + * Enforce full ordering for operations with BPF_FETCH by emitting a 'sync' + * before and after the operation. + * + * This is a requirement in the Linux Kernel Memory Model. + * See __cmpxchg_u64() in asm/cmpxchg.h as an example. + */ + if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP)) + EMIT(PPC_RAW_SYNC()); + + *tmp_idx = ctx->idx; + + /* load value from memory into TMP_REG_2 */ + if (size == BPF_DW) + EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg, 0)); + else + EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg, 0)); + /* Save old value in _R0 */ + if (imm & BPF_FETCH) + EMIT(PPC_RAW_MR(_R0, tmp2_reg)); + + switch (imm) { + case BPF_ADD: + case BPF_ADD | BPF_FETCH: + EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg)); + break; + case BPF_AND: + case BPF_AND | BPF_FETCH: + EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg)); + break; + case BPF_OR: + case BPF_OR | BPF_FETCH: + EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg)); + break; + case BPF_XOR: + case BPF_XOR | BPF_FETCH: + EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg)); + break; + case BPF_CMPXCHG: + /* + * Return old value in BPF_REG_0 for BPF_CMPXCHG & + * in src_reg for other cases. + */ + ret_reg = bpf_to_ppc(BPF_REG_0); + + /* Compare with old value in BPF_R0 */ + if (size == BPF_DW) + EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg)); + else + EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg)); + /* Don't set if different from old value */ + PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4); + fallthrough; + case BPF_XCHG: + save_reg = src_reg; + break; + default: + return -EOPNOTSUPP; + } + + /* store new value */ + if (size == BPF_DW) + EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg, dst_reg)); + else + EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg, dst_reg)); + /* we're done if this succeeded */ + PPC_BCC_SHORT(COND_NE, *tmp_idx * 4); + fixup_idx = ctx->idx; + + if (imm & BPF_FETCH) { + /* Emit 'sync' to enforce full ordering */ + if (IS_ENABLED(CONFIG_SMP)) + EMIT(PPC_RAW_SYNC()); + EMIT(PPC_RAW_MR(ret_reg, _R0)); + /* + * Skip unnecessary zero-extension for 32-bit cmpxchg. + * For context, see commit 39491867ace5. + */ + if (size != BPF_DW && imm == BPF_CMPXCHG && + insn_is_zext(insn + 1)) + *addrp = ctx->idx * 4; + } + + *jmp_off = (fixup_idx - *tmp_idx) * 4; + + return 0; +} + static int bpf_jit_emit_probe_mem_store(struct codegen_context *ctx, u32 src_reg, s16 off, u32 code, u32 *image) { @@ -538,7 +643,6 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code u32 size = BPF_SIZE(code); u32 tmp1_reg = bpf_to_ppc(TMP_REG_1); u32 tmp2_reg = bpf_to_ppc(TMP_REG_2); - u32 save_reg, ret_reg; s16 off = insn[i].off; s32 imm = insn[i].imm; bool func_addr_fixed; @@ -546,6 +650,7 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code u64 imm64; u32 true_cond; u32 tmp_idx; + u32 jmp_off;
/* * addrs[] maps a BPF bytecode address into a real offset from @@ -1080,93 +1185,15 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code return -EOPNOTSUPP; }
- save_reg = tmp2_reg; - ret_reg = src_reg; - - /* Get offset into TMP_REG_1 */ - EMIT(PPC_RAW_LI(tmp1_reg, off)); - /* - * Enforce full ordering for operations with BPF_FETCH by emitting a 'sync' - * before and after the operation. - * - * This is a requirement in the Linux Kernel Memory Model. - * See __cmpxchg_u64() in asm/cmpxchg.h as an example. - */ - if ((imm & BPF_FETCH) && IS_ENABLED(CONFIG_SMP)) - EMIT(PPC_RAW_SYNC()); - tmp_idx = ctx->idx * 4; - /* load value from memory into TMP_REG_2 */ - if (size == BPF_DW) - EMIT(PPC_RAW_LDARX(tmp2_reg, tmp1_reg, dst_reg, 0)); - else - EMIT(PPC_RAW_LWARX(tmp2_reg, tmp1_reg, dst_reg, 0)); - - /* Save old value in _R0 */ - if (imm & BPF_FETCH) - EMIT(PPC_RAW_MR(_R0, tmp2_reg)); - - switch (imm) { - case BPF_ADD: - case BPF_ADD | BPF_FETCH: - EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg)); - break; - case BPF_AND: - case BPF_AND | BPF_FETCH: - EMIT(PPC_RAW_AND(tmp2_reg, tmp2_reg, src_reg)); - break; - case BPF_OR: - case BPF_OR | BPF_FETCH: - EMIT(PPC_RAW_OR(tmp2_reg, tmp2_reg, src_reg)); - break; - case BPF_XOR: - case BPF_XOR | BPF_FETCH: - EMIT(PPC_RAW_XOR(tmp2_reg, tmp2_reg, src_reg)); - break; - case BPF_CMPXCHG: - /* - * Return old value in BPF_REG_0 for BPF_CMPXCHG & - * in src_reg for other cases. - */ - ret_reg = bpf_to_ppc(BPF_REG_0); - - /* Compare with old value in BPF_R0 */ - if (size == BPF_DW) - EMIT(PPC_RAW_CMPD(bpf_to_ppc(BPF_REG_0), tmp2_reg)); - else - EMIT(PPC_RAW_CMPW(bpf_to_ppc(BPF_REG_0), tmp2_reg)); - /* Don't set if different from old value */ - PPC_BCC_SHORT(COND_NE, (ctx->idx + 3) * 4); - fallthrough; - case BPF_XCHG: - save_reg = src_reg; - break; - default: - pr_err_ratelimited( - "eBPF filter atomic op code %02x (@%d) unsupported\n", - code, i); - return -EOPNOTSUPP; - } - - /* store new value */ - if (size == BPF_DW) - EMIT(PPC_RAW_STDCX(save_reg, tmp1_reg, dst_reg)); - else - EMIT(PPC_RAW_STWCX(save_reg, tmp1_reg, dst_reg)); - /* we're done if this succeeded */ - PPC_BCC_SHORT(COND_NE, tmp_idx); - - if (imm & BPF_FETCH) { - /* Emit 'sync' to enforce full ordering */ - if (IS_ENABLED(CONFIG_SMP)) - EMIT(PPC_RAW_SYNC()); - EMIT(PPC_RAW_MR(ret_reg, _R0)); - /* - * Skip unnecessary zero-extension for 32-bit cmpxchg. - * For context, see commit 39491867ace5. - */ - if (size != BPF_DW && imm == BPF_CMPXCHG && - insn_is_zext(&insn[i + 1])) - addrs[++i] = ctx->idx * 4; + ret = bpf_jit_emit_atomic_ops(image, ctx, &insn[i], + &jmp_off, &tmp_idx, &addrs[i + 1]); + if (ret) { + if (ret == -EOPNOTSUPP) { + pr_err_ratelimited( + "eBPF filter atomic op code %02x (@%d) unsupported\n", + code, i); + } + return ret; } break;
powerpc supports BPF atomic operations using a loop around Load-And-Reserve(LDARX/LWARX) and Store-Conditional(STDCX/STWCX) instructions gated by sync instructions to enforce full ordering.
To implement arena_atomics, arena vm start address is added to the dst_reg to be used for both the LDARX/LWARX and STDCX/STWCX instructions. Further, an exception table entry is added for LDARX/LWARX instruction to land after the loop on fault. At the end of sequence, dst_reg is restored by subtracting arena vm start address.
bpf_jit_supports_insn() is introduced to selectively enable instruction support as in other architectures like x86 and arm64.
Reviewed-by: Hari Bathini hbathini@linux.ibm.com Tested-by: Venkat Rao Bagalkote venkat88@linux.ibm.com Signed-off-by: Saket Kumar Bhaskar skb99@linux.ibm.com --- arch/powerpc/net/bpf_jit_comp.c | 16 ++++++++++++++++ arch/powerpc/net/bpf_jit_comp64.c | 26 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+)
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index cfa84cab0a18..88ad5ba7b87f 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -450,6 +450,22 @@ bool bpf_jit_supports_far_kfunc_call(void) return IS_ENABLED(CONFIG_PPC64); }
+bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) +{ + if (!in_arena) + return true; + switch (insn->code) { + case BPF_STX | BPF_ATOMIC | BPF_H: + case BPF_STX | BPF_ATOMIC | BPF_B: + case BPF_STX | BPF_ATOMIC | BPF_W: + case BPF_STX | BPF_ATOMIC | BPF_DW: + if (bpf_atomic_is_load_store(insn)) + return false; + return IS_ENABLED(CONFIG_PPC64); + } + return true; +} + void *arch_alloc_bpf_trampoline(unsigned int size) { return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns); diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index cb4d1e954961..1fe37128c876 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -1163,6 +1163,32 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code
break;
+ /* + * BPF_STX PROBE_ATOMIC (arena atomic ops) + */ + case BPF_STX | BPF_PROBE_ATOMIC | BPF_W: + case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW: + EMIT(PPC_RAW_ADD(dst_reg, dst_reg, bpf_to_ppc(ARENA_VM_START))); + ret = bpf_jit_emit_atomic_ops(image, ctx, &insn[i], + &jmp_off, &tmp_idx, &addrs[i + 1]); + if (ret) { + if (ret == -EOPNOTSUPP) { + pr_err_ratelimited( + "eBPF filter atomic op code %02x (@%d) unsupported\n", + code, i); + } + return ret; + } + /* LDARX/LWARX should land here on exception. */ + ret = bpf_add_extable_entry(fp, image, fimage, pass, ctx, + tmp_idx, jmp_off, dst_reg, code); + if (ret) + return ret; + + /* Retrieve the dst_reg */ + EMIT(PPC_RAW_SUB(dst_reg, dst_reg, bpf_to_ppc(ARENA_VM_START))); + break; + /* * BPF_STX ATOMIC (atomic ops) */
linux-kselftest-mirror@lists.linaro.org
-
Saket Kumar Bhaskar