From: Abhishek Dubey adubey@linux.ibm.com
The trampoline mechanism sets up its own stack frame and an additional dummy frame. We need to have additional JIT instructions handling tailcall dereferencing in the trampoline's context.
We don't add the two stack frames pointed above, rather add space for conventional 'non-volatile register save area' and tail_call_info in trampoline's frame for ppc64. This makes the trampoline's frame consistent with layout of all other frames.
Signed-off-by: Abhishek Dubey adubey@linux.ibm.com --- arch/powerpc/net/bpf_jit_comp.c | 48 ++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 6 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 069a8822c30d..4aaa0a287a45 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -606,15 +606,42 @@ static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context return 0; }
-static void bpf_trampoline_setup_tail_call_cnt(u32 *image, struct codegen_context *ctx, - int func_frame_offset, int r4_off) +/* + * Refer the label 'Generated stack layout' in this file for actual stack + * layout during trampoline invocation. + * + * Refer __arch_prepare_bpf_trampoline() for stack component details. + * + * The tailcall count/reference is present in caller's stack frame. Its required + * to copy the content of tail_call_info before calling the actual function + * to which the trampoline is attached. + * + */ + +static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_context *ctx, + int func_frame_offset, + int bpf_dummy_frame_size, int r4_off) { if (IS_ENABLED(CONFIG_PPC64)) { /* See bpf_jit_stack_tailcallinfo_offset() */ - int tailcallcnt_offset = 7 * 8; + int tailcallinfo_offset = BPF_PPC_STACK_SAVE + SZL; + /* + * func_frame_offset = + * bpf_dummy_frame_size + trampoline_frame_size + */ + EMIT(PPC_RAW_LD(_R4, _R1, func_frame_offset)); + EMIT(PPC_RAW_LD(_R3, _R4, -tailcallinfo_offset)); + + /* + * Setting the tail_call_info in trampoline's frame + * depending on if previous frame had value or reference. + */ + EMIT(PPC_RAW_CMPLWI(_R3, MAX_TAIL_CALL_CNT)); + PPC_COND_BRANCH(COND_GT, CTX_NIA(ctx) + 8); + EMIT(PPC_RAW_ADDI(_R3, _R4, bpf_jit_stack_tailcallinfo_offset(ctx))); + EMIT(PPC_RAW_STL(_R3, _R1, func_frame_offset + - bpf_dummy_frame_size - tailcallinfo_offset));
- EMIT(PPC_RAW_LL(_R3, _R1, func_frame_offset - tailcallcnt_offset)); - EMIT(PPC_RAW_STL(_R3, _R1, -tailcallcnt_offset)); } else { /* See bpf_jit_stack_offsetof() and BPF_PPC_TC */ EMIT(PPC_RAW_LL(_R4, _R1, r4_off)); @@ -721,6 +748,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * [ r0 save (32-bit) ] | * dummy frame for unwind [ back chain 1 ] -- * [ padding ] align stack frame + * [ r26..r31 ] nvr save : BPF_PPC_STACK_SAVE + * [ tail_call_info ] non optional - 64-bit powerpc * r4_off [ r4 (tailcallcnt) ] optional - 32-bit powerpc * alt_lr_off [ real lr (ool stub)] optional - actual lr * [ r26 ] @@ -801,6 +830,12 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } }
+ /* Room for 64-bit tail_call_cnt */ + bpf_frame_size += SZL; + + /* Room for nvr save area */ + bpf_frame_size += BPF_PPC_STACK_SAVE; + /* Padding to align stack frame, if any */ bpf_frame_size = round_up(bpf_frame_size, SZL * 2);
@@ -902,7 +937,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
/* Replicate tail_call_cnt before calling the original BPF prog */ if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) - bpf_trampoline_setup_tail_call_cnt(image, ctx, func_frame_offset, r4_off); + bpf_trampoline_setup_tail_call_info(image, ctx, func_frame_offset, + bpf_dummy_frame_size, r4_off);
/* Restore args */ bpf_trampoline_restore_args_stack(image, ctx, func_frame_offset, nr_regs, regs_off);