From: Abhishek Dubey adubey@linux.ibm.com
The modified prologue/epilogue generation code now enables exception-callback to use the stack frame of the program marked as exception boundary, where callee saved registers are stored.
As per ppc64 ABIv2 documentation[1], r14-r31 are callee saved registers. BPF programs on ppc64 already saves r26-r31 registers. Saving the remaining set of callee saved registers(r14-r25) is handled in the next patch.
[1] https://ftp.rtems.org/pub/rtems/people/sebh/ABI64BitOpenPOWERv1.1_16July2015...
Following is exceptions selftest result on ppc64le:
# ./test_progs -t exceptions #100/1 exceptions/exception_throw_always_1:OK #100/2 exceptions/exception_throw_always_2:OK #100/3 exceptions/exception_throw_unwind_1:OK #100/4 exceptions/exception_throw_unwind_2:OK #100/5 exceptions/exception_throw_default:OK #100/6 exceptions/exception_throw_default_value:OK #100/7 exceptions/exception_tail_call:OK #100/8 exceptions/exception_ext:OK #100/9 exceptions/exception_ext_mod_cb_runtime:OK #100/10 exceptions/exception_throw_subprog:OK #100/11 exceptions/exception_assert_nz_gfunc:OK #100/12 exceptions/exception_assert_zero_gfunc:OK #100/13 exceptions/exception_assert_neg_gfunc:OK #100/14 exceptions/exception_assert_pos_gfunc:OK #100/15 exceptions/exception_assert_negeq_gfunc:OK #100/16 exceptions/exception_assert_poseq_gfunc:OK #100/17 exceptions/exception_assert_nz_gfunc_with:OK #100/18 exceptions/exception_assert_zero_gfunc_with:OK #100/19 exceptions/exception_assert_neg_gfunc_with:OK #100/20 exceptions/exception_assert_pos_gfunc_with:OK #100/21 exceptions/exception_assert_negeq_gfunc_with:OK #100/22 exceptions/exception_assert_poseq_gfunc_with:OK #100/23 exceptions/exception_bad_assert_nz_gfunc:OK #100/24 exceptions/exception_bad_assert_zero_gfunc:OK #100/25 exceptions/exception_bad_assert_neg_gfunc:OK #100/26 exceptions/exception_bad_assert_pos_gfunc:OK #100/27 exceptions/exception_bad_assert_negeq_gfunc:OK #100/28 exceptions/exception_bad_assert_poseq_gfunc:OK #100/29 exceptions/exception_bad_assert_nz_gfunc_with:OK #100/30 exceptions/exception_bad_assert_zero_gfunc_with:OK #100/31 exceptions/exception_bad_assert_neg_gfunc_with:OK #100/32 exceptions/exception_bad_assert_pos_gfunc_with:OK #100/33 exceptions/exception_bad_assert_negeq_gfunc_with:OK #100/34 exceptions/exception_bad_assert_poseq_gfunc_with:OK #100/35 exceptions/exception_assert_range:OK #100/36 exceptions/exception_assert_range_with:OK #100/37 exceptions/exception_bad_assert_range:OK #100/38 exceptions/exception_bad_assert_range_with:OK #100/39 exceptions/non-throwing fentry -> exception_cb:OK #100/40 exceptions/throwing fentry -> exception_cb:OK #100/41 exceptions/non-throwing fexit -> exception_cb:OK #100/42 exceptions/throwing fexit -> exception_cb:OK #100/43 exceptions/throwing extension (with custom cb) -> exception_cb:OK #100/44 exceptions/throwing extension -> global func in exception_cb:OK #100/45 exceptions/exception_ext_mod_cb_runtime:OK #100/46 exceptions/throwing extension (with custom cb) -> global func in exception_cb:OK #100/47 exceptions/exception_ext:OK #100/48 exceptions/non-throwing fentry -> non-throwing subprog:OK #100/49 exceptions/throwing fentry -> non-throwing subprog:OK #100/50 exceptions/non-throwing fentry -> throwing subprog:OK #100/51 exceptions/throwing fentry -> throwing subprog:OK #100/52 exceptions/non-throwing fexit -> non-throwing subprog:OK #100/53 exceptions/throwing fexit -> non-throwing subprog:OK #100/54 exceptions/non-throwing fexit -> throwing subprog:OK #100/55 exceptions/throwing fexit -> throwing subprog:OK #100/56 exceptions/non-throwing fmod_ret -> non-throwing subprog:OK #100/57 exceptions/non-throwing fmod_ret -> non-throwing global subprog:OK #100/58 exceptions/non-throwing extension -> non-throwing subprog:OK #100/59 exceptions/non-throwing extension -> throwing subprog:OK #100/60 exceptions/non-throwing extension -> non-throwing subprog:OK #100/61 exceptions/non-throwing extension -> throwing global subprog:OK #100/62 exceptions/throwing extension -> throwing global subprog:OK #100/63 exceptions/throwing extension -> non-throwing global subprog:OK #100/64 exceptions/non-throwing extension -> main subprog:OK #100/65 exceptions/throwing extension -> main subprog:OK #100/66 exceptions/reject_exception_cb_type_1:OK #100/67 exceptions/reject_exception_cb_type_2:OK #100/68 exceptions/reject_exception_cb_type_3:OK #100/69 exceptions/reject_exception_cb_type_4:OK #100/70 exceptions/reject_async_callback_throw:OK #100/71 exceptions/reject_with_lock:OK #100/72 exceptions/reject_subprog_with_lock:OK #100/73 exceptions/reject_with_rcu_read_lock:OK #100/74 exceptions/reject_subprog_with_rcu_read_lock:OK #100/75 exceptions/reject_with_rbtree_add_throw:OK #100/76 exceptions/reject_with_reference:OK #100/77 exceptions/reject_with_cb_reference:OK #100/78 exceptions/reject_with_cb:OK #100/79 exceptions/reject_with_subprog_reference:OK #100/80 exceptions/reject_throwing_exception_cb:OK #100/81 exceptions/reject_exception_cb_call_global_func:OK #100/82 exceptions/reject_exception_cb_call_static_func:OK #100/83 exceptions/reject_multiple_exception_cb:OK #100/84 exceptions/reject_exception_throw_cb:OK #100/85 exceptions/reject_exception_throw_cb_diff:OK #100/86 exceptions/reject_set_exception_cb_bad_ret1:OK #100/87 exceptions/reject_set_exception_cb_bad_ret2:OK #100/88 exceptions/check_assert_eq_int_min:OK #100/89 exceptions/check_assert_eq_int_max:OK #100/90 exceptions/check_assert_eq_zero:OK #100/91 exceptions/check_assert_eq_llong_min:OK #100/92 exceptions/check_assert_eq_llong_max:OK #100/93 exceptions/check_assert_lt_pos:OK #100/94 exceptions/check_assert_lt_zero:OK #100/95 exceptions/check_assert_lt_neg:OK #100/96 exceptions/check_assert_le_pos:OK #100/97 exceptions/check_assert_le_zero:OK #100/98 exceptions/check_assert_le_neg:OK #100/99 exceptions/check_assert_gt_pos:OK #100/100 exceptions/check_assert_gt_zero:OK #100/101 exceptions/check_assert_gt_neg:OK #100/102 exceptions/check_assert_ge_pos:OK #100/103 exceptions/check_assert_ge_zero:OK #100/104 exceptions/check_assert_ge_neg:OK #100/105 exceptions/check_assert_range_s64:OK #100/106 exceptions/check_assert_range_u64:OK #100/107 exceptions/check_assert_single_range_s64:OK #100/108 exceptions/check_assert_single_range_u64:OK #100/109 exceptions/check_assert_generic:OK #100/110 exceptions/check_assert_with_return:OK #100 exceptions:OK Summary: 1/110 PASSED, 0 SKIPPED, 0 FAILED
Signed-off-by: Abhishek Dubey adubey@linux.ibm.com --- arch/powerpc/net/bpf_jit.h | 2 ++ arch/powerpc/net/bpf_jit_comp.c | 7 ++++ arch/powerpc/net/bpf_jit_comp64.c | 53 +++++++++++++++++++++---------- 3 files changed, 45 insertions(+), 17 deletions(-)
diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h index 98e8b1f9c2f9..b9316780a501 100644 --- a/arch/powerpc/net/bpf_jit.h +++ b/arch/powerpc/net/bpf_jit.h @@ -177,6 +177,8 @@ struct codegen_context { u64 arena_vm_start; u64 user_vm_start; bool is_subprog; + bool exception_boundary; + bool exception_cb; };
#define bpf_to_ppc(r) (ctx->b2p[r]) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index b09d294084d4..3c030a7d8e73 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -207,6 +207,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); cgctx.is_subprog = bpf_is_subprog(fp); + cgctx.exception_boundary = fp->aux->exception_boundary; + cgctx.exception_cb = fp->aux->exception_cb;
/* Scouting faux-generate pass 0 */ if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { @@ -436,6 +438,11 @@ void bpf_jit_free(struct bpf_prog *fp) bpf_prog_unlock_free(fp); }
+bool bpf_jit_supports_exceptions(void) +{ + return IS_ENABLED(CONFIG_PPC64); +} + bool bpf_jit_supports_subprog_tailcalls(void) { return IS_ENABLED(CONFIG_PPC64); diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 0f3af67914d6..5ec8e3654098 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -85,7 +85,9 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx) * - the bpf program uses its stack area * The latter condition is deduced from the usage of BPF_REG_FP */ - return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP)); + return ctx->seen & SEEN_FUNC || + bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP)) || + ctx->exception_cb; }
/* @@ -180,23 +182,32 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) EMIT(PPC_RAW_STDU(_R1, _R1, -(BPF_PPC_STACKFRAME + ctx->stack_size))); }
- /* - * Back up non-volatile regs -- BPF registers 6-10 - * If we haven't created our own stack frame, we save these - * in the protected zone below the previous stack frame - */ - for (i = BPF_REG_6; i <= BPF_REG_10; i++) - if (bpf_is_seen_register(ctx, bpf_to_ppc(i))) - EMIT(PPC_RAW_STD(bpf_to_ppc(i), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i)))); + if (!ctx->exception_cb) { + /* + * Back up non-volatile regs -- BPF registers 6-10 + * If we haven't created our own stack frame, we save these + * in the protected zone below the previous stack frame + */ + for (i = BPF_REG_6; i <= BPF_REG_10; i++) + if (ctx->exception_boundary || bpf_is_seen_register(ctx, bpf_to_ppc(i))) + EMIT(PPC_RAW_STD(bpf_to_ppc(i), _R1, + bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i))));
- if (ctx->arena_vm_start) - EMIT(PPC_RAW_STD(bpf_to_ppc(ARENA_VM_START), _R1, + if (ctx->exception_boundary || ctx->arena_vm_start) + EMIT(PPC_RAW_STD(bpf_to_ppc(ARENA_VM_START), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(ARENA_VM_START))));
- /* Setup frame pointer to point to the bpf stack area */ - if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP))) - EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1, + /* Setup frame pointer to point to the bpf stack area */ + if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP))) + EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1, STACK_FRAME_MIN_SIZE + ctx->stack_size)); + } else { + /* + * Exception callback receives Frame Pointer of main + * program as third arg + */ + EMIT(PPC_RAW_MR(_R1, _R5)); + }
if (ctx->arena_vm_start) PPC_LI64(bpf_to_ppc(ARENA_VM_START), ctx->arena_vm_start); @@ -208,17 +219,25 @@ static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx
/* Restore NVRs */ for (i = BPF_REG_6; i <= BPF_REG_10; i++) - if (bpf_is_seen_register(ctx, bpf_to_ppc(i))) + if (ctx->exception_cb || bpf_is_seen_register(ctx, bpf_to_ppc(i))) EMIT(PPC_RAW_LD(bpf_to_ppc(i), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i))));
- if (ctx->arena_vm_start) + if (ctx->exception_cb || ctx->arena_vm_start) EMIT(PPC_RAW_LD(bpf_to_ppc(ARENA_VM_START), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(ARENA_VM_START))));
+ if (ctx->exception_cb) { + /* + * LR value from boundary-frame is received as second parameter + * in exception callback. + */ + EMIT(PPC_RAW_MTLR(_R4)); + } + /* Tear down our stack frame */ if (bpf_has_stack_frame(ctx)) { EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME + ctx->stack_size)); - if (ctx->seen & SEEN_FUNC) { + if (ctx->seen & SEEN_FUNC || ctx->exception_cb) { EMIT(PPC_RAW_LD(_R0, _R1, PPC_LR_STKOFF)); EMIT(PPC_RAW_MTLR(_R0)); }