This patch implements arm64 specific part to support function tracers, such as function (CONFIG_FUNCTION_TRACER), function_graph (CONFIG_FUNCTION_GRAPH_TRACER) and function profiler (CONFIG_FUNCTION_PROFILER).

With 'function' tracer, all the functions in the kernel are traced with timestamps in ${sysfs}/tracing/trace. If function_graph tracer is specified, call graph is generated.

The kernel must be compiled with -pg option so that _mcount() is inserted at the beginning of functions. This function is called on every function's entry as long as tracing is enabled. In addition, function_graph tracer also needs to be able to probe function's exit. ftrace_graph_caller() & return_to_handler do this by faking link register's value to intercept function's return path.

More details on architecture specific requirements are described in Documentation/trace/ftrace-design.txt.

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/Kconfig | 2 + arch/arm64/include/asm/ftrace.h | 23 +++++ arch/arm64/kernel/Makefile | 6 ++ arch/arm64/kernel/arm64ksyms.c | 4 + arch/arm64/kernel/entry-ftrace.S | 172 ++++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/ftrace.c | 64 ++++++++++++++ 6 files changed, 271 insertions(+) create mode 100644 arch/arm64/include/asm/ftrace.h create mode 100644 arch/arm64/kernel/entry-ftrace.S create mode 100644 arch/arm64/kernel/ftrace.c

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index dd4327f..332e50c 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -33,6 +33,8 @@ config ARM64 select HAVE_DMA_ATTRS select HAVE_DMA_CONTIGUOUS select HAVE_EFFICIENT_UNALIGNED_ACCESS + select HAVE_FUNCTION_TRACER + select HAVE_FUNCTION_GRAPH_TRACER select HAVE_GENERIC_DMA_COHERENT select HAVE_HW_BREAKPOINT if PERF_EVENTS select HAVE_MEMBLOCK diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h new file mode 100644 index 0000000..0d5dfdb --- /dev/null +++ b/arch/arm64/include/asm/ftrace.h @@ -0,0 +1,23 @@ +/* + * arch/arm64/include/asm/ftrace.h + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef __ASM_FTRACE_H +#define __ASM_FTRACE_H + +#ifdef CONFIG_FUNCTION_TRACER +#define MCOUNT_ADDR ((unsigned long)_mcount) +#define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */ + +#ifndef __ASSEMBLY__ +extern void _mcount(unsigned long); +#endif /* __ASSEMBLY__ */ +#endif /* CONFIG_FUNCTION_TRACER */ + +#endif /* __ASM_FTRACE_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 2d4554b..ca921fb 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -5,6 +5,11 @@ CPPFLAGS_vmlinux.lds := -DTEXT_OFFSET=$(TEXT_OFFSET) AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET)

+ifdef CONFIG_FUNCTION_TRACER +CFLAGS_REMOVE_ftrace.o = -pg +CFLAGS_REMOVE_insn.o = -pg +endif + # Object file lists. arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ @@ -13,6 +18,7 @@ arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \

arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.o +arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c index 338b568..7f0512f 100644 --- a/arch/arm64/kernel/arm64ksyms.c +++ b/arch/arm64/kernel/arm64ksyms.c @@ -56,3 +56,7 @@ EXPORT_SYMBOL(clear_bit); EXPORT_SYMBOL(test_and_clear_bit); EXPORT_SYMBOL(change_bit); EXPORT_SYMBOL(test_and_change_bit); + +#ifdef CONFIG_FUNCTION_TRACER +EXPORT_SYMBOL(_mcount); +#endif diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S new file mode 100644 index 0000000..065f426 --- /dev/null +++ b/arch/arm64/kernel/entry-ftrace.S @@ -0,0 +1,172 @@ +/* + * arch/arm64/kernel/entry-ftrace.S + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/ftrace.h> + +/* + * Gcc with -pg will put the following code in the beginning of each function: + * mov x0, x30 + * bl _mcount + * [function's body ...] + * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic + * ftrace is enabled. + * + * Please note that x0 as an argument will not be used here because we can + * get lr(x30) of insturmented function at any time by winding up call stack + * as long as the kernel is compiled without -fomit-frame-pointer. + * (or CONFIG_FRAME_POINTER, this is forced on arm64) + * + * stack layout after mcount_enter: + * + * 0 ---------------------------------------- sp of _mcount() + * x29: fp of instrumented function fp is not winded + * -------------------- + * x30: lr of _mcount() (= instrumented function's pc) + * +16 ---------------------------------------- sp of instrumented function + * + * .... + * + * +xx ---------------------------------------- fp of instrumented function + * x29: fp of parent + * -------------------- + * x30: lr of insturmented function (= parent's pc) + * -------------------- + * xxx + */ + + .macro mcount_enter + stp x29, x30, [sp, #-48]! + .endm + + .macro mcount_exit + ldp x29, x30, [sp], #48 + ret + .endm + + .macro mcount_adjust_addr rd, rn + sub \rd, \rn, #MCOUNT_INSN_SIZE + .endm + + /* for instrumented function's parent */ + .macro mcount_get_parent_fp reg + ldr \reg, [sp] + ldr \reg, [\reg] + .endm + + /* for instrumented function */ + .macro mcount_get_pc0 reg + mcount_adjust_addr \reg, x30 + .endm + + .macro mcount_get_pc reg + ldr \reg, [sp, #8] + mcount_adjust_addr \reg, \reg + .endm + + .macro mcount_get_lr reg + ldr \reg, [sp] + ldr \reg, [\reg, #8] + mcount_adjust_addr \reg, \reg + .endm + + .macro mcount_get_saved_lr_addr reg + ldr \reg, [sp] + add \reg, \reg, #8 + .endm + +/* + * void _mcount(unsigned long return_address) + * @return_address: return address to instrumented function + * + * This function makes calls, if enabled, to: + * - tracer function to probe instrumented function's entry, + * - ftrace_graph_caller to set up an exit hook + */ +ENTRY(_mcount) +#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST + ldr x0, =ftrace_trace_stop + ldr x0, [x0] // if ftrace_trace_stop + ret // return; +#endif + mcount_enter + + ldr x0, =ftrace_trace_function + ldr x2, [x0] + adr x0, ftrace_stub + cmp x0, x2 // if (ftrace_trace_function + b.eq skip_ftrace_call // != ftrace_stub) { + + mcount_get_pc x0 // function's pc + mcount_get_lr x1 // function's lr (= parent's pc) + blr x2 // (*ftrace_trace_function)(pc, lr); + +#ifndef CONFIG_FUNCTION_GRAPH_TRACER +skip_ftrace_call: // return; + mcount_exit // } +#else + mcount_exit // return; + // } +skip_ftrace_call: + ldr x1, =ftrace_graph_return + ldr x2, [x1] // if ((ftrace_graph_return + cmp x0, x2 // != ftrace_stub) + b.ne ftrace_graph_caller + + ldr x1, =ftrace_graph_entry // || (ftrace_graph_entry + ldr x2, [x1] // != ftrace_graph_entry_stub)) + ldr x0, =ftrace_graph_entry_stub + cmp x0, x2 + b.ne ftrace_graph_caller // ftrace_graph_caller(); + + mcount_exit +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ +ENDPROC(_mcount) + +ENTRY(ftrace_stub) + ret +ENDPROC(ftrace_stub) + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/* + * void ftrace_graph_caller(void) + * + * Called from _mcount() or ftrace_caller() when function_graph tracer is + * selected. + * This function w/ prepare_ftrace_return() fakes link register's value on + * the call stack in order to intercept instrumented function's return path + * and run return_to_handler() later on its exit. + */ +ENTRY(ftrace_graph_caller) + mcount_get_saved_lr_addr x0 // pointer to function's saved lr + mcount_get_pc x1 // function's pc + mcount_get_parent_fp x2 // parent's fp + bl prepare_ftrace_return // prepare_ftrace_return(&lr, pc, fp) + + mcount_exit +ENDPROC(ftrace_graph_caller) + +/* + * void return_to_handler(void) + * + * Run ftrace_return_to_handler() before going back to parent. + * @fp is checked against the value passed by ftrace_graph_caller() + * only when CONFIG_FUNCTION_GRAPH_FP_TEST is enabled. + */ + .global return_to_handler +return_to_handler: + str x0, [sp, #-16]! + mov x0, x29 // parent's fp + bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp); + mov x30, x0 // restore the original return address + ldr x0, [sp], #16 + ret +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c new file mode 100644 index 0000000..a559ab8 --- /dev/null +++ b/arch/arm64/kernel/ftrace.c @@ -0,0 +1,64 @@ +/* + * arch/arm64/kernel/ftrace.c + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/ftrace.h> +#include <linux/swab.h> +#include <linux/uaccess.h> + +#include <asm/cacheflush.h> +#include <asm/ftrace.h> +#include <asm/insn.h> + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/* + * function_graph tracer expects ftrace_return_to_handler() to be called + * on the way back to parent. For this purpose, this function is called + * in _mcount() or ftrace_caller() to replace return address (*parent) on + * the call stack to return_to_handler. + * + * Note that @frame_pointer is used only for sanity check later. + */ +void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, + unsigned long frame_pointer) +{ + unsigned long return_hooker = (unsigned long)&return_to_handler; + unsigned long old; + struct ftrace_graph_ent trace; + int err; + + if (unlikely(atomic_read(&current->tracing_graph_pause))) + return; + + /* + * Note: + * No protection against faulting at *parent, which may be seen + * on other archs. It's unlikely on AArch64. + */ + old = *parent; + *parent = return_hooker; + + trace.func = self_addr; + trace.depth = current->curr_ret_stack + 1; + + /* Only trace if the calling function expects to */ + if (!ftrace_graph_entry(&trace)) { + *parent = old; + return; + } + + err = ftrace_push_return_trace(old, self_addr, &trace.depth, + frame_pointer); + if (err == -EBUSY) { + *parent = old; + return; + } +} +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

On Mon, 10 Feb 2014 16:03:21 +0100 Arnd Bergmann arnd@arndb.de wrote:

On Friday 07 February 2014, AKASHI Takahiro wrote:

...

@@ -0,0 +1,23 @@

...

+#ifdef CONFIG_FUNCTION_TRACER +#define MCOUNT_ADDR ((unsigned long)_mcount) +#define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */

+#ifndef __ASSEMBLY__ +extern void _mcount(unsigned long); +#endif /* __ASSEMBLY__ */ +#endif /* CONFIG_FUNCTION_TRACER */

We generally like to have as few #ifdef as possible, and I think the #ifdef CONFIG_FUNCTION_TRACER here is not needed.

...

+#endif /* __ASM_FTRACE_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 2d4554b..ca921fb 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -5,6 +5,11 @@ CPPFLAGS_vmlinux.lds := -DTEXT_OFFSET=$(TEXT_OFFSET) AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET) +ifdef CONFIG_FUNCTION_TRACER +CFLAGS_REMOVE_ftrace.o = -pg +CFLAGS_REMOVE_insn.o = -pg +endif

Same here. It never hurts to have the CFLAGS_REMOVE_* statements here, since you will not want to build these files for profiling.

I agree to this. I will admit, this was probably just a copy from my code as I did the same in other Makefiles. I think I added the ifdef statements as more of documenting what the REMOVE was for, when ftrace was just being added to the kernel, and not well known.

I should probably go back and remove the ifdef's from other Makefiles too.

A comment about what the -pg is for wouldn't hurt, though:

# For files that should not have any function tracing done to them, # we must remove the -pg flag.

Something like that.

...

diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c index 338b568..7f0512f 100644 --- a/arch/arm64/kernel/arm64ksyms.c +++ b/arch/arm64/kernel/arm64ksyms.c @@ -56,3 +56,7 @@ EXPORT_SYMBOL(clear_bit); EXPORT_SYMBOL(test_and_clear_bit); EXPORT_SYMBOL(change_bit); EXPORT_SYMBOL(test_and_change_bit);

+#ifdef CONFIG_FUNCTION_TRACER +EXPORT_SYMBOL(_mcount); +#endif

This one is clearly needed of course.

...

+/*

• • Gcc with -pg will put the following code in the beginning of each function:

• •  mov x0, x30
    

• •  bl _mcount
    

• • [function's body ...]
• • "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic
• • ftrace is enabled.

Unrelated to your patch, I have run into problems with this on arm32, maybe someone on Cc to this mail has an idea what to do about it:

If I build a large "randconfig" kernel or "allyesconfig", the kernel image size exceeds 32MB, which means I can no longer link callers with a 26 bit signed immediate argument like the "bl _mcount" here. For any other function calls, "gcc -mlong-calls" can be used to work around this, but this particular instance is created by inserting assembly statements into the output without considering the long-call options. Is there a way to get the kernel to link anyway?

I wonder if we play linker games and move the _mcount and ftrace_caller and friends into the middle of the kernel? If it is only missing it by a slight overflow of a 32bit jump, then maybe moving it will work.

...

+#ifdef CONFIG_FUNCTION_GRAPH_TRACER

Here, again, you an remove the #ifdef by making the entire file built only for CONFIG_FUNCTION_GRAPH_TRACER.

Interesting. Other archs (at least x86 and powerpc) required this file for function tracing too. Seems this is only needed for function graph tracer.

-- Steve

On 02/18/2014 03:12 AM, Will Deacon wrote:

On Fri, Feb 07, 2014 at 10:18:51AM +0000, AKASHI Takahiro wrote:

...

This patch implements arm64 specific part to support function tracers, such as function (CONFIG_FUNCTION_TRACER), function_graph (CONFIG_FUNCTION_GRAPH_TRACER) and function profiler (CONFIG_FUNCTION_PROFILER).

With 'function' tracer, all the functions in the kernel are traced with timestamps in ${sysfs}/tracing/trace. If function_graph tracer is specified, call graph is generated.

The kernel must be compiled with -pg option so that _mcount() is inserted at the beginning of functions. This function is called on every function's entry as long as tracing is enabled. In addition, function_graph tracer also needs to be able to probe function's exit. ftrace_graph_caller() & return_to_handler do this by faking link register's value to intercept function's return path.

More details on architecture specific requirements are described in Documentation/trace/ftrace-design.txt.

[...]

...

diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h new file mode 100644 index 0000000..0d5dfdb --- /dev/null +++ b/arch/arm64/include/asm/ftrace.h @@ -0,0 +1,23 @@ +/*

• • arch/arm64/include/asm/ftrace.h
• • Copyright (C) 2013 Linaro Limited
• • Author: AKASHI Takahiro takahiro.akashi@linaro.org
• • This program is free software; you can redistribute it and/or modify
• • it under the terms of the GNU General Public License version 2 as
• • published by the Free Software Foundation.
• */

+#ifndef __ASM_FTRACE_H +#define __ASM_FTRACE_H

+#ifdef CONFIG_FUNCTION_TRACER +#define MCOUNT_ADDR ((unsigned long)_mcount) +#define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */

You can use AARCH64_INSN_SIZE here.

OK, but MCOUNT_INSN_SIZE can't be removed here because it is also used in kernel/trace/ftrace.c. So I will redefine it as #define MCOUNT_INSN_SIZE AARCH64_INSN_SIZE

In this case, I need to add __ASSEMBLY__ in asm/insn.h because asm/ftrace.h (and so asm/insn.h) is included in entry-ftrace.S.

Is it OK?

...

+#include <linux/linkage.h> +#include <asm/ftrace.h>

+/*

• • Gcc with -pg will put the following code in the beginning of each function:

• •  mov x0, x30
    

• •  bl _mcount
    

• • [function's body ...]
    

• • "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic
• • ftrace is enabled.
• • Please note that x0 as an argument will not be used here because we can
• • get lr(x30) of insturmented function at any time by winding up call stack

instrumented

I will fix two occurences of "insturmented" :-) Just a question: Is there any better word than "instrument"?

...

• • as long as the kernel is compiled without -fomit-frame-pointer.
• • (or CONFIG_FRAME_POINTER, this is forced on arm64)
• • stack layout after mcount_enter:
• • 0 ---------------------------------------- sp of _mcount()

• •  x29: fp of instrumented function       fp is not winded
    

• • --------------------
    

• •  x30: lr of _mcount() (= instrumented function's pc)
    

• • +16 ---------------------------------------- sp of instrumented function

• • ....
    

• • +xx ---------------------------------------- fp of instrumented function

• •  x29: fp of parent
    

• • --------------------
    

• •  x30: lr of insturmented function (= parent's pc)
    

• • --------------------
    

• •  xxx
    

• */

•   .macro mcount_enter
  

•   stp     x29, x30, [sp, #-48]!
  

•   .endm
  

Can you elaborate in your comment about where this 48 comes from please? I can't join it up with your ascii art.

Right. It should be -16 as shown in the stack layout. When I removed an operation of saving/restoring x0-x3 at v3 patch, this value should also have been changed.

-Takahiro AKASHI

Will

CALLER_ADDRx returns caller's address at specified level in call stacks. They are used for several tracers like irqsoff and preemptoff. Strange to say, however, they are refered even without FTRACE.

Please note that this implementation assumes that we have frame pointers. (which means kernel should be compiled with -fno-omit-frame-pointer.)

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/include/asm/ftrace.h | 14 +++++++++ arch/arm64/kernel/Makefile | 3 +- arch/arm64/kernel/return_address.c | 55 ++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/kernel/return_address.c

diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h index 37a379d..4c6ac03 100644 --- a/arch/arm64/include/asm/ftrace.h +++ b/arch/arm64/include/asm/ftrace.h @@ -37,4 +37,18 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) #endif /* __ASSEMBLY__ */ #endif /* CONFIG_FUNCTION_TRACER */

+#ifndef __ASSEMBLY__ +#define HAVE_ARCH_CALLER_ADDR + +extern void *return_address(unsigned int); + +#define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0)) +#define CALLER_ADDR1 ((unsigned long)return_address(1)) +#define CALLER_ADDR2 ((unsigned long)return_address(2)) +#define CALLER_ADDR3 ((unsigned long)return_address(3)) +#define CALLER_ADDR4 ((unsigned long)return_address(4)) +#define CALLER_ADDR5 ((unsigned long)return_address(5)) +#define CALLER_ADDR6 ((unsigned long)return_address(6)) +#endif /* ifndef __ASSEMBLY__ */ + #endif /* __ASM_FTRACE_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index ca921fb..55e0b6e 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -9,12 +9,13 @@ ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_ftrace.o = -pg CFLAGS_REMOVE_insn.o = -pg endif +CFLAGS_REMOVE_return_address.o = -pg

# Object file lists. arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \ - hyp-stub.o psci.o cpu_ops.o insn.o + hyp-stub.o psci.o cpu_ops.o insn.o return_address.o

arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.o diff --git a/arch/arm64/kernel/return_address.c b/arch/arm64/kernel/return_address.c new file mode 100644 index 0000000..89102a6 --- /dev/null +++ b/arch/arm64/kernel/return_address.c @@ -0,0 +1,55 @@ +/* + * arch/arm64/kernel/return_address.c + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/export.h> +#include <linux/ftrace.h> + +#include <asm/stacktrace.h> + +struct return_address_data { + unsigned int level; + void *addr; +}; + +static int save_return_addr(struct stackframe *frame, void *d) +{ + struct return_address_data *data = d; + + if (!data->level) { + data->addr = (void *)frame->pc; + return 1; + } else { + --data->level; + return 0; + } +} + +void *return_address(unsigned int level) +{ + struct return_address_data data; + struct stackframe frame; + register unsigned long current_sp asm ("sp"); + + data.level = level + 2; + data.addr = NULL; + + frame.fp = (unsigned long)__builtin_frame_address(0); + frame.sp = current_sp; + frame.pc = (unsigned long)return_address; /* dummy */ + + walk_stackframe(&frame, save_return_addr, &data); + + if (!data.level) + return data.addr; + else + return NULL; +} +EXPORT_SYMBOL_GPL(return_address);

This patch allows system call entry or exit to be traced as ftrace events, ie. sys_enter_*/sys_exit_*, if CONFIG_FTRACE_SYSCALLS is enabled. Those events appear and can be controlled under ${sysfs}/tracing/events/syscalls/

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/syscall.h | 1 + arch/arm64/include/asm/unistd.h | 2 ++ arch/arm64/kernel/ptrace.c | 5 +++++ 4 files changed, 9 insertions(+)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 43fce76..9bdaf5c 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -42,6 +42,7 @@ config ARM64 select HAVE_HW_BREAKPOINT if PERF_EVENTS select HAVE_MEMBLOCK select HAVE_PERF_EVENTS + select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN select MODULES_USE_ELF_RELA select NO_BOOTMEM diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h index 70ba9d4..383771e 100644 --- a/arch/arm64/include/asm/syscall.h +++ b/arch/arm64/include/asm/syscall.h @@ -18,6 +18,7 @@

#include <linux/err.h>

+extern const void *sys_call_table[];

static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 82ce217..c335479 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -28,3 +28,5 @@ #endif #define __ARCH_WANT_SYS_CLONE #include <uapi/asm/unistd.h> + +#define NR_syscalls (__NR_syscalls) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 64ce39f..f4c3c4e 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -41,6 +41,9 @@ #include <asm/traps.h> #include <asm/system_misc.h>

+#define CREATE_TRACE_POINTS +#include <trace/events/syscalls.h> + /* * TODO: does not yet catch signals sent when the child dies. * in exit.c or in signal.c. @@ -1076,10 +1079,12 @@ asmlinkage int syscall_trace(int dir, struct pt_regs *regs) }

if (dir) { + trace_sys_exit(regs, regs->syscallno); tracehook_report_syscall_exit(regs, 0); } else { if (tracehook_report_syscall_entry(regs)) regs->syscallno = ~0UL; + trace_sys_enter(regs, regs->syscallno); }

if (is_compat_task())

On 02/18/2014 04:29 AM, Will Deacon wrote:

On Fri, Feb 07, 2014 at 10:18:55AM +0000, AKASHI Takahiro wrote:

...

This patch allows system call entry or exit to be traced as ftrace events, ie. sys_enter_*/sys_exit_*, if CONFIG_FTRACE_SYSCALLS is enabled. Those events appear and can be controlled under ${sysfs}/tracing/events/syscalls/

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org

arch/arm64/Kconfig | 1 + arch/arm64/include/asm/syscall.h | 1 + arch/arm64/include/asm/unistd.h | 2 ++ arch/arm64/kernel/ptrace.c | 5 +++++ 4 files changed, 9 insertions(+)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 43fce76..9bdaf5c 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -42,6 +42,7 @@ config ARM64 select HAVE_HW_BREAKPOINT if PERF_EVENTS select HAVE_MEMBLOCK select HAVE_PERF_EVENTS

• select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN select MODULES_USE_ELF_RELA select NO_BOOTMEM

diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h index 70ba9d4..383771e 100644 --- a/arch/arm64/include/asm/syscall.h +++ b/arch/arm64/include/asm/syscall.h @@ -18,6 +18,7 @@

#include <linux/err.h>

+extern const void *sys_call_table[];

static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 82ce217..c335479 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -28,3 +28,5 @@ #endif #define __ARCH_WANT_SYS_CLONE #include <uapi/asm/unistd.h>

+#define NR_syscalls (__NR_syscalls) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 64ce39f..f4c3c4e 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -41,6 +41,9 @@ #include <asm/traps.h> #include <asm/system_misc.h>

+#define CREATE_TRACE_POINTS +#include <trace/events/syscalls.h>

• /*
  • TODO: does not yet catch signals sent when the child dies.
  • in exit.c or in signal.c.

@@ -1076,10 +1079,12 @@ asmlinkage int syscall_trace(int dir, struct pt_regs *regs) }

if (dir) {

• trace_sys_exit(regs, regs->syscallno);
  

  tracehook_report_syscall_exit(regs, 0); } else { if (tracehook_report_syscall_entry(regs)) regs->syscallno = ~0UL;

• trace_sys_enter(regs, regs->syscallno);
  

Shouldn't these calls be guarded by a TIF_SYSCALL_TRACEPOINT check?

Yes. I will add the guard.

-Takahiro AKASHI

Will

This patch implements arm64 specific part to support function tracers, such as function (CONFIG_FUNCTION_TRACER), function_graph (CONFIG_FUNCTION_GRAPH_TRACER) and function profiler (CONFIG_FUNCTION_PROFILER).

With 'function' tracer, all the functions in the kernel are traced with timestamps in ${sysfs}/tracing/trace. If function_graph tracer is specified, call graph is generated.

The kernel must be compiled with -pg option so that _mcount() is inserted at the beginning of functions. This function is called on every function's entry as long as tracing is enabled. In addition, function_graph tracer also needs to be able to probe function's exit. ftrace_graph_caller() & return_to_handler do this by faking link register's value to intercept function's return path.

More details on architecture specific requirements are described in Documentation/trace/ftrace-design.txt.

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/Kconfig | 2 + arch/arm64/include/asm/ftrace.h | 23 +++++ arch/arm64/kernel/Makefile | 4 + arch/arm64/kernel/arm64ksyms.c | 4 + arch/arm64/kernel/entry-ftrace.S | 173 ++++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/ftrace.c | 64 ++++++++++++++ 6 files changed, 270 insertions(+) create mode 100644 arch/arm64/include/asm/ftrace.h create mode 100644 arch/arm64/kernel/entry-ftrace.S create mode 100644 arch/arm64/kernel/ftrace.c

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 27bbcfc..5783641 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -33,6 +33,8 @@ config ARM64 select HAVE_DMA_ATTRS select HAVE_DMA_CONTIGUOUS select HAVE_EFFICIENT_UNALIGNED_ACCESS + select HAVE_FUNCTION_TRACER + select HAVE_FUNCTION_GRAPH_TRACER select HAVE_GENERIC_DMA_COHERENT select HAVE_HW_BREAKPOINT if PERF_EVENTS select HAVE_MEMBLOCK diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h new file mode 100644 index 0000000..58ea595 --- /dev/null +++ b/arch/arm64/include/asm/ftrace.h @@ -0,0 +1,23 @@ +/* + * arch/arm64/include/asm/ftrace.h + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef __ASM_FTRACE_H +#define __ASM_FTRACE_H + +#include <asm/insn.h> + +#define MCOUNT_ADDR ((unsigned long)_mcount) +#define MCOUNT_INSN_SIZE AARCH64_INSN_SIZE + +#ifndef __ASSEMBLY__ +extern void _mcount(unsigned long); +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_FTRACE_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 2d4554b..ac67fd0 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -5,6 +5,9 @@ CPPFLAGS_vmlinux.lds := -DTEXT_OFFSET=$(TEXT_OFFSET) AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET)

+CFLAGS_REMOVE_ftrace.o = -pg +CFLAGS_REMOVE_insn.o = -pg + # Object file lists. arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ @@ -13,6 +16,7 @@ arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \

arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.o +arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c index 338b568..7f0512f 100644 --- a/arch/arm64/kernel/arm64ksyms.c +++ b/arch/arm64/kernel/arm64ksyms.c @@ -56,3 +56,7 @@ EXPORT_SYMBOL(clear_bit); EXPORT_SYMBOL(test_and_clear_bit); EXPORT_SYMBOL(change_bit); EXPORT_SYMBOL(test_and_change_bit); + +#ifdef CONFIG_FUNCTION_TRACER +EXPORT_SYMBOL(_mcount); +#endif diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S new file mode 100644 index 0000000..2e8162e --- /dev/null +++ b/arch/arm64/kernel/entry-ftrace.S @@ -0,0 +1,173 @@ +/* + * arch/arm64/kernel/entry-ftrace.S + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/ftrace.h> +#include <asm/insn.h> + +/* + * Gcc with -pg will put the following code in the beginning of each function: + * mov x0, x30 + * bl _mcount + * [function's body ...] + * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic + * ftrace is enabled. + * + * Please note that x0 as an argument will not be used here because we can + * get lr(x30) of instrumented function at any time by winding up call stack + * as long as the kernel is compiled without -fomit-frame-pointer. + * (or CONFIG_FRAME_POINTER, this is forced on arm64) + * + * stack layout after mcount_enter: + * + * 0 ---------------------------------------- sp of _mcount() + * x29: fp of instrumented function fp is not winded + * -------------------- + * x30: lr of _mcount() (= instrumented function's pc) + * +16 ---------------------------------------- sp of instrumented function + * + * .... + * + * +xx ---------------------------------------- fp of instrumented function + * x29: fp of parent + * -------------------- + * x30: lr of instrumented function (= parent's pc) + * -------------------- + * xxx + */ + + .macro mcount_enter + stp x29, x30, [sp, #-16]! + .endm + + .macro mcount_exit + ldp x29, x30, [sp], #16 + ret + .endm + + .macro mcount_adjust_addr rd, rn + sub \rd, \rn, #AARCH64_INSN_SIZE + .endm + + /* for instrumented function's parent */ + .macro mcount_get_parent_fp reg + ldr \reg, [sp] + ldr \reg, [\reg] + .endm + + /* for instrumented function */ + .macro mcount_get_pc0 reg + mcount_adjust_addr \reg, x30 + .endm + + .macro mcount_get_pc reg + ldr \reg, [sp, #8] + mcount_adjust_addr \reg, \reg + .endm + + .macro mcount_get_lr reg + ldr \reg, [sp] + ldr \reg, [\reg, #8] + mcount_adjust_addr \reg, \reg + .endm + + .macro mcount_get_saved_lr_addr reg + ldr \reg, [sp] + add \reg, \reg, #8 + .endm + +/* + * void _mcount(unsigned long return_address) + * @return_address: return address to instrumented function + * + * This function makes calls, if enabled, to: + * - tracer function to probe instrumented function's entry, + * - ftrace_graph_caller to set up an exit hook + */ +ENTRY(_mcount) +#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST + ldr x0, =ftrace_trace_stop + ldr x0, [x0] // if ftrace_trace_stop + ret // return; +#endif + mcount_enter + + ldr x0, =ftrace_trace_function + ldr x2, [x0] + adr x0, ftrace_stub + cmp x0, x2 // if (ftrace_trace_function + b.eq skip_ftrace_call // != ftrace_stub) { + + mcount_get_pc x0 // function's pc + mcount_get_lr x1 // function's lr (= parent's pc) + blr x2 // (*ftrace_trace_function)(pc, lr); + +#ifndef CONFIG_FUNCTION_GRAPH_TRACER +skip_ftrace_call: // return; + mcount_exit // } +#else + mcount_exit // return; + // } +skip_ftrace_call: + ldr x1, =ftrace_graph_return + ldr x2, [x1] // if ((ftrace_graph_return + cmp x0, x2 // != ftrace_stub) + b.ne ftrace_graph_caller + + ldr x1, =ftrace_graph_entry // || (ftrace_graph_entry + ldr x2, [x1] // != ftrace_graph_entry_stub)) + ldr x0, =ftrace_graph_entry_stub + cmp x0, x2 + b.ne ftrace_graph_caller // ftrace_graph_caller(); + + mcount_exit +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ +ENDPROC(_mcount) + +ENTRY(ftrace_stub) + ret +ENDPROC(ftrace_stub) + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/* + * void ftrace_graph_caller(void) + * + * Called from _mcount() or ftrace_caller() when function_graph tracer is + * selected. + * This function w/ prepare_ftrace_return() fakes link register's value on + * the call stack in order to intercept instrumented function's return path + * and run return_to_handler() later on its exit. + */ +ENTRY(ftrace_graph_caller) + mcount_get_saved_lr_addr x0 // pointer to function's saved lr + mcount_get_pc x1 // function's pc + mcount_get_parent_fp x2 // parent's fp + bl prepare_ftrace_return // prepare_ftrace_return(&lr, pc, fp) + + mcount_exit +ENDPROC(ftrace_graph_caller) + +/* + * void return_to_handler(void) + * + * Run ftrace_return_to_handler() before going back to parent. + * @fp is checked against the value passed by ftrace_graph_caller() + * only when CONFIG_FUNCTION_GRAPH_FP_TEST is enabled. + */ + .global return_to_handler +return_to_handler: + str x0, [sp, #-16]! + mov x0, x29 // parent's fp + bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp); + mov x30, x0 // restore the original return address + ldr x0, [sp], #16 + ret +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c new file mode 100644 index 0000000..a559ab8 --- /dev/null +++ b/arch/arm64/kernel/ftrace.c @@ -0,0 +1,64 @@ +/* + * arch/arm64/kernel/ftrace.c + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/ftrace.h> +#include <linux/swab.h> +#include <linux/uaccess.h> + +#include <asm/cacheflush.h> +#include <asm/ftrace.h> +#include <asm/insn.h> + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/* + * function_graph tracer expects ftrace_return_to_handler() to be called + * on the way back to parent. For this purpose, this function is called + * in _mcount() or ftrace_caller() to replace return address (*parent) on + * the call stack to return_to_handler. + * + * Note that @frame_pointer is used only for sanity check later. + */ +void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, + unsigned long frame_pointer) +{ + unsigned long return_hooker = (unsigned long)&return_to_handler; + unsigned long old; + struct ftrace_graph_ent trace; + int err; + + if (unlikely(atomic_read(&current->tracing_graph_pause))) + return; + + /* + * Note: + * No protection against faulting at *parent, which may be seen + * on other archs. It's unlikely on AArch64. + */ + old = *parent; + *parent = return_hooker; + + trace.func = self_addr; + trace.depth = current->curr_ret_stack + 1; + + /* Only trace if the calling function expects to */ + if (!ftrace_graph_entry(&trace)) { + *parent = old; + return; + } + + err = ftrace_push_return_trace(old, self_addr, &trace.depth, + frame_pointer); + if (err == -EBUSY) { + *parent = old; + return; + } +} +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

Ganapatrao,

On 02/26/2014 12:38 AM, Kulkarni, Ganapatrao wrote:

---

From: AKASHI Takahiro takahiro.akashi@linaro.org Sent: Tuesday, February 25, 2014 2:53 PM To: rostedt@goodmis.org; fweisbec@gmail.com; mingo@redhat.com; catalin.marinas@arm.com; will.deacon@arm.com; tim.bird@sonymobile.com Cc: Kulkarni, Ganapatrao; dsaxena@linaro.org; arndb@arndb.de; linux-arm-kernel@lists.infradead.org; linaro-kernel@lists.linaro.org; linux-kernel@vger.kernel.org; AKASHI Takahiro Subject: [PATCH v4 1/7] arm64: Add ftrace support

This patch implements arm64 specific part to support function tracers, such as function (CONFIG_FUNCTION_TRACER), function_graph (CONFIG_FUNCTION_GRAPH_TRACER) and function profiler (CONFIG_FUNCTION_PROFILER).

With 'function' tracer, all the functions in the kernel are traced with timestamps in ${sysfs}/tracing/trace. If function_graph tracer is specified, call graph is generated.

The kernel must be compiled with -pg option so that _mcount() is inserted at the beginning of functions. This function is called on every function's entry as long as tracing is enabled. In addition, function_graph tracer also needs to be able to probe function's exit. ftrace_graph_caller() & return_to_handler do this by faking link register's value to intercept function's return path.

More details on architecture specific requirements are described in Documentation/trace/ftrace-design.txt.

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org

arch/arm64/Kconfig | 2 + arch/arm64/include/asm/ftrace.h | 23 +++++ arch/arm64/kernel/Makefile | 4 + arch/arm64/kernel/arm64ksyms.c | 4 + arch/arm64/kernel/entry-ftrace.S | 173 ++++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/ftrace.c | 64 ++++++++++++++ 6 files changed, 270 insertions(+) create mode 100644 arch/arm64/include/asm/ftrace.h create mode 100644 arch/arm64/kernel/entry-ftrace.S create mode 100644 arch/arm64/kernel/ftrace.c

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 27bbcfc..5783641 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -33,6 +33,8 @@ config ARM64 select HAVE_DMA_ATTRS select HAVE_DMA_CONTIGUOUS select HAVE_EFFICIENT_UNALIGNED_ACCESS

•   select HAVE_FUNCTION_TRACER
  

•   select HAVE_FUNCTION_GRAPH_TRACER
     select HAVE_GENERIC_DMA_COHERENT
     select HAVE_HW_BREAKPOINT if PERF_EVENTS
     select HAVE_MEMBLOCK
  

diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h new file mode 100644 index 0000000..58ea595 --- /dev/null +++ b/arch/arm64/include/asm/ftrace.h @@ -0,0 +1,23 @@ +/*

• • arch/arm64/include/asm/ftrace.h
• • Copyright (C) 2013 Linaro Limited
• • Author: AKASHI Takahiro takahiro.akashi@linaro.org
• • This program is free software; you can redistribute it and/or modify
• • it under the terms of the GNU General Public License version 2 as
• • published by the Free Software Foundation.
• */

+#ifndef __ASM_FTRACE_H +#define __ASM_FTRACE_H

+#include <asm/insn.h>

+#define MCOUNT_ADDR ((unsigned long)_mcount) +#define MCOUNT_INSN_SIZE AARCH64_INSN_SIZE

+#ifndef __ASSEMBLY__ +extern void _mcount(unsigned long); +#endif /* __ASSEMBLY__ */

+#endif /* __ASM_FTRACE_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 2d4554b..ac67fd0 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -5,6 +5,9 @@ CPPFLAGS_vmlinux.lds := -DTEXT_OFFSET=$(TEXT_OFFSET) AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET)

+CFLAGS_REMOVE_ftrace.o = -pg +CFLAGS_REMOVE_insn.o = -pg

• # Object file lists. arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \

@@ -13,6 +16,7 @@ arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \

arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.o +arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c index 338b568..7f0512f 100644 --- a/arch/arm64/kernel/arm64ksyms.c +++ b/arch/arm64/kernel/arm64ksyms.c @@ -56,3 +56,7 @@ EXPORT_SYMBOL(clear_bit); EXPORT_SYMBOL(test_and_clear_bit); EXPORT_SYMBOL(change_bit); EXPORT_SYMBOL(test_and_change_bit);

+#ifdef CONFIG_FUNCTION_TRACER +EXPORT_SYMBOL(_mcount); +#endif diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S new file mode 100644 index 0000000..2e8162e --- /dev/null +++ b/arch/arm64/kernel/entry-ftrace.S @@ -0,0 +1,173 @@ +/*

• • arch/arm64/kernel/entry-ftrace.S
• • Copyright (C) 2013 Linaro Limited
• • Author: AKASHI Takahiro takahiro.akashi@linaro.org
• • This program is free software; you can redistribute it and/or modify
• • it under the terms of the GNU General Public License version 2 as
• • published by the Free Software Foundation.
• */

+#include <linux/linkage.h> +#include <asm/ftrace.h> +#include <asm/insn.h>

+/*

• • Gcc with -pg will put the following code in the beginning of each function:

• •  mov x0, x30
    

• •  bl _mcount
    

• • [function's body ...]
    

• • "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic
• • ftrace is enabled.
• • Please note that x0 as an argument will not be used here because we can
• • get lr(x30) of instrumented function at any time by winding up call stack
• • as long as the kernel is compiled without -fomit-frame-pointer.
• • (or CONFIG_FRAME_POINTER, this is forced on arm64)
• • stack layout after mcount_enter:
• • 0 ---------------------------------------- sp of _mcount()

• •  x29: fp of instrumented function       fp is not winded
    

• • --------------------
    

• •  x30: lr of _mcount() (= instrumented function's pc)
    

• • +16 ---------------------------------------- sp of instrumented function

• • ....
    

• • +xx ---------------------------------------- fp of instrumented function

• •  x29: fp of parent
    

• • --------------------
    

• •  x30: lr of instrumented function (= parent's pc)
    

• • --------------------
    

• •  xxx
    

• */

can stack layout be more readable as below?

Stack layout of _mcount after mcount_enter

    |------------------| sp + 16   <= frame pointer of _mcount.
    |                       |
    |       x30          | <=  lr of _mcount (instrumented func's pc)
    |                       |
    |------------------| sp + 8
    |                       |
    |       x29          | <= value of x29 is sp of instrumented function.

not sp, but fp. Anyway pls check with my new patch v5.

Thanks, -Takahiro AKASHI

    |                       |
    |------------------| sp   <= stack pointer of _mcount

Stack layout of instrumented function. . . . . |------------------| sp + 16 | | | x30 | <= lr of instrumented function (= parent's pc) | | |------------------| sp + 8 | | | x29 | <= sp of parent. | | |------------------| sp <= this address is stored at sp of _mcount

•   .macro mcount_enter
  

•   stp     x29, x30, [sp, #-16]!
  

•   .endm
  

•   .macro mcount_exit
  

•   ldp     x29, x30, [sp], #16
  

•   ret
  

•   .endm
  

•   .macro mcount_adjust_addr rd, rn
  

•   sub     \rd, \rn, #AARCH64_INSN_SIZE
  

•   .endm
  

•   /* for instrumented function's parent */
  

•   .macro mcount_get_parent_fp reg
  

•   ldr     \reg, [sp]
  

•   ldr     \reg, [\reg]
  

•   .endm
  

•   /* for instrumented function */
  

•   .macro mcount_get_pc0 reg
  

•   mcount_adjust_addr      \reg, x30
  

•   .endm
  

•   .macro mcount_get_pc reg
  

•   ldr     \reg, [sp, #8]
  

•   mcount_adjust_addr      \reg, \reg
  

•   .endm
  

•   .macro mcount_get_lr reg
  

•   ldr     \reg, [sp]
  

•   ldr     \reg, [\reg, #8]
  

•   mcount_adjust_addr      \reg, \reg
  

•   .endm
  

•   .macro mcount_get_saved_lr_addr reg
  

•   ldr     \reg, [sp]
  

•   add     \reg, \reg, #8
  

•   .endm
  

+/*

• • void _mcount(unsigned long return_address)
• • @return_address: return address to instrumented function
• • This function makes calls, if enabled, to:

• • - tracer function to probe instrumented function's entry,
    

• • - ftrace_graph_caller to set up an exit hook
    

• */

+ENTRY(_mcount) +#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST

•   ldr     x0, =ftrace_trace_stop
  

•   ldr     x0, [x0]                // if ftrace_trace_stop
  

•   ret                             //   return;
  

+#endif

•   mcount_enter
  

•   ldr     x0, =ftrace_trace_function
  

•   ldr     x2, [x0]
  

•   adr     x0, ftrace_stub
  

•   cmp     x0, x2                  // if (ftrace_trace_function
  

•   b.eq    skip_ftrace_call        //     != ftrace_stub) {
  

•   mcount_get_pc   x0              //       function's pc
  

•   mcount_get_lr   x1              //       function's lr (= parent's pc)
  

•   blr     x2                      //   (*ftrace_trace_function)(pc, lr);
  

+#ifndef CONFIG_FUNCTION_GRAPH_TRACER +skip_ftrace_call: // return;

•   mcount_exit                     // }
  

+#else

•   mcount_exit                     //   return;
  

•                                   // }
  

+skip_ftrace_call:

•   ldr     x1, =ftrace_graph_return
  

•   ldr     x2, [x1]                //   if ((ftrace_graph_return
  

•   cmp     x0, x2                  //        != ftrace_stub)
  

•   b.ne    ftrace_graph_caller
  

•   ldr     x1, =ftrace_graph_entry //     || (ftrace_graph_entry
  

•   ldr     x2, [x1]                //        != ftrace_graph_entry_stub))
  

•   ldr     x0, =ftrace_graph_entry_stub
  

•   cmp     x0, x2
  

•   b.ne    ftrace_graph_caller     //     ftrace_graph_caller();
  

•   mcount_exit
  

+#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ +ENDPROC(_mcount)

+ENTRY(ftrace_stub)

•   ret
  

+ENDPROC(ftrace_stub)

+#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/*

• • void ftrace_graph_caller(void)
• • Called from _mcount() or ftrace_caller() when function_graph tracer is
• • selected.
• • This function w/ prepare_ftrace_return() fakes link register's value on
• • the call stack in order to intercept instrumented function's return path
• • and run return_to_handler() later on its exit.
• */

+ENTRY(ftrace_graph_caller)

•   mcount_get_saved_lr_addr  x0    //     pointer to function's saved lr
  

•   mcount_get_pc             x1    //     function's pc
  

•   mcount_get_parent_fp      x2    //     parent's fp
  

•   bl      prepare_ftrace_return   // prepare_ftrace_return(&lr, pc, fp)
  

•   mcount_exit
  

+ENDPROC(ftrace_graph_caller)

+/*

• • void return_to_handler(void)
• • Run ftrace_return_to_handler() before going back to parent.
• • @fp is checked against the value passed by ftrace_graph_caller()
• • only when CONFIG_FUNCTION_GRAPH_FP_TEST is enabled.
• */

•   .global return_to_handler
  

+return_to_handler:

•   str     x0, [sp, #-16]!
  

•   mov     x0, x29                 //     parent's fp
  

•   bl      ftrace_return_to_handler// addr = ftrace_return_to_hander(fp);
  

•   mov     x30, x0                 // restore the original return address
  

•   ldr     x0, [sp], #16
  

•   ret
  

+#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c new file mode 100644 index 0000000..a559ab8 --- /dev/null +++ b/arch/arm64/kernel/ftrace.c @@ -0,0 +1,64 @@ +/*

• • arch/arm64/kernel/ftrace.c
• • Copyright (C) 2013 Linaro Limited
• • Author: AKASHI Takahiro takahiro.akashi@linaro.org
• • This program is free software; you can redistribute it and/or modify
• • it under the terms of the GNU General Public License version 2 as
• • published by the Free Software Foundation.
• */

+#include <linux/ftrace.h> +#include <linux/swab.h> +#include <linux/uaccess.h>

+#include <asm/cacheflush.h> +#include <asm/ftrace.h> +#include <asm/insn.h>

+#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/*

• • function_graph tracer expects ftrace_return_to_handler() to be called
• • on the way back to parent. For this purpose, this function is called
• • in _mcount() or ftrace_caller() to replace return address (*parent) on
• • the call stack to return_to_handler.
• • Note that @frame_pointer is used only for sanity check later.
• */

+void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,

•                      unsigned long frame_pointer)
  

+{

•   unsigned long return_hooker = (unsigned long)&return_to_handler;
  

•   unsigned long old;
  

•   struct ftrace_graph_ent trace;
  

•   int err;
  

•   if (unlikely(atomic_read(&current->tracing_graph_pause)))
  

•           return;
  

•   /*
  

•    * Note:
  

•    * No protection against faulting at *parent, which may be seen
  

•    * on other archs. It's unlikely on AArch64.
  

•    */
  

•   old = *parent;
  

•   *parent = return_hooker;
  

•   trace.func = self_addr;
  

•   trace.depth = current->curr_ret_stack + 1;
  

•   /* Only trace if the calling function expects to */
  

•   if (!ftrace_graph_entry(&trace)) {
  

•           *parent = old;
  

•           return;
  

•   }
  

•   err = ftrace_push_return_trace(old, self_addr, &trace.depth,
  

•                                  frame_pointer);
  

•   if (err == -EBUSY) {
  

•           *parent = old;
  

•           return;
  

•   }
  

+}

+#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

1.7.9.5

CALLER_ADDRx returns caller's address at specified level in call stacks. They are used for several tracers like irqsoff and preemptoff. Strange to say, however, they are refered even without FTRACE.

Please note that this implementation assumes that we have frame pointers. (which means kernel should be compiled with -fno-omit-frame-pointer.)

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/include/asm/ftrace.h | 14 +++++++++ arch/arm64/kernel/Makefile | 3 +- arch/arm64/kernel/return_address.c | 55 ++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/kernel/return_address.c

diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h index ed5c448..f93da72 100644 --- a/arch/arm64/include/asm/ftrace.h +++ b/arch/arm64/include/asm/ftrace.h @@ -35,4 +35,18 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) } #endif /* __ASSEMBLY__ */

+#ifndef __ASSEMBLY__ +#define HAVE_ARCH_CALLER_ADDR + +extern void *return_address(unsigned int); + +#define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0)) +#define CALLER_ADDR1 ((unsigned long)return_address(1)) +#define CALLER_ADDR2 ((unsigned long)return_address(2)) +#define CALLER_ADDR3 ((unsigned long)return_address(3)) +#define CALLER_ADDR4 ((unsigned long)return_address(4)) +#define CALLER_ADDR5 ((unsigned long)return_address(5)) +#define CALLER_ADDR6 ((unsigned long)return_address(6)) +#endif /* ifndef __ASSEMBLY__ */ + #endif /* __ASM_FTRACE_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index ac67fd0..b5bfa7f 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -7,12 +7,13 @@ AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET)

CFLAGS_REMOVE_ftrace.o = -pg CFLAGS_REMOVE_insn.o = -pg +CFLAGS_REMOVE_return_address.o = -pg

# Object file lists. arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \ - hyp-stub.o psci.o cpu_ops.o insn.o + hyp-stub.o psci.o cpu_ops.o insn.o return_address.o

arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.o diff --git a/arch/arm64/kernel/return_address.c b/arch/arm64/kernel/return_address.c new file mode 100644 index 0000000..89102a6 --- /dev/null +++ b/arch/arm64/kernel/return_address.c @@ -0,0 +1,55 @@ +/* + * arch/arm64/kernel/return_address.c + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/export.h> +#include <linux/ftrace.h> + +#include <asm/stacktrace.h> + +struct return_address_data { + unsigned int level; + void *addr; +}; + +static int save_return_addr(struct stackframe *frame, void *d) +{ + struct return_address_data *data = d; + + if (!data->level) { + data->addr = (void *)frame->pc; + return 1; + } else { + --data->level; + return 0; + } +} + +void *return_address(unsigned int level) +{ + struct return_address_data data; + struct stackframe frame; + register unsigned long current_sp asm ("sp"); + + data.level = level + 2; + data.addr = NULL; + + frame.fp = (unsigned long)__builtin_frame_address(0); + frame.sp = current_sp; + frame.pc = (unsigned long)return_address; /* dummy */ + + walk_stackframe(&frame, save_return_addr, &data); + + if (!data.level) + return data.addr; + else + return NULL; +} +EXPORT_SYMBOL_GPL(return_address);

This patch allows system call entry or exit to be traced as ftrace events, ie. sys_enter_*/sys_exit_*, if CONFIG_FTRACE_SYSCALLS is enabled. Those events appear and can be controlled under ${sysfs}/tracing/events/syscalls/

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/syscall.h | 1 + arch/arm64/include/asm/unistd.h | 2 ++ arch/arm64/kernel/ptrace.c | 9 +++++++++ 4 files changed, 13 insertions(+)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 6954959..b1dcdb4 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -43,6 +43,7 @@ config ARM64 select HAVE_MEMBLOCK select HAVE_PATA_PLATFORM select HAVE_PERF_EVENTS + select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN select MODULES_USE_ELF_RELA select NO_BOOTMEM diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h index 70ba9d4..383771e 100644 --- a/arch/arm64/include/asm/syscall.h +++ b/arch/arm64/include/asm/syscall.h @@ -18,6 +18,7 @@

#include <linux/err.h>

+extern const void *sys_call_table[];

static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 82ce217..c335479 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -28,3 +28,5 @@ #endif #define __ARCH_WANT_SYS_CLONE #include <uapi/asm/unistd.h> + +#define NR_syscalls (__NR_syscalls) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index c70133e..dc90f9f 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -41,6 +41,9 @@ #include <asm/traps.h> #include <asm/system_misc.h>

+#define CREATE_TRACE_POINTS +#include <trace/events/syscalls.h> + /* * TODO: does not yet catch signals sent when the child dies. * in exit.c or in signal.c. @@ -1062,6 +1065,9 @@ asmlinkage int syscall_trace(int dir, struct pt_regs *regs) { unsigned long saved_reg;

+ if (dir && unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) + trace_sys_exit(regs, regs_return_value(regs)); + if (test_thread_flag(TIF_SYSCALL_TRACE)) { if (is_compat_task()) { /* AArch32 uses ip (r12) for scratch */ @@ -1087,5 +1093,8 @@ asmlinkage int syscall_trace(int dir, struct pt_regs *regs) regs->regs[7] = saved_reg; }

+ if (!dir && unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) + trace_sys_enter(regs, regs->syscallno); + return regs->syscallno; }

Since insn.h is indirectly included in asm/entry-ftrace.S, we need to exclude some declarations by __ASSEMBLY__.

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/include/asm/insn.h | 2 ++ 1 file changed, 2 insertions(+)

diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index c44ad39..dc1f73b 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -21,6 +21,7 @@ /* A64 instructions are always 32 bits. */ #define AARCH64_INSN_SIZE 4

+#ifndef __ASSEMBLY__ /* * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a * Section C3.1 "A64 instruction index by encoding": @@ -104,5 +105,6 @@ bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn); int aarch64_insn_patch_text_nosync(void *addr, u32 insn); int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt); int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt); +#endif /* __ASSEMBLY__ */

#endif /* __ASM_INSN_H */

This patch implements arm64 specific part to support function tracers, such as function (CONFIG_FUNCTION_TRACER), function_graph (CONFIG_FUNCTION_GRAPH_TRACER) and function profiler (CONFIG_FUNCTION_PROFILER).

With 'function' tracer, all the functions in the kernel are traced with timestamps in ${sysfs}/tracing/trace. If function_graph tracer is specified, call graph is generated.

The kernel must be compiled with -pg option so that _mcount() is inserted at the beginning of functions. This function is called on every function's entry as long as tracing is enabled. In addition, function_graph tracer also needs to be able to probe function's exit. ftrace_graph_caller() & return_to_handler do this by faking link register's value to intercept function's return path.

More details on architecture specific requirements are described in Documentation/trace/ftrace-design.txt.

Reviewed-by: Ganapatrao Kulkarni ganapatrao.kulkarni@cavium.com Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/Kconfig | 2 + arch/arm64/include/asm/ftrace.h | 23 +++++ arch/arm64/kernel/Makefile | 4 + arch/arm64/kernel/arm64ksyms.c | 4 + arch/arm64/kernel/entry-ftrace.S | 174 ++++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/ftrace.c | 64 ++++++++++++++ 6 files changed, 271 insertions(+) create mode 100644 arch/arm64/include/asm/ftrace.h create mode 100644 arch/arm64/kernel/entry-ftrace.S create mode 100644 arch/arm64/kernel/ftrace.c

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 27bbcfc..5783641 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -33,6 +33,8 @@ config ARM64 select HAVE_DMA_ATTRS select HAVE_DMA_CONTIGUOUS select HAVE_EFFICIENT_UNALIGNED_ACCESS + select HAVE_FUNCTION_TRACER + select HAVE_FUNCTION_GRAPH_TRACER select HAVE_GENERIC_DMA_COHERENT select HAVE_HW_BREAKPOINT if PERF_EVENTS select HAVE_MEMBLOCK diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h new file mode 100644 index 0000000..58ea595 --- /dev/null +++ b/arch/arm64/include/asm/ftrace.h @@ -0,0 +1,23 @@ +/* + * arch/arm64/include/asm/ftrace.h + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef __ASM_FTRACE_H +#define __ASM_FTRACE_H + +#include <asm/insn.h> + +#define MCOUNT_ADDR ((unsigned long)_mcount) +#define MCOUNT_INSN_SIZE AARCH64_INSN_SIZE + +#ifndef __ASSEMBLY__ +extern void _mcount(unsigned long); +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_FTRACE_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 2d4554b..ac67fd0 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -5,6 +5,9 @@ CPPFLAGS_vmlinux.lds := -DTEXT_OFFSET=$(TEXT_OFFSET) AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET)

+CFLAGS_REMOVE_ftrace.o = -pg +CFLAGS_REMOVE_insn.o = -pg + # Object file lists. arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ @@ -13,6 +16,7 @@ arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \

arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.o +arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c index 338b568..7f0512f 100644 --- a/arch/arm64/kernel/arm64ksyms.c +++ b/arch/arm64/kernel/arm64ksyms.c @@ -56,3 +56,7 @@ EXPORT_SYMBOL(clear_bit); EXPORT_SYMBOL(test_and_clear_bit); EXPORT_SYMBOL(change_bit); EXPORT_SYMBOL(test_and_change_bit); + +#ifdef CONFIG_FUNCTION_TRACER +EXPORT_SYMBOL(_mcount); +#endif diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S new file mode 100644 index 0000000..0e5b8b0 --- /dev/null +++ b/arch/arm64/kernel/entry-ftrace.S @@ -0,0 +1,174 @@ +/* + * arch/arm64/kernel/entry-ftrace.S + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/ftrace.h> +#include <asm/insn.h> + +/* + * Gcc with -pg will put the following code in the beginning of each function: + * mov x0, x30 + * bl _mcount + * [function's body ...] + * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic + * ftrace is enabled. + * + * Please note that x0 as an argument will not be used here because we can + * get lr(x30) of instrumented function at any time by winding up call stack + * as long as the kernel is compiled without -fomit-frame-pointer. + * (or CONFIG_FRAME_POINTER, this is forced on arm64) + * + * stack layout after mcount_enter in _mcount(): + * + * current sp => 0:+-----+ + * | x29 | -> instrumented function's fp + * +-----+ + * | x30 | -> _mcount()'s lr (= instrumented function's pc) + * old sp => +16:+-----+ + * when instrumented | | + * function calls | ... | + * _mcount() | | + * | | + * instrumented => +xx:+-----+ + * function's fp | x29 | -> parent's fp + * +-----+ + * | x30 | -> instrumented function's lr (= parent's pc) + * +-----+ + * | ... | + */ + + .macro mcount_enter + stp x29, x30, [sp, #-16]! + .endm + + .macro mcount_exit + ldp x29, x30, [sp], #16 + ret + .endm + + .macro mcount_adjust_addr rd, rn + sub \rd, \rn, #AARCH64_INSN_SIZE + .endm + + /* for instrumented function's parent */ + .macro mcount_get_parent_fp reg + ldr \reg, [sp] + ldr \reg, [\reg] + .endm + + /* for instrumented function */ + .macro mcount_get_pc0 reg + mcount_adjust_addr \reg, x30 + .endm + + .macro mcount_get_pc reg + ldr \reg, [sp, #8] + mcount_adjust_addr \reg, \reg + .endm + + .macro mcount_get_lr reg + ldr \reg, [sp] + ldr \reg, [\reg, #8] + mcount_adjust_addr \reg, \reg + .endm + + .macro mcount_get_lr_addr reg + ldr \reg, [sp] + add \reg, \reg, #8 + .endm + +/* + * void _mcount(unsigned long return_address) + * @return_address: return address to instrumented function + * + * This function makes calls, if enabled, to: + * - tracer function to probe instrumented function's entry, + * - ftrace_graph_caller to set up an exit hook + */ +ENTRY(_mcount) +#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST + ldr x0, =ftrace_trace_stop + ldr x0, [x0] // if ftrace_trace_stop + ret // return; +#endif + mcount_enter + + ldr x0, =ftrace_trace_function + ldr x2, [x0] + adr x0, ftrace_stub + cmp x0, x2 // if (ftrace_trace_function + b.eq skip_ftrace_call // != ftrace_stub) { + + mcount_get_pc x0 // function's pc + mcount_get_lr x1 // function's lr (= parent's pc) + blr x2 // (*ftrace_trace_function)(pc, lr); + +#ifndef CONFIG_FUNCTION_GRAPH_TRACER +skip_ftrace_call: // return; + mcount_exit // } +#else + mcount_exit // return; + // } +skip_ftrace_call: + ldr x1, =ftrace_graph_return + ldr x2, [x1] // if ((ftrace_graph_return + cmp x0, x2 // != ftrace_stub) + b.ne ftrace_graph_caller + + ldr x1, =ftrace_graph_entry // || (ftrace_graph_entry + ldr x2, [x1] // != ftrace_graph_entry_stub)) + ldr x0, =ftrace_graph_entry_stub + cmp x0, x2 + b.ne ftrace_graph_caller // ftrace_graph_caller(); + + mcount_exit +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ +ENDPROC(_mcount) + +ENTRY(ftrace_stub) + ret +ENDPROC(ftrace_stub) + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/* + * void ftrace_graph_caller(void) + * + * Called from _mcount() or ftrace_caller() when function_graph tracer is + * selected. + * This function w/ prepare_ftrace_return() fakes link register's value on + * the call stack in order to intercept instrumented function's return path + * and run return_to_handler() later on its exit. + */ +ENTRY(ftrace_graph_caller) + mcount_get_lr_addr x0 // pointer to function's saved lr + mcount_get_pc x1 // function's pc + mcount_get_parent_fp x2 // parent's fp + bl prepare_ftrace_return // prepare_ftrace_return(&lr, pc, fp) + + mcount_exit +ENDPROC(ftrace_graph_caller) + +/* + * void return_to_handler(void) + * + * Run ftrace_return_to_handler() before going back to parent. + * @fp is checked against the value passed by ftrace_graph_caller() + * only when CONFIG_FUNCTION_GRAPH_FP_TEST is enabled. + */ + .global return_to_handler +return_to_handler: + str x0, [sp, #-16]! + mov x0, x29 // parent's fp + bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp); + mov x30, x0 // restore the original return address + ldr x0, [sp], #16 + ret +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c new file mode 100644 index 0000000..a559ab8 --- /dev/null +++ b/arch/arm64/kernel/ftrace.c @@ -0,0 +1,64 @@ +/* + * arch/arm64/kernel/ftrace.c + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/ftrace.h> +#include <linux/swab.h> +#include <linux/uaccess.h> + +#include <asm/cacheflush.h> +#include <asm/ftrace.h> +#include <asm/insn.h> + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/* + * function_graph tracer expects ftrace_return_to_handler() to be called + * on the way back to parent. For this purpose, this function is called + * in _mcount() or ftrace_caller() to replace return address (*parent) on + * the call stack to return_to_handler. + * + * Note that @frame_pointer is used only for sanity check later. + */ +void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, + unsigned long frame_pointer) +{ + unsigned long return_hooker = (unsigned long)&return_to_handler; + unsigned long old; + struct ftrace_graph_ent trace; + int err; + + if (unlikely(atomic_read(&current->tracing_graph_pause))) + return; + + /* + * Note: + * No protection against faulting at *parent, which may be seen + * on other archs. It's unlikely on AArch64. + */ + old = *parent; + *parent = return_hooker; + + trace.func = self_addr; + trace.depth = current->curr_ret_stack + 1; + + /* Only trace if the calling function expects to */ + if (!ftrace_graph_entry(&trace)) { + *parent = old; + return; + } + + err = ftrace_push_return_trace(old, self_addr, &trace.depth, + frame_pointer); + if (err == -EBUSY) { + *parent = old; + return; + } +} +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

CALLER_ADDRx returns caller's address at specified level in call stacks. They are used for several tracers like irqsoff and preemptoff. Strange to say, however, they are refered even without FTRACE.

Please note that this implementation assumes that we have frame pointers. (which means kernel should be compiled with -fno-omit-frame-pointer.)

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/include/asm/ftrace.h | 14 +++++++++ arch/arm64/kernel/Makefile | 3 +- arch/arm64/kernel/return_address.c | 55 ++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/kernel/return_address.c

diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h index ed5c448..f93da72 100644 --- a/arch/arm64/include/asm/ftrace.h +++ b/arch/arm64/include/asm/ftrace.h @@ -35,4 +35,18 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) } #endif /* __ASSEMBLY__ */

+#ifndef __ASSEMBLY__ +#define HAVE_ARCH_CALLER_ADDR + +extern void *return_address(unsigned int); + +#define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0)) +#define CALLER_ADDR1 ((unsigned long)return_address(1)) +#define CALLER_ADDR2 ((unsigned long)return_address(2)) +#define CALLER_ADDR3 ((unsigned long)return_address(3)) +#define CALLER_ADDR4 ((unsigned long)return_address(4)) +#define CALLER_ADDR5 ((unsigned long)return_address(5)) +#define CALLER_ADDR6 ((unsigned long)return_address(6)) +#endif /* ifndef __ASSEMBLY__ */ + #endif /* __ASM_FTRACE_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index ac67fd0..b5bfa7f 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -7,12 +7,13 @@ AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET)

CFLAGS_REMOVE_ftrace.o = -pg CFLAGS_REMOVE_insn.o = -pg +CFLAGS_REMOVE_return_address.o = -pg

# Object file lists. arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \ - hyp-stub.o psci.o cpu_ops.o insn.o + hyp-stub.o psci.o cpu_ops.o insn.o return_address.o

arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.o diff --git a/arch/arm64/kernel/return_address.c b/arch/arm64/kernel/return_address.c new file mode 100644 index 0000000..89102a6 --- /dev/null +++ b/arch/arm64/kernel/return_address.c @@ -0,0 +1,55 @@ +/* + * arch/arm64/kernel/return_address.c + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/export.h> +#include <linux/ftrace.h> + +#include <asm/stacktrace.h> + +struct return_address_data { + unsigned int level; + void *addr; +}; + +static int save_return_addr(struct stackframe *frame, void *d) +{ + struct return_address_data *data = d; + + if (!data->level) { + data->addr = (void *)frame->pc; + return 1; + } else { + --data->level; + return 0; + } +} + +void *return_address(unsigned int level) +{ + struct return_address_data data; + struct stackframe frame; + register unsigned long current_sp asm ("sp"); + + data.level = level + 2; + data.addr = NULL; + + frame.fp = (unsigned long)__builtin_frame_address(0); + frame.sp = current_sp; + frame.pc = (unsigned long)return_address; /* dummy */ + + walk_stackframe(&frame, save_return_addr, &data); + + if (!data.level) + return data.addr; + else + return NULL; +} +EXPORT_SYMBOL_GPL(return_address);

This patch allows system call entry or exit to be traced as ftrace events, ie. sys_enter_*/sys_exit_*, if CONFIG_FTRACE_SYSCALLS is enabled. Those events appear and can be controlled under ${sysfs}/tracing/events/syscalls/

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/syscall.h | 1 + arch/arm64/include/asm/unistd.h | 2 ++ arch/arm64/kernel/ptrace.c | 52 ++++++++++++++++++++++---------------- 4 files changed, 34 insertions(+), 22 deletions(-)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 6954959..b1dcdb4 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -43,6 +43,7 @@ config ARM64 select HAVE_MEMBLOCK select HAVE_PATA_PLATFORM select HAVE_PERF_EVENTS + select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN select MODULES_USE_ELF_RELA select NO_BOOTMEM diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h index 70ba9d4..383771e 100644 --- a/arch/arm64/include/asm/syscall.h +++ b/arch/arm64/include/asm/syscall.h @@ -18,6 +18,7 @@

#include <linux/err.h>

+extern const void *sys_call_table[];

static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 82ce217..c335479 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -28,3 +28,5 @@ #endif #define __ARCH_WANT_SYS_CLONE #include <uapi/asm/unistd.h> + +#define NR_syscalls (__NR_syscalls) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 6a8928b..dc90f9f 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -41,6 +41,9 @@ #include <asm/traps.h> #include <asm/system_misc.h>

+#define CREATE_TRACE_POINTS +#include <trace/events/syscalls.h> + /* * TODO: does not yet catch signals sent when the child dies. * in exit.c or in signal.c. @@ -1062,31 +1065,36 @@ asmlinkage int syscall_trace(int dir, struct pt_regs *regs) { unsigned long saved_reg;

- if (!test_thread_flag(TIF_SYSCALL_TRACE)) - return regs->syscallno; + if (dir && unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) + trace_sys_exit(regs, regs_return_value(regs)); + + if (test_thread_flag(TIF_SYSCALL_TRACE)) { + if (is_compat_task()) { + /* AArch32 uses ip (r12) for scratch */ + saved_reg = regs->regs[12]; + regs->regs[12] = dir; + } else { + /* + * Save X7. X7 is used to denote syscall entry/exit: + * X7 = 0 -> entry, = 1 -> exit + */ + saved_reg = regs->regs[7]; + regs->regs[7] = dir; + }

- if (is_compat_task()) { - /* AArch32 uses ip (r12) for scratch */ - saved_reg = regs->regs[12]; - regs->regs[12] = dir; - } else { - /* - * Save X7. X7 is used to denote syscall entry/exit: - * X7 = 0 -> entry, = 1 -> exit - */ - saved_reg = regs->regs[7]; - regs->regs[7] = dir; - } + if (dir) + tracehook_report_syscall_exit(regs, 0); + else if (tracehook_report_syscall_entry(regs)) + regs->syscallno = ~0UL;

- if (dir) - tracehook_report_syscall_exit(regs, 0); - else if (tracehook_report_syscall_entry(regs)) - regs->syscallno = ~0UL; + if (is_compat_task()) + regs->regs[12] = saved_reg; + else + regs->regs[7] = saved_reg; + }

- if (is_compat_task()) - regs->regs[12] = saved_reg; - else - regs->regs[7] = saved_reg; + if (!dir && unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) + trace_sys_enter(regs, regs->syscallno);

return regs->syscallno; }

Since insn.h is indirectly included in asm/entry-ftrace.S, we need to exclude some declarations by __ASSEMBLY__.

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/include/asm/insn.h | 2 ++ 1 file changed, 2 insertions(+)

diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index c44ad39..dc1f73b 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -21,6 +21,7 @@ /* A64 instructions are always 32 bits. */ #define AARCH64_INSN_SIZE 4

+#ifndef __ASSEMBLY__ /* * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a * Section C3.1 "A64 instruction index by encoding": @@ -104,5 +105,6 @@ bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn); int aarch64_insn_patch_text_nosync(void *addr, u32 insn); int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt); int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt); +#endif /* __ASSEMBLY__ */

#endif /* __ASM_INSN_H */

Since insn.h is indirectly included in asm/entry-ftrace.S, we need to exclude some declarations by __ASSEMBLY__.

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/include/asm/insn.h | 2 ++ 1 file changed, 2 insertions(+)

diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index c44ad39..dc1f73b 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -21,6 +21,7 @@ /* A64 instructions are always 32 bits. */ #define AARCH64_INSN_SIZE 4

+#ifndef __ASSEMBLY__ /* * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a * Section C3.1 "A64 instruction index by encoding": @@ -104,5 +105,6 @@ bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn); int aarch64_insn_patch_text_nosync(void *addr, u32 insn); int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt); int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt); +#endif /* __ASSEMBLY__ */

#endif /* __ASM_INSN_H */

walk_stackframe() calls unwind_frame(), and if walk_stackframe() is "notrace", unwind_frame() should be also "notrace".

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/kernel/stacktrace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index c3b6c63..54122c4 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c @@ -35,7 +35,7 @@ * ldp x29, x30, [sp] * add sp, sp, #0x10 */ -int unwind_frame(struct stackframe *frame) +int notrace unwind_frame(struct stackframe *frame) { unsigned long high, low; unsigned long fp = frame->fp;

Recordmcount utility under scripts is run, after compiling each object, to find out all the locations of calling _mcount() and put them into specific seciton named __mcount_loc. Then linker collects all such information into a table in the kernel image (between __start_mcount_loc and __stop_mcount_loc) for later use by ftrace.

This patch adds arm64 specific definitions to identify such locations. There are two types of implementation, C and Perl. On arm64, only C version is used to build the kernel now that CONFIG_HAVE_C_RECORDMCOUNT is on. But Perl version is also maintained.

This patch also contains a workaround just in case where a header file, elf.h, on host machine doesn't have definitions of EM_AARCH64 nor R_AARCH64_ABS64. Without them, compiling C version of recordmcount will fail.

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/Kconfig | 2 ++ scripts/recordmcount.c | 9 +++++++++ scripts/recordmcount.pl | 5 +++++ 3 files changed, 16 insertions(+)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 27bbcfc..340e344 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -27,12 +27,14 @@ config ARM64 select HARDIRQS_SW_RESEND select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_TRACEHOOK + select HAVE_C_RECORDMCOUNT select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_API_DEBUG select HAVE_DMA_ATTRS select HAVE_DMA_CONTIGUOUS select HAVE_EFFICIENT_UNALIGNED_ACCESS + select HAVE_FTRACE_MCOUNT_RECORD select HAVE_GENERIC_DMA_COHERENT select HAVE_HW_BREAKPOINT if PERF_EVENTS select HAVE_MEMBLOCK diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c index 9c22317..047c8cd 100644 --- a/scripts/recordmcount.c +++ b/scripts/recordmcount.c @@ -40,6 +40,11 @@ #define R_METAG_NONE 3 #endif

+#ifndef EM_AARCH64 +#define EM_AARCH64 183 +#define R_AARCH64_ABS64 257 +#endif + static int fd_map; /* File descriptor for file being modified. */ static int mmap_failed; /* Boolean flag. */ static void *ehdr_curr; /* current ElfXX_Ehdr * for resource cleanup */ @@ -347,6 +352,10 @@ do_file(char const *const fname) case EM_ARM: reltype = R_ARM_ABS32; altmcount = "__gnu_mcount_nc"; break; + case EM_AARCH64: + reltype = R_AARCH64_ABS64; + altmcount = "_mcount"; + break; case EM_IA_64: reltype = R_IA64_IMM64; gpfx = '_'; break; case EM_METAG: reltype = R_METAG_ADDR32; altmcount = "_mcount_wrapper"; diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl index 91280b8..397b6b8 100755 --- a/scripts/recordmcount.pl +++ b/scripts/recordmcount.pl @@ -279,6 +279,11 @@ if ($arch eq "x86_64") { $mcount_regex = "^\s*([0-9a-fA-F]+):\s*R_ARM_(CALL|PC24|THM_CALL)" . "\s+(__gnu_mcount_nc|mcount)$";

+} elsif ($arch eq "arm64") { + $alignment = 3; + $section_type = '%progbits'; + $mcount_regex = "^\s*([0-9a-fA-F]+):\s*R_AARCH64_CALL26\s+_mcount$"; + $type = ".quad"; } elsif ($arch eq "ia64") { $mcount_regex = "^\s*([0-9a-fA-F]+):.*\s_mcount$"; $type = "data8";

On 03/13/2014 09:38 PM, Will Deacon wrote:

On Thu, Mar 13, 2014 at 10:13:46AM +0000, AKASHI Takahiro wrote:

...

Recordmcount utility under scripts is run, after compiling each object, to find out all the locations of calling _mcount() and put them into specific seciton named __mcount_loc. Then linker collects all such information into a table in the kernel image (between __start_mcount_loc and __stop_mcount_loc) for later use by ftrace.

This patch adds arm64 specific definitions to identify such locations. There are two types of implementation, C and Perl. On arm64, only C version is used to build the kernel now that CONFIG_HAVE_C_RECORDMCOUNT is on. But Perl version is also maintained.

This patch also contains a workaround just in case where a header file, elf.h, on host machine doesn't have definitions of EM_AARCH64 nor R_AARCH64_ABS64. Without them, compiling C version of recordmcount will fail.

[...]

...

diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c index 9c22317..047c8cd 100644 --- a/scripts/recordmcount.c +++ b/scripts/recordmcount.c @@ -40,6 +40,11 @@ #define R_METAG_NONE 3 #endif

+#ifndef EM_AARCH64 +#define EM_AARCH64 183 +#define R_AARCH64_ABS64 257 +#endif

• static int fd_map; /* File descriptor for file being modified. */ static int mmap_failed; /* Boolean flag. */ static void *ehdr_curr; /* current ElfXX_Ehdr * for resource cleanup */

@@ -347,6 +352,10 @@ do_file(char const *const fname) case EM_ARM: reltype = R_ARM_ABS32; altmcount = "__gnu_mcount_nc"; break;

• case EM_AARCH64:

• 	 reltype = R_AARCH64_ABS64;
  

• 	 altmcount = "_mcount";
  

Shouldn't you be setting gpfx = '_' instead of overriding altmcount like this?

Yep. Fix it.

...

• 	 break;
  

  case EM_IA_64: reltype = R_IA64_IMM64; gpfx = '_'; break; case EM_METAG: reltype = R_METAG_ADDR32; altmcount = "_mcount_wrapper";

diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl index 91280b8..397b6b8 100755 --- a/scripts/recordmcount.pl +++ b/scripts/recordmcount.pl @@ -279,6 +279,11 @@ if ($arch eq "x86_64") { $mcount_regex = "^\s*([0-9a-fA-F]+):\s*R_ARM_(CALL|PC24|THM_CALL)" . "\s+(__gnu_mcount_nc|mcount)$";

+} elsif ($arch eq "arm64") {

• $alignment = 3;
• $section_type = '%progbits';
• $mcount_regex = "^\s*([0-9a-fA-F]+):\s*R_AARCH64_CALL26\s+_mcount$";

What about "mcount" (i.e. no underscore)?

I don't think it works since such regex doesn't match to "CALL26 _mcount".

Thanks, -Takahiro AKASHI

Will

This patch implements arm64 specific part to support function tracers, such as function (CONFIG_FUNCTION_TRACER), function_graph (CONFIG_FUNCTION_GRAPH_TRACER) and function profiler (CONFIG_FUNCTION_PROFILER).

With 'function' tracer, all the functions in the kernel are traced with timestamps in ${sysfs}/tracing/trace. If function_graph tracer is specified, call graph is generated.

The kernel must be compiled with -pg option so that _mcount() is inserted at the beginning of functions. This function is called on every function's entry as long as tracing is enabled. In addition, function_graph tracer also needs to be able to probe function's exit. ftrace_graph_caller() & return_to_handler do this by faking link register's value to intercept function's return path.

More details on architecture specific requirements are described in Documentation/trace/ftrace-design.txt.

Reviewed-by: Ganapatrao Kulkarni ganapatrao.kulkarni@cavium.com Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/Kconfig | 2 + arch/arm64/include/asm/ftrace.h | 23 +++++ arch/arm64/kernel/Makefile | 4 + arch/arm64/kernel/arm64ksyms.c | 4 + arch/arm64/kernel/entry-ftrace.S | 175 ++++++++++++++++++++++++++++++++++++++ arch/arm64/kernel/ftrace.c | 64 ++++++++++++++ 6 files changed, 272 insertions(+) create mode 100644 arch/arm64/include/asm/ftrace.h create mode 100644 arch/arm64/kernel/entry-ftrace.S create mode 100644 arch/arm64/kernel/ftrace.c

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 340e344..6b3fef6 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -35,6 +35,8 @@ config ARM64 select HAVE_DMA_CONTIGUOUS select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_FTRACE_MCOUNT_RECORD + select HAVE_FUNCTION_TRACER + select HAVE_FUNCTION_GRAPH_TRACER select HAVE_GENERIC_DMA_COHERENT select HAVE_HW_BREAKPOINT if PERF_EVENTS select HAVE_MEMBLOCK diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h new file mode 100644 index 0000000..58ea595 --- /dev/null +++ b/arch/arm64/include/asm/ftrace.h @@ -0,0 +1,23 @@ +/* + * arch/arm64/include/asm/ftrace.h + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef __ASM_FTRACE_H +#define __ASM_FTRACE_H + +#include <asm/insn.h> + +#define MCOUNT_ADDR ((unsigned long)_mcount) +#define MCOUNT_INSN_SIZE AARCH64_INSN_SIZE + +#ifndef __ASSEMBLY__ +extern void _mcount(unsigned long); +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_FTRACE_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 2d4554b..ac67fd0 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -5,6 +5,9 @@ CPPFLAGS_vmlinux.lds := -DTEXT_OFFSET=$(TEXT_OFFSET) AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET)

+CFLAGS_REMOVE_ftrace.o = -pg +CFLAGS_REMOVE_insn.o = -pg + # Object file lists. arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ @@ -13,6 +16,7 @@ arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \

arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.o +arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c index 338b568..7f0512f 100644 --- a/arch/arm64/kernel/arm64ksyms.c +++ b/arch/arm64/kernel/arm64ksyms.c @@ -56,3 +56,7 @@ EXPORT_SYMBOL(clear_bit); EXPORT_SYMBOL(test_and_clear_bit); EXPORT_SYMBOL(change_bit); EXPORT_SYMBOL(test_and_change_bit); + +#ifdef CONFIG_FUNCTION_TRACER +EXPORT_SYMBOL(_mcount); +#endif diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S new file mode 100644 index 0000000..0ac31c8 --- /dev/null +++ b/arch/arm64/kernel/entry-ftrace.S @@ -0,0 +1,175 @@ +/* + * arch/arm64/kernel/entry-ftrace.S + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/ftrace.h> +#include <asm/insn.h> + +/* + * Gcc with -pg will put the following code in the beginning of each function: + * mov x0, x30 + * bl _mcount + * [function's body ...] + * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic + * ftrace is enabled. + * + * Please note that x0 as an argument will not be used here because we can + * get lr(x30) of instrumented function at any time by winding up call stack + * as long as the kernel is compiled without -fomit-frame-pointer. + * (or CONFIG_FRAME_POINTER, this is forced on arm64) + * + * stack layout after mcount_enter in _mcount(): + * + * current sp/fp => 0:+-----+ + * in _mcount() | x29 | -> instrumented function's fp + * +-----+ + * | x30 | -> _mcount()'s lr (= instrumented function's pc) + * old sp => +16:+-----+ + * when instrumented | | + * function calls | ... | + * _mcount() | | + * | | + * instrumented => +xx:+-----+ + * function's fp | x29 | -> parent's fp + * +-----+ + * | x30 | -> instrumented function's lr (= parent's pc) + * +-----+ + * | ... | + */ + + .macro mcount_enter + stp x29, x30, [sp, #-16]! + mov x29, sp + .endm + + .macro mcount_exit + ldp x29, x30, [sp], #16 + ret + .endm + + .macro mcount_adjust_addr rd, rn + sub \rd, \rn, #AARCH64_INSN_SIZE + .endm + + /* for instrumented function's parent */ + .macro mcount_get_parent_fp reg + ldr \reg, [x29] + ldr \reg, [\reg] + .endm + + /* for instrumented function */ + .macro mcount_get_pc0 reg + mcount_adjust_addr \reg, x30 + .endm + + .macro mcount_get_pc reg + ldr \reg, [x29, #8] + mcount_adjust_addr \reg, \reg + .endm + + .macro mcount_get_lr reg + ldr \reg, [x29] + ldr \reg, [\reg, #8] + mcount_adjust_addr \reg, \reg + .endm + + .macro mcount_get_lr_addr reg + ldr \reg, [x29] + add \reg, \reg, #8 + .endm + +/* + * void _mcount(unsigned long return_address) + * @return_address: return address to instrumented function + * + * This function makes calls, if enabled, to: + * - tracer function to probe instrumented function's entry, + * - ftrace_graph_caller to set up an exit hook + */ +ENTRY(_mcount) +#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST + ldr x0, =ftrace_trace_stop + ldr x0, [x0] // if ftrace_trace_stop + ret // return; +#endif + mcount_enter + + ldr x0, =ftrace_trace_function + ldr x2, [x0] + adr x0, ftrace_stub + cmp x0, x2 // if (ftrace_trace_function + b.eq skip_ftrace_call // != ftrace_stub) { + + mcount_get_pc x0 // function's pc + mcount_get_lr x1 // function's lr (= parent's pc) + blr x2 // (*ftrace_trace_function)(pc, lr); + +#ifndef CONFIG_FUNCTION_GRAPH_TRACER +skip_ftrace_call: // return; + mcount_exit // } +#else + mcount_exit // return; + // } +skip_ftrace_call: + ldr x1, =ftrace_graph_return + ldr x2, [x1] // if ((ftrace_graph_return + cmp x0, x2 // != ftrace_stub) + b.ne ftrace_graph_caller + + ldr x1, =ftrace_graph_entry // || (ftrace_graph_entry + ldr x2, [x1] // != ftrace_graph_entry_stub)) + ldr x0, =ftrace_graph_entry_stub + cmp x0, x2 + b.ne ftrace_graph_caller // ftrace_graph_caller(); + + mcount_exit +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ +ENDPROC(_mcount) + +ENTRY(ftrace_stub) + ret +ENDPROC(ftrace_stub) + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/* + * void ftrace_graph_caller(void) + * + * Called from _mcount() or ftrace_caller() when function_graph tracer is + * selected. + * This function w/ prepare_ftrace_return() fakes link register's value on + * the call stack in order to intercept instrumented function's return path + * and run return_to_handler() later on its exit. + */ +ENTRY(ftrace_graph_caller) + mcount_get_lr_addr x0 // pointer to function's saved lr + mcount_get_pc x1 // function's pc + mcount_get_parent_fp x2 // parent's fp + bl prepare_ftrace_return // prepare_ftrace_return(&lr, pc, fp) + + mcount_exit +ENDPROC(ftrace_graph_caller) + +/* + * void return_to_handler(void) + * + * Run ftrace_return_to_handler() before going back to parent. + * @fp is checked against the value passed by ftrace_graph_caller() + * only when CONFIG_FUNCTION_GRAPH_FP_TEST is enabled. + */ + .global return_to_handler +return_to_handler: + str x0, [sp, #-16]! + mov x0, x29 // parent's fp + bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp); + mov x30, x0 // restore the original return address + ldr x0, [sp], #16 + ret +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c new file mode 100644 index 0000000..a559ab8 --- /dev/null +++ b/arch/arm64/kernel/ftrace.c @@ -0,0 +1,64 @@ +/* + * arch/arm64/kernel/ftrace.c + * + * Copyright (C) 2013 Linaro Limited + * Author: AKASHI Takahiro takahiro.akashi@linaro.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/ftrace.h> +#include <linux/swab.h> +#include <linux/uaccess.h> + +#include <asm/cacheflush.h> +#include <asm/ftrace.h> +#include <asm/insn.h> + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +/* + * function_graph tracer expects ftrace_return_to_handler() to be called + * on the way back to parent. For this purpose, this function is called + * in _mcount() or ftrace_caller() to replace return address (*parent) on + * the call stack to return_to_handler. + * + * Note that @frame_pointer is used only for sanity check later. + */ +void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, + unsigned long frame_pointer) +{ + unsigned long return_hooker = (unsigned long)&return_to_handler; + unsigned long old; + struct ftrace_graph_ent trace; + int err; + + if (unlikely(atomic_read(&current->tracing_graph_pause))) + return; + + /* + * Note: + * No protection against faulting at *parent, which may be seen + * on other archs. It's unlikely on AArch64. + */ + old = *parent; + *parent = return_hooker; + + trace.func = self_addr; + trace.depth = current->curr_ret_stack + 1; + + /* Only trace if the calling function expects to */ + if (!ftrace_graph_entry(&trace)) { + *parent = old; + return; + } + + err = ftrace_push_return_trace(old, self_addr, &trace.depth, + frame_pointer); + if (err == -EBUSY) { + *parent = old; + return; + } +} +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

This patch allows "dynamic ftrace" if CONFIG_DYNAMIC_FTRACE is enabled. Here we can turn on and off tracing dynamically per-function base.

On arm64, this is done by patching single branch instruction to _mcount() inserted by gcc -pg option. The branch is replaced to NOP initially at kernel start up, and later on, NOP to branch to ftrace_caller() when enabled or branch to NOP when disabled. Please note that ftrace_caller() is a counterpart of _mcount() in case of 'static' ftrace.

More details on architecture specific requirements are described in Documentation/trace/ftrace-design.txt.

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/ftrace.h | 15 +++++ arch/arm64/kernel/entry-ftrace.S | 43 +++++++++++++++ arch/arm64/kernel/ftrace.c | 113 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 172 insertions(+)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 6b3fef6..6954959 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -33,6 +33,7 @@ config ARM64 select HAVE_DMA_API_DEBUG select HAVE_DMA_ATTRS select HAVE_DMA_CONTIGUOUS + select HAVE_DYNAMIC_FTRACE select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_TRACER diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h index 58ea595..ed5c448 100644 --- a/arch/arm64/include/asm/ftrace.h +++ b/arch/arm64/include/asm/ftrace.h @@ -18,6 +18,21 @@

#ifndef __ASSEMBLY__ extern void _mcount(unsigned long); + +struct dyn_arch_ftrace { + /* No extra data needed for arm64 */ +}; + +extern unsigned long ftrace_graph_call; + +static inline unsigned long ftrace_call_adjust(unsigned long addr) +{ + /* + * addr is the address of the mcount call instruction. + * recordmcount does the necessary offset calculation. + */ + return addr; +} #endif /* __ASSEMBLY__ */

#endif /* __ASM_FTRACE_H */ diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S index 0ac31c8..c0fbe10 100644 --- a/arch/arm64/kernel/entry-ftrace.S +++ b/arch/arm64/kernel/entry-ftrace.S @@ -86,6 +86,7 @@ add \reg, \reg, #8 .endm

+#ifndef CONFIG_DYNAMIC_FTRACE /* * void _mcount(unsigned long return_address) * @return_address: return address to instrumented function @@ -134,6 +135,48 @@ skip_ftrace_call: #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ ENDPROC(_mcount)

+#else /* CONFIG_DYNAMIC_FTRACE */ +/* + * _mcount() is used to build the kernel with -pg option, but all the branch + * instructions to _mcount() are replaced to NOP initially at kernel start up, + * and later on, NOP to branch to ftrace_caller() when enabled or branch to + * NOP when disabled per-function base. + */ +ENTRY(_mcount) + ret +ENDPROC(_mcount) + +/* + * void ftrace_caller(unsigned long return_address) + * @return_address: return address to instrumented function + * + * This function is a counterpart of _mcount() in 'static' ftrace, and + * makes calls to: + * - tracer function to probe instrumented function's entry, + * - ftrace_graph_caller to set up an exit hook + */ +ENTRY(ftrace_caller) + mcount_enter + + mcount_get_pc0 x0 // function's pc + mcount_get_lr x1 // function's lr + + .global ftrace_call +ftrace_call: // tracer(pc, lr); + nop // This will be replaced with "bl xxx" + // where xxx can be any kind of tracer. + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + .global ftrace_graph_call +ftrace_graph_call: // ftrace_graph_caller(); + nop // If enabled, this will be replaced + // "b ftrace_graph_caller" +#endif + + mcount_exit +ENDPROC(ftrace_caller) +#endif /* CONFIG_DYNAMIC_FTRACE */ + ENTRY(ftrace_stub) ret ENDPROC(ftrace_stub) diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index a559ab8..8c26476 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c @@ -17,6 +17,89 @@ #include <asm/ftrace.h> #include <asm/insn.h>

+#ifdef CONFIG_DYNAMIC_FTRACE +/* + * Replace a single instruction, which may be a branch or NOP. + * If @validate == true, a replaced instruction is checked against 'old'. + */ +static int ftrace_modify_code(unsigned long pc, unsigned int old, + unsigned int new, bool validate) +{ + unsigned int replaced; + + /* + * Note: + * Due to modules and __init, code can disappear and change, + * we need to protect against faulting as well as code changing. + * We do this by aarch64_insn_*() which use the probe_kernel_*(). + * + * No lock is held here because all the modifications are run + * through stop_machine(). + */ + if (validate) { + if (aarch64_insn_read((void *)pc, &replaced)) + return -EFAULT; + + if (replaced != old) + return -EINVAL; + } + if (aarch64_insn_patch_text_nosync((void *)pc, new)) + return -EPERM; + + return 0; +} + +/* + * Replace tracer function in ftrace_caller() + */ +int ftrace_update_ftrace_func(ftrace_func_t func) +{ + unsigned long pc; + unsigned int new; + + pc = (unsigned long)&ftrace_call; + new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func, true); + + return ftrace_modify_code(pc, 0, new, false); +} + +/* + * Turn on the call to ftrace_caller() in instrumented function + */ +int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) +{ + unsigned long pc = rec->ip; + unsigned int old, new; + + old = aarch64_insn_gen_nop(); + new = aarch64_insn_gen_branch_imm(pc, addr, true); + + return ftrace_modify_code(pc, old, new, true); +} + +/* + * Turn off the call to ftrace_caller() in instrumented function + */ +int ftrace_make_nop(struct module *mod, + struct dyn_ftrace *rec, unsigned long addr) +{ + unsigned long pc = rec->ip; + unsigned int old, new; + + old = aarch64_insn_gen_branch_imm(pc, addr, true); + new = aarch64_insn_gen_nop(); + + return ftrace_modify_code(pc, old, new, true); +} + +int __init ftrace_dyn_arch_init(void *data) +{ + *(unsigned long *)data = 0; + + return 0; +} +#endif /* CONFIG_DYNAMIC_FTRACE */ + #ifdef CONFIG_FUNCTION_GRAPH_TRACER /* * function_graph tracer expects ftrace_return_to_handler() to be called @@ -61,4 +144,34 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, return; } } + +#ifdef CONFIG_DYNAMIC_FTRACE +/* + * Turn on/off the call to ftrace_graph_caller() in ftrace_caller() + * depending on @enable. + */ +static int ftrace_modify_graph_caller(bool enable) +{ + unsigned long pc = (unsigned long)&ftrace_graph_call; + unsigned int branch, nop, old, new; + + branch = aarch64_insn_gen_branch_imm(pc, + (unsigned long)ftrace_graph_caller, false); + nop = aarch64_insn_gen_nop(); + old = enable ? nop : branch; + new = enable ? branch : nop; + + return ftrace_modify_code(pc, old, new, true); +} + +int ftrace_enable_ftrace_graph_caller(void) +{ + return ftrace_modify_graph_caller(true); +} + +int ftrace_disable_ftrace_graph_caller(void) +{ + return ftrace_modify_graph_caller(false); +} +#endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* CONFIG_FUNCTION_GRAPH_TRACER */

On Thu, 2014-03-13 at 18:10 +0000, Will Deacon wrote:

...

+#else /* CONFIG_DYNAMIC_FTRACE */ +/*

• • _mcount() is used to build the kernel with -pg option, but all the branch
• • instructions to _mcount() are replaced to NOP initially at kernel start up,
• • and later on, NOP to branch to ftrace_caller() when enabled or branch to
• • NOP when disabled per-function base.
• */

+ENTRY(_mcount)

• ret

+ENDPROC(_mcount)

Judging by your comment then, this should never be called. Is that right? If so, we could add a BUG-equivalent so we know if we missed an mcount during patching.

Actually, it can be called before the change to nops are done in early boot. This is done very early, but everything before ftrace_init() in init/main.c can still call _mcount.

...

• /*

• * Note:
  

• * Due to modules and __init, code can disappear and change,
  

• * we need to protect against faulting as well as code changing.
  

• * We do this by aarch64_insn_*() which use the probe_kernel_*().
  

• *
  

• * No lock is held here because all the modifications are run
  

• * through stop_machine().
  

• */
  

• if (validate) {

• if (aarch64_insn_read((void *)pc, &replaced))
  

• 	return -EFAULT;
  

• if (replaced != old)
  

• 	return -EINVAL;
  

• }
• if (aarch64_insn_patch_text_nosync((void *)pc, new))

• return -EPERM;
  

I think you're better off propagating the errors here, rather than overriding them with EFAULT/EINVAL/EPERM.

The ftrace generic code expects to see these specific errors. Look at ftrace_bug() in kernel/trace/ftrace.c.

...

• return 0;

+}

+/*

• • Replace tracer function in ftrace_caller()
• */

+int ftrace_update_ftrace_func(ftrace_func_t func) +{

• unsigned long pc;
• unsigned int new;
• pc = (unsigned long)&ftrace_call;
• new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func, true);
• return ftrace_modify_code(pc, 0, new, false);

+}

+/*

• • Turn on the call to ftrace_caller() in instrumented function
• */

+int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) +{

• unsigned long pc = rec->ip;
• unsigned int old, new;
• old = aarch64_insn_gen_nop();
• new = aarch64_insn_gen_branch_imm(pc, addr, true);
• return ftrace_modify_code(pc, old, new, true);

+}

+/*

• • Turn off the call to ftrace_caller() in instrumented function
• */

+int ftrace_make_nop(struct module *mod,

•     struct dyn_ftrace *rec, unsigned long addr)
  

+{

• unsigned long pc = rec->ip;
• unsigned int old, new;
• old = aarch64_insn_gen_branch_imm(pc, addr, true);
• new = aarch64_insn_gen_nop();
• return ftrace_modify_code(pc, old, new, true);

+}

+int __init ftrace_dyn_arch_init(void *data) +{

• *(unsigned long *)data = 0;
• return 0;

+} +#endif /* CONFIG_DYNAMIC_FTRACE */

#ifdef CONFIG_FUNCTION_GRAPH_TRACER /*

• function_graph tracer expects ftrace_return_to_handler() to be called

@@ -61,4 +144,34 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, return; } }

+#ifdef CONFIG_DYNAMIC_FTRACE +/*

• • Turn on/off the call to ftrace_graph_caller() in ftrace_caller()
• • depending on @enable.
• */

+static int ftrace_modify_graph_caller(bool enable) +{

• unsigned long pc = (unsigned long)&ftrace_graph_call;
• unsigned int branch, nop, old, new;
• branch = aarch64_insn_gen_branch_imm(pc,

• 	(unsigned long)ftrace_graph_caller, false);
  

• nop = aarch64_insn_gen_nop();
• old = enable ? nop : branch;
• new = enable ? branch : nop;
• return ftrace_modify_code(pc, old, new, true);

You could rewrite this as:

if (enable) return ftrace_modify_code(pc, nop, branch, true); else return ftrace_modify_code(pc, branch, nop, true);

which I find easier to read.

Heh, maybe that could be updated in other archs too. I'll have to think about that one.

-- Steve

On 03/14/2014 03:10 AM, Will Deacon wrote:

On Thu, Mar 13, 2014 at 10:13:48AM +0000, AKASHI Takahiro wrote:

...

This patch allows "dynamic ftrace" if CONFIG_DYNAMIC_FTRACE is enabled. Here we can turn on and off tracing dynamically per-function base.

On arm64, this is done by patching single branch instruction to _mcount() inserted by gcc -pg option. The branch is replaced to NOP initially at kernel start up, and later on, NOP to branch to ftrace_caller() when enabled or branch to NOP when disabled. Please note that ftrace_caller() is a counterpart of _mcount() in case of 'static' ftrace.

More details on architecture specific requirements are described in Documentation/trace/ftrace-design.txt.

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org

arch/arm64/Kconfig | 1 + arch/arm64/include/asm/ftrace.h | 15 +++++ arch/arm64/kernel/entry-ftrace.S | 43 +++++++++++++++ arch/arm64/kernel/ftrace.c | 113 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 172 insertions(+)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 6b3fef6..6954959 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -33,6 +33,7 @@ config ARM64 select HAVE_DMA_API_DEBUG select HAVE_DMA_ATTRS select HAVE_DMA_CONTIGUOUS

• select HAVE_DYNAMIC_FTRACE select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_FTRACE_MCOUNT_RECORD select HAVE_FUNCTION_TRACER

diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h index 58ea595..ed5c448 100644 --- a/arch/arm64/include/asm/ftrace.h +++ b/arch/arm64/include/asm/ftrace.h @@ -18,6 +18,21 @@

#ifndef __ASSEMBLY__ extern void _mcount(unsigned long);

+struct dyn_arch_ftrace {

• /* No extra data needed for arm64 */

+};

+extern unsigned long ftrace_graph_call;

+static inline unsigned long ftrace_call_adjust(unsigned long addr) +{

• /*

• * addr is the address of the mcount call instruction.
  

• * recordmcount does the necessary offset calculation.
  

• */
  

• return addr;

+}

You could just as easily implement this as a dummy macro, but I guess it doesn't matter either way.

FYI, all archs define this as an inline function. Leave it as it is.

...

#endif /* __ASSEMBLY__ */

#endif /* __ASM_FTRACE_H */ diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S index 0ac31c8..c0fbe10 100644 --- a/arch/arm64/kernel/entry-ftrace.S +++ b/arch/arm64/kernel/entry-ftrace.S @@ -86,6 +86,7 @@ add \reg, \reg, #8 .endm

+#ifndef CONFIG_DYNAMIC_FTRACE /*

• void _mcount(unsigned long return_address)
• @return_address: return address to instrumented function

@@ -134,6 +135,48 @@ skip_ftrace_call: #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ ENDPROC(_mcount)

+#else /* CONFIG_DYNAMIC_FTRACE */ +/*

• • _mcount() is used to build the kernel with -pg option, but all the branch
• • instructions to _mcount() are replaced to NOP initially at kernel start up,
• • and later on, NOP to branch to ftrace_caller() when enabled or branch to
• • NOP when disabled per-function base.
• */

+ENTRY(_mcount)

• ret

+ENDPROC(_mcount)

Judging by your comment then, this should never be called. Is that right? If so, we could add a BUG-equivalent so we know if we missed an mcount during patching.

Steven explained this.

...

+/*

• • void ftrace_caller(unsigned long return_address)
• • @return_address: return address to instrumented function
• • This function is a counterpart of _mcount() in 'static' ftrace, and
• • makes calls to:

• • - tracer function to probe instrumented function's entry,
    

• • - ftrace_graph_caller to set up an exit hook
    

• */

+ENTRY(ftrace_caller)

• mcount_enter
• mcount_get_pc0 x0 // function's pc
• mcount_get_lr x1 // function's lr
• .global ftrace_call

+ftrace_call: // tracer(pc, lr);

• nop // This will be replaced with "bl xxx"

• 			// where xxx can be any kind of tracer.
  

+#ifdef CONFIG_FUNCTION_GRAPH_TRACER

• .global ftrace_graph_call

+ftrace_graph_call: // ftrace_graph_caller();

• nop // If enabled, this will be replaced

• 			// "b ftrace_graph_caller"
  

+#endif

• mcount_exit

+ENDPROC(ftrace_caller) +#endif /* CONFIG_DYNAMIC_FTRACE */

• ENTRY(ftrace_stub) ret ENDPROC(ftrace_stub)

diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index a559ab8..8c26476 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c @@ -17,6 +17,89 @@ #include <asm/ftrace.h> #include <asm/insn.h>

+#ifdef CONFIG_DYNAMIC_FTRACE +/*

• • Replace a single instruction, which may be a branch or NOP.
• • If @validate == true, a replaced instruction is checked against 'old'.
• */

+static int ftrace_modify_code(unsigned long pc, unsigned int old,

• 	      unsigned int new, bool validate)
  

+{

• unsigned int replaced;

u32 is a bit clearer for instructions.

Fix it.

...

• /*

• * Note:
  

• * Due to modules and __init, code can disappear and change,
  

• * we need to protect against faulting as well as code changing.
  

• * We do this by aarch64_insn_*() which use the probe_kernel_*().
  

• *
  

• * No lock is held here because all the modifications are run
  

• * through stop_machine().
  

• */
  

• if (validate) {

• if (aarch64_insn_read((void *)pc, &replaced))
  

• 	return -EFAULT;
  

• if (replaced != old)
  

• 	return -EINVAL;
  

• }
• if (aarch64_insn_patch_text_nosync((void *)pc, new))

• return -EPERM;
  

I think you're better off propagating the errors here, rather than overriding them with EFAULT/EINVAL/EPERM.

Steven explained this.

...

• return 0;

+}

+/*

• • Replace tracer function in ftrace_caller()
• */

+int ftrace_update_ftrace_func(ftrace_func_t func) +{

• unsigned long pc;
• unsigned int new;
• pc = (unsigned long)&ftrace_call;
• new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func, true);
• return ftrace_modify_code(pc, 0, new, false);

+}

+/*

• • Turn on the call to ftrace_caller() in instrumented function
• */

+int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) +{

• unsigned long pc = rec->ip;
• unsigned int old, new;
• old = aarch64_insn_gen_nop();
• new = aarch64_insn_gen_branch_imm(pc, addr, true);
• return ftrace_modify_code(pc, old, new, true);

+}

+/*

• • Turn off the call to ftrace_caller() in instrumented function
• */

+int ftrace_make_nop(struct module *mod,

•     struct dyn_ftrace *rec, unsigned long addr)
  

+{

• unsigned long pc = rec->ip;
• unsigned int old, new;
• old = aarch64_insn_gen_branch_imm(pc, addr, true);
• new = aarch64_insn_gen_nop();
• return ftrace_modify_code(pc, old, new, true);

+}

+int __init ftrace_dyn_arch_init(void *data) +{

• *(unsigned long *)data = 0;
• return 0;

+} +#endif /* CONFIG_DYNAMIC_FTRACE */

• #ifdef CONFIG_FUNCTION_GRAPH_TRACER /*
  • function_graph tracer expects ftrace_return_to_handler() to be called

@@ -61,4 +144,34 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, return; } }

+#ifdef CONFIG_DYNAMIC_FTRACE +/*

• • Turn on/off the call to ftrace_graph_caller() in ftrace_caller()
• • depending on @enable.
• */

+static int ftrace_modify_graph_caller(bool enable) +{

• unsigned long pc = (unsigned long)&ftrace_graph_call;
• unsigned int branch, nop, old, new;
• branch = aarch64_insn_gen_branch_imm(pc,

• 	(unsigned long)ftrace_graph_caller, false);
  

• nop = aarch64_insn_gen_nop();
• old = enable ? nop : branch;
• new = enable ? branch : nop;
• return ftrace_modify_code(pc, old, new, true);

You could rewrite this as:

if (enable) return ftrace_modify_code(pc, nop, branch, true); else return ftrace_modify_code(pc, branch, nop, true);

which I find easier to read.

Yep, fix it.

-Takahiro AKASHI

Will

On Thu, Mar 13, 2014 at 10:13:49AM +0000, AKASHI Takahiro wrote:

CALLER_ADDRx returns caller's address at specified level in call stacks. They are used for several tracers like irqsoff and preemptoff. Strange to say, however, they are refered even without FTRACE.

Please note that this implementation assumes that we have frame pointers. (which means kernel should be compiled with -fno-omit-frame-pointer.)

How do you ensure that -fno-omit-frame-pointer is passed?

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org

arch/arm64/include/asm/ftrace.h | 13 ++++++++- arch/arm64/kernel/Makefile | 3 +- arch/arm64/kernel/return_address.c | 55 ++++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/kernel/return_address.c

diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h index ed5c448..c44c4b1 100644 --- a/arch/arm64/include/asm/ftrace.h +++ b/arch/arm64/include/asm/ftrace.h @@ -18,6 +18,7 @@ #ifndef __ASSEMBLY__ extern void _mcount(unsigned long); +extern void *return_address(unsigned int); struct dyn_arch_ftrace { /* No extra data needed for arm64 */ @@ -33,6 +34,16 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) */ return addr; } -#endif /* __ASSEMBLY__ */

+#define HAVE_ARCH_CALLER_ADDR

+#define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0)) +#define CALLER_ADDR1 ((unsigned long)return_address(1)) +#define CALLER_ADDR2 ((unsigned long)return_address(2)) +#define CALLER_ADDR3 ((unsigned long)return_address(3)) +#define CALLER_ADDR4 ((unsigned long)return_address(4)) +#define CALLER_ADDR5 ((unsigned long)return_address(5)) +#define CALLER_ADDR6 ((unsigned long)return_address(6))

Could we change the core definitions of these macros (in linux/ftrace.h) to use return_address, then provide an overridable version of return_address that defaults to __builtin_return_address, instead of copy-pasting this sequence?

diff --git a/arch/arm64/kernel/return_address.c b/arch/arm64/kernel/return_address.c new file mode 100644 index 0000000..89102a6 --- /dev/null +++ b/arch/arm64/kernel/return_address.c @@ -0,0 +1,55 @@ +/*

• • arch/arm64/kernel/return_address.c
• • Copyright (C) 2013 Linaro Limited
• • Author: AKASHI Takahiro takahiro.akashi@linaro.org
• • This program is free software; you can redistribute it and/or modify
• • it under the terms of the GNU General Public License version 2 as
• • published by the Free Software Foundation.
• */

+#include <linux/export.h> +#include <linux/ftrace.h>

+#include <asm/stacktrace.h>

+struct return_address_data {

• unsigned int level;
• void *addr;

+};

+static int save_return_addr(struct stackframe *frame, void *d) +{

• struct return_address_data *data = d;
• if (!data->level) {

• data->addr = (void *)frame->pc;
  

• return 1;
  

• } else {

• --data->level;
  

• return 0;
  

• }

+}

+void *return_address(unsigned int level) +{

• struct return_address_data data;
• struct stackframe frame;
• register unsigned long current_sp asm ("sp");
• data.level = level + 2;
• data.addr = NULL;
• frame.fp = (unsigned long)__builtin_frame_address(0);
• frame.sp = current_sp;
• frame.pc = (unsigned long)return_address; /* dummy */
• walk_stackframe(&frame, save_return_addr, &data);
• if (!data.level)

• return data.addr;
  

• else

• return NULL;
  

+} +EXPORT_SYMBOL_GPL(return_address);

This whole file is basically copied from arch/arm/, but it's not too much code. Ideally the toolchain would have made use of the frame pointer, but it looks like it doesn't bother.

Will

On Fri, Mar 14, 2014 at 03:00:14AM +0000, AKASHI Takahiro wrote:

On 03/14/2014 12:54 AM, Will Deacon wrote:

...

On Thu, Mar 13, 2014 at 10:13:49AM +0000, AKASHI Takahiro wrote:

...

CALLER_ADDRx returns caller's address at specified level in call stacks. They are used for several tracers like irqsoff and preemptoff. Strange to say, however, they are refered even without FTRACE.

Please note that this implementation assumes that we have frame pointers. (which means kernel should be compiled with -fno-omit-frame-pointer.)

How do you ensure that -fno-omit-frame-pointer is passed?

arm64 selects ARCH_WANT_FRAME_POINTERS, then FRAME_POINTER is on (lib/Kconfig.debug) and so -fno-omit-frame-pointer is appended (${TOP}/Makefile). (stacktrace.c also assumes FRAME_POINTER.)

Do you think I should remove the comment above?

Yes please, it sounds like everything is taken care of automatically, so there's no need to scare people in the commit message!

...

...

diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h index ed5c448..c44c4b1 100644 --- a/arch/arm64/include/asm/ftrace.h +++ b/arch/arm64/include/asm/ftrace.h @@ -18,6 +18,7 @@

#ifndef __ASSEMBLY__ extern void _mcount(unsigned long); +extern void *return_address(unsigned int);

struct dyn_arch_ftrace { /* No extra data needed for arm64 */ @@ -33,6 +34,16 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) */ return addr; } -#endif /* __ASSEMBLY__ */

+#define HAVE_ARCH_CALLER_ADDR

+#define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0)) +#define CALLER_ADDR1 ((unsigned long)return_address(1)) +#define CALLER_ADDR2 ((unsigned long)return_address(2)) +#define CALLER_ADDR3 ((unsigned long)return_address(3)) +#define CALLER_ADDR4 ((unsigned long)return_address(4)) +#define CALLER_ADDR5 ((unsigned long)return_address(5)) +#define CALLER_ADDR6 ((unsigned long)return_address(6))

Could we change the core definitions of these macros (in linux/ftrace.h) to use return_address, then provide an overridable version of return_address that defaults to __builtin_return_address, instead of copy-pasting this sequence?

I think I understand what you mean, and will try to post a separate RFC, but I also want to hold off this change on this patch since such a change may raise a small controversy from other archs' maintainers.

I don't see anything controversial here, but ok. Steve already posted something you can get started with.

Will

This patch allows system call entry or exit to be traced as ftrace events, ie. sys_enter_*/sys_exit_*, if CONFIG_FTRACE_SYSCALLS is enabled. Those events appear and can be controlled under ${sysfs}/tracing/events/syscalls/

Please note that we can't trace compat system calls here because AArch32 mode does not share the same syscall table with AArch64. Just define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS in order to avoid unexpected results (bogus syscalls reported or even hang-up).

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/Kconfig | 1 + arch/arm64/include/asm/ftrace.h | 20 ++++++++++++++++ arch/arm64/include/asm/syscall.h | 1 + arch/arm64/include/asm/unistd.h | 2 ++ arch/arm64/kernel/ptrace.c | 48 ++++++++++++++++++++++---------------- 5 files changed, 52 insertions(+), 20 deletions(-)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 6954959..b1dcdb4 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -43,6 +43,7 @@ config ARM64 select HAVE_MEMBLOCK select HAVE_PATA_PLATFORM select HAVE_PERF_EVENTS + select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN select MODULES_USE_ELF_RELA select NO_BOOTMEM diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h index c44c4b1..4ef06f1 100644 --- a/arch/arm64/include/asm/ftrace.h +++ b/arch/arm64/include/asm/ftrace.h @@ -44,6 +44,26 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) #define CALLER_ADDR4 ((unsigned long)return_address(4)) #define CALLER_ADDR5 ((unsigned long)return_address(5)) #define CALLER_ADDR6 ((unsigned long)return_address(6)) + +#include <asm/compat.h> + +/* + * Because AArch32 mode does not share the same syscall table with AArch64, + * tracing compat syscalls may result in reporting bogus syscalls or even + * hang-up, so just do not trace them. + * See kernel/trace/trace_syscalls.c + * + * x86 code says: + * If the user realy wants these, then they should use the + * raw syscall tracepoints with filtering. + */ +#define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS 1 +static inline bool arch_trace_is_compat_syscall(struct pt_regs *regs) +{ + if (is_compat_task()) + return true; + return false; +} #endif /* ifndef __ASSEMBLY__ */

#endif /* __ASM_FTRACE_H */ diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h index 70ba9d4..383771e 100644 --- a/arch/arm64/include/asm/syscall.h +++ b/arch/arm64/include/asm/syscall.h @@ -18,6 +18,7 @@

#include <linux/err.h>

+extern const void *sys_call_table[];

static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 82ce217..c335479 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -28,3 +28,5 @@ #endif #define __ARCH_WANT_SYS_CLONE #include <uapi/asm/unistd.h> + +#define NR_syscalls (__NR_syscalls) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 9993a8f..9c52b3e 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -41,6 +41,9 @@ #include <asm/traps.h> #include <asm/system_misc.h>

+#define CREATE_TRACE_POINTS +#include <trace/events/syscalls.h> + /* * TODO: does not yet catch signals sent when the child dies. * in exit.c or in signal.c. @@ -1062,29 +1065,31 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs) { unsigned long saved_reg;

- if (!test_thread_flag(TIF_SYSCALL_TRACE)) - return regs->syscallno; + if (test_thread_flag(TIF_SYSCALL_TRACE)) { + /* + * A scrach register (ip(r12) on AArch32, x7 on AArch64) is + * used to denote syscall entry/exit: + * 0 -> entry + */ + if (is_compat_task()) { + saved_reg = regs->regs[12]; + regs->regs[12] = 0; + } else { + saved_reg = regs->regs[7]; + regs->regs[7] = 0; + }

- /* - * A scrach register (ip(r12) on AArch32, x7 on AArch64) is - * used to denote syscall entry/exit: - * 0 -> entry - */ - if (is_compat_task()) { - saved_reg = regs->regs[12]; - regs->regs[12] = 0; - } else { - saved_reg = regs->regs[7]; - regs->regs[7] = 0; - } + if (tracehook_report_syscall_entry(regs)) + regs->syscallno = ~0UL;

- if (tracehook_report_syscall_entry(regs)) - regs->syscallno = ~0UL; + if (is_compat_task()) + regs->regs[12] = saved_reg; + else + regs->regs[7] = saved_reg; + }

- if (is_compat_task()) - regs->regs[12] = saved_reg; - else - regs->regs[7] = saved_reg; + if (test_thread_flag(TIF_SYSCALL_TRACEPOINT)) + trace_sys_enter(regs, regs->syscallno);

return regs->syscallno; } @@ -1093,6 +1098,9 @@ asmlinkage void syscall_trace_exit(struct pt_regs *regs) { unsigned long saved_reg;

+ if (test_thread_flag(TIF_SYSCALL_TRACEPOINT)) + trace_sys_exit(regs, regs_return_value(regs)); + if (!test_thread_flag(TIF_SYSCALL_TRACE)) return;

On 03/14/2014 01:25 AM, Will Deacon wrote:

On Thu, Mar 13, 2014 at 10:13:50AM +0000, AKASHI Takahiro wrote:

...

This patch allows system call entry or exit to be traced as ftrace events, ie. sys_enter_*/sys_exit_*, if CONFIG_FTRACE_SYSCALLS is enabled. Those events appear and can be controlled under ${sysfs}/tracing/events/syscalls/

Please note that we can't trace compat system calls here because AArch32 mode does not share the same syscall table with AArch64. Just define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS in order to avoid unexpected results (bogus syscalls reported or even hang-up).

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org

arch/arm64/Kconfig | 1 + arch/arm64/include/asm/ftrace.h | 20 ++++++++++++++++ arch/arm64/include/asm/syscall.h | 1 + arch/arm64/include/asm/unistd.h | 2 ++ arch/arm64/kernel/ptrace.c | 48 ++++++++++++++++++++++---------------- 5 files changed, 52 insertions(+), 20 deletions(-)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 6954959..b1dcdb4 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -43,6 +43,7 @@ config ARM64 select HAVE_MEMBLOCK select HAVE_PATA_PLATFORM select HAVE_PERF_EVENTS

• select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN select MODULES_USE_ELF_RELA select NO_BOOTMEM

diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h index c44c4b1..4ef06f1 100644 --- a/arch/arm64/include/asm/ftrace.h +++ b/arch/arm64/include/asm/ftrace.h @@ -44,6 +44,26 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) #define CALLER_ADDR4 ((unsigned long)return_address(4)) #define CALLER_ADDR5 ((unsigned long)return_address(5)) #define CALLER_ADDR6 ((unsigned long)return_address(6))

+#include <asm/compat.h>

+/*

• • Because AArch32 mode does not share the same syscall table with AArch64,
• • tracing compat syscalls may result in reporting bogus syscalls or even
• • hang-up, so just do not trace them.
• • See kernel/trace/trace_syscalls.c
• • x86 code says:
• • If the user realy wants these, then they should use the
• • raw syscall tracepoints with filtering.

Fair enough.

...

• */

+#define ARCH_TRACE_IGNORE_COMPAT_SYSCALLS 1

You don't need the '1' here.

OK.

...

+static inline bool arch_trace_is_compat_syscall(struct pt_regs *regs) +{

•    if (is_compat_task())
  

•            return true;
  

•    return false;
  

+}

return is_compat_task();

Fix it.

...

#endif /* ifndef __ASSEMBLY__ */

#endif /* __ASM_FTRACE_H */ diff --git a/arch/arm64/include/asm/syscall.h b/arch/arm64/include/asm/syscall.h index 70ba9d4..383771e 100644 --- a/arch/arm64/include/asm/syscall.h +++ b/arch/arm64/include/asm/syscall.h @@ -18,6 +18,7 @@

#include <linux/err.h>

+extern const void *sys_call_table[];

static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 82ce217..c335479 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -28,3 +28,5 @@ #endif #define __ARCH_WANT_SYS_CLONE #include <uapi/asm/unistd.h>

+#define NR_syscalls (__NR_syscalls) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 9993a8f..9c52b3e 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -41,6 +41,9 @@ #include <asm/traps.h> #include <asm/system_misc.h>

+#define CREATE_TRACE_POINTS +#include <trace/events/syscalls.h>

• /*
  • TODO: does not yet catch signals sent when the child dies.
  • in exit.c or in signal.c.

@@ -1062,29 +1065,31 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs) { unsigned long saved_reg;

• if (!test_thread_flag(TIF_SYSCALL_TRACE))

• return regs->syscallno;
  

• if (test_thread_flag(TIF_SYSCALL_TRACE)) {

• /*
  

•  * A scrach register (ip(r12) on AArch32, x7 on AArch64) is
  

•  * used to denote syscall entry/exit:
  

•  *   0 -> entry
  

•  */
  

• if (is_compat_task()) {
  

if (arch_trace_is_compat_syscall())

I don't mind either way, but this part of code comes from the original syscall_trace() (ie. ptrace stuff), and has nothing to do with ftrace events. (You know, arch_trace_is_compat_syscall() is currently defined in asm/ftrace.h.) So I'd like to keep it unchanged unless you really want.

With those changes:

Acked-by: Will Deacon will.deacon@arm.com

Thank you, -Takahiro AKASHI

Will

Since insn.h is indirectly included in asm/entry-ftrace.S, we need to exclude some declarations by __ASSEMBLY__.

Acked-by: Will Deacon will.deacon@arm.com Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/include/asm/insn.h | 2 ++ 1 file changed, 2 insertions(+)

diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index c44ad39..dc1f73b 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -21,6 +21,7 @@ /* A64 instructions are always 32 bits. */ #define AARCH64_INSN_SIZE 4

+#ifndef __ASSEMBLY__ /* * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a * Section C3.1 "A64 instruction index by encoding": @@ -104,5 +105,6 @@ bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn); int aarch64_insn_patch_text_nosync(void *addr, u32 insn); int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt); int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt); +#endif /* __ASSEMBLY__ */

#endif /* __ASM_INSN_H */

Recordmcount utility under scripts is run, after compiling each object, to find out all the locations of calling _mcount() and put them into specific seciton named __mcount_loc. Then linker collects all such information into a table in the kernel image (between __start_mcount_loc and __stop_mcount_loc) for later use by ftrace.

This patch adds arm64 specific definitions to identify such locations. There are two types of implementation, C and Perl. On arm64, only C version is used to build the kernel now that CONFIG_HAVE_C_RECORDMCOUNT is on. But Perl version is also maintained.

This patch also contains a workaround just in case where a header file, elf.h, on host machine doesn't have definitions of EM_AARCH64 nor R_AARCH64_ABS64. Without them, compiling C version of recordmcount will fail.

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org --- arch/arm64/Kconfig | 2 ++ scripts/recordmcount.c | 7 +++++++ scripts/recordmcount.pl | 5 +++++ 3 files changed, 14 insertions(+)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 27bbcfc..340e344 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -27,12 +27,14 @@ config ARM64 select HARDIRQS_SW_RESEND select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_TRACEHOOK + select HAVE_C_RECORDMCOUNT select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_API_DEBUG select HAVE_DMA_ATTRS select HAVE_DMA_CONTIGUOUS select HAVE_EFFICIENT_UNALIGNED_ACCESS + select HAVE_FTRACE_MCOUNT_RECORD select HAVE_GENERIC_DMA_COHERENT select HAVE_HW_BREAKPOINT if PERF_EVENTS select HAVE_MEMBLOCK diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c index 9c22317..e11aa4a 100644 --- a/scripts/recordmcount.c +++ b/scripts/recordmcount.c @@ -40,6 +40,11 @@ #define R_METAG_NONE 3 #endif

+#ifndef EM_AARCH64 +#define EM_AARCH64 183 +#define R_AARCH64_ABS64 257 +#endif + static int fd_map; /* File descriptor for file being modified. */ static int mmap_failed; /* Boolean flag. */ static void *ehdr_curr; /* current ElfXX_Ehdr * for resource cleanup */ @@ -347,6 +352,8 @@ do_file(char const *const fname) case EM_ARM: reltype = R_ARM_ABS32; altmcount = "__gnu_mcount_nc"; break; + case EM_AARCH64: + reltype = R_AARCH64_ABS64; gpfx = '_'; break; case EM_IA_64: reltype = R_IA64_IMM64; gpfx = '_'; break; case EM_METAG: reltype = R_METAG_ADDR32; altmcount = "_mcount_wrapper"; diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl index 91280b8..397b6b8 100755 --- a/scripts/recordmcount.pl +++ b/scripts/recordmcount.pl @@ -279,6 +279,11 @@ if ($arch eq "x86_64") { $mcount_regex = "^\s*([0-9a-fA-F]+):\s*R_ARM_(CALL|PC24|THM_CALL)" . "\s+(__gnu_mcount_nc|mcount)$";

+} elsif ($arch eq "arm64") { + $alignment = 3; + $section_type = '%progbits'; + $mcount_regex = "^\s*([0-9a-fA-F]+):\s*R_AARCH64_CALL26\s+_mcount$"; + $type = ".quad"; } elsif ($arch eq "ia64") { $mcount_regex = "^\s*([0-9a-fA-F]+):.*\s_mcount$"; $type = "data8";

Hi Akashi,

On Wed, Apr 30, 2014 at 10:54:29AM +0100, AKASHI Takahiro wrote:

Most archs with HAVE_ARCH_CALLER_ADDR have the almost same definitions of CALLER_ADDRx(n), and so put them into linux/ftrace.h.

Signed-off-by: AKASHI Takahiro takahiro.akashi@linaro.org

arch/arm/include/asm/ftrace.h | 10 +--------- arch/blackfin/include/asm/ftrace.h | 11 +---------- arch/parisc/include/asm/ftrace.h | 10 +--------- arch/sh/include/asm/ftrace.h | 10 +--------- arch/xtensa/include/asm/ftrace.h | 14 ++++---------- include/linux/ftrace.h | 34 ++++++++++++++++++---------------- 6 files changed, 26 insertions(+), 63 deletions(-)

This one looks a bit too widespread to be merged via the arm64 tree. I wonder if the ftrace maintainers would consider taking it as a cleanup?

Will

diff --git a/arch/arm/include/asm/ftrace.h b/arch/arm/include/asm/ftrace.h index f89515a..eb577f4 100644 --- a/arch/arm/include/asm/ftrace.h +++ b/arch/arm/include/asm/ftrace.h @@ -52,15 +52,7 @@ extern inline void *return_address(unsigned int level) #endif -#define HAVE_ARCH_CALLER_ADDR

-#define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0)) -#define CALLER_ADDR1 ((unsigned long)return_address(1)) -#define CALLER_ADDR2 ((unsigned long)return_address(2)) -#define CALLER_ADDR3 ((unsigned long)return_address(3)) -#define CALLER_ADDR4 ((unsigned long)return_address(4)) -#define CALLER_ADDR5 ((unsigned long)return_address(5)) -#define CALLER_ADDR6 ((unsigned long)return_address(6)) +#define ftrace_return_addr(n) return_address(n) #endif /* ifndef __ASSEMBLY__ */ diff --git a/arch/blackfin/include/asm/ftrace.h b/arch/blackfin/include/asm/ftrace.h index 8a02950..2f1c3c2 100644 --- a/arch/blackfin/include/asm/ftrace.h +++ b/arch/blackfin/include/asm/ftrace.h @@ -66,16 +66,7 @@ extern inline void *return_address(unsigned int level) #endif /* CONFIG_FRAME_POINTER */ -#define HAVE_ARCH_CALLER_ADDR

-/* inline function or macro may lead to unexpected result */ -#define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0)) -#define CALLER_ADDR1 ((unsigned long)return_address(1)) -#define CALLER_ADDR2 ((unsigned long)return_address(2)) -#define CALLER_ADDR3 ((unsigned long)return_address(3)) -#define CALLER_ADDR4 ((unsigned long)return_address(4)) -#define CALLER_ADDR5 ((unsigned long)return_address(5)) -#define CALLER_ADDR6 ((unsigned long)return_address(6)) +#define ftrace_return_address(n) return_address(n) #endif /* __ASSEMBLY__ */ diff --git a/arch/parisc/include/asm/ftrace.h b/arch/parisc/include/asm/ftrace.h index 72c0faf..544ed8e 100644 --- a/arch/parisc/include/asm/ftrace.h +++ b/arch/parisc/include/asm/ftrace.h @@ -24,15 +24,7 @@ extern void return_to_handler(void); extern unsigned long return_address(unsigned int); -#define HAVE_ARCH_CALLER_ADDR

-#define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0)) -#define CALLER_ADDR1 return_address(1) -#define CALLER_ADDR2 return_address(2) -#define CALLER_ADDR3 return_address(3) -#define CALLER_ADDR4 return_address(4) -#define CALLER_ADDR5 return_address(5) -#define CALLER_ADDR6 return_address(6) +#define ftrace_return_address(n) return_address(n) #endif /* __ASSEMBLY__ */ diff --git a/arch/sh/include/asm/ftrace.h b/arch/sh/include/asm/ftrace.h index 13e9966..e79fb6e 100644 --- a/arch/sh/include/asm/ftrace.h +++ b/arch/sh/include/asm/ftrace.h @@ -40,15 +40,7 @@ static inline unsigned long ftrace_call_adjust(unsigned long addr) /* arch/sh/kernel/return_address.c */ extern void *return_address(unsigned int); -#define HAVE_ARCH_CALLER_ADDR

-#define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0)) -#define CALLER_ADDR1 ((unsigned long)return_address(1)) -#define CALLER_ADDR2 ((unsigned long)return_address(2)) -#define CALLER_ADDR3 ((unsigned long)return_address(3)) -#define CALLER_ADDR4 ((unsigned long)return_address(4)) -#define CALLER_ADDR5 ((unsigned long)return_address(5)) -#define CALLER_ADDR6 ((unsigned long)return_address(6)) +#define ftrace_return_address(n) return_address(n) #endif /* __ASSEMBLY__ */ diff --git a/arch/xtensa/include/asm/ftrace.h b/arch/xtensa/include/asm/ftrace.h index 736b9d2..6c6d9a9 100644 --- a/arch/xtensa/include/asm/ftrace.h +++ b/arch/xtensa/include/asm/ftrace.h @@ -12,24 +12,18 @@ #include <asm/processor.h> -#define HAVE_ARCH_CALLER_ADDR #ifndef __ASSEMBLY__ -#define CALLER_ADDR0 ({ unsigned long a0, a1; \ +#define ftrace_return_address0 ({ unsigned long a0, a1; \ __asm__ __volatile__ ( \ "mov %0, a0\n" \ "mov %1, a1\n" \ : "=r"(a0), "=r"(a1)); \ MAKE_PC_FROM_RA(a0, a1); })

#ifdef CONFIG_FRAME_POINTER extern unsigned long return_address(unsigned level); -#define CALLER_ADDR1 return_address(1) -#define CALLER_ADDR2 return_address(2) -#define CALLER_ADDR3 return_address(3) -#else /* CONFIG_FRAME_POINTER */ -#define CALLER_ADDR1 (0) -#define CALLER_ADDR2 (0) -#define CALLER_ADDR3 (0) -#endif /* CONFIG_FRAME_POINTER */ +#define ftrace_return_address(n) return_address(n) +#endif #endif /* __ASSEMBLY__ */ #ifdef CONFIG_FUNCTION_TRACER diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 9212b01..18f1ae7 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -614,25 +614,27 @@ static inline void __ftrace_enabled_restore(int enabled) #endif } -#ifndef HAVE_ARCH_CALLER_ADDR +/* All archs should have this, but we define it for consistency */ +#ifndef ftrace_return_address0 +# define ftrace_return_address0 __builtin_return_address(0) +#endif

+/* Archs may use other ways for ADDR1 and beyond */ +#ifndef ftrace_return_address # ifdef CONFIG_FRAME_POINTER -# define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0)) -# define CALLER_ADDR1 ((unsigned long)__builtin_return_address(1)) -# define CALLER_ADDR2 ((unsigned long)__builtin_return_address(2)) -# define CALLER_ADDR3 ((unsigned long)__builtin_return_address(3)) -# define CALLER_ADDR4 ((unsigned long)__builtin_return_address(4)) -# define CALLER_ADDR5 ((unsigned long)__builtin_return_address(5)) -# define CALLER_ADDR6 ((unsigned long)__builtin_return_address(6)) +# define ftrace_return_address(n) __builtin_return_address(n) # else -# define CALLER_ADDR0 ((unsigned long)__builtin_return_address(0)) -# define CALLER_ADDR1 0UL -# define CALLER_ADDR2 0UL -# define CALLER_ADDR3 0UL -# define CALLER_ADDR4 0UL -# define CALLER_ADDR5 0UL -# define CALLER_ADDR6 0UL +# define ftrace_return_address(n) 0UL # endif -#endif /* ifndef HAVE_ARCH_CALLER_ADDR */ +#endif

+#define CALLER_ADDR0 ((unsigned long)ftrace_return_address0) +#define CALLER_ADDR1 ((unsigned long)ftrace_return_address(1)) +#define CALLER_ADDR2 ((unsigned long)ftrace_return_address(2)) +#define CALLER_ADDR3 ((unsigned long)ftrace_return_address(3)) +#define CALLER_ADDR4 ((unsigned long)ftrace_return_address(4)) +#define CALLER_ADDR5 ((unsigned long)ftrace_return_address(5)) +#define CALLER_ADDR6 ((unsigned long)ftrace_return_address(6)) #ifdef CONFIG_IRQSOFF_TRACER extern void time_hardirqs_on(unsigned long a0, unsigned long a1); -- 1.7.9.5