The BTF dumper code currently displays arrays of characters as just that - arrays, with each character formatted individually. Sometimes this is what makes sense, but it's nice to be able to treat that array as a string.
This change adds a special case to the btf_dump functionality to allow 0-terminated arrays of single-byte integer values to be printed as character strings. Characters for which isprint() returns false are printed as hex-escaped values. This is enabled when the new ".emit_strings" is set to 1 in the btf_dump_type_data_opts structure.
As an example, here's what it looks like to dump the string "hello" using a few different field values for btf_dump_type_data_opts (.compact = 1):
- .emit_strings = 0, .skip_names = 0: (char[6])['h','e','l','l','o',] - .emit_strings = 0, .skip_names = 1: ['h','e','l','l','o',] - .emit_strings = 1, .skip_names = 0: (char[6])"hello" - .emit_strings = 1, .skip_names = 1: "hello"
Here's the string "h\xff", dumped with .compact = 1 and .skip_names = 1:
- .emit_strings = 0: ['h',-1,] - .emit_strings = 1: "h\xff"
Signed-off-by: Blake Jones blakejones@google.com --- tools/lib/bpf/btf.h | 3 ++- tools/lib/bpf/btf_dump.c | 55 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-)
diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index 4392451d634b..ccfd905f03df 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -326,9 +326,10 @@ struct btf_dump_type_data_opts { bool compact; /* no newlines/indentation */ bool skip_names; /* skip member/type names */ bool emit_zeroes; /* show 0-valued fields */ + bool emit_strings; /* print char arrays as strings */ size_t :0; }; -#define btf_dump_type_data_opts__last_field emit_zeroes +#define btf_dump_type_data_opts__last_field emit_strings
LIBBPF_API int btf_dump__dump_type_data(struct btf_dump *d, __u32 id, diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 460c3e57fadb..7c2f1f13f958 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -68,6 +68,7 @@ struct btf_dump_data { bool compact; bool skip_names; bool emit_zeroes; + bool emit_strings; __u8 indent_lvl; /* base indent level */ char indent_str[BTF_DATA_INDENT_STR_LEN]; /* below are used during iteration */ @@ -2028,6 +2029,52 @@ static int btf_dump_var_data(struct btf_dump *d, return btf_dump_dump_type_data(d, NULL, t, type_id, data, 0, 0); }
+static int btf_dump_string_data(struct btf_dump *d, + const struct btf_type *t, + __u32 id, + const void *data) +{ + const struct btf_array *array = btf_array(t); + const char *chars = data; + __u32 i; + + /* Make sure it is a NUL-terminated string. */ + for (i = 0; i < array->nelems; i++) { + if ((void *)(chars + i) >= d->typed_dump->data_end) + return -E2BIG; + if (chars[i] == '\0') + break; + } + if (i == array->nelems) { + /* The caller will print this as a regular array. */ + return -EINVAL; + } + + btf_dump_data_pfx(d); + btf_dump_printf(d, """); + + for (i = 0; i < array->nelems; i++) { + char c = chars[i]; + + if (c == '\0') { + /* + * When printing character arrays as strings, NUL bytes + * are always treated as string terminators; they are + * never printed. + */ + break; + } + if (isprint(c)) + btf_dump_printf(d, "%c", c); + else + btf_dump_printf(d, "\x%02x", (__u8)c); + } + + btf_dump_printf(d, """); + + return 0; +} + static int btf_dump_array_data(struct btf_dump *d, const struct btf_type *t, __u32 id, @@ -2055,8 +2102,13 @@ static int btf_dump_array_data(struct btf_dump *d, * char arrays, so if size is 1 and element is * printable as a char, we'll do that. */ - if (elem_size == 1) + if (elem_size == 1) { + if (d->typed_dump->emit_strings && + btf_dump_string_data(d, t, id, data) == 0) { + return 0; + } d->typed_dump->is_array_char = true; + } }
/* note that we increment depth before calling btf_dump_print() below; @@ -2544,6 +2596,7 @@ int btf_dump__dump_type_data(struct btf_dump *d, __u32 id, d->typed_dump->compact = OPTS_GET(opts, compact, false); d->typed_dump->skip_names = OPTS_GET(opts, skip_names, false); d->typed_dump->emit_zeroes = OPTS_GET(opts, emit_zeroes, false); + d->typed_dump->emit_strings = OPTS_GET(opts, emit_strings, false);
ret = btf_dump_dump_type_data(d, NULL, t, id, data, 0, 0);
When this mode is turned on, "emit_zeroes" and "compact" have no effect, and embedded NUL characters always terminate printing of an array.
Signed-off-by: Blake Jones blakejones@google.com --- .../selftests/bpf/prog_tests/btf_dump.c | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+)
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index c0a776feec23..82903585c870 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -879,6 +879,122 @@ static void test_btf_dump_var_data(struct btf *btf, struct btf_dump *d, "static int bpf_cgrp_storage_busy = (int)2", 2); }
+struct btf_dump_string_ctx { + struct btf *btf; + struct btf_dump *d; + char *str; + struct btf_dump_type_data_opts *opts; + int array_id; +}; + +static int btf_dump_one_string(struct btf_dump_string_ctx *ctx, + char *ptr, size_t ptr_sz, + const char *expected_val) +{ + size_t type_sz; + int ret; + + ctx->str[0] = '\0'; + type_sz = btf__resolve_size(ctx->btf, ctx->array_id); + ret = btf_dump__dump_type_data(ctx->d, ctx->array_id, ptr, ptr_sz, ctx->opts); + if (type_sz <= ptr_sz) { + if (!ASSERT_EQ(ret, type_sz, "failed/unexpected type_sz")) + return -EINVAL; + } + if (!ASSERT_STREQ(ctx->str, expected_val, "ensure expected/actual match")) + return -EFAULT; + return 0; +} + +static void btf_dump_strings(struct btf_dump_string_ctx *ctx) +{ + struct btf_dump_type_data_opts *opts = ctx->opts; + + opts->emit_strings = true; + + opts->compact = true; + opts->emit_zeroes = false; + + opts->skip_names = false; + btf_dump_one_string(ctx, "foo", 4, "(char[4])"foo""); + + opts->skip_names = true; + btf_dump_one_string(ctx, "foo", 4, ""foo""); + + /* This should have no effect. */ + opts->emit_zeroes = false; + btf_dump_one_string(ctx, "foo", 4, ""foo""); + + /* This should have no effect. */ + opts->compact = false; + btf_dump_one_string(ctx, "foo", 4, ""foo""); + + /* Non-printable characters come out as hex. */ + btf_dump_one_string(ctx, "fo\xff", 4, ""fo\xff""); + btf_dump_one_string(ctx, "fo\x7", 4, ""fo\x07""); + + /* + * Strings that are too long for the specified type ("char[4]") + * should fall back to the current behavior. + */ + opts->compact = true; + btf_dump_one_string(ctx, "abcde", 6, "['a','b','c','d',]"); + + /* + * Strings that are too short for the specified type ("char[4]") + * should work normally. + */ + btf_dump_one_string(ctx, "ab", 3, ""ab""); + + /* Non-NUL-terminated arrays don't get printed as strings. */ + char food[4] = { 'f', 'o', 'o', 'd' }; + char bye[3] = { 'b', 'y', 'e' }; + + btf_dump_one_string(ctx, food, 4, "['f','o','o','d',]"); + btf_dump_one_string(ctx, bye, 3, "['b','y','e',]"); + + /* The embedded NUL should terminate the string. */ + char embed[4] = { 'f', 'o', '\0', 'd' }; + + btf_dump_one_string(ctx, embed, 4, ""fo""); +} + +static void test_btf_dump_string_data(void) +{ + struct test_ctx t = {}; + char str[STRSIZE]; + struct btf_dump *d; + DECLARE_LIBBPF_OPTS(btf_dump_type_data_opts, opts); + struct btf_dump_string_ctx ctx; + int char_id, int_id, array_id; + + if (test_ctx__init(&t)) + return; + + d = btf_dump__new(t.btf, btf_dump_snprintf, str, NULL); + if (!ASSERT_OK_PTR(d, "could not create BTF dump")) + return; + + /* Generate BTF for a four-element char array. */ + char_id = btf__add_int(t.btf, "char", 1, BTF_INT_CHAR); + ASSERT_EQ(char_id, 1, "char_id"); + int_id = btf__add_int(t.btf, "int", 4, BTF_INT_SIGNED); + ASSERT_EQ(int_id, 2, "int_id"); + array_id = btf__add_array(t.btf, int_id, char_id, 4); + ASSERT_EQ(array_id, 3, "array_id"); + + ctx.btf = t.btf; + ctx.d = d; + ctx.str = str; + ctx.opts = &opts; + ctx.array_id = array_id; + + btf_dump_strings(&ctx); + + btf_dump__free(d); + test_ctx__free(&t); +} + static void test_btf_datasec(struct btf *btf, struct btf_dump *d, char *str, const char *name, const char *expected_val, void *data, size_t data_sz) @@ -970,6 +1086,8 @@ void test_btf_dump() { test_btf_dump_struct_data(btf, d, str); if (test__start_subtest("btf_dump: var_data")) test_btf_dump_var_data(btf, d, str); + if (test__start_subtest("btf_dump: string_data")) + test_btf_dump_string_data(); btf_dump__free(d); btf__free(btf);
On Tue, Jun 3, 2025 at 1:37 PM Blake Jones blakejones@google.com wrote:
When this mode is turned on, "emit_zeroes" and "compact" have no effect, and embedded NUL characters always terminate printing of an array.
Signed-off-by: Blake Jones blakejones@google.com
.../selftests/bpf/prog_tests/btf_dump.c | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+)
Nice set of tests, thank you! Applied to bpf-next.
[...]
Hello:
This series was applied to bpf/bpf-next.git (master) by Andrii Nakryiko andrii@kernel.org:
On Tue, 3 Jun 2025 13:37:00 -0700 you wrote:
The BTF dumper code currently displays arrays of characters as just that - arrays, with each character formatted individually. Sometimes this is what makes sense, but it's nice to be able to treat that array as a string.
This change adds a special case to the btf_dump functionality to allow 0-terminated arrays of single-byte integer values to be printed as character strings. Characters for which isprint() returns false are printed as hex-escaped values. This is enabled when the new ".emit_strings" is set to 1 in the btf_dump_type_data_opts structure.
[...]
Here is the summary with links: - [v3,1/2] libbpf: add support for printing BTF character arrays as strings https://git.kernel.org/bpf/bpf-next/c/87c9c79a02b4 - [v3,2/2] Tests for the ".emit_strings" functionality in the BTF dumper. https://git.kernel.org/bpf/bpf-next/c/a570f386f3d1
You are awesome, thank you!
On Thu, Jun 5, 2025 at 2:00 PM patchwork-bot+netdevbpf@kernel.org wrote:
Hello:
This series was applied to bpf/bpf-next.git (master) by Andrii Nakryiko andrii@kernel.org:
On Tue, 3 Jun 2025 13:37:00 -0700 you wrote:
The BTF dumper code currently displays arrays of characters as just that - arrays, with each character formatted individually. Sometimes this is what makes sense, but it's nice to be able to treat that array as a string.
This change adds a special case to the btf_dump functionality to allow 0-terminated arrays of single-byte integer values to be printed as character strings. Characters for which isprint() returns false are printed as hex-escaped values. This is enabled when the new ".emit_strings" is set to 1 in the btf_dump_type_data_opts structure.
[...]
Here is the summary with links:
- [v3,1/2] libbpf: add support for printing BTF character arrays as strings https://git.kernel.org/bpf/bpf-next/c/87c9c79a02b4
- [v3,2/2] Tests for the ".emit_strings" functionality in the BTF dumper. https://git.kernel.org/bpf/bpf-next/c/a570f386f3d1
You are awesome, thank you!
I believe this patch is responsible for segvs occurring in v6.17 in various perf tests when the perf tests run in parallel. There's lots of BPF things happening in parallel in the test but the failures are happening in a shell and I did get to attach a debugger. I've not seen this problem earlier as the patches weren't in the perf-tools-next tree. Through bisection I was able to blame the patch and I came up with this minimal fix: ``` diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index ccfd905f03df..71e198b30c5f 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -326,10 +326,10 @@ struct btf_dump_type_data_opts { bool compact; /* no newlines/indentation */ bool skip_names; /* skip member/type names */ bool emit_zeroes; /* show 0-valued fields */ - bool emit_strings; /* print char arrays as strings */ + //bool emit_strings; /* print char arrays as strings */ size_t :0; }; -#define btf_dump_type_data_opts__last_field emit_strings +#define btf_dump_type_data_opts__last_field emit_zeroes
LIBBPF_API int btf_dump__dump_type_data(struct btf_dump *d, __u32 id, diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index f09f25eccf3c..c7b5a376642f 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -2599,7 +2599,7 @@ int btf_dump__dump_type_data(struct btf_dump *d, __u32 id, d->typed_dump->compact = OPTS_GET(opts, compact, false); d->typed_dump->skip_names = OPTS_GET(opts, skip_names, false); d->typed_dump->emit_zeroes = OPTS_GET(opts, emit_zeroes, false); - d->typed_dump->emit_strings = OPTS_GET(opts, emit_strings, false); + d->typed_dump->emit_strings = true; // OPTS_GET(opts, emit_strings, false);
ret = btf_dump_dump_type_data(d, NULL, t, id, data, 0, 0);
``` So I think the problem relates to modifying struct btf_dump_type_data_opts. Given I'm statically linking libbpf into perf I'm not sure on the exact route of the segv, no doubt this report will be enough for someone else to figure it out.
Given this is a regression what should the fix be?
Thanks, Ian
On 8/29/25 10:19 PM, Ian Rogers wrote:
On Thu, Jun 5, 2025 at 2:00 PM patchwork-bot+netdevbpf@kernel.org wrote:
Hello:
This series was applied to bpf/bpf-next.git (master) by Andrii Nakryiko andrii@kernel.org:
On Tue, 3 Jun 2025 13:37:00 -0700 you wrote:
The BTF dumper code currently displays arrays of characters as just that - arrays, with each character formatted individually. Sometimes this is what makes sense, but it's nice to be able to treat that array as a string.
This change adds a special case to the btf_dump functionality to allow 0-terminated arrays of single-byte integer values to be printed as character strings. Characters for which isprint() returns false are printed as hex-escaped values. This is enabled when the new ".emit_strings" is set to 1 in the btf_dump_type_data_opts structure.
[...]
Here is the summary with links:
- [v3,1/2] libbpf: add support for printing BTF character arrays as strings https://git.kernel.org/bpf/bpf-next/c/87c9c79a02b4
- [v3,2/2] Tests for the ".emit_strings" functionality in the BTF dumper. https://git.kernel.org/bpf/bpf-next/c/a570f386f3d1
You are awesome, thank you!
I believe this patch is responsible for segvs occurring in v6.17 in various perf tests when the perf tests run in parallel. There's lots
Could you share the command line to reproduce this failure? This will help debugging. Thanks!
of BPF things happening in parallel in the test but the failures are happening in a shell and I did get to attach a debugger. I've not seen this problem earlier as the patches weren't in the perf-tools-next tree. Through bisection I was able to blame the patch and I came up with this minimal fix:
diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index ccfd905f03df..71e198b30c5f 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -326,10 +326,10 @@ struct btf_dump_type_data_opts { bool compact; /* no newlines/indentation */ bool skip_names; /* skip member/type names */ bool emit_zeroes; /* show 0-valued fields */ - bool emit_strings; /* print char arrays as strings */ + //bool emit_strings; /* print char arrays as strings */ size_t :0; }; -#define btf_dump_type_data_opts__last_field emit_strings +#define btf_dump_type_data_opts__last_field emit_zeroes LIBBPF_API int btf_dump__dump_type_data(struct btf_dump *d, __u32 id, diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index f09f25eccf3c..c7b5a376642f 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -2599,7 +2599,7 @@ int btf_dump__dump_type_data(struct btf_dump *d, __u32 id, d->typed_dump->compact = OPTS_GET(opts, compact, false); d->typed_dump->skip_names = OPTS_GET(opts, skip_names, false); d->typed_dump->emit_zeroes = OPTS_GET(opts, emit_zeroes, false); - d->typed_dump->emit_strings = OPTS_GET(opts, emit_strings, false); + d->typed_dump->emit_strings = true; // OPTS_GET(opts, emit_strings, false); ret = btf_dump_dump_type_data(d, NULL, t, id, data, 0, 0);
So I think the problem relates to modifying struct btf_dump_type_data_opts. Given I'm statically linking libbpf into perf I'm not sure on the exact route of the segv, no doubt this report will be enough for someone else to figure it out.
Given this is a regression what should the fix be?
Thanks, Ian
On Sun, Aug 31, 2025 at 9:17 PM Yonghong Song yonghong.song@linux.dev wrote:
On 8/29/25 10:19 PM, Ian Rogers wrote:
On Thu, Jun 5, 2025 at 2:00 PM patchwork-bot+netdevbpf@kernel.org wrote:
Hello:
This series was applied to bpf/bpf-next.git (master) by Andrii Nakryiko andrii@kernel.org:
On Tue, 3 Jun 2025 13:37:00 -0700 you wrote:
The BTF dumper code currently displays arrays of characters as just that - arrays, with each character formatted individually. Sometimes this is what makes sense, but it's nice to be able to treat that array as a string.
This change adds a special case to the btf_dump functionality to allow 0-terminated arrays of single-byte integer values to be printed as character strings. Characters for which isprint() returns false are printed as hex-escaped values. This is enabled when the new ".emit_strings" is set to 1 in the btf_dump_type_data_opts structure.
[...]
Here is the summary with links:
- [v3,1/2] libbpf: add support for printing BTF character arrays as strings https://git.kernel.org/bpf/bpf-next/c/87c9c79a02b4
- [v3,2/2] Tests for the ".emit_strings" functionality in the BTF dumper. https://git.kernel.org/bpf/bpf-next/c/a570f386f3d1
You are awesome, thank you!
I believe this patch is responsible for segvs occurring in v6.17 in various perf tests when the perf tests run in parallel. There's lots
Could you share the command line to reproduce this failure? This will help debugging. Thanks!
Sure: ``` $ git clone git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git ... $ cd linux $ mkdir /tmp/perf $ make -C tools/perf O=/tmp/perf make: Entering directory 'linux/tools/perf' BUILD: Doing 'make -j28' parallel build Warning: Kernel ABI header differences: diff -u tools/arch/arm64/include/asm/cputype.h arch/arm64/include/asm/cputype.h
Auto-detecting system features: ... libdw: [ on ] ... glibc: [ on ] ... libelf: [ on ] ... libnuma: [ on ] ... numa_num_possible_cpus: [ on ] ... libperl: [ on ] ... libpython: [ on ] ... libcapstone: [ on ] ... llvm-perf: [ on ] ... zlib: [ on ] ... lzma: [ on ] ... get_cpuid: [ on ] ... bpf: [ on ] ... libaio: [ on ] ... libzstd: [ on ]
GEN /tmp/perf/common-cmds.h ... LINK /tmp/perf/perf GEN /tmp/perf/python/perf.cpython-313-x86_64-linux-gnu.so make: Leaving directory 'linux/tools/perf' $ git describe v6.17-rc4 $ sudo /tmp/perf/perf test -v 1: vmlinux symtab matches kallsyms : Skip 2: Detect openat syscall event : Ok ... 79: build id cache operations : Ok --- start --- test child forked, pid 2546565 test daemon list FAILED: wrong name FAILED: wrong run FAILED: wrong base FAILED: wrong output FAILED: wrong control FAILED: wrong ack test daemon reconfig FAILED: Timeout waiting for daemon to ping ---- end(-1) ---- 80: daemon operations : FAILED! 81: perf diff tests : Ok 82: DRM PMU : Ok 83: perf ftrace tests : Ok --- start --- test child forked, pid 2546573 Test perf header file perf: Segmentation fault linux/tools/perf/tests/shell/header.sh: line 51: 2546664 Segmentation fault (core dumped ) perf record -o "${perfdata}" -- perf test -w noploop Unexpected signal in test_file ---- end(-1) ---- 84: perf header tests : FAILED! 85: perf list tests : Ok ... 92: perf sched tests : Ok --- start --- test child forked, pid 2546593 DB test perf: Segmentation fault /tmp/x/linux/tools/perf/tests/shell/script.sh: line 35: 2546822 Segmentation fault (core dumped ) perf record $cmd_flags -o "${perfdatafile}" true --- Cleaning up --- ---- end(-1) ---- 93: perf script tests : FAILED! ... $ /tmp/perf/perf version --build-options perf version 6.17.rc4.gb320789d6883 aio: [ on ] # HAVE_AIO_SUPPORT bpf: [ on ] # HAVE_LIBBPF_SUPPORT bpf_skeletons: [ on ] # HAVE_BPF_SKEL debuginfod: [ on ] # HAVE_DEBUGINFOD_SUPPORT dwarf: [ on ] # HAVE_LIBDW_SUPPORT dwarf_getlocations: [ on ] # HAVE_LIBDW_SUPPORT dwarf-unwind: [ on ] # HAVE_DWARF_UNWIND_SUPPORT auxtrace: [ on ] # HAVE_AUXTRACE_SUPPORT libbfd: [ OFF ] # HAVE_LIBBFD_SUPPORT ( tip: Deprecated, license incompatibility, u se BUILD_NONDISTRO=1 and install binutils-dev[el] ) libbpf-strings: [ on ] # HAVE_LIBBPF_STRINGS_SUPPORT libcapstone: [ on ] # HAVE_LIBCAPSTONE_SUPPORT libdw-dwarf-unwind: [ on ] # HAVE_LIBDW_SUPPORT libelf: [ on ] # HAVE_LIBELF_SUPPORT libnuma: [ on ] # HAVE_LIBNUMA_SUPPORT libopencsd: [ OFF ] # HAVE_CSTRACE_SUPPORT libperl: [ on ] # HAVE_LIBPERL_SUPPORT libpfm4: [ on ] # HAVE_LIBPFM libpython: [ on ] # HAVE_LIBPYTHON_SUPPORT libslang: [ on ] # HAVE_SLANG_SUPPORT libtraceevent: [ on ] # HAVE_LIBTRACEEVENT libunwind: [ OFF ] # HAVE_LIBUNWIND_SUPPORT ( tip: Deprecated, use LIBUNWIND=1 and ins tall libunwind-dev[el] to build with it ) lzma: [ on ] # HAVE_LZMA_SUPPORT numa_num_possible_cpus: [ on ] # HAVE_LIBNUMA_SUPPORT zlib: [ on ] # HAVE_ZLIB_SUPPORT zstd: [ on ] # HAVE_ZSTD_SUPPORT ```
Thanks, Ian
of BPF things happening in parallel in the test but the failures are happening in a shell and I did get to attach a debugger. I've not seen this problem earlier as the patches weren't in the perf-tools-next tree. Through bisection I was able to blame the patch and I came up with this minimal fix:
diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index ccfd905f03df..71e198b30c5f 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -326,10 +326,10 @@ struct btf_dump_type_data_opts { bool compact; /* no newlines/indentation */ bool skip_names; /* skip member/type names */ bool emit_zeroes; /* show 0-valued fields */ - bool emit_strings; /* print char arrays as strings */ + //bool emit_strings; /* print char arrays as strings */ size_t :0; }; -#define btf_dump_type_data_opts__last_field emit_strings +#define btf_dump_type_data_opts__last_field emit_zeroes LIBBPF_API int btf_dump__dump_type_data(struct btf_dump *d, __u32 id, diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index f09f25eccf3c..c7b5a376642f 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -2599,7 +2599,7 @@ int btf_dump__dump_type_data(struct btf_dump *d, __u32 id, d->typed_dump->compact = OPTS_GET(opts, compact, false); d->typed_dump->skip_names = OPTS_GET(opts, skip_names, false); d->typed_dump->emit_zeroes = OPTS_GET(opts, emit_zeroes, false); - d->typed_dump->emit_strings = OPTS_GET(opts, emit_strings, false); + d->typed_dump->emit_strings = true; // OPTS_GET(opts, emit_strings, false); ret = btf_dump_dump_type_data(d, NULL, t, id, data, 0, 0);
So I think the problem relates to modifying struct btf_dump_type_data_opts. Given I'm statically linking libbpf into perf I'm not sure on the exact route of the segv, no doubt this report will be enough for someone else to figure it out.
Given this is a regression what should the fix be?
Thanks, Ian
Hello,
On Sun, Aug 31, 2025 at 09:17:34PM -0700, Yonghong Song wrote:
On 8/29/25 10:19 PM, Ian Rogers wrote:
On Thu, Jun 5, 2025 at 2:00 PM patchwork-bot+netdevbpf@kernel.org wrote:
Hello:
This series was applied to bpf/bpf-next.git (master) by Andrii Nakryiko andrii@kernel.org:
On Tue, 3 Jun 2025 13:37:00 -0700 you wrote:
The BTF dumper code currently displays arrays of characters as just that - arrays, with each character formatted individually. Sometimes this is what makes sense, but it's nice to be able to treat that array as a string.
This change adds a special case to the btf_dump functionality to allow 0-terminated arrays of single-byte integer values to be printed as character strings. Characters for which isprint() returns false are printed as hex-escaped values. This is enabled when the new ".emit_strings" is set to 1 in the btf_dump_type_data_opts structure.
[...]
Here is the summary with links:
- [v3,1/2] libbpf: add support for printing BTF character arrays as strings https://git.kernel.org/bpf/bpf-next/c/87c9c79a02b4
- [v3,2/2] Tests for the ".emit_strings" functionality in the BTF dumper. https://git.kernel.org/bpf/bpf-next/c/a570f386f3d1
You are awesome, thank you!
I believe this patch is responsible for segvs occurring in v6.17 in various perf tests when the perf tests run in parallel. There's lots
Could you share the command line to reproduce this failure? This will help debugging. Thanks!
My reproducer is below:
terminal 1: run perf trace in a loop.
$ while true; do sudo ./perf trace true; done
terminal 2: run perf record in a loop until hit the segfault.
$ while true; do sudo ./perf record true || break; done ... perf: Segmentation fault #0 0x560b2db790e4 in dump_stack debug.c:366 #1 0x560b2db7915a in sighandler_dump_stack debug.c:378 #2 0x560b2d973b1b in sigsegv_handler builtin-record.c:722 #3 0x7f975f249df0 in __restore_rt libc_sigaction.c:0 #4 0x560b2dca1ee6 in snprintf_hex bpf-event.c:39 #5 0x560b2dca2306 in synthesize_bpf_prog_name bpf-event.c:144 #6 0x560b2dca2d92 in bpf_metadata_create bpf-event.c:401 #7 0x560b2dca3838 in perf_event__synthesize_one_bpf_prog bpf-event.c:673 #8 0x560b2dca3dd5 in perf_event__synthesize_bpf_events bpf-event.c:798 #9 0x560b2d977ef5 in record__synthesize builtin-record.c:2131 #10 0x560b2d9797c1 in __cmd_record builtin-record.c:2581 #11 0x560b2d97db30 in cmd_record builtin-record.c:4376 #12 0x560b2da0672e in run_builtin perf.c:349 #13 0x560b2da069c6 in handle_internal_command perf.c:401 #14 0x560b2da06b1f in run_argv perf.c:448 #15 0x560b2da06e68 in main perf.c:555 #16 0x7f975f233ca8 in __libc_start_call_main libc_start_call_main.h:74 #17 0x7f975f233d65 in __libc_start_main_alias_2 libc-start.c:128 #18 0x560b2d959b11 in _start perf[4cb11]
I manually ran it with gdb to get some more hints.
Thread 1 "perf" received signal SIGSEGV, Segmentation fault. 0x00005555558e8ee6 in snprintf_hex (buf=0x5555562c1d79 "", size=503, data=0x40 <error: Cannot access memory at address 0x40>, len=8) at util/bpf-event.c:39 39 ret += snprintf(buf + ret, size - ret, "%02x", data[i]);
The data is bpf_prog_info->prog_tags and it's called from synthesize_bpf_prog_name().
(gdb) bt #0 0x00005555558e8ee6 in snprintf_hex (buf=0x5555562c1d79 "", size=503, data=0x40 <error: Cannot access memory at address 0x40>, len=8) at util/bpf-event.c:39 #1 0x00005555558e9306 in synthesize_bpf_prog_name (buf=0x5555562c1d70 "bpf_prog_", size=512, info=0x55555665e400, btf=0x5555562c5630, sub_id=0) at util/bpf-event.c:144 #2 0x00005555558e9db5 in bpf_metadata_create (info=0x55555665e400) at util/bpf-event.c:403 #3 0x00005555558ea85b in perf_event__synthesize_one_bpf_prog (session=0x555556178510, process=0x5555555ba7ab <process_synthesized_event>, machine=0x555556178728, fd=25, event=0x5555561b73a0, opts=0x5555560d33a8 <record+328>) at util/bpf-event.c:674 #4 0x00005555558eadf8 in perf_event__synthesize_bpf_events (session=0x555556178510, process=0x5555555ba7ab <process_synthesized_event>, machine=0x555556178728, opts=0x5555560d33a8 <record+328>) at util/bpf-event.c:799 #5 0x00005555555beef5 in record__synthesize (rec=0x5555560d3260 <record>, tail=false) at builtin-record.c:2131 #6 0x00005555555c07c1 in __cmd_record (rec=0x5555560d3260 <record>, argc=1, argv=0x7fffffffe2e0) at builtin-record.c:2581 #7 0x00005555555c4b30 in cmd_record (argc=1, argv=0x7fffffffe2e0) at builtin-record.c:4376 #8 0x000055555564d72e in run_builtin (p=0x5555560d63c0 <commands+288>, argc=6, argv=0x7fffffffe2e0) at perf.c:349 #9 0x000055555564d9c6 in handle_internal_command (argc=6, argv=0x7fffffffe2e0) at perf.c:401 #10 0x000055555564db1f in run_argv (argcp=0x7fffffffe0dc, argv=0x7fffffffe0d0) at perf.c:445 #11 0x000055555564de68 in main (argc=6, argv=0x7fffffffe2e0) at perf.c:553
I seems bpf_prog_info is broken for some reason.
(gdb) up #1 0x00005555558e9306 in synthesize_bpf_prog_name (buf=0x5555563305b0 "bpf_prog_", size=512, info=0x55555664e1d0, btf=0x55555637ad40, sub_id=0) at util/bpf-event.c:144 144 name_len += snprintf_hex(buf + name_len, size - name_len,
(gdb) p *info $1 = {type = 68, id = 80, tag = "\\000\000\000\214\000\000", jited_prog_len = 152, xlated_prog_len = 164, jited_prog_insns = 824633721012, xlated_prog_insns = 1185410973912, load_time = 1305670058276, created_by_uid = 352, nr_map_ids = 364, map_ids = 1975684956608, name = "\330\001\000\000\350\001\000\000$\002\000\0004\002\000", ifindex = 576, gpl_compatible = 0, netns_dev = 2697239462496, netns_ino = 2834678416000, nr_jited_ksyms = 756, nr_jited_func_lens = 768, jited_ksyms = 3418793968396, jited_func_lens = 3573412791092, btf_id = 844, func_info_rec_size = 880, func_info = 3934190044028, nr_func_info = 928, nr_line_info = 952, line_info = 4294967296988, jited_line_info = 4449586119680, nr_jited_line_info = 1060, line_info_rec_size = 1076, jited_line_info_rec_size = 1092, nr_prog_tags = 1108, prog_tags = 4861902980192, run_time_ns = 5085241279632, run_cnt = 5257039971512, recursion_misses = 5360119186644, verified_insns = 1264, attach_btf_obj_id = 1288, attach_btf_id = 1312}
Thanks, Namhyung
On Mon, Sep 1, 2025 at 12:57 AM Namhyung Kim namhyung@kernel.org wrote:
Hello,
On Sun, Aug 31, 2025 at 09:17:34PM -0700, Yonghong Song wrote:
On 8/29/25 10:19 PM, Ian Rogers wrote:
On Thu, Jun 5, 2025 at 2:00 PM patchwork-bot+netdevbpf@kernel.org wrote:
Hello:
This series was applied to bpf/bpf-next.git (master) by Andrii Nakryiko andrii@kernel.org:
On Tue, 3 Jun 2025 13:37:00 -0700 you wrote:
The BTF dumper code currently displays arrays of characters as just that - arrays, with each character formatted individually. Sometimes this is what makes sense, but it's nice to be able to treat that array as a string.
This change adds a special case to the btf_dump functionality to allow 0-terminated arrays of single-byte integer values to be printed as character strings. Characters for which isprint() returns false are printed as hex-escaped values. This is enabled when the new ".emit_strings" is set to 1 in the btf_dump_type_data_opts structure.
[...]
Here is the summary with links:
- [v3,1/2] libbpf: add support for printing BTF character arrays as strings https://git.kernel.org/bpf/bpf-next/c/87c9c79a02b4
- [v3,2/2] Tests for the ".emit_strings" functionality in the BTF dumper. https://git.kernel.org/bpf/bpf-next/c/a570f386f3d1
You are awesome, thank you!
I believe this patch is responsible for segvs occurring in v6.17 in various perf tests when the perf tests run in parallel. There's lots
Could you share the command line to reproduce this failure? This will help debugging. Thanks!
My reproducer is below:
terminal 1: run perf trace in a loop.
$ while true; do sudo ./perf trace true; done
terminal 2: run perf record in a loop until hit the segfault.
$ while true; do sudo ./perf record true || break; done ... perf: Segmentation fault #0 0x560b2db790e4 in dump_stack debug.c:366 #1 0x560b2db7915a in sighandler_dump_stack debug.c:378 #2 0x560b2d973b1b in sigsegv_handler builtin-record.c:722 #3 0x7f975f249df0 in __restore_rt libc_sigaction.c:0 #4 0x560b2dca1ee6 in snprintf_hex bpf-event.c:39 #5 0x560b2dca2306 in synthesize_bpf_prog_name bpf-event.c:144 #6 0x560b2dca2d92 in bpf_metadata_create bpf-event.c:401 #7 0x560b2dca3838 in perf_event__synthesize_one_bpf_prog bpf-event.c:673 #8 0x560b2dca3dd5 in perf_event__synthesize_bpf_events bpf-event.c:798 #9 0x560b2d977ef5 in record__synthesize builtin-record.c:2131 #10 0x560b2d9797c1 in __cmd_record builtin-record.c:2581 #11 0x560b2d97db30 in cmd_record builtin-record.c:4376 #12 0x560b2da0672e in run_builtin perf.c:349 #13 0x560b2da069c6 in handle_internal_command perf.c:401 #14 0x560b2da06b1f in run_argv perf.c:448 #15 0x560b2da06e68 in main perf.c:555 #16 0x7f975f233ca8 in __libc_start_call_main libc_start_call_main.h:74 #17 0x7f975f233d65 in __libc_start_main_alias_2 libc-start.c:128 #18 0x560b2d959b11 in _start perf[4cb11]
I manually ran it with gdb to get some more hints.
Thread 1 "perf" received signal SIGSEGV, Segmentation fault. 0x00005555558e8ee6 in snprintf_hex (buf=0x5555562c1d79 "", size=503, data=0x40 <error: Cannot access memory at address 0x40>, len=8) at util/bpf-event.c:39 39 ret += snprintf(buf + ret, size - ret, "%02x", data[i]);
The data is bpf_prog_info->prog_tags and it's called from synthesize_bpf_prog_name().
(gdb) bt #0 0x00005555558e8ee6 in snprintf_hex (buf=0x5555562c1d79 "", size=503, data=0x40 <error: Cannot access memory at address 0x40>, len=8) at util/bpf-event.c:39 #1 0x00005555558e9306 in synthesize_bpf_prog_name (buf=0x5555562c1d70 "bpf_prog_", size=512, info=0x55555665e400, btf=0x5555562c5630, sub_id=0) at util/bpf-event.c:144 #2 0x00005555558e9db5 in bpf_metadata_create (info=0x55555665e400) at util/bpf-event.c:403 #3 0x00005555558ea85b in perf_event__synthesize_one_bpf_prog (session=0x555556178510, process=0x5555555ba7ab <process_synthesized_event>, machine=0x555556178728, fd=25, event=0x5555561b73a0, opts=0x5555560d33a8 <record+328>) at util/bpf-event.c:674 #4 0x00005555558eadf8 in perf_event__synthesize_bpf_events (session=0x555556178510, process=0x5555555ba7ab <process_synthesized_event>, machine=0x555556178728, opts=0x5555560d33a8 <record+328>) at util/bpf-event.c:799 #5 0x00005555555beef5 in record__synthesize (rec=0x5555560d3260 <record>, tail=false) at builtin-record.c:2131 #6 0x00005555555c07c1 in __cmd_record (rec=0x5555560d3260 <record>, argc=1, argv=0x7fffffffe2e0) at builtin-record.c:2581 #7 0x00005555555c4b30 in cmd_record (argc=1, argv=0x7fffffffe2e0) at builtin-record.c:4376 #8 0x000055555564d72e in run_builtin (p=0x5555560d63c0 <commands+288>, argc=6, argv=0x7fffffffe2e0) at perf.c:349 #9 0x000055555564d9c6 in handle_internal_command (argc=6, argv=0x7fffffffe2e0) at perf.c:401 #10 0x000055555564db1f in run_argv (argcp=0x7fffffffe0dc, argv=0x7fffffffe0d0) at perf.c:445 #11 0x000055555564de68 in main (argc=6, argv=0x7fffffffe2e0) at perf.c:553
I seems bpf_prog_info is broken for some reason.
(gdb) up #1 0x00005555558e9306 in synthesize_bpf_prog_name (buf=0x5555563305b0 "bpf_prog_", size=512, info=0x55555664e1d0, btf=0x55555637ad40, sub_id=0) at util/bpf-event.c:144 144 name_len += snprintf_hex(buf + name_len, size - name_len,
(gdb) p *info $1 = {type = 68, id = 80, tag = "\\000\000\000\214\000\000", jited_prog_len = 152, xlated_prog_len = 164, jited_prog_insns = 824633721012, xlated_prog_insns = 1185410973912, load_time = 1305670058276, created_by_uid = 352, nr_map_ids = 364, map_ids = 1975684956608, name = "\330\001\000\000\350\001\000\000$\002\000\0004\002\000", ifindex = 576, gpl_compatible = 0, netns_dev = 2697239462496, netns_ino = 2834678416000, nr_jited_ksyms = 756, nr_jited_func_lens = 768, jited_ksyms = 3418793968396, jited_func_lens = 3573412791092, btf_id = 844, func_info_rec_size = 880, func_info = 3934190044028, nr_func_info = 928, nr_line_info = 952, line_info = 4294967296988, jited_line_info = 4449586119680, nr_jited_line_info = 1060, line_info_rec_size = 1076, jited_line_info_rec_size = 1092, nr_prog_tags = 1108, prog_tags = 4861902980192, run_time_ns = 5085241279632, run_cnt = 5257039971512, recursion_misses = 5360119186644, verified_insns = 1264, attach_btf_obj_id = 1288, attach_btf_id = 1312}
Thanks Namhyung!
So it looks like my "fix" was breaking the tools/perf build feature test for btf_dump_type_data_opts opts.emit_strings and that was avoiding this code.
Having terminal 1 run perf trace is going to be loading/unloading a BPF program for system call augmentation. This must be creating the race condition that is causing perf record to segv when it is inspecting the bpf_prog_info.
The cast in: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/... ``` static int synthesize_bpf_prog_name(char *buf, int size, struct bpf_prog_info *info, struct btf *btf, u32 sub_id) { u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags); ``` looks concerning given the bad address comes from: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/... ``` name_len += snprintf_hex(buf + name_len, size - name_len, prog_tags[sub_id], BPF_TAG_SIZE); ``` Checking git blame this code has existed since 2019, commit 7b612e291a5a ("perf tools: Synthesize PERF_RECORD_* for loaded BPF programs"): http://lkml.kernel.org/r/20190117161521.1341602-8-songliubraving@fb.com it was refactored in 2019 to a single memory allocation commit ("a742258af131 perf bpf: Synthesize bpf events with bpf_program__get_prog_info_linear()") http://lkml.kernel.org/r/20190312053051.2690567-5-songliubraving@fb.com
There seems like a potential race here: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/... ``` /* step 1: get array dimensions */ err = bpf_obj_get_info_by_fd(fd, &info, &info_len); ``` and later: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/... ``` /* step 5: call syscall again to get required arrays */ err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); ``` There's a verification step that looks to cover issues with the race. I thought making those warnings fatal may help, but no: ``` --- a/tools/perf/util/bpf-utils.c +++ b/tools/perf/util/bpf-utils.c @@ -202,14 +202,20 @@ get_bpf_prog_info_linear(int fd, __u64 arrays) v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); v2 = bpf_prog_info_read_offset_u32(&info_linear->info, desc->count_offset); - if (v1 != v2) + if (v1 != v2) { pr_warning("%s: mismatch in element count\n", __func__); + free(info_linear); + return ERR_PTR(-EFAULT); + }
v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); v2 = bpf_prog_info_read_offset_u32(&info_linear->info, desc->size_offset); - if (v1 != v2) + if (v1 != v2) { pr_warning("%s: mismatch in rec size\n", __func__); + free(info_linear); + return ERR_PTR(-EFAULT); + } }
/* step 7: update info_len and data_len */ ```
Fwiw, the address of "data=0x40" in the stack trace makes it looks like an offset has been applied to NULL. 0x40 is 64 which corresponds with "name" info a bpf_prog_info by way of pahole: ``` struct bpf_prog_info { __u32 type; /* 0 4 */ /* --- cacheline 1 boundary (64 bytes) --- */ ... char name[16]; /* 64 16 */ ```
I feel we're relatively close to discovering a proper fix for the issue, if others could lend a hand as I'm not overly familiar with the BPF code. I'm wondering if the second bpf_obj_get_info_by_fd could be filling in offsets relative to NULL rather than returning an error, but this would be (I believe) a kernel issue :-(
Thanks, Ian
On Mon, Sep 1, 2025 at 10:04 AM Ian Rogers irogers@google.com wrote:
On Mon, Sep 1, 2025 at 12:57 AM Namhyung Kim namhyung@kernel.org wrote:
Hello,
On Sun, Aug 31, 2025 at 09:17:34PM -0700, Yonghong Song wrote:
On 8/29/25 10:19 PM, Ian Rogers wrote:
On Thu, Jun 5, 2025 at 2:00 PM patchwork-bot+netdevbpf@kernel.org wrote:
Hello:
This series was applied to bpf/bpf-next.git (master) by Andrii Nakryiko andrii@kernel.org:
On Tue, 3 Jun 2025 13:37:00 -0700 you wrote:
The BTF dumper code currently displays arrays of characters as just that - arrays, with each character formatted individually. Sometimes this is what makes sense, but it's nice to be able to treat that array as a string.
This change adds a special case to the btf_dump functionality to allow 0-terminated arrays of single-byte integer values to be printed as character strings. Characters for which isprint() returns false are printed as hex-escaped values. This is enabled when the new ".emit_strings" is set to 1 in the btf_dump_type_data_opts structure.
[...]
Here is the summary with links:
- [v3,1/2] libbpf: add support for printing BTF character arrays as strings https://git.kernel.org/bpf/bpf-next/c/87c9c79a02b4
- [v3,2/2] Tests for the ".emit_strings" functionality in the BTF dumper. https://git.kernel.org/bpf/bpf-next/c/a570f386f3d1
You are awesome, thank you!
I believe this patch is responsible for segvs occurring in v6.17 in various perf tests when the perf tests run in parallel. There's lots
Could you share the command line to reproduce this failure? This will help debugging. Thanks!
My reproducer is below:
terminal 1: run perf trace in a loop.
$ while true; do sudo ./perf trace true; done
terminal 2: run perf record in a loop until hit the segfault.
$ while true; do sudo ./perf record true || break; done ... perf: Segmentation fault #0 0x560b2db790e4 in dump_stack debug.c:366 #1 0x560b2db7915a in sighandler_dump_stack debug.c:378 #2 0x560b2d973b1b in sigsegv_handler builtin-record.c:722 #3 0x7f975f249df0 in __restore_rt libc_sigaction.c:0 #4 0x560b2dca1ee6 in snprintf_hex bpf-event.c:39 #5 0x560b2dca2306 in synthesize_bpf_prog_name bpf-event.c:144 #6 0x560b2dca2d92 in bpf_metadata_create bpf-event.c:401 #7 0x560b2dca3838 in perf_event__synthesize_one_bpf_prog bpf-event.c:673 #8 0x560b2dca3dd5 in perf_event__synthesize_bpf_events bpf-event.c:798 #9 0x560b2d977ef5 in record__synthesize builtin-record.c:2131 #10 0x560b2d9797c1 in __cmd_record builtin-record.c:2581 #11 0x560b2d97db30 in cmd_record builtin-record.c:4376 #12 0x560b2da0672e in run_builtin perf.c:349 #13 0x560b2da069c6 in handle_internal_command perf.c:401 #14 0x560b2da06b1f in run_argv perf.c:448 #15 0x560b2da06e68 in main perf.c:555 #16 0x7f975f233ca8 in __libc_start_call_main libc_start_call_main.h:74 #17 0x7f975f233d65 in __libc_start_main_alias_2 libc-start.c:128 #18 0x560b2d959b11 in _start perf[4cb11]
I manually ran it with gdb to get some more hints.
Thread 1 "perf" received signal SIGSEGV, Segmentation fault. 0x00005555558e8ee6 in snprintf_hex (buf=0x5555562c1d79 "", size=503, data=0x40 <error: Cannot access memory at address 0x40>, len=8) at util/bpf-event.c:39 39 ret += snprintf(buf + ret, size - ret, "%02x", data[i]);
The data is bpf_prog_info->prog_tags and it's called from synthesize_bpf_prog_name().
(gdb) bt #0 0x00005555558e8ee6 in snprintf_hex (buf=0x5555562c1d79 "", size=503, data=0x40 <error: Cannot access memory at address 0x40>, len=8) at util/bpf-event.c:39 #1 0x00005555558e9306 in synthesize_bpf_prog_name (buf=0x5555562c1d70 "bpf_prog_", size=512, info=0x55555665e400, btf=0x5555562c5630, sub_id=0) at util/bpf-event.c:144 #2 0x00005555558e9db5 in bpf_metadata_create (info=0x55555665e400) at util/bpf-event.c:403 #3 0x00005555558ea85b in perf_event__synthesize_one_bpf_prog (session=0x555556178510, process=0x5555555ba7ab <process_synthesized_event>, machine=0x555556178728, fd=25, event=0x5555561b73a0, opts=0x5555560d33a8 <record+328>) at util/bpf-event.c:674 #4 0x00005555558eadf8 in perf_event__synthesize_bpf_events (session=0x555556178510, process=0x5555555ba7ab <process_synthesized_event>, machine=0x555556178728, opts=0x5555560d33a8 <record+328>) at util/bpf-event.c:799 #5 0x00005555555beef5 in record__synthesize (rec=0x5555560d3260 <record>, tail=false) at builtin-record.c:2131 #6 0x00005555555c07c1 in __cmd_record (rec=0x5555560d3260 <record>, argc=1, argv=0x7fffffffe2e0) at builtin-record.c:2581 #7 0x00005555555c4b30 in cmd_record (argc=1, argv=0x7fffffffe2e0) at builtin-record.c:4376 #8 0x000055555564d72e in run_builtin (p=0x5555560d63c0 <commands+288>, argc=6, argv=0x7fffffffe2e0) at perf.c:349 #9 0x000055555564d9c6 in handle_internal_command (argc=6, argv=0x7fffffffe2e0) at perf.c:401 #10 0x000055555564db1f in run_argv (argcp=0x7fffffffe0dc, argv=0x7fffffffe0d0) at perf.c:445 #11 0x000055555564de68 in main (argc=6, argv=0x7fffffffe2e0) at perf.c:553
I seems bpf_prog_info is broken for some reason.
(gdb) up #1 0x00005555558e9306 in synthesize_bpf_prog_name (buf=0x5555563305b0 "bpf_prog_", size=512, info=0x55555664e1d0, btf=0x55555637ad40, sub_id=0) at util/bpf-event.c:144 144 name_len += snprintf_hex(buf + name_len, size - name_len,
(gdb) p *info $1 = {type = 68, id = 80, tag = "\\000\000\000\214\000\000", jited_prog_len = 152, xlated_prog_len = 164, jited_prog_insns = 824633721012, xlated_prog_insns = 1185410973912, load_time = 1305670058276, created_by_uid = 352, nr_map_ids = 364, map_ids = 1975684956608, name = "\330\001\000\000\350\001\000\000$\002\000\0004\002\000", ifindex = 576, gpl_compatible = 0, netns_dev = 2697239462496, netns_ino = 2834678416000, nr_jited_ksyms = 756, nr_jited_func_lens = 768, jited_ksyms = 3418793968396, jited_func_lens = 3573412791092, btf_id = 844, func_info_rec_size = 880, func_info = 3934190044028, nr_func_info = 928, nr_line_info = 952, line_info = 4294967296988, jited_line_info = 4449586119680, nr_jited_line_info = 1060, line_info_rec_size = 1076, jited_line_info_rec_size = 1092, nr_prog_tags = 1108, prog_tags = 4861902980192, run_time_ns = 5085241279632, run_cnt = 5257039971512, recursion_misses = 5360119186644, verified_insns = 1264, attach_btf_obj_id = 1288, attach_btf_id = 1312}
Thanks Namhyung!
So it looks like my "fix" was breaking the tools/perf build feature test for btf_dump_type_data_opts opts.emit_strings and that was avoiding this code.
Having terminal 1 run perf trace is going to be loading/unloading a BPF program for system call augmentation. This must be creating the race condition that is causing perf record to segv when it is inspecting the bpf_prog_info.
The cast in: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/...
static int synthesize_bpf_prog_name(char *buf, int size, struct bpf_prog_info *info, struct btf *btf, u32 sub_id) { u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags);
looks concerning given the bad address comes from: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/...
name_len += snprintf_hex(buf + name_len, size - name_len, prog_tags[sub_id], BPF_TAG_SIZE);
Checking git blame this code has existed since 2019, commit 7b612e291a5a ("perf tools: Synthesize PERF_RECORD_* for loaded BPF programs"): http://lkml.kernel.org/r/20190117161521.1341602-8-songliubraving@fb.com it was refactored in 2019 to a single memory allocation commit ("a742258af131 perf bpf: Synthesize bpf events with bpf_program__get_prog_info_linear()") http://lkml.kernel.org/r/20190312053051.2690567-5-songliubraving@fb.com
There seems like a potential race here: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/...
/* step 1: get array dimensions */ err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
and later: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/...
/* step 5: call syscall again to get required arrays */ err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
There's a verification step that looks to cover issues with the race. I thought making those warnings fatal may help, but no:
--- a/tools/perf/util/bpf-utils.c +++ b/tools/perf/util/bpf-utils.c @@ -202,14 +202,20 @@ get_bpf_prog_info_linear(int fd, __u64 arrays) v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); v2 = bpf_prog_info_read_offset_u32(&info_linear->info, desc->count_offset); - if (v1 != v2) + if (v1 != v2) { pr_warning("%s: mismatch in element count\n", __func__); + free(info_linear); + return ERR_PTR(-EFAULT); + } v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); v2 = bpf_prog_info_read_offset_u32(&info_linear->info, desc->size_offset); - if (v1 != v2) + if (v1 != v2) { pr_warning("%s: mismatch in rec size\n", __func__); + free(info_linear); + return ERR_PTR(-EFAULT); + } } /* step 7: update info_len and data_len */
Fwiw, the address of "data=0x40" in the stack trace makes it looks like an offset has been applied to NULL. 0x40 is 64 which corresponds with "name" info a bpf_prog_info by way of pahole:
struct bpf_prog_info { __u32 type; /* 0 4 */ /* --- cacheline 1 boundary (64 bytes) --- */ ... char name[16]; /* 64 16 */
I feel we're relatively close to discovering a proper fix for the issue, if others could lend a hand as I'm not overly familiar with the BPF code. I'm wondering if the second bpf_obj_get_info_by_fd could be filling in offsets relative to NULL rather than returning an error, but this would be (I believe) a kernel issue :-(
It's a use-after-free perf bug: https://web.git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git... ``` /* save bpf_prog_info to env */ info_node = malloc(sizeof(struct bpf_prog_info_node)); if (!info_node) { err = -1; goto out; }
info_node->info_linear = info_linear; info_node->metadata = NULL; if (!perf_env__insert_bpf_prog_info(env, info_node)) { free(info_linear); free(info_node); } info_linear = NULL; ... /* Synthesize PERF_RECORD_BPF_METADATA */ metadata = bpf_metadata_create(info); ``` The insert can fail due to a duplicated BPF program (the sideband data thread is racing with the synthesizing thread in perf) leading to info_linear to be freed which the info points into. The later use of the freed memory leads to the crash.
I hope that's all it is and I'll work on a fix.
Thanks, Ian
On Tue, Sep 2, 2025 at 9:47 AM Ian Rogers irogers@google.com wrote:
On Mon, Sep 1, 2025 at 10:04 AM Ian Rogers irogers@google.com wrote:
On Mon, Sep 1, 2025 at 12:57 AM Namhyung Kim namhyung@kernel.org wrote:
Hello,
On Sun, Aug 31, 2025 at 09:17:34PM -0700, Yonghong Song wrote:
On 8/29/25 10:19 PM, Ian Rogers wrote:
On Thu, Jun 5, 2025 at 2:00 PM patchwork-bot+netdevbpf@kernel.org wrote:
Hello:
This series was applied to bpf/bpf-next.git (master) by Andrii Nakryiko andrii@kernel.org:
On Tue, 3 Jun 2025 13:37:00 -0700 you wrote: > The BTF dumper code currently displays arrays of characters as just that - > arrays, with each character formatted individually. Sometimes this is what > makes sense, but it's nice to be able to treat that array as a string. > > This change adds a special case to the btf_dump functionality to allow > 0-terminated arrays of single-byte integer values to be printed as > character strings. Characters for which isprint() returns false are > printed as hex-escaped values. This is enabled when the new ".emit_strings" > is set to 1 in the btf_dump_type_data_opts structure. > > [...] Here is the summary with links:
- [v3,1/2] libbpf: add support for printing BTF character arrays as strings https://git.kernel.org/bpf/bpf-next/c/87c9c79a02b4
- [v3,2/2] Tests for the ".emit_strings" functionality in the BTF dumper. https://git.kernel.org/bpf/bpf-next/c/a570f386f3d1
You are awesome, thank you!
I believe this patch is responsible for segvs occurring in v6.17 in various perf tests when the perf tests run in parallel. There's lots
Could you share the command line to reproduce this failure? This will help debugging. Thanks!
My reproducer is below:
terminal 1: run perf trace in a loop.
$ while true; do sudo ./perf trace true; done
terminal 2: run perf record in a loop until hit the segfault.
$ while true; do sudo ./perf record true || break; done ... perf: Segmentation fault #0 0x560b2db790e4 in dump_stack debug.c:366 #1 0x560b2db7915a in sighandler_dump_stack debug.c:378 #2 0x560b2d973b1b in sigsegv_handler builtin-record.c:722 #3 0x7f975f249df0 in __restore_rt libc_sigaction.c:0 #4 0x560b2dca1ee6 in snprintf_hex bpf-event.c:39 #5 0x560b2dca2306 in synthesize_bpf_prog_name bpf-event.c:144 #6 0x560b2dca2d92 in bpf_metadata_create bpf-event.c:401 #7 0x560b2dca3838 in perf_event__synthesize_one_bpf_prog bpf-event.c:673 #8 0x560b2dca3dd5 in perf_event__synthesize_bpf_events bpf-event.c:798 #9 0x560b2d977ef5 in record__synthesize builtin-record.c:2131 #10 0x560b2d9797c1 in __cmd_record builtin-record.c:2581 #11 0x560b2d97db30 in cmd_record builtin-record.c:4376 #12 0x560b2da0672e in run_builtin perf.c:349 #13 0x560b2da069c6 in handle_internal_command perf.c:401 #14 0x560b2da06b1f in run_argv perf.c:448 #15 0x560b2da06e68 in main perf.c:555 #16 0x7f975f233ca8 in __libc_start_call_main libc_start_call_main.h:74 #17 0x7f975f233d65 in __libc_start_main_alias_2 libc-start.c:128 #18 0x560b2d959b11 in _start perf[4cb11]
I manually ran it with gdb to get some more hints.
Thread 1 "perf" received signal SIGSEGV, Segmentation fault. 0x00005555558e8ee6 in snprintf_hex (buf=0x5555562c1d79 "", size=503, data=0x40 <error: Cannot access memory at address 0x40>, len=8) at util/bpf-event.c:39 39 ret += snprintf(buf + ret, size - ret, "%02x", data[i]);
The data is bpf_prog_info->prog_tags and it's called from synthesize_bpf_prog_name().
(gdb) bt #0 0x00005555558e8ee6 in snprintf_hex (buf=0x5555562c1d79 "", size=503, data=0x40 <error: Cannot access memory at address 0x40>, len=8) at util/bpf-event.c:39 #1 0x00005555558e9306 in synthesize_bpf_prog_name (buf=0x5555562c1d70 "bpf_prog_", size=512, info=0x55555665e400, btf=0x5555562c5630, sub_id=0) at util/bpf-event.c:144 #2 0x00005555558e9db5 in bpf_metadata_create (info=0x55555665e400) at util/bpf-event.c:403 #3 0x00005555558ea85b in perf_event__synthesize_one_bpf_prog (session=0x555556178510, process=0x5555555ba7ab <process_synthesized_event>, machine=0x555556178728, fd=25, event=0x5555561b73a0, opts=0x5555560d33a8 <record+328>) at util/bpf-event.c:674 #4 0x00005555558eadf8 in perf_event__synthesize_bpf_events (session=0x555556178510, process=0x5555555ba7ab <process_synthesized_event>, machine=0x555556178728, opts=0x5555560d33a8 <record+328>) at util/bpf-event.c:799 #5 0x00005555555beef5 in record__synthesize (rec=0x5555560d3260 <record>, tail=false) at builtin-record.c:2131 #6 0x00005555555c07c1 in __cmd_record (rec=0x5555560d3260 <record>, argc=1, argv=0x7fffffffe2e0) at builtin-record.c:2581 #7 0x00005555555c4b30 in cmd_record (argc=1, argv=0x7fffffffe2e0) at builtin-record.c:4376 #8 0x000055555564d72e in run_builtin (p=0x5555560d63c0 <commands+288>, argc=6, argv=0x7fffffffe2e0) at perf.c:349 #9 0x000055555564d9c6 in handle_internal_command (argc=6, argv=0x7fffffffe2e0) at perf.c:401 #10 0x000055555564db1f in run_argv (argcp=0x7fffffffe0dc, argv=0x7fffffffe0d0) at perf.c:445 #11 0x000055555564de68 in main (argc=6, argv=0x7fffffffe2e0) at perf.c:553
I seems bpf_prog_info is broken for some reason.
(gdb) up #1 0x00005555558e9306 in synthesize_bpf_prog_name (buf=0x5555563305b0 "bpf_prog_", size=512, info=0x55555664e1d0, btf=0x55555637ad40, sub_id=0) at util/bpf-event.c:144 144 name_len += snprintf_hex(buf + name_len, size - name_len,
(gdb) p *info $1 = {type = 68, id = 80, tag = "\\000\000\000\214\000\000", jited_prog_len = 152, xlated_prog_len = 164, jited_prog_insns = 824633721012, xlated_prog_insns = 1185410973912, load_time = 1305670058276, created_by_uid = 352, nr_map_ids = 364, map_ids = 1975684956608, name = "\330\001\000\000\350\001\000\000$\002\000\0004\002\000", ifindex = 576, gpl_compatible = 0, netns_dev = 2697239462496, netns_ino = 2834678416000, nr_jited_ksyms = 756, nr_jited_func_lens = 768, jited_ksyms = 3418793968396, jited_func_lens = 3573412791092, btf_id = 844, func_info_rec_size = 880, func_info = 3934190044028, nr_func_info = 928, nr_line_info = 952, line_info = 4294967296988, jited_line_info = 4449586119680, nr_jited_line_info = 1060, line_info_rec_size = 1076, jited_line_info_rec_size = 1092, nr_prog_tags = 1108, prog_tags = 4861902980192, run_time_ns = 5085241279632, run_cnt = 5257039971512, recursion_misses = 5360119186644, verified_insns = 1264, attach_btf_obj_id = 1288, attach_btf_id = 1312}
Thanks Namhyung!
So it looks like my "fix" was breaking the tools/perf build feature test for btf_dump_type_data_opts opts.emit_strings and that was avoiding this code.
Having terminal 1 run perf trace is going to be loading/unloading a BPF program for system call augmentation. This must be creating the race condition that is causing perf record to segv when it is inspecting the bpf_prog_info.
The cast in: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/...
static int synthesize_bpf_prog_name(char *buf, int size, struct bpf_prog_info *info, struct btf *btf, u32 sub_id) { u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags);
looks concerning given the bad address comes from: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/...
name_len += snprintf_hex(buf + name_len, size - name_len, prog_tags[sub_id], BPF_TAG_SIZE);
Checking git blame this code has existed since 2019, commit 7b612e291a5a ("perf tools: Synthesize PERF_RECORD_* for loaded BPF programs"): http://lkml.kernel.org/r/20190117161521.1341602-8-songliubraving@fb.com it was refactored in 2019 to a single memory allocation commit ("a742258af131 perf bpf: Synthesize bpf events with bpf_program__get_prog_info_linear()") http://lkml.kernel.org/r/20190312053051.2690567-5-songliubraving@fb.com
There seems like a potential race here: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/...
/* step 1: get array dimensions */ err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
and later: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/...
/* step 5: call syscall again to get required arrays */ err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
There's a verification step that looks to cover issues with the race. I thought making those warnings fatal may help, but no:
--- a/tools/perf/util/bpf-utils.c +++ b/tools/perf/util/bpf-utils.c @@ -202,14 +202,20 @@ get_bpf_prog_info_linear(int fd, __u64 arrays) v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); v2 = bpf_prog_info_read_offset_u32(&info_linear->info, desc->count_offset); - if (v1 != v2) + if (v1 != v2) { pr_warning("%s: mismatch in element count\n", __func__); + free(info_linear); + return ERR_PTR(-EFAULT); + } v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); v2 = bpf_prog_info_read_offset_u32(&info_linear->info, desc->size_offset); - if (v1 != v2) + if (v1 != v2) { pr_warning("%s: mismatch in rec size\n", __func__); + free(info_linear); + return ERR_PTR(-EFAULT); + } } /* step 7: update info_len and data_len */
Fwiw, the address of "data=0x40" in the stack trace makes it looks like an offset has been applied to NULL. 0x40 is 64 which corresponds with "name" info a bpf_prog_info by way of pahole:
struct bpf_prog_info { __u32 type; /* 0 4 */ /* --- cacheline 1 boundary (64 bytes) --- */ ... char name[16]; /* 64 16 */
I feel we're relatively close to discovering a proper fix for the issue, if others could lend a hand as I'm not overly familiar with the BPF code. I'm wondering if the second bpf_obj_get_info_by_fd could be filling in offsets relative to NULL rather than returning an error, but this would be (I believe) a kernel issue :-(
It's a use-after-free perf bug: https://web.git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git...
/* save bpf_prog_info to env */ info_node = malloc(sizeof(struct bpf_prog_info_node)); if (!info_node) { err = -1; goto out; } info_node->info_linear = info_linear; info_node->metadata = NULL; if (!perf_env__insert_bpf_prog_info(env, info_node)) { free(info_linear); free(info_node); } info_linear = NULL; ... /* Synthesize PERF_RECORD_BPF_METADATA */ metadata = bpf_metadata_create(info);
The insert can fail due to a duplicated BPF program (the sideband data thread is racing with the synthesizing thread in perf) leading to info_linear to be freed which the info points into. The later use of the freed memory leads to the crash.
I hope that's all it is and I'll work on a fix.
That was all. Fixes sent in: https://lore.kernel.org/all/20250902181713.309797-1-irogers@google.com/
Thanks, Ian
On Tue, Jun 03, 2025 at 01:37:00PM -0700, Blake Jones wrote:
The BTF dumper code currently displays arrays of characters as just that - arrays, with each character formatted individually. Sometimes this is what makes sense, but it's nice to be able to treat that array as a string.
This change adds a special case to the btf_dump functionality to allow 0-terminated arrays of single-byte integer values to be printed as character strings. Characters for which isprint() returns false are printed as hex-escaped values. This is enabled when the new ".emit_strings" is set to 1 in the btf_dump_type_data_opts structure.
As an example, here's what it looks like to dump the string "hello" using a few different field values for btf_dump_type_data_opts (.compact = 1):
- .emit_strings = 0, .skip_names = 0: (char[6])['h','e','l','l','o',]
- .emit_strings = 0, .skip_names = 1: ['h','e','l','l','o',]
- .emit_strings = 1, .skip_names = 0: (char[6])"hello"
- .emit_strings = 1, .skip_names = 1: "hello"
hi, could this be used in bpftool map dump? ;-) I checked, but it looks like bpftool map dump is using something else to dump data.. I admit I haven't spent much on time that
thanks, jirka
Here's the string "h\xff", dumped with .compact = 1 and .skip_names = 1:
- .emit_strings = 0: ['h',-1,]
- .emit_strings = 1: "h\xff"
Signed-off-by: Blake Jones blakejones@google.com
tools/lib/bpf/btf.h | 3 ++- tools/lib/bpf/btf_dump.c | 55 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-)
diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index 4392451d634b..ccfd905f03df 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -326,9 +326,10 @@ struct btf_dump_type_data_opts { bool compact; /* no newlines/indentation */ bool skip_names; /* skip member/type names */ bool emit_zeroes; /* show 0-valued fields */
- bool emit_strings; /* print char arrays as strings */ size_t :0;
}; -#define btf_dump_type_data_opts__last_field emit_zeroes +#define btf_dump_type_data_opts__last_field emit_strings LIBBPF_API int btf_dump__dump_type_data(struct btf_dump *d, __u32 id, diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 460c3e57fadb..7c2f1f13f958 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -68,6 +68,7 @@ struct btf_dump_data { bool compact; bool skip_names; bool emit_zeroes;
- bool emit_strings; __u8 indent_lvl; /* base indent level */ char indent_str[BTF_DATA_INDENT_STR_LEN]; /* below are used during iteration */
@@ -2028,6 +2029,52 @@ static int btf_dump_var_data(struct btf_dump *d, return btf_dump_dump_type_data(d, NULL, t, type_id, data, 0, 0); } +static int btf_dump_string_data(struct btf_dump *d,
const struct btf_type *t,
__u32 id,
const void *data)
+{
- const struct btf_array *array = btf_array(t);
- const char *chars = data;
- __u32 i;
- /* Make sure it is a NUL-terminated string. */
- for (i = 0; i < array->nelems; i++) {
if ((void *)(chars + i) >= d->typed_dump->data_end)
return -E2BIG;
if (chars[i] == '\0')
break;
- }
- if (i == array->nelems) {
/* The caller will print this as a regular array. */
return -EINVAL;
- }
- btf_dump_data_pfx(d);
- btf_dump_printf(d, """);
- for (i = 0; i < array->nelems; i++) {
char c = chars[i];
if (c == '\0') {
/*
* When printing character arrays as strings, NUL bytes
* are always treated as string terminators; they are
* never printed.
*/
break;
}
if (isprint(c))
btf_dump_printf(d, "%c", c);
else
btf_dump_printf(d, "\\x%02x", (__u8)c);
- }
- btf_dump_printf(d, """);
- return 0;
+}
static int btf_dump_array_data(struct btf_dump *d, const struct btf_type *t, __u32 id, @@ -2055,8 +2102,13 @@ static int btf_dump_array_data(struct btf_dump *d, * char arrays, so if size is 1 and element is * printable as a char, we'll do that. */
if (elem_size == 1)
if (elem_size == 1) {
if (d->typed_dump->emit_strings &&
btf_dump_string_data(d, t, id, data) == 0) {
return 0;
} d->typed_dump->is_array_char = true;
}}
/* note that we increment depth before calling btf_dump_print() below; @@ -2544,6 +2596,7 @@ int btf_dump__dump_type_data(struct btf_dump *d, __u32 id, d->typed_dump->compact = OPTS_GET(opts, compact, false); d->typed_dump->skip_names = OPTS_GET(opts, skip_names, false); d->typed_dump->emit_zeroes = OPTS_GET(opts, emit_zeroes, false);
- d->typed_dump->emit_strings = OPTS_GET(opts, emit_strings, false);
ret = btf_dump_dump_type_data(d, NULL, t, id, data, 0, 0); -- 2.49.0.1204.g71687c7c1d-goog
Hi Jiri,
On Wed, Jun 11, 2025 at 3:34 AM Jiri Olsa olsajiri@gmail.com wrote:
could this be used in bpftool map dump? ;-) I checked, but it looks like bpftool map dump is using something else to dump data.. I admit I haven't spent much on time that
I actually started this work by looking at bpftool's dumper, and only after a bit of time did I realize that there was a more general dumper in libbpf. I suspect that part of why no one has updated bpftool to use libbpf for this is that bpftool's dumper also has a "display in JSON" mode which libbpf doesn't support. So yes, it could be done, but it would take some work.
Blake
linux-kselftest-mirror@lists.linaro.org