Andrii suggested to remove this abstraction layer and have the percpu handling more explicit[1].

This patch also updates the tests that relied on the macros.

[1] https://lore.kernel.org/bpf/CAEf4BzYmj_ZPDq8Zi4dbntboJKRPU2TVopysBNrdd9foHTf...

Suggested-by: Andrii Nakryiko andrii@kernel.org Signed-off-by: Pedro Tammela pctammela@mojatatu.com --- tools/testing/selftests/bpf/bpf_util.h | 7 -- .../bpf/map_tests/htab_map_batch_ops.c | 71 ++++++++-------- .../selftests/bpf/prog_tests/map_init.c | 9 +- tools/testing/selftests/bpf/test_maps.c | 84 +++++++++++-------- 4 files changed, 89 insertions(+), 82 deletions(-)

diff --git a/tools/testing/selftests/bpf/bpf_util.h b/tools/testing/selftests/bpf/bpf_util.h index a3352a64c067..105db3120ab4 100644 --- a/tools/testing/selftests/bpf/bpf_util.h +++ b/tools/testing/selftests/bpf/bpf_util.h @@ -20,13 +20,6 @@ static inline unsigned int bpf_num_possible_cpus(void) return possible_cpus; }

-#define __bpf_percpu_val_align __attribute__((__aligned__(8))) - -#define BPF_DECLARE_PERCPU(type, name) \ - struct { type v; /* padding */ } __bpf_percpu_val_align \ - name[bpf_num_possible_cpus()] -#define bpf_percpu(name, cpu) name[(cpu)].v - #ifndef ARRAY_SIZE # define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) #endif diff --git a/tools/testing/selftests/bpf/map_tests/htab_map_batch_ops.c b/tools/testing/selftests/bpf/map_tests/htab_map_batch_ops.c index 976bf415fbdd..16e64bd89b1a 100644 --- a/tools/testing/selftests/bpf/map_tests/htab_map_batch_ops.c +++ b/tools/testing/selftests/bpf/map_tests/htab_map_batch_ops.c @@ -7,65 +7,63 @@ #include <bpf/bpf.h> #include <bpf/libbpf.h>

-#include <bpf_util.h> #include <test_maps.h>

static void map_batch_update(int map_fd, __u32 max_entries, int *keys, - void *values, bool is_pcpu) + __s64 *values, bool is_pcpu) { - typedef BPF_DECLARE_PERCPU(int, value); - value *v = NULL; + int nr_cpus = libbpf_num_possible_cpus(); int i, j, err; + int offset = 0; DECLARE_LIBBPF_OPTS(bpf_map_batch_opts, opts, .elem_flags = 0, .flags = 0, );

- if (is_pcpu) - v = (value *)values; + CHECK(nr_cpus < 0, "nr_cpus checking", + "error: get possible cpus failed");

for (i = 0; i < max_entries; i++) { keys[i] = i + 1; if (is_pcpu) - for (j = 0; j < bpf_num_possible_cpus(); j++) - bpf_percpu(v[i], j) = i + 2 + j; + for (j = 0; j < nr_cpus; j++) + (values + offset)[j] = i + 2 + j; else - ((int *)values)[i] = i + 2; + values[i] = i + 2; + offset += nr_cpus; }

err = bpf_map_update_batch(map_fd, keys, values, &max_entries, &opts); CHECK(err, "bpf_map_update_batch()", "error:%s\n", strerror(errno)); }

-static void map_batch_verify(int *visited, __u32 max_entries, - int *keys, void *values, bool is_pcpu) +static void map_batch_verify(int *visited, __u32 max_entries, int *keys, + __s64 *values, bool is_pcpu) { - typedef BPF_DECLARE_PERCPU(int, value); - value *v = NULL; + int nr_cpus = libbpf_num_possible_cpus(); int i, j; + int offset = 0;

- if (is_pcpu) - v = (value *)values; + CHECK(nr_cpus < 0, "nr_cpus checking", + "error: get possible cpus failed");

memset(visited, 0, max_entries * sizeof(*visited)); for (i = 0; i < max_entries; i++) { - if (is_pcpu) { - for (j = 0; j < bpf_num_possible_cpus(); j++) { - CHECK(keys[i] + 1 + j != bpf_percpu(v[i], j), + for (j = 0; j < nr_cpus; j++) { + __s64 value = (values + offset)[j]; + CHECK(keys[i] + 1 + j != value, "key/value checking", - "error: i %d j %d key %d value %d\n", - i, j, keys[i], bpf_percpu(v[i], j)); + "error: i %d j %d key %d value %d\n", i, + j, keys[i], value); } } else { - CHECK(keys[i] + 1 != ((int *)values)[i], - "key/value checking", + CHECK(keys[i] + 1 != values[i], "key/value checking", "error: i %d key %d value %d\n", i, keys[i], - ((int *)values)[i]); + values[i]); } - + offset += nr_cpus; visited[i] = 1; - } for (i = 0; i < max_entries; i++) { CHECK(visited[i] != 1, "visited checking", @@ -75,11 +73,10 @@ static void map_batch_verify(int *visited, __u32 max_entries,

void __test_map_lookup_and_delete_batch(bool is_pcpu) { + int nr_cpus = libbpf_num_possible_cpus(); __u32 batch, count, total, total_success; - typedef BPF_DECLARE_PERCPU(int, value); int map_fd, *keys, *visited, key; const __u32 max_entries = 10; - value pcpu_values[max_entries]; int err, step, value_size; bool nospace_err; void *values; @@ -88,24 +85,27 @@ void __test_map_lookup_and_delete_batch(bool is_pcpu) .map_type = is_pcpu ? BPF_MAP_TYPE_PERCPU_HASH : BPF_MAP_TYPE_HASH, .key_size = sizeof(int), - .value_size = sizeof(int), + .value_size = sizeof(__s64), }; DECLARE_LIBBPF_OPTS(bpf_map_batch_opts, opts, .elem_flags = 0, .flags = 0, );

+ CHECK(nr_cpus < 0, "nr_cpus checking", + "error: get possible cpus failed"); + xattr.max_entries = max_entries; map_fd = bpf_create_map_xattr(&xattr); CHECK(map_fd == -1, "bpf_create_map_xattr()", "error:%s\n", strerror(errno));

- value_size = is_pcpu ? sizeof(value) : sizeof(int); - keys = malloc(max_entries * sizeof(int)); + value_size = sizeof(__s64); if (is_pcpu) - values = pcpu_values; - else - values = malloc(max_entries * sizeof(int)); + value_size *= nr_cpus; + + keys = malloc(max_entries * sizeof(int)); + values = calloc(max_entries, value_size); visited = malloc(max_entries * sizeof(int)); CHECK(!keys || !values || !visited, "malloc()", "error:%s\n", strerror(errno)); @@ -203,7 +203,7 @@ void __test_map_lookup_and_delete_batch(bool is_pcpu) CHECK(total != max_entries, "delete with steps", "total = %u, max_entries = %u\n", total, max_entries);

- /* check map is empty, errono == ENOENT */ + /* check map is empty, errno == ENOENT */ err = bpf_map_get_next_key(map_fd, NULL, &key); CHECK(!err || errno != ENOENT, "bpf_map_get_next_key()", "error: %s\n", strerror(errno)); @@ -260,8 +260,7 @@ void __test_map_lookup_and_delete_batch(bool is_pcpu) "unexpected failure\n"); free(keys); free(visited); - if (!is_pcpu) - free(values); + free(values); }

void htab_map_batch_ops(void) diff --git a/tools/testing/selftests/bpf/prog_tests/map_init.c b/tools/testing/selftests/bpf/prog_tests/map_init.c index 14a31109dd0e..49386d0aa684 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_init.c +++ b/tools/testing/selftests/bpf/prog_tests/map_init.c @@ -12,10 +12,7 @@ static int duration;

typedef unsigned long long map_key_t; typedef unsigned long long map_value_t; -typedef struct { - map_value_t v; /* padding */ -} __bpf_percpu_val_align pcpu_map_value_t; - +typedef __s64 pcpu_map_value_t;

static int map_populate(int map_fd, int num) { @@ -24,7 +21,7 @@ static int map_populate(int map_fd, int num) map_key_t key;

for (i = 0; i < nr_cpus; i++) - bpf_percpu(value, i) = FILL_VALUE; + value[i] = FILL_VALUE;

for (key = 1; key <= num; key++) { err = bpf_map_update_elem(map_fd, &key, value, BPF_NOEXIST); @@ -103,7 +100,7 @@ static int check_values_one_cpu(pcpu_map_value_t *value, map_value_t expected) map_value_t val;

for (i = 0; i < nr_cpus; i++) { - val = bpf_percpu(value, i); + val = value[i]; if (val) { if (CHECK(val != expected, "map value", "unexpected for cpu %d: 0x%llx\n", i, val)) diff --git a/tools/testing/selftests/bpf/test_maps.c b/tools/testing/selftests/bpf/test_maps.c index 51adc42b2b40..b8ce837a7ada 100644 --- a/tools/testing/selftests/bpf/test_maps.c +++ b/tools/testing/selftests/bpf/test_maps.c @@ -146,63 +146,69 @@ static void test_hashmap_sizes(unsigned int task, void *data)

static void test_hashmap_percpu(unsigned int task, void *data) { - unsigned int nr_cpus = bpf_num_possible_cpus(); - BPF_DECLARE_PERCPU(long, value); + int nr_cpus = libbpf_num_possible_cpus(); + __s64 *values; long long key, next_key, first_key; int expected_key_mask = 0; int fd, i;

+ if (nr_cpus < 0) { + printf("Failed get possible cpus\n"); + exit(1); + } + + values = alloca(nr_cpus * sizeof(__s64)); + fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_HASH, sizeof(key), - sizeof(bpf_percpu(value, 0)), 2, map_flags); + sizeof(*values), 2, map_flags); if (fd < 0) { printf("Failed to create hashmap '%s'!\n", strerror(errno)); exit(1); }

for (i = 0; i < nr_cpus; i++) - bpf_percpu(value, i) = i + 100; + values[i] = i + 100;

key = 1; /* Insert key=1 element. */ assert(!(expected_key_mask & key)); - assert(bpf_map_update_elem(fd, &key, value, BPF_ANY) == 0); + assert(bpf_map_update_elem(fd, &key, values, BPF_ANY) == 0); expected_key_mask |= key;

/* BPF_NOEXIST means add new element if it doesn't exist. */ - assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == -1 && + assert(bpf_map_update_elem(fd, &key, values, BPF_NOEXIST) == -1 && /* key=1 already exists. */ errno == EEXIST);

/* -1 is an invalid flag. */ - assert(bpf_map_update_elem(fd, &key, value, -1) == -1 && + assert(bpf_map_update_elem(fd, &key, values, -1) == -1 && errno == EINVAL);

/* Check that key=1 can be found. Value could be 0 if the lookup * was run from a different CPU. */ - bpf_percpu(value, 0) = 1; - assert(bpf_map_lookup_elem(fd, &key, value) == 0 && - bpf_percpu(value, 0) == 100); + values[0] = 1; + assert(bpf_map_lookup_elem(fd, &key, values) == 0 && values[0] == 100);

key = 2; /* Check that key=2 is not found. */ - assert(bpf_map_lookup_elem(fd, &key, value) == -1 && errno == ENOENT); + assert(bpf_map_lookup_elem(fd, &key, values) == -1 && errno == ENOENT);

/* BPF_EXIST means update existing element. */ - assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) == -1 && + assert(bpf_map_update_elem(fd, &key, values, BPF_EXIST) == -1 && /* key=2 is not there. */ errno == ENOENT);

/* Insert key=2 element. */ assert(!(expected_key_mask & key)); - assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == 0); + assert(bpf_map_update_elem(fd, &key, values, BPF_NOEXIST) == 0); expected_key_mask |= key;

/* key=1 and key=2 were inserted, check that key=0 cannot be * inserted due to max_entries limit. */ key = 0; - assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == -1 && + assert(bpf_map_update_elem(fd, &key, values, BPF_NOEXIST) == -1 && errno == E2BIG);

/* Check that key = 0 doesn't exist. */ @@ -219,10 +225,10 @@ static void test_hashmap_percpu(unsigned int task, void *data) assert((expected_key_mask & next_key) == next_key); expected_key_mask &= ~next_key;

- assert(bpf_map_lookup_elem(fd, &next_key, value) == 0); + assert(bpf_map_lookup_elem(fd, &next_key, values) == 0);

for (i = 0; i < nr_cpus; i++) - assert(bpf_percpu(value, i) == i + 100); + assert(values[i] == i + 100);

key = next_key; } @@ -230,7 +236,7 @@ static void test_hashmap_percpu(unsigned int task, void *data)

/* Update with BPF_EXIST. */ key = 1; - assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) == 0); + assert(bpf_map_update_elem(fd, &key, values, BPF_EXIST) == 0);

/* Delete both elements. */ key = 1; @@ -399,37 +405,42 @@ static void test_arraymap(unsigned int task, void *data)

static void test_arraymap_percpu(unsigned int task, void *data) { - unsigned int nr_cpus = bpf_num_possible_cpus(); - BPF_DECLARE_PERCPU(long, values); + int nr_cpus = libbpf_num_possible_cpus(); + __s64 *values; int key, next_key, fd, i;

+ if (nr_cpus < 0) { + printf("Failed get possible cpus\n"); + exit(1); + } + + values = alloca(nr_cpus * sizeof(__s64)); + fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_ARRAY, sizeof(key), - sizeof(bpf_percpu(values, 0)), 2, 0); + sizeof(*values), 2, 0); if (fd < 0) { printf("Failed to create arraymap '%s'!\n", strerror(errno)); exit(1); }

for (i = 0; i < nr_cpus; i++) - bpf_percpu(values, i) = i + 100; + values[i] = i + 100;

key = 1; /* Insert key=1 element. */ assert(bpf_map_update_elem(fd, &key, values, BPF_ANY) == 0);

- bpf_percpu(values, 0) = 0; + values[0] = 0; assert(bpf_map_update_elem(fd, &key, values, BPF_NOEXIST) == -1 && errno == EEXIST);

/* Check that key=1 can be found. */ - assert(bpf_map_lookup_elem(fd, &key, values) == 0 && - bpf_percpu(values, 0) == 100); + assert(bpf_map_lookup_elem(fd, &key, values) == 0 && values[0] == 100);

key = 0; /* Check that key=0 is also found and zero initialized. */ - assert(bpf_map_lookup_elem(fd, &key, values) == 0 && - bpf_percpu(values, 0) == 0 && - bpf_percpu(values, nr_cpus - 1) == 0); + assert(bpf_map_lookup_elem(fd, &key, values) == 0 && values[0] == 0 && + values[nr_cpus - 1] == 0);

/* Check that key=2 cannot be inserted due to max_entries limit. */ key = 2; @@ -458,16 +469,23 @@ static void test_arraymap_percpu(unsigned int task, void *data)

static void test_arraymap_percpu_many_keys(void) { - unsigned int nr_cpus = bpf_num_possible_cpus(); - BPF_DECLARE_PERCPU(long, values); + unsigned int nr_cpus = libbpf_num_possible_cpus(); + __s64 *values; /* nr_keys is not too large otherwise the test stresses percpu * allocator more than anything else */ unsigned int nr_keys = 2000; int key, fd, i;

+ if (nr_cpus < 0) { + printf("Failed get possible cpus\n"); + exit(1); + } + + values = alloca(nr_cpus * sizeof(__s64)); + fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_ARRAY, sizeof(key), - sizeof(bpf_percpu(values, 0)), nr_keys, 0); + sizeof(*values), nr_keys, 0); if (fd < 0) { printf("Failed to create per-cpu arraymap '%s'!\n", strerror(errno)); @@ -475,19 +493,19 @@ static void test_arraymap_percpu_many_keys(void) }

for (i = 0; i < nr_cpus; i++) - bpf_percpu(values, i) = i + 10; + values[i] = i + 10;

for (key = 0; key < nr_keys; key++) assert(bpf_map_update_elem(fd, &key, values, BPF_ANY) == 0);

for (key = 0; key < nr_keys; key++) { for (i = 0; i < nr_cpus; i++) - bpf_percpu(values, i) = 0; + values[i] = 0;

assert(bpf_map_lookup_elem(fd, &key, values) == 0);

for (i = 0; i < nr_cpus; i++) - assert(bpf_percpu(values, i) == i + 10); + assert(values[i] == i + 10); }

close(fd);