This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "".

The branch, api-next has been updated via 07bee0be1f22f8a136982935ee06313e5bbc6f3a (commit) via 1594671cc63d7d1ac0192acb0a8496e7fec9f4a2 (commit) via b08633c5740311374f509cef5245f19bd53e55f7 (commit) via 6147511e8b2d8c5ae0a3fda161f346ba40203569 (commit) via ab82e67f3ea25b392c90557dfc4cad43750c8d94 (commit) via 6ec6cf6347efec4bf4268c6db37f287b9de35e30 (commit) via 442b93274d8f24f31557fe678245f65a2c499f97 (commit) via a4dd0bade1ca02eb91178834a159c9d26f140c38 (commit) via 4871852cc509ca998802616fadfbb5619e3324da (commit) via 48ae68961873bb818a75aa8aa1b93d697fe39517 (commit) via 85c4f0bd3f18900e484b0e71670fc18c555e75b1 (commit) via 57962f1b9c3a5a41a7dd6146f9666da18e62ec9c (commit) via 9a45802c94f58d9dd443c10e6dc204a58fa0f9ce (commit) from 0293146428ef9c6d8d2b24e0bdd6c781486904a5 (commit)

Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below.

- Log ----------------------------------------------------------------- commit 07bee0be1f22f8a136982935ee06313e5bbc6f3a Author: Christophe Milard christophe.milard@linaro.org Date: Sat Aug 20 09:46:03 2016 +0200

test: validation: drv: shmem: stress tests

Stress tests, randomly allocating memory are added: the test is based on a group of odp threads allocating, mapping and freeing each-other memory.

Signed-off-by: Christophe Milard christophe.milard@linaro.org Reviewed-and-tested-by: Bill Fischofer bill.fischofer@linaro.org Reviewed-by: Brian Brooks brian.brooks@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/test/common_plat/validation/drv/drvshmem/drvshmem.c b/test/common_plat/validation/drv/drvshmem/drvshmem.c index 9ca81fc..5e6d2e5 100644 --- a/test/common_plat/validation/drv/drvshmem/drvshmem.c +++ b/test/common_plat/validation/drv/drvshmem/drvshmem.c @@ -17,6 +17,25 @@ #define SMALL_MEM 10 #define MEDIUM_MEM 4096 #define BIG_MEM 16777216 +#define STRESS_SIZE 32 /* power of 2 and <=256 */ +#define STRESS_RANDOM_SZ 5 +#define STRESS_ITERATION 5000 + +typedef enum { + STRESS_FREE, /* entry is free and can be allocated */ + STRESS_BUSY, /* entry is being processed: don't touch */ + STRESS_ALLOC /* entry is allocated and can be freed */ +} stress_state_t; + +typedef struct { + stress_state_t state; + odpdrv_shm_t shm; + void *address; + uint32_t flags; + uint32_t size; + uint64_t align; + uint8_t data_val; +} stress_data_t;

typedef struct { odpdrv_barrier_t test_barrier1; @@ -29,6 +48,8 @@ typedef struct { uint32_t nb_threads; odpdrv_shm_t shm[MAX_WORKERS]; void *address[MAX_WORKERS]; + odp_spinlock_t stress_lock; + stress_data_t stress[STRESS_SIZE]; } shared_test_data_t;

/* memory stuff expected to fit in a single page */ @@ -543,10 +564,211 @@ void drvshmem_test_singleva_after_fork(void) CU_ASSERT(odpdrv_shm_print_all("Test completion") == base); }

+/* + * thread part for the drvshmem_test_stress + */ +static int run_test_stress(void *arg ODP_UNUSED) +{ + odpdrv_shm_t shm; + uint8_t *address; + shared_test_data_t *glob_data; + uint8_t random_bytes[STRESS_RANDOM_SZ]; + uint32_t index; + uint32_t size; + uint64_t align; + uint32_t flags; + uint8_t data; + uint32_t iter; + uint32_t i; + + shm = odpdrv_shm_lookup_by_name(MEM_NAME); + glob_data = odpdrv_shm_addr(shm); + CU_ASSERT_PTR_NOT_NULL(glob_data); + + /* wait for general GO! */ + odpdrv_barrier_wait(&glob_data->test_barrier1); + /* + + * at each iteration: pick up a random index for + * glob_data->stress[index]: If the entry is free, allocated mem + * randomly. If it is already allocated, make checks and free it: + * Note that different tread cann allocate or free a given block + */ + for (iter = 0; iter < STRESS_ITERATION; iter++) { + /* get 4 random bytes from which index, size ,align, flags + * and data will be derived: + */ + odp_random_data(random_bytes, STRESS_RANDOM_SZ, 0); + index = random_bytes[0] & (STRESS_SIZE - 1); + + odp_spinlock_lock(&glob_data->stress_lock); + + switch (glob_data->stress[index].state) { + case STRESS_FREE: + /* allocated a new block for this entry */ + + glob_data->stress[index].state = STRESS_BUSY; + odp_spinlock_unlock(&glob_data->stress_lock); + + size = (random_bytes[1] + 1) << 6; /* up to 16Kb */ + /* we just play with the VA flag. randomly setting + * the mlock flag may exceed user ulimit -l + */ + flags = random_bytes[2] & ODPDRV_SHM_SINGLE_VA; + align = (random_bytes[3] + 1) << 6;/* up to 16Kb */ + data = random_bytes[4]; + + shm = odpdrv_shm_reserve(NULL, size, align, flags); + glob_data->stress[index].shm = shm; + if (shm == ODPDRV_SHM_INVALID) { /* out of mem ? */ + odp_spinlock_lock(&glob_data->stress_lock); + glob_data->stress[index].state = STRESS_ALLOC; + odp_spinlock_unlock(&glob_data->stress_lock); + continue; + } + + address = odpdrv_shm_addr(shm); + CU_ASSERT_PTR_NOT_NULL(address); + glob_data->stress[index].address = address; + glob_data->stress[index].flags = flags; + glob_data->stress[index].size = size; + glob_data->stress[index].align = align; + glob_data->stress[index].data_val = data; + + /* write some data: writing each byte would be a + * waste of time: just make sure each page is reached */ + for (i = 0; i < size; i += 256) + address[i] = (data++) & 0xFF; + odp_spinlock_lock(&glob_data->stress_lock); + glob_data->stress[index].state = STRESS_ALLOC; + odp_spinlock_unlock(&glob_data->stress_lock); + + break; + + case STRESS_ALLOC: + /* free the block for this entry */ + + glob_data->stress[index].state = STRESS_BUSY; + odp_spinlock_unlock(&glob_data->stress_lock); + shm = glob_data->stress[index].shm; + + if (shm == ODPDRV_SHM_INVALID) { /* out of mem ? */ + odp_spinlock_lock(&glob_data->stress_lock); + glob_data->stress[index].state = STRESS_FREE; + odp_spinlock_unlock(&glob_data->stress_lock); + continue; + } + + CU_ASSERT(odpdrv_shm_lookup_by_handle(shm) != 0); + + address = odpdrv_shm_addr(shm); + CU_ASSERT_PTR_NOT_NULL(address); + + align = glob_data->stress[index].align; + if (align) { + align = glob_data->stress[index].align; + CU_ASSERT(((uintptr_t)address & (align - 1)) + == 0) + } + + flags = glob_data->stress[index].flags; + if (flags & ODPDRV_SHM_SINGLE_VA) + CU_ASSERT(glob_data->stress[index].address == + address) + + /* check that data is reachable and correct: */ + data = glob_data->stress[index].data_val; + size = glob_data->stress[index].size; + for (i = 0; i < size; i += 256) { + CU_ASSERT(address[i] == (data & 0xFF)); + data++; + } + + if (flags & ODPDRV_SHM_SINGLE_VA) { + CU_ASSERT(!odpdrv_shm_free_by_address(address)); + } else { + CU_ASSERT(!odpdrv_shm_free_by_handle(shm)); + } + + odp_spinlock_lock(&glob_data->stress_lock); + glob_data->stress[index].state = STRESS_FREE; + odp_spinlock_unlock(&glob_data->stress_lock); + + break; + + case STRESS_BUSY: + default: + odp_spinlock_unlock(&glob_data->stress_lock); + break; + } + } + + fflush(stdout); + return CU_get_number_of_failures(); +} + +/* + * stress tests + */ +void drvshmem_test_stress(void) +{ + pthrd_arg thrdarg; + odpdrv_shm_t shm; + shared_test_data_t *glob_data; + odp_cpumask_t unused; + int base; /* number of blocks already allocated at start of test */ + uint32_t i; + + base = odpdrv_shm_print_all("Before thread tests"); + + shm = odpdrv_shm_reserve(MEM_NAME, sizeof(shared_test_data_t), + 0, ODPDRV_SHM_LOCK); + CU_ASSERT(ODPDRV_SHM_INVALID != shm); + glob_data = odpdrv_shm_addr(shm); + CU_ASSERT_PTR_NOT_NULL(glob_data); + + thrdarg.numthrds = odp_cpumask_default_worker(&unused, 0); + if (thrdarg.numthrds > MAX_WORKERS) + thrdarg.numthrds = MAX_WORKERS; + + glob_data->nb_threads = thrdarg.numthrds; + odpdrv_barrier_init(&glob_data->test_barrier1, thrdarg.numthrds); + odp_spinlock_init(&glob_data->stress_lock); + + /* before starting the threads, mark all entries as free: */ + for (i = 0; i < STRESS_SIZE; i++) + glob_data->stress[i].state = STRESS_FREE; + + /* create threads */ + odp_cunit_thread_create(run_test_stress, &thrdarg); + + /* wait for all thread endings: */ + CU_ASSERT(odp_cunit_thread_exit(&thrdarg) >= 0); + + odpdrv_shm_print_all("Middle"); + + /* release left overs: */ + for (i = 0; i < STRESS_SIZE; i++) { + shm = glob_data->stress[i].shm; + if ((glob_data->stress[i].state == STRESS_ALLOC) && + (glob_data->stress[i].shm != ODPDRV_SHM_INVALID)) { + CU_ASSERT(odpdrv_shm_lookup_by_handle(shm) != + NULL); + CU_ASSERT(!odpdrv_shm_free_by_handle(shm)); + } + } + + CU_ASSERT(0 == odpdrv_shm_free_by_name(MEM_NAME)); + + /* check that no memory is left over: */ + CU_ASSERT(odpdrv_shm_print_all("After stress tests") == base); +} + odp_testinfo_t drvshmem_suite[] = { ODP_TEST_INFO(drvshmem_test_basic), ODP_TEST_INFO(drvshmem_test_reserve_after_fork), ODP_TEST_INFO(drvshmem_test_singleva_after_fork), + ODP_TEST_INFO(drvshmem_test_stress), ODP_TEST_INFO_NULL, };

diff --git a/test/common_plat/validation/drv/drvshmem/drvshmem.h b/test/common_plat/validation/drv/drvshmem/drvshmem.h index 3f9f96e..f4c26a1 100644 --- a/test/common_plat/validation/drv/drvshmem/drvshmem.h +++ b/test/common_plat/validation/drv/drvshmem/drvshmem.h @@ -13,6 +13,7 @@ void drvshmem_test_basic(void); void drvshmem_test_reserve_after_fork(void); void drvshmem_test_singleva_after_fork(void); +void drvshmem_test_stress(void);

/* test arrays: */ extern odp_testinfo_t drvshmem_suite[];

commit 1594671cc63d7d1ac0192acb0a8496e7fec9f4a2 Author: Christophe Milard christophe.milard@linaro.org Date: Sat Aug 20 09:46:02 2016 +0200

test: validation: drv: shmem: testing SINGLE_VA flag

Basic test for different drvshm_reserve() time in conjunction with the usage of SINGLE_VA flag is added.

Signed-off-by: Christophe Milard christophe.milard@linaro.org Reviewed-and-tested-by: Bill Fischofer bill.fischofer@linaro.org Reviewed-by: Brian Brooks brian.brooks@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/test/common_plat/validation/drv/drvshmem/drvshmem.c b/test/common_plat/validation/drv/drvshmem/drvshmem.c index 42ad844..9ca81fc 100644 --- a/test/common_plat/validation/drv/drvshmem/drvshmem.c +++ b/test/common_plat/validation/drv/drvshmem/drvshmem.c @@ -21,10 +21,14 @@ typedef struct { odpdrv_barrier_t test_barrier1; odpdrv_barrier_t test_barrier2; + odpdrv_barrier_t test_barrier3; + odpdrv_barrier_t test_barrier4; uint32_t foo; uint32_t bar; odpdrv_atomic_u32_t index; + uint32_t nb_threads; odpdrv_shm_t shm[MAX_WORKERS]; + void *address[MAX_WORKERS]; } shared_test_data_t;

/* memory stuff expected to fit in a single page */ @@ -323,9 +327,226 @@ void drvshmem_test_reserve_after_fork(void) CU_ASSERT(odpdrv_shm_print_all("Test completion") == base); }

+/* + * thread part for the drvshmem_test_singleva_after_fork + */ +static int run_test_singleva_after_fork(void *arg ODP_UNUSED) +{ + odpdrv_shm_t shm; + shared_test_data_t *glob_data; + int thr; + int thr_index; + char *name; + int name_len; + int size; + shared_test_data_small_t *pattern_small; + shared_test_data_medium_t *pattern_medium; + shared_test_data_big_t *pattern_big; + uint32_t i; + int ret; + + thr = odp_thread_id(); + printf("Thread %i starts\n", thr); + + shm = odpdrv_shm_lookup_by_name(MEM_NAME); + glob_data = odpdrv_shm_addr(shm); + + /* + * odp_thread_id are not guaranteed to be consecutive, so we create + * a consecutive ID + */ + thr_index = odpdrv_atomic_fetch_inc_u32(&glob_data->index); + + /* allocate some memory (of different sizes) and fill with pattern */ + name_len = strlen(MEM_NAME) + 20; + name = malloc(name_len); + snprintf(name, name_len, "%s-%09d", MEM_NAME, thr_index); + switch (thr_index % 3) { + case 0: + size = sizeof(shared_test_data_small_t); + shm = odpdrv_shm_reserve(name, size, 0, ODPDRV_SHM_SINGLE_VA); + CU_ASSERT(ODPDRV_SHM_INVALID != shm); + glob_data->shm[thr_index] = shm; + pattern_small = odpdrv_shm_addr(shm); + CU_ASSERT_PTR_NOT_NULL(pattern_small); + glob_data->address[thr_index] = (void *)pattern_small; + for (i = 0; i < SMALL_MEM; i++) + pattern_small->data[i] = i; + break; + case 1: + size = sizeof(shared_test_data_medium_t); + shm = odpdrv_shm_reserve(name, size, 0, ODPDRV_SHM_SINGLE_VA); + CU_ASSERT(ODPDRV_SHM_INVALID != shm); + glob_data->shm[thr_index] = shm; + pattern_medium = odpdrv_shm_addr(shm); + CU_ASSERT_PTR_NOT_NULL(pattern_medium); + glob_data->address[thr_index] = (void *)pattern_medium; + for (i = 0; i < MEDIUM_MEM; i++) + pattern_medium->data[i] = (i << 2); + break; + case 2: + size = sizeof(shared_test_data_big_t); + shm = odpdrv_shm_reserve(name, size, 0, ODPDRV_SHM_SINGLE_VA); + CU_ASSERT(ODPDRV_SHM_INVALID != shm); + glob_data->shm[thr_index] = shm; + pattern_big = odpdrv_shm_addr(shm); + CU_ASSERT_PTR_NOT_NULL(pattern_big); + glob_data->address[thr_index] = (void *)pattern_big; + for (i = 0; i < BIG_MEM; i++) + pattern_big->data[i] = (i >> 2); + break; + } + free(name); + + /* print block address */ + printf("In thread: Block index: %d mapped at %lx\n", + thr_index, (long int)odpdrv_shm_addr(shm)); + + odpdrv_barrier_wait(&glob_data->test_barrier1); + odpdrv_barrier_wait(&glob_data->test_barrier2); + + /* map each-other block, checking common address: */ + for (i = 0; i < glob_data->nb_threads; i++) { + shm = odpdrv_shm_lookup_by_address(glob_data->address[i]); + CU_ASSERT(shm == glob_data->shm[i]); + CU_ASSERT(odpdrv_shm_addr(shm) == glob_data->address[i]); + } + + /* wait for main control task and free the allocated block */ + odpdrv_barrier_wait(&glob_data->test_barrier3); + odpdrv_barrier_wait(&glob_data->test_barrier4); + ret = odpdrv_shm_free_by_address(glob_data->address[thr_index]); + CU_ASSERT(ret == 0); + + fflush(stdout); + return CU_get_number_of_failures(); +} + +/* + * test sharing memory reserved after odp_thread creation (e.g. fork()): + * with single VA flag. + */ +void drvshmem_test_singleva_after_fork(void) +{ + int base; /* memory usage when test starts */ + pthrd_arg thrdarg; + odpdrv_shm_t shm; + odpdrv_shm_t thr_shm; + shared_test_data_t *glob_data; + odp_cpumask_t unused; + char *name; + int name_len; + int thr_index; + int i; + void *address; + shared_test_data_small_t *pattern_small; + shared_test_data_medium_t *pattern_medium; + shared_test_data_big_t *pattern_big; + + base = odpdrv_shm_print_all("Before drvshmem_test_singleva_after_fork"); + + shm = odpdrv_shm_reserve(MEM_NAME, sizeof(shared_test_data_t), + 0, ODPDRV_SHM_LOCK); + CU_ASSERT(ODPDRV_SHM_INVALID != shm); + glob_data = odpdrv_shm_addr(shm); + CU_ASSERT_PTR_NOT_NULL(glob_data); + + thrdarg.numthrds = odp_cpumask_default_worker(&unused, 0); + if (thrdarg.numthrds > MAX_WORKERS) + thrdarg.numthrds = MAX_WORKERS; + + glob_data->nb_threads = thrdarg.numthrds; + odpdrv_barrier_init(&glob_data->test_barrier1, thrdarg.numthrds + 1); + odpdrv_barrier_init(&glob_data->test_barrier2, thrdarg.numthrds + 1); + odpdrv_barrier_init(&glob_data->test_barrier3, thrdarg.numthrds + 1); + odpdrv_barrier_init(&glob_data->test_barrier4, thrdarg.numthrds + 1); + odpdrv_atomic_store_u32(&glob_data->index, 0); + + odp_cunit_thread_create(run_test_singleva_after_fork, &thrdarg); + + /* wait until all threads have made their shm_reserve: */ + odpdrv_barrier_wait(&glob_data->test_barrier1); + CU_ASSERT(odpdrv_shm_print_all("After thread reserve") + == base + thrdarg.numthrds + 1); + + /* perform a lookup of all memories, by handle or name: */ + name_len = strlen(MEM_NAME) + 20; + name = malloc(name_len); + for (thr_index = 0; thr_index < thrdarg.numthrds; thr_index++) { + if (thr_index % 2) { + snprintf(name, name_len, "%s-%09d", + MEM_NAME, thr_index); + thr_shm = odpdrv_shm_lookup_by_name(name); + CU_ASSERT(thr_shm == glob_data->shm[thr_index]); + } else { + odpdrv_shm_lookup_by_handle(glob_data->shm[thr_index]); + } + } + free(name); + + /* check that the patterns are correct: */ + for (thr_index = 0; thr_index < thrdarg.numthrds; thr_index++) { + switch (thr_index % 3) { + case 0: + pattern_small = + odpdrv_shm_addr(glob_data->shm[thr_index]); + CU_ASSERT_PTR_NOT_NULL(pattern_small); + for (i = 0; i < SMALL_MEM; i++) + CU_ASSERT(pattern_small->data[i] == i); + break; + case 1: + pattern_medium = + odpdrv_shm_addr(glob_data->shm[thr_index]); + CU_ASSERT_PTR_NOT_NULL(pattern_medium); + for (i = 0; i < MEDIUM_MEM; i++) + CU_ASSERT(pattern_medium->data[i] == (i << 2)); + break; + case 2: + pattern_big = + odpdrv_shm_addr(glob_data->shm[thr_index]); + CU_ASSERT_PTR_NOT_NULL(pattern_big); + for (i = 0; i < BIG_MEM; i++) + CU_ASSERT(pattern_big->data[i] == (i >> 2)); + break; + } + } + + /* + * check that the mapping address is common to all (SINGLE_VA): + */ + for (thr_index = 0; thr_index < thrdarg.numthrds; thr_index++) { + address = odpdrv_shm_addr(glob_data->shm[thr_index]); + CU_ASSERT(glob_data->address[thr_index] == address); + } + + CU_ASSERT(odpdrv_shm_print_all("After local lookup") + == base + thrdarg.numthrds + 1); + + /* unblock the threads and let them map each-other blocks: */ + odpdrv_barrier_wait(&glob_data->test_barrier2); + + /* then check mem status */ + odpdrv_barrier_wait(&glob_data->test_barrier3); + CU_ASSERT(odpdrv_shm_print_all("After mutual lookup") + == base + thrdarg.numthrds + 1); + + /* unblock the threads and let them free all thread blocks: */ + odpdrv_barrier_wait(&glob_data->test_barrier4); + + /* wait for all thread endings: */ + CU_ASSERT(odp_cunit_thread_exit(&thrdarg) >= 0); + + /* just glob_data should remain: */ + CU_ASSERT(odpdrv_shm_print_all("After threads free") == base + 1); + + CU_ASSERT(0 == odpdrv_shm_free_by_name(MEM_NAME)); + CU_ASSERT(odpdrv_shm_print_all("Test completion") == base); +} + odp_testinfo_t drvshmem_suite[] = { ODP_TEST_INFO(drvshmem_test_basic), ODP_TEST_INFO(drvshmem_test_reserve_after_fork), + ODP_TEST_INFO(drvshmem_test_singleva_after_fork), ODP_TEST_INFO_NULL, };

diff --git a/test/common_plat/validation/drv/drvshmem/drvshmem.h b/test/common_plat/validation/drv/drvshmem/drvshmem.h index 12d738e..3f9f96e 100644 --- a/test/common_plat/validation/drv/drvshmem/drvshmem.h +++ b/test/common_plat/validation/drv/drvshmem/drvshmem.h @@ -12,6 +12,7 @@ /* test functions: */ void drvshmem_test_basic(void); void drvshmem_test_reserve_after_fork(void); +void drvshmem_test_singleva_after_fork(void);

/* test arrays: */ extern odp_testinfo_t drvshmem_suite[];

commit b08633c5740311374f509cef5245f19bd53e55f7 Author: Christophe Milard christophe.milard@linaro.org Date: Sat Aug 20 09:46:01 2016 +0200

test: validation: drv: shmem: basic tests

Adding simple tests for shmem operations performed on the driver (drv) interface. Does not to assume pthread as odpthreads (the barrier is now part of the reserved area, rather than a global which only works for pthreads). Tests simple functionanlity, but regardless of reserve() time (i.e. memory reservation occurs both before and after odpthread creation here).

Signed-off-by: Christophe Milard christophe.milard@linaro.org Reviewed-and-tested-by: Bill Fischofer bill.fischofer@linaro.org Reviewed-by: Brian Brooks brian.brooks@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/test/common_plat/m4/configure.m4 b/test/common_plat/m4/configure.m4 index 20c3a3b..1fc350d 100644 --- a/test/common_plat/m4/configure.m4 +++ b/test/common_plat/m4/configure.m4 @@ -32,4 +32,5 @@ AC_CONFIG_FILES([test/common_plat/Makefile test/common_plat/validation/api/timer/Makefile test/common_plat/validation/api/traffic_mngr/Makefile test/common_plat/validation/drv/Makefile - test/common_plat/validation/drv/drvatomic/Makefile]) + test/common_plat/validation/drv/drvatomic/Makefile + test/common_plat/validation/drv/drvshmem/Makefile]) diff --git a/test/common_plat/validation/drv/Makefile.am b/test/common_plat/validation/drv/Makefile.am index eedbad5..bcdb92e 100644 --- a/test/common_plat/validation/drv/Makefile.am +++ b/test/common_plat/validation/drv/Makefile.am @@ -1,4 +1,5 @@ -ODPDRV_MODULES = drvatomic +ODPDRV_MODULES = drvatomic \ + drvshmem

SUBDIRS = $(ODPDRV_MODULES)

diff --git a/test/common_plat/validation/drv/drvshmem/.gitignore b/test/common_plat/validation/drv/drvshmem/.gitignore new file mode 100644 index 0000000..726dea3 --- /dev/null +++ b/test/common_plat/validation/drv/drvshmem/.gitignore @@ -0,0 +1 @@ +drvshmem_main diff --git a/test/common_plat/validation/drv/drvshmem/Makefile.am b/test/common_plat/validation/drv/drvshmem/Makefile.am new file mode 100644 index 0000000..816a5e6 --- /dev/null +++ b/test/common_plat/validation/drv/drvshmem/Makefile.am @@ -0,0 +1,10 @@ +include ../Makefile.inc + +noinst_LTLIBRARIES = libtestdrvshmem.la +libtestdrvshmem_la_SOURCES = drvshmem.c + +test_PROGRAMS = drvshmem_main$(EXEEXT) +dist_drvshmem_main_SOURCES = drvshmem_main.c +drvshmem_main_LDADD = libtestdrvshmem.la $(LIBCUNIT_COMMON) $(LIBODP) + +EXTRA_DIST = drvshmem.h diff --git a/test/common_plat/validation/drv/drvshmem/drvshmem.c b/test/common_plat/validation/drv/drvshmem/drvshmem.c new file mode 100644 index 0000000..42ad844 --- /dev/null +++ b/test/common_plat/validation/drv/drvshmem/drvshmem.c @@ -0,0 +1,351 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_drv.h> +#include <odp_api.h> +#include <odp_cunit_common.h> +#include "drvshmem.h" +#include <stdlib.h> + +#define ALIGN_SIZE (128) +#define MEM_NAME "test_shmem" +#define TEST_SHARE_FOO (0xf0f0f0f0) +#define TEST_SHARE_BAR (0xf0f0f0f) +#define SMALL_MEM 10 +#define MEDIUM_MEM 4096 +#define BIG_MEM 16777216 + +typedef struct { + odpdrv_barrier_t test_barrier1; + odpdrv_barrier_t test_barrier2; + uint32_t foo; + uint32_t bar; + odpdrv_atomic_u32_t index; + odpdrv_shm_t shm[MAX_WORKERS]; +} shared_test_data_t; + +/* memory stuff expected to fit in a single page */ +typedef struct { + int data[SMALL_MEM]; +} shared_test_data_small_t; + +/* memory stuff expected to fit in a huge page */ +typedef struct { + int data[MEDIUM_MEM]; +} shared_test_data_medium_t; + +/* memory stuff expected to fit in many huge pages */ +typedef struct { + int data[BIG_MEM]; +} shared_test_data_big_t; + +/* + * thread part for the drvshmem_test_basic test + */ +static int run_test_basic_thread(void *arg ODP_UNUSED) +{ + odpdrv_shm_info_t info; + odpdrv_shm_t shm; + shared_test_data_t *shared_test_data; + int thr; + + thr = odp_thread_id(); + printf("Thread %i starts\n", thr); + + shm = odpdrv_shm_lookup_by_name(MEM_NAME); + CU_ASSERT(ODPDRV_SHM_INVALID != shm); + shared_test_data = odpdrv_shm_addr(shm); + CU_ASSERT(NULL != shared_test_data); + + odpdrv_barrier_wait(&shared_test_data->test_barrier1); + odpdrv_shm_print_all("(from thread) After lookup for the global shmem"); + CU_ASSERT(TEST_SHARE_FOO == shared_test_data->foo); + CU_ASSERT(TEST_SHARE_BAR == shared_test_data->bar); + CU_ASSERT(0 == odpdrv_shm_info(shm, &info)); + CU_ASSERT(0 == strcmp(MEM_NAME, info.name)); + CU_ASSERT(0 == info.flags); + CU_ASSERT(shared_test_data == info.addr); + CU_ASSERT(sizeof(shared_test_data_t) <= info.size); +#ifdef MAP_HUGETLB + CU_ASSERT(odp_sys_huge_page_size() == info.page_size); +#else + CU_ASSERT(odp_sys_page_size() == info.page_size); +#endif + odpdrv_shm_print_all("(from thread) About to end"); + + fflush(stdout); + return CU_get_number_of_failures(); +} + +/* + * test basic things: shmem creation, info, share, and free + */ +void drvshmem_test_basic(void) +{ + int base; /* memory usage when test starts */ + pthrd_arg thrdarg; + odpdrv_shm_t shm; + shared_test_data_t *shared_test_data; + odp_cpumask_t unused; + + base = odpdrv_shm_print_all("Before drvshmem_test_basic"); + shm = odpdrv_shm_reserve(MEM_NAME, + sizeof(shared_test_data_t), ALIGN_SIZE, 0); + CU_ASSERT(ODPDRV_SHM_INVALID != shm); + CU_ASSERT(odpdrv_shm_to_u64(shm) != + odpdrv_shm_to_u64(ODPDRV_SHM_INVALID)); + + CU_ASSERT(0 == odpdrv_shm_free_by_handle(shm)); + CU_ASSERT(ODPDRV_SHM_INVALID == odpdrv_shm_lookup_by_name(MEM_NAME)); + + shm = odpdrv_shm_reserve(MEM_NAME, + sizeof(shared_test_data_t), ALIGN_SIZE, 0); + CU_ASSERT(ODPDRV_SHM_INVALID != shm); + + shared_test_data = odpdrv_shm_addr(shm); + CU_ASSERT_FATAL(NULL != shared_test_data); + shared_test_data->foo = TEST_SHARE_FOO; + shared_test_data->bar = TEST_SHARE_BAR; + + CU_ASSERT(odpdrv_shm_free_by_address((char *)shared_test_data - 1) < 0); + + thrdarg.numthrds = odp_cpumask_default_worker(&unused, 0); + + if (thrdarg.numthrds > MAX_WORKERS) + thrdarg.numthrds = MAX_WORKERS; + + odpdrv_barrier_init(&shared_test_data->test_barrier1, thrdarg.numthrds); + odp_cunit_thread_create(run_test_basic_thread, &thrdarg); + CU_ASSERT(odp_cunit_thread_exit(&thrdarg) >= 0); + + CU_ASSERT(0 == odpdrv_shm_free_by_handle(shm)); + CU_ASSERT(odpdrv_shm_print_all("Test completion") == base); +} + +/* + * thread part for the drvshmem_test_reserve_after_fork + */ +static int run_test_reserve_after_fork(void *arg ODP_UNUSED) +{ + odpdrv_shm_t shm; + shared_test_data_t *glob_data; + int thr; + int thr_index; + char *name; + int name_len; + int size; + shared_test_data_small_t *pattern_small; + shared_test_data_medium_t *pattern_medium; + shared_test_data_big_t *pattern_big; + int i; + + thr = odp_thread_id(); + printf("Thread %i starts\n", thr); + + shm = odpdrv_shm_lookup_by_name(MEM_NAME); + glob_data = odpdrv_shm_addr(shm); + + /* + * odp_thread_id are not guaranteed to be consecutive, so we create + * a consecutive ID + */ + thr_index = odpdrv_atomic_fetch_inc_u32(&glob_data->index); + + /* allocate some memory (of different sizes) and fill with pattern */ + name_len = strlen(MEM_NAME) + 20; + name = malloc(name_len); + snprintf(name, name_len, "%s-%09d", MEM_NAME, thr_index); + switch (thr_index % 3) { + case 0: + size = sizeof(shared_test_data_small_t); + shm = odpdrv_shm_reserve(name, size, 0, 0); + CU_ASSERT(ODPDRV_SHM_INVALID != shm); + glob_data->shm[thr_index] = shm; + pattern_small = odpdrv_shm_addr(shm); + CU_ASSERT_PTR_NOT_NULL(pattern_small); + for (i = 0; i < SMALL_MEM; i++) + pattern_small->data[i] = i; + break; + case 1: + size = sizeof(shared_test_data_medium_t); + shm = odpdrv_shm_reserve(name, size, 0, 0); + CU_ASSERT(ODPDRV_SHM_INVALID != shm); + glob_data->shm[thr_index] = shm; + pattern_medium = odpdrv_shm_addr(shm); + CU_ASSERT_PTR_NOT_NULL(pattern_medium); + for (i = 0; i < MEDIUM_MEM; i++) + pattern_medium->data[i] = (i << 2); + break; + case 2: + size = sizeof(shared_test_data_big_t); + shm = odpdrv_shm_reserve(name, size, 0, 0); + CU_ASSERT(ODPDRV_SHM_INVALID != shm); + glob_data->shm[thr_index] = shm; + pattern_big = odpdrv_shm_addr(shm); + CU_ASSERT_PTR_NOT_NULL(pattern_big); + for (i = 0; i < BIG_MEM; i++) + pattern_big->data[i] = (i >> 2); + break; + } + free(name); + + /* print block address */ + printf("In thread: Block index: %d mapped at %lx\n", + thr_index, (long int)odpdrv_shm_addr(shm)); + + odpdrv_barrier_wait(&glob_data->test_barrier1); + odpdrv_barrier_wait(&glob_data->test_barrier2); + + fflush(stdout); + return CU_get_number_of_failures(); +} + +/* + * test sharing memory reserved after odp_thread creation (e.g. fork()): + */ +void drvshmem_test_reserve_after_fork(void) +{ + int base; /* memory usage when test starts */ + pthrd_arg thrdarg; + odpdrv_shm_t shm; + odpdrv_shm_t thr_shm; + shared_test_data_t *glob_data; + odp_cpumask_t unused; + char *name; + int name_len; + int thr_index; + int i; + void *address; + shared_test_data_small_t *pattern_small; + shared_test_data_medium_t *pattern_medium; + shared_test_data_big_t *pattern_big; + + base = odpdrv_shm_print_all("Before drvshmem_test_reserve_after_fork"); + shm = odpdrv_shm_reserve(MEM_NAME, sizeof(shared_test_data_t), 0, 0); + CU_ASSERT(ODPDRV_SHM_INVALID != shm); + glob_data = odpdrv_shm_addr(shm); + CU_ASSERT_PTR_NOT_NULL(glob_data); + + thrdarg.numthrds = odp_cpumask_default_worker(&unused, 0); + if (thrdarg.numthrds > MAX_WORKERS) + thrdarg.numthrds = MAX_WORKERS; + + odpdrv_barrier_init(&glob_data->test_barrier1, thrdarg.numthrds + 1); + odpdrv_barrier_init(&glob_data->test_barrier2, thrdarg.numthrds + 1); + odpdrv_atomic_store_u32(&glob_data->index, 0); + + odp_cunit_thread_create(run_test_reserve_after_fork, &thrdarg); + + /* wait until all threads have made their shm_reserve: */ + odpdrv_barrier_wait(&glob_data->test_barrier1); + CU_ASSERT(odpdrv_shm_print_all("After all thread reserve") + == base + thrdarg.numthrds + 1); + + /* perform a lookup of all memories, by handle or name: */ + name_len = strlen(MEM_NAME) + 20; + name = malloc(name_len); + for (thr_index = 0; thr_index < thrdarg.numthrds; thr_index++) { + if (thr_index % 2) { + snprintf(name, name_len, "%s-%09d", + MEM_NAME, thr_index); + thr_shm = odpdrv_shm_lookup_by_name(name); + CU_ASSERT(thr_shm == glob_data->shm[thr_index]); + } else { + odpdrv_shm_lookup_by_handle(glob_data->shm[thr_index]); + } + } + + /* check that the patterns are correct: */ + for (thr_index = 0; thr_index < thrdarg.numthrds; thr_index++) { + switch (thr_index % 3) { + case 0: + pattern_small = + odpdrv_shm_addr(glob_data->shm[thr_index]); + CU_ASSERT_PTR_NOT_NULL(pattern_small); + for (i = 0; i < SMALL_MEM; i++) + CU_ASSERT(pattern_small->data[i] == i); + break; + case 1: + pattern_medium = + odpdrv_shm_addr(glob_data->shm[thr_index]); + CU_ASSERT_PTR_NOT_NULL(pattern_medium); + for (i = 0; i < MEDIUM_MEM; i++) + CU_ASSERT(pattern_medium->data[i] == (i << 2)); + break; + case 2: + pattern_big = + odpdrv_shm_addr(glob_data->shm[thr_index]); + CU_ASSERT_PTR_NOT_NULL(pattern_big); + for (i = 0; i < BIG_MEM; i++) + CU_ASSERT(pattern_big->data[i] == (i >> 2)); + break; + } + } + + /* + * print the mapping address of the blocks + */ + for (thr_index = 0; thr_index < thrdarg.numthrds; thr_index++) { + address = odpdrv_shm_addr(glob_data->shm[thr_index]); + printf("In main Block index: %d mapped at %lx\n", + thr_index, (long int)address); + } + + CU_ASSERT(odpdrv_shm_print_all("After main lookup of thread shmem") + == base + thrdarg.numthrds + 1); + + /* unblock the threads and let them terminate (no free is done): */ + odpdrv_barrier_wait(&glob_data->test_barrier2); + + /* at the same time, (race),free of all memories, by handle or name: */ + for (thr_index = 0; thr_index < thrdarg.numthrds; thr_index++) { + if (thr_index % 2) { + thr_shm = glob_data->shm[thr_index]; + CU_ASSERT(odpdrv_shm_free_by_handle(thr_shm) == 0); + } else { + snprintf(name, name_len, "%s-%09d", + MEM_NAME, thr_index); + CU_ASSERT(odpdrv_shm_free_by_name(name) == 0); + } + } + free(name); + + /* wait for all thread endings: */ + CU_ASSERT(odp_cunit_thread_exit(&thrdarg) >= 0); + + /* just glob_data should remain: */ + CU_ASSERT(odpdrv_shm_print_all("After all threads end") == base + 1); + + CU_ASSERT(0 == odpdrv_shm_free_by_handle(shm)); + CU_ASSERT(odpdrv_shm_print_all("Test completion") == base); +} + +odp_testinfo_t drvshmem_suite[] = { + ODP_TEST_INFO(drvshmem_test_basic), + ODP_TEST_INFO(drvshmem_test_reserve_after_fork), + ODP_TEST_INFO_NULL, +}; + +odp_suiteinfo_t drvshmem_suites[] = { + {"Shared Memory", NULL, NULL, drvshmem_suite}, + ODP_SUITE_INFO_NULL, +}; + +int drvshmem_main(int argc, char *argv[]) +{ + int ret; + + /* parse common options: */ + if (odp_cunit_parse_options(argc, argv)) + return -1; + + ret = odp_cunit_register(drvshmem_suites); + + if (ret == 0) + ret = odp_cunit_run(); + + return ret; +} diff --git a/test/common_plat/validation/drv/drvshmem/drvshmem.h b/test/common_plat/validation/drv/drvshmem/drvshmem.h new file mode 100644 index 0000000..12d738e --- /dev/null +++ b/test/common_plat/validation/drv/drvshmem/drvshmem.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _ODP_TEST_DRVSHMEM_H_ +#define _ODP_TEST_DRVSHMEM_H_ + +#include <odp_cunit_common.h> + +/* test functions: */ +void drvshmem_test_basic(void); +void drvshmem_test_reserve_after_fork(void); + +/* test arrays: */ +extern odp_testinfo_t drvshmem_suite[]; + +/* test registry: */ +extern odp_suiteinfo_t drvshmem_suites[]; + +/* main test program: */ +int drvshmem_main(int argc, char *argv[]); + +#endif diff --git a/test/common_plat/validation/drv/drvshmem/drvshmem_main.c b/test/common_plat/validation/drv/drvshmem/drvshmem_main.c new file mode 100644 index 0000000..566ccec --- /dev/null +++ b/test/common_plat/validation/drv/drvshmem/drvshmem_main.c @@ -0,0 +1,12 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include "drvshmem.h" + +int main(int argc, char *argv[]) +{ + return drvshmem_main(argc, argv); +} diff --git a/test/linux-generic/Makefile.am b/test/linux-generic/Makefile.am index 23b8e03..2b7cbc3 100644 --- a/test/linux-generic/Makefile.am +++ b/test/linux-generic/Makefile.am @@ -35,6 +35,7 @@ TESTS = validation/api/pktio/pktio_run.sh \ $(ALL_API_VALIDATION_DIR)/shmem/shmem_main$(EXEEXT) \ $(ALL_API_VALIDATION_DIR)/system/system_main$(EXEEXT) \ $(ALL_DRV_VALIDATION_DIR)/drvatomic/drvatomic_main$(EXEEXT) \ + $(ALL_DRV_VALIDATION_DIR)/drvshmem/drvshmem_main$(EXEEXT) \ ring/ring_main$(EXEEXT)

SUBDIRS += validation/api/pktio\

commit 6147511e8b2d8c5ae0a3fda161f346ba40203569 Author: Christophe Milard christophe.milard@linaro.org Date: Sat Aug 20 09:46:00 2016 +0200

linux-gen: drv: shm

The shared memory DRV interface is created, mostly as a wrapper around the internal memory allocator, ishm.

Signed-off-by: Christophe Milard christophe.milard@linaro.org Reviewed-and-tested-by: Bill Fischofer bill.fischofer@linaro.org Reviewed-by: Brian Brooks brian.brooks@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/include/odp_drv.h b/include/odp_drv.h index 31c620e..0959879 100644 --- a/include/odp_drv.h +++ b/include/odp_drv.h @@ -23,6 +23,7 @@ extern C { #include <odp/drv/barrier.h> #include <odp/drv/byteorder.h> #include <odp/drv/compiler.h> +#include <odp/drv/shm.h> #include <odp/drv/spinlock.h> #include <odp/drv/std_types.h> #include <odp/drv/sync.h> diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index ac2b2da..4d9b84b 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -99,6 +99,7 @@ odpdrvinclude_HEADERS = \ $(srcdir)/include/odp/drv/barrier.h \ $(srcdir)/include/odp/drv/byteorder.h \ $(srcdir)/include/odp/drv/compiler.h \ + $(srcdir)/include/odp/drv/shm.h \ $(srcdir)/include/odp/drv/spinlock.h \ $(srcdir)/include/odp/drv/std_types.h \ $(srcdir)/include/odp/drv/sync.h @@ -108,6 +109,7 @@ odpdrvplatinclude_HEADERS = \ $(srcdir)/include/odp/drv/plat/atomic_types.h \ $(srcdir)/include/odp/drv/plat/barrier_types.h \ $(srcdir)/include/odp/drv/plat/byteorder_types.h \ + $(srcdir)/include/odp/drv/plat/shm_types.h \ $(srcdir)/include/odp/drv/plat/spinlock_types.h \ $(srcdir)/include/odp/drv/plat/strong_types.h

@@ -214,6 +216,7 @@ __LIB__libodp_linux_la_SOURCES = \ odp_weak.c \ drv_atomic.c \ drv_barrier.c \ + drv_shm.c \ drv_spinlock.c \ arch/@ARCH_DIR@/odp_cpu_arch.c \ arch/@ARCH_DIR@/odp_sysinfo_parse.c diff --git a/platform/linux-generic/_ishm.c b/platform/linux-generic/_ishm.c index c79c58e..f4aa6d3 100644 --- a/platform/linux-generic/_ishm.c +++ b/platform/linux-generic/_ishm.c @@ -52,6 +52,7 @@ #include <odp/api/align.h> #include <odp/api/system_info.h> #include <odp/api/debug.h> +#include <odp/drv/shm.h> #include <odp_shm_internal.h> #include <odp_debug_internal.h> #include <odp_align_internal.h> @@ -79,13 +80,12 @@ * if some of the block ownwers never procsync() after free). This number * should take that into account) */ -#define ISHM_MAX_NB_BLOCKS 128 +#define ISHM_MAX_NB_BLOCKS ODPDRV_CONFIG_SHM_BLOCKS

/* * Maximum internal shared memory block name length in chars - * probably taking the same number as SHM name size make sense at this stage */ -#define ISHM_NAME_MAXLEN 32 +#define ISHM_NAME_MAXLEN ODPDRV_SHM_NAME_LEN

/* * Linux underlying file name: <directory>/odp-<odp_pid>-ishm-<name> diff --git a/platform/linux-generic/drv_shm.c b/platform/linux-generic/drv_shm.c new file mode 100644 index 0000000..9b2560d --- /dev/null +++ b/platform/linux-generic/drv_shm.c @@ -0,0 +1,102 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_config_internal.h> +#include <odp/api/std_types.h> +#include <odp/drv/shm.h> +#include <_ishm_internal.h> + +static inline uint32_t from_handle(odpdrv_shm_t shm) +{ + return _odpdrv_typeval(shm) - 1; +} + +static inline odpdrv_shm_t to_handle(uint32_t index) +{ + return _odpdrv_cast_scalar(odpdrv_shm_t, index + 1); +} + +int odpdrv_shm_capability(odpdrv_shm_capability_t *capa) +{ + capa->max_blocks = ODPDRV_CONFIG_SHM_BLOCKS; + capa->max_size = 0; + capa->max_align = 0; + + return 0; +} + +odpdrv_shm_t odpdrv_shm_reserve(const char *name, uint64_t size, uint64_t align, + uint32_t flags) +{ + int block_index; + int flgs = 0; /* internal ishm flags */ + + /* set internal ishm flags according to API flags: */ + flgs |= (flags & ODPDRV_SHM_SINGLE_VA) ? _ODP_ISHM_SINGLE_VA : 0; + flgs |= (flags & ODPDRV_SHM_LOCK) ? _ODP_ISHM_LOCK : 0; + + block_index = _odp_ishm_reserve(name, size, -1, align, flgs, flags); + if (block_index >= 0) + return to_handle(block_index); + else + return ODPDRV_SHM_INVALID; +} + +int odpdrv_shm_free_by_handle(odpdrv_shm_t shm) +{ + return _odp_ishm_free_by_index(from_handle(shm)); +} + +int odpdrv_shm_free_by_name(const char *name) +{ + return _odp_ishm_free_by_name(name); +} + +int odpdrv_shm_free_by_address(void *address) +{ + return _odp_ishm_free_by_address(address); +} + +void *odpdrv_shm_lookup_by_handle(odpdrv_shm_t shm) +{ + return _odp_ishm_lookup_by_index(from_handle(shm)); +} + +odpdrv_shm_t odpdrv_shm_lookup_by_name(const char *name) +{ + return to_handle(_odp_ishm_lookup_by_name(name)); +} + +odpdrv_shm_t odpdrv_shm_lookup_by_address(void *address) +{ + return to_handle(_odp_ishm_lookup_by_address(address)); +} + +void *odpdrv_shm_addr(odpdrv_shm_t shm) +{ + return _odp_ishm_address(from_handle(shm)); +} + +int odpdrv_shm_info(odpdrv_shm_t shm, odpdrv_shm_info_t *info) +{ + _odp_ishm_info_t ishm_info; + + if (_odp_ishm_info(from_handle(shm), &ishm_info)) + return -1; + + info->name = ishm_info.name; + info->addr = ishm_info.addr; + info->size = ishm_info.size; + info->page_size = ishm_info.page_size; + info->flags = ishm_info.user_flags; + + return 0; +} + +int odpdrv_shm_print_all(const char *title) +{ + return _odp_ishm_status(title); +} diff --git a/platform/linux-generic/include/odp/drv/plat/shm_types.h b/platform/linux-generic/include/odp/drv/plat/shm_types.h new file mode 100644 index 0000000..c48eeca --- /dev/null +++ b/platform/linux-generic/include/odp/drv/plat/shm_types.h @@ -0,0 +1,46 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP shared memory + */ + +#ifndef ODPDRV_SHM_TYPES_H_ +#define ODPDRV_SHM_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/drv/std_types.h> +#include <odp/drv/plat/strong_types.h> + +/** @addtogroup odpdrv_shm ODPDRV SHARED MEMORY + * Operations on driver shared memory. + * @{ + */ + +typedef ODPDRV_HANDLE_T(odpdrv_shm_t); + +#define ODPDRV_SHM_INVALID _odpdrv_cast_scalar(odpdrv_shm_t, 0) + +/** Get printable format of odpdrv_shm_t */ +static inline uint64_t odpdrv_shm_to_u64(odpdrv_shm_t hdl) +{ + return _odpdrv_pri(hdl); +} + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/drv/shm.h b/platform/linux-generic/include/odp/drv/shm.h new file mode 100644 index 0000000..644b036 --- /dev/null +++ b/platform/linux-generic/include/odp/drv/shm.h @@ -0,0 +1,36 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODPDRV shared memory + */ + +#ifndef ODPDRV_PLAT_SHM_H_ +#define ODPDRV_PLAT_SHM_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/drv/plat/shm_types.h> + +/** @ingroup odpdrv_shm + * @{ + */ + +/** + * @} + */ + +#include <odp/drv/spec/shm.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 3eabc7b..7d5dbeb 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -120,6 +120,12 @@ extern "C" { */ #define ODP_CONFIG_ISHM_VA_PREALLOC_SZ (536870912L)

+/* Maximum number of shared memory blocks available on the driver interface. + * + * This the the number of separate SHM areas that can be reserved concurrently + */ +#define ODPDRV_CONFIG_SHM_BLOCKS 48 + #ifdef __cplusplus } #endif

commit ab82e67f3ea25b392c90557dfc4cad43750c8d94 Author: Christophe Milard christophe.milard@linaro.org Date: Sat Aug 20 09:45:59 2016 +0200

linux-gen: ishm: adding debug function

A debug function, printing the internal mem alloc status is added.

Signed-off-by: Christophe Milard christophe.milard@linaro.org Reviewed-and-tested-by: Bill Fischofer bill.fischofer@linaro.org Reviewed-by: Brian Brooks brian.brooks@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/_ishm.c b/platform/linux-generic/_ishm.c index c8156aa..c79c58e 100644 --- a/platform/linux-generic/_ishm.c +++ b/platform/linux-generic/_ishm.c @@ -1333,3 +1333,128 @@ int _odp_ishm_term_local(void) odp_spinlock_unlock(&ishm_tbl->lock); return 0; } + +/* + * Print the current ishm status (allocated blocks and VA space map) + * Return the number of allocated blocks (including those not mapped + * by the current odp thread). Also perform a number of sanity check. + * For debug. + */ +int _odp_ishm_status(const char *title) +{ + int i; + char flags[3]; + char huge; + int proc_index; + ishm_fragment_t *fragmnt; + int consecutive_unallocated = 0; /* should never exceed 1 */ + uintptr_t last_address = 0; + ishm_fragment_t *previous = NULL; + int nb_used_frgments = 0; + int nb_unused_frgments = 0; /* nb frag describing a VA area */ + int nb_allocated_frgments = 0; /* nb frag describing an allocated VA */ + int nb_blocks = 0; + int single_va_blocks = 0; + + odp_spinlock_lock(&ishm_tbl->lock); + procsync(); + + ODP_DBG("ishm blocks allocated at: %s\n", title); + + /* display block table: 1 line per entry +1 extra line if mapped here */ + for (i = 0; i < ISHM_MAX_NB_BLOCKS; i++) { + if (ishm_tbl->block[i].len <= 0) + continue; /* unused block */ + + nb_blocks++; + if (ishm_tbl->block[i].flags & _ODP_ISHM_SINGLE_VA) + single_va_blocks++; + + flags[0] = (ishm_tbl->block[i].flags & _ODP_ISHM_SINGLE_VA) ? + 'S' : '.'; + flags[1] = (ishm_tbl->block[i].flags & _ODP_ISHM_LOCK) ? + 'L' : '.'; + flags[2] = 0; + huge = (ishm_tbl->block[i].huge) ? 'H' : '.'; + proc_index = procfind_block(i); + ODP_DBG("%-3d: name:%-.24s file:%-.24s tid:%-3d" + " flags:%s,%c len:0x%-08lx" + " user_len:%-8ld seq:%-3ld refcnt:%-4d\n", + i, + ishm_tbl->block[i].name, + ishm_tbl->block[i].filename, + ishm_tbl->block[i].main_odpthread, + flags, huge, + ishm_tbl->block[i].len, + ishm_tbl->block[i].user_len, + ishm_tbl->block[i].seq, + ishm_tbl->block[i].refcnt); + + if (proc_index < 0) + continue; + + ODP_DBG(" start:%-08lx fd:%-3d\n", + ishm_proctable->entry[proc_index].start, + ishm_proctable->entry[proc_index].fd); + } + + /* display the virtual space allocations... : */ + ODP_DBG("ishm virtual space:\n"); + for (fragmnt = ishm_ftbl->used_fragmnts; + fragmnt; fragmnt = fragmnt->next) { + if (fragmnt->block_index >= 0) { + nb_allocated_frgments++; + ODP_DBG(" %08p - %08p: ALLOCATED by block:%d\n", + (uintptr_t)fragmnt->start, + (uintptr_t)fragmnt->start + fragmnt->len - 1, + fragmnt->block_index); + consecutive_unallocated = 0; + } else { + ODP_DBG(" %08p - %08p: NOT ALLOCATED\n", + (uintptr_t)fragmnt->start, + (uintptr_t)fragmnt->start + fragmnt->len - 1); + if (consecutive_unallocated++) + ODP_ERR("defragmentation error\n"); + } + + /* some other sanity checks: */ + if (fragmnt->prev != previous) + ODP_ERR("chaining error\n"); + + if (fragmnt != ishm_ftbl->used_fragmnts) { + if ((uintptr_t)fragmnt->start != last_address + 1) + ODP_ERR("lost space error\n"); + } + + last_address = (uintptr_t)fragmnt->start + fragmnt->len - 1; + previous = fragmnt; + nb_used_frgments++; + } + + /* + * the number of blocks with the single_VA flag set should match + * the number of used fragments: + */ + if (single_va_blocks != nb_allocated_frgments) + ODP_ERR("single_va_blocks != nb_allocated_fragments!\n"); + + /* compute the number of unused fragments*/ + for (fragmnt = ishm_ftbl->unused_fragmnts; + fragmnt; fragmnt = fragmnt->next) + nb_unused_frgments++; + + ODP_DBG("ishm: %d fragment used. %d fragements unused. (total=%d)\n", + nb_used_frgments, nb_unused_frgments, + nb_used_frgments + nb_unused_frgments); + + if ((nb_used_frgments + nb_unused_frgments) != ISHM_NB_FRAGMNTS) + ODP_ERR("lost fragments!\n"); + + if (nb_blocks < ishm_proctable->nb_entries) + ODP_ERR("process known block cannot exceed main total sum!\n"); + + ODP_DBG("\n"); + + odp_spinlock_unlock(&ishm_tbl->lock); + return nb_blocks; +} diff --git a/platform/linux-generic/include/_ishm_internal.h b/platform/linux-generic/include/_ishm_internal.h index 7d27477..3231b60 100644 --- a/platform/linux-generic/include/_ishm_internal.h +++ b/platform/linux-generic/include/_ishm_internal.h @@ -37,6 +37,7 @@ int _odp_ishm_lookup_by_name(const char *name); int _odp_ishm_lookup_by_address(void *addr); void *_odp_ishm_address(int block_index); int _odp_ishm_info(int block_index, _odp_ishm_info_t *info); +int _odp_ishm_status(const char *title);

#ifdef __cplusplus }

commit 6ec6cf6347efec4bf4268c6db37f287b9de35e30 Author: Christophe Milard christophe.milard@linaro.org Date: Sat Aug 20 09:45:58 2016 +0200

linux-gen: ishm: internal shared memory allocator (ishm) added

A new ODP internal memory allocator, called ishm (for internal shmem) is introduced here. This memory allocator enables the following: - works for odpthreads being linux processes, regardless for fork time. - guarantees the uniqueness of the virtual space mapping address over all ODP threads (even processes and regardless of fork time), when the required _ODP_ISHM_SINGLE_VA flag is used.

Signed-off-by: Christophe Milard christophe.milard@linaro.org Reviewed-and-tested-by: Bill Fischofer bill.fischofer@linaro.org Reviewed-by: Brian Brooks brian.brooks@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index cb7c8e9..ac2b2da 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -113,6 +113,8 @@ odpdrvplatinclude_HEADERS = \

noinst_HEADERS = \ ${srcdir}/include/_fdserver_internal.h \ + ${srcdir}/include/_ishm_internal.h \ + ${srcdir}/include/_ishmphy_internal.h \ ${srcdir}/include/odp_align_internal.h \ ${srcdir}/include/odp_atomic_internal.h \ ${srcdir}/include/odp_buffer_inlines.h \ @@ -156,6 +158,8 @@ noinst_HEADERS = \

__LIB__libodp_linux_la_SOURCES = \ _fdserver.c \ + _ishm.c \ + _ishmphy.c \ odp_atomic.c \ odp_barrier.c \ odp_buffer.c \ diff --git a/platform/linux-generic/_ishm.c b/platform/linux-generic/_ishm.c new file mode 100644 index 0000000..c8156aa --- /dev/null +++ b/platform/linux-generic/_ishm.c @@ -0,0 +1,1335 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* This file handles the internal shared memory: internal shared memory + * is memory which is sharable by all ODP threads regardless of how the + * ODP thread is implemented (pthread or process) and regardless of fork() + * time. + * Moreover, when reserved with the _ODP_ISHM_SINGLE_VA flag, + * internal shared memory is guaranteed to always be located at the same virtual + * address, i.e. pointers to internal shared memory are fully shareable + * between odp threads (regardless of thread type or fork time) in that case. + * Internal shared memory is mainly meant to be used internaly within ODP + * (hence its name), but may also be allocated by odp applications and drivers, + * in the future (through these interfaces). + * To guarrentee this full pointer shareability (when reserved with the + * _ODP_ISHM_SINGLE_VA flag) internal shared memory is handled as follows: + * At global_init time, a huge virtual address space reservation is performed. + * Note that this is just reserving virtual space, not physical memory. + * Because all ODP threads (pthreads or processes) are descendants of the ODP + * instantiation process, this VA space is inherited by all ODP threads. + * When internal shmem reservation actually occurs, and + * when reserved with the _ODP_ISHM_SINGLE_VA flag, physical memory is + * allocated, and mapped (MAP_FIXED) to some part in the huge preallocated + * address space area: + * because this virtual address space is common to all ODP threads, we + * know this mapping will succeed, and not clash with anything else. + * Hence, an ODP threads which perform a lookup for the same ishm block + * can map it at the same VA address. + * When internal shared memory is released, the physical memory is released + * and the corresponding virtual space returned to its "pool" of preallocated + * virtual space (assuming it was allocated from there). + * Note, though, that, if 2 linux processes share the same ishm block, + * the virtual space is marked as released as soon as one of the processes + * releases the ishm block, but the physical memory space is actually released + * by the kernel once all processes have done a ishm operation (i,e. a sync). + * This is due to the fact that linux does not contain any syscall to unmap + * memory from a different process. + * + * This file contains functions to handle the VA area (handling fragmentation + * and defragmentation resulting from different allocs/release) and also + * define the functions to allocate, release and lookup internal shared + * memory: + * _odp_ishm_reserve(), _odp_ishm_free*() and _odp_ishm_lookup*()... + */ +#include <odp_posix_extensions.h> +#include <odp_config_internal.h> +#include <odp_internal.h> +#include <odp/api/spinlock.h> +#include <odp/api/align.h> +#include <odp/api/system_info.h> +#include <odp/api/debug.h> +#include <odp_shm_internal.h> +#include <odp_debug_internal.h> +#include <odp_align_internal.h> +#include <_fdserver_internal.h> +#include <_ishm_internal.h> +#include <_ishmphy_internal.h> +#include <stdlib.h> +#include <stdio.h> +#include <unistd.h> +#include <string.h> +#include <errno.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/syscall.h> +#include <fcntl.h> +#include <sys/types.h> +#include <inttypes.h> +#include <sys/wait.h> + +/* + * Maximum number of internal shared memory blocks. + * + * This the the number of separate ISHM areas that can be reserved concurrently + * (Note that freeing such blocks may take time, or possibly never happen + * if some of the block ownwers never procsync() after free). This number + * should take that into account) + */ +#define ISHM_MAX_NB_BLOCKS 128 + +/* + * Maximum internal shared memory block name length in chars + * probably taking the same number as SHM name size make sense at this stage + */ +#define ISHM_NAME_MAXLEN 32 + +/* + * Linux underlying file name: <directory>/odp-<odp_pid>-ishm-<name> + * The <name> part may be replaced by a sequence number if no specific + * name is given at reserve time + * <directory> is either /tmp or the hugepagefs mount point for default size. + * (searched at init time) + */ +#define ISHM_FILENAME_MAXLEN (ISHM_NAME_MAXLEN + 64) +#define ISHM_FILENAME_FORMAT "%s/odp-%d-ishm-%s" +#define ISHM_FILENAME_NORMAL_PAGE_DIR "/tmp" + +/* + * At worse case the virtual space gets so fragmented that there is + * a unallocated fragment between each allocated fragment: + * In that case, the number of fragments to take care of is twice the + * number of ISHM blocks + 1. + */ +#define ISHM_NB_FRAGMNTS (ISHM_MAX_NB_BLOCKS * 2 + 1) + +/* + * A fragment describes a piece of the shared virtual address space, + * and is allocated only when allocation is done with the _ODP_ISHM_SINGLE_VA + * flag: + * A fragment is said to be used when it actually does represent some + * portion of the virtual address space, and is said to be unused when + * it does not (so at start, one single fragment is used -describing the + * whole address space as unallocated-, and all others are unused). + * Fragments get used as address space fragmentation increases. + * A fragment is allocated if the piece of address space it + * describes is actually used by a shared memory block. + * Allocated fragments get their block_index set >=0. + */ +typedef struct ishm_fragment { + struct ishm_fragment *prev; /* not used when the fragment is unused */ + struct ishm_fragment *next; + void *start; /* start of segment (VA) */ + uintptr_t len; /* length of segment. multiple of page size */ + int block_index; /* -1 for unallocated fragments */ +} ishm_fragment_t; + +/* + * A block describes a piece of reserved memory: Any successful ishm_reserve() + * will allocate a block. A ishm_reserve() with the _ODP_ISHM_SINGLE_VA flag set + * will allocate both a block and a fragment. + * Blocks contain only global data common to all processes. + */ +typedef struct ishm_block { + char name[ISHM_NAME_MAXLEN]; /* name for the ishm block (if any) */ + char filename[ISHM_FILENAME_MAXLEN]; /* name of the .../odp-* file */ + int main_odpthread; /* The thread which did the initial reserve*/ + uint32_t user_flags; /* any flags the user want to remember. */ + uint32_t flags; /* block creation flags. */ + uint64_t user_len; /* length, as requested at reserve time. */ + void *start; /* only valid if _ODP_ISHM_SINGLE_VA is set*/ + uint64_t len; /* length. multiple of page size. 0 if free*/ + ishm_fragment_t *fragment; /* used when _ODP_ISHM_SINGLE_VA is used */ + int huge; /* true if this segment is mapped using huge pages */ + uint64_t seq; /* sequence number, incremented on alloc and free */ + uint64_t refcnt;/* number of linux processes mapping this block */ +} ishm_block_t; + +/* + * Table of blocks describing allocated internal shared memory + * This table is visible to every ODP thread (linux process or pthreads). + * (it is allocated shared at odp init time and is therefore inherited by all) + * Table index is used as handle, so it cannot move!. Entry is regarded as + * free when len==0 + */ +typedef struct { + odp_spinlock_t lock; + uint64_t dev_seq; /* used when creating device names */ + ishm_block_t block[ISHM_MAX_NB_BLOCKS]; +} ishm_table_t; +static ishm_table_t *ishm_tbl; + +/* + * Process local table containing the list of (believed) allocated blocks seen + * from the current process. There is one such table per linux process. linux + * threads within a process shares this table. + * The contents within this table may become obsolete when other processes + * reserve/free ishm blocks. This is what the procsync() function + * catches by comparing the block sequence number with the one in this table. + * This table is filled at ishm_reserve and ishm_lookup time. + * Entries are removed at ishm_free or procsync time. + * Note that flags and len are present in this table and seems to be redundant + * with those present in the ishm block table: but this is not fully true: + * When ishm_sync() detects obsolete mappings and tries to remove them, + * the entry in the ishm block table is then obsolete, and the values which are + * found in this table must be used to perform the ummap. + * (and the values in the block tables are needed at lookup time...) + */ +typedef struct { + int thrd_refcnt; /* number of pthreads in this process, really */ + struct { + int block_index; /* entry in the ishm_tbl */ + uint32_t flags; /* flags used at creation time */ + uint64_t seq; + void *start; /* start of block (VA) */ + uint64_t len; /* length of block. multiple of page size */ + int fd; /* file descriptor used for this block */ + } entry[ISHM_MAX_NB_BLOCKS]; + int nb_entries; +} ishm_proctable_t; +static ishm_proctable_t *ishm_proctable; + +/* + * Table of fragments describing the common virtual address space: + * This table is visible to every ODP thread (linux process or pthreads). + * (it is allocated at odp init time and is therefore inherited by all) + */ +typedef struct { + ishm_fragment_t fragment[ISHM_NB_FRAGMNTS]; + ishm_fragment_t *used_fragmnts; /* ordered by increasing start addr */ + ishm_fragment_t *unused_fragmnts; +} ishm_ftable_t; +static ishm_ftable_t *ishm_ftbl; + +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif + +/* prototypes: */ +static void procsync(void); + +/* + * Take a piece of the preallocated virtual space to fit "size" bytes. + * (best fit). Size must be rounded up to an integer number of pages size. + * Possibly split the fragment to keep track of remaining space. + * Returns the allocated fragment (best_fragmnt) and the corresponding address. + * External caller must ensure mutex before the call! + */ +static void *alloc_fragment(uintptr_t size, int block_index, intptr_t align, + ishm_fragment_t **best_fragmnt) +{ + ishm_fragment_t *fragmnt; + *best_fragmnt = NULL; + ishm_fragment_t *rem_fragmnt; + uintptr_t border;/* possible start of new fragment (next alignement) */ + intptr_t left; /* room remaining after, if the segment is allocated */ + uintptr_t remainder = ODP_CONFIG_ISHM_VA_PREALLOC_SZ; + + /* + * search for the best bit, i.e. search for the unallocated fragment + * would give less remainder if the new fragment was allocated within + * it: + */ + for (fragmnt = ishm_ftbl->used_fragmnts; + fragmnt; fragmnt = fragmnt->next) { + /* skip allocated segment: */ + if (fragmnt->block_index >= 0) + continue; + /* skip too short segment: */ + border = ((uintptr_t)fragmnt->start + align - 1) & (-align); + left = + ((uintptr_t)fragmnt->start + fragmnt->len) - (border + size); + if (left < 0) + continue; + /* remember best fit: */ + if ((uintptr_t)left < remainder) { + remainder = left; /* best, so far */ + *best_fragmnt = fragmnt; + } + } + + if (!(*best_fragmnt)) { + ODP_ERR("unable to get virtual address for shmem block!\n."); + return NULL; + } + + (*best_fragmnt)->block_index = block_index; + border = ((uintptr_t)(*best_fragmnt)->start + align - 1) & (-align); + + /* + * if there is room between previous fragment and new one, (due to + * alignement requirement) then fragment (split) the space between + * the end of the previous fragment and the beginning of the new one: + */ + if (border - (uintptr_t)(*best_fragmnt)->start > 0) { + /* frangment space, i.e. take a new fragment descriptor... */ + rem_fragmnt = ishm_ftbl->unused_fragmnts; + if (!rem_fragmnt) { + ODP_ERR("unable to get shmem fragment descriptor!\n."); + return NULL; + } + ishm_ftbl->unused_fragmnts = rem_fragmnt->next; + + /* and link it between best_fragmnt->prev and best_fragmnt */ + if ((*best_fragmnt)->prev) + (*best_fragmnt)->prev->next = rem_fragmnt; + else + ishm_ftbl->used_fragmnts = rem_fragmnt; + rem_fragmnt->prev = (*best_fragmnt)->prev; + (*best_fragmnt)->prev = rem_fragmnt; + rem_fragmnt->next = (*best_fragmnt); + + /* update length: rem_fragmnt getting space before border */ + rem_fragmnt->block_index = -1; + rem_fragmnt->start = (*best_fragmnt)->start; + rem_fragmnt->len = border - (uintptr_t)(*best_fragmnt)->start; + (*best_fragmnt)->start = + (void *)((uintptr_t)rem_fragmnt->start + rem_fragmnt->len); + (*best_fragmnt)->len -= rem_fragmnt->len; + } + + /* if this was a perfect fit, i.e. no free space follows, we are done */ + if (remainder == 0) + return (*best_fragmnt)->start; + + /* otherwise, frangment space, i.e. take a new fragment descriptor... */ + rem_fragmnt = ishm_ftbl->unused_fragmnts; + if (!rem_fragmnt) { + ODP_ERR("unable to get shmem fragment descriptor!\n."); + return (*best_fragmnt)->start; + } + ishm_ftbl->unused_fragmnts = rem_fragmnt->next; + + /* ... double link it... */ + rem_fragmnt->next = (*best_fragmnt)->next; + rem_fragmnt->prev = (*best_fragmnt); + if ((*best_fragmnt)->next) + (*best_fragmnt)->next->prev = rem_fragmnt; + (*best_fragmnt)->next = rem_fragmnt; + + /* ... and keep track of the remainder */ + (*best_fragmnt)->len = size; + rem_fragmnt->len = remainder; + rem_fragmnt->start = (void *)((char *)(*best_fragmnt)->start + size); + rem_fragmnt->block_index = -1; + + return (*best_fragmnt)->start; +} + +/* + * Free a portion of virtual space. + * Possibly defragment, if the freed fragment is adjacent to another + * free virtual fragment. + * External caller must ensure mutex before the call! + */ +static void free_fragment(ishm_fragment_t *fragmnt) +{ + ishm_fragment_t *prev_f; + ishm_fragment_t *next_f; + + /* sanity check */ + if (!fragmnt) + return; + + prev_f = fragmnt->prev; + next_f = fragmnt->next; + + /* free the fragment */ + fragmnt->block_index = -1; + + /* check if the previous fragment is also free: if so, defragment */ + if (prev_f && (prev_f->block_index < 0)) { + fragmnt->start = prev_f->start; + fragmnt->len += prev_f->len; + if (prev_f->prev) { + prev_f->prev->next = fragmnt; + } else { + if (ishm_ftbl->used_fragmnts == prev_f) + ishm_ftbl->used_fragmnts = fragmnt; + else + ODP_ERR("corrupted fragment list!.\n"); + } + fragmnt->prev = prev_f->prev; + + /* put removed fragment in free list */ + prev_f->prev = NULL; + prev_f->next = ishm_ftbl->unused_fragmnts; + ishm_ftbl->unused_fragmnts = prev_f; + } + + /* check if the next fragment is also free: if so, defragment */ + if (next_f && (next_f->block_index < 0)) { + fragmnt->len += next_f->len; + if (next_f->next) + next_f->next->prev = fragmnt; + fragmnt->next = next_f->next; + + /* put removed fragment in free list */ + next_f->prev = NULL; + next_f->next = ishm_ftbl->unused_fragmnts; + ishm_ftbl->unused_fragmnts = next_f; + } +} + +/* + * Create file with size len. returns -1 on error + * Creates a file to /tmp/odp-<pid>-<sequence_or_name> (for normal pages) + * or /mnt/huge/odp-<pid>-<sequence_or_name> (for huge pages) + * Return the new file descriptor, or -1 on error. + */ +static int create_file(int block_index, int huge, uint64_t len) +{ + char *name; + int fd; + ishm_block_t *new_block; /* entry in the main block table */ + char seq_string[ISHM_FILENAME_MAXLEN]; /* used to construct filename*/ + char filename[ISHM_FILENAME_MAXLEN];/* filename in /tmp/ or /mnt/huge */ + int oflag = O_RDWR | O_CREAT | O_TRUNC; /* flags for open */ + + new_block = &ishm_tbl->block[block_index]; + name = new_block->name; + + /* create the filename: */ + snprintf(seq_string, ISHM_FILENAME_MAXLEN, "%08" PRIu64, + ishm_tbl->dev_seq++); + + /* huge dir must be known to create files there!: */ + if (huge && !odp_global_data.hugepage_info.default_huge_page_dir) + return -1; + + if (huge) + snprintf(filename, ISHM_FILENAME_MAXLEN, + ISHM_FILENAME_FORMAT, + odp_global_data.hugepage_info.default_huge_page_dir, + odp_global_data.main_pid, + (name && name[0]) ? name : seq_string); + else + snprintf(filename, ISHM_FILENAME_MAXLEN, + ISHM_FILENAME_FORMAT, + ISHM_FILENAME_NORMAL_PAGE_DIR, + odp_global_data.main_pid, + (name && name[0]) ? name : seq_string); + + fd = open(filename, oflag, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (fd < 0) { + ODP_ERR("open failed for %s: %s.\n", + filename, strerror(errno)); + return -1; + } + + if (ftruncate(fd, len) == -1) { + ODP_ERR("ftruncate failed: fd=%d, err=%s.\n", + fd, strerror(errno)); + close(fd); + return -1; + } + + strncpy(new_block->filename, filename, ISHM_FILENAME_MAXLEN - 1); + + return fd; +} + +/* + * performs the mapping, possibly allocating a fragment of the pre-reserved + * VA space if the _ODP_ISHM_SINGLE_VA flag was given. + * Sets fd, and returns the mapping address. + * This funstion will also set the _ODP_ISHM_SINGLE_VA flag if the alignment + * requires it + * Mutex must be assured by the caller. + */ +static void *do_map(int block_index, uint64_t len, uint32_t align, + uint32_t flags, int huge, int *fd) +{ + ishm_block_t *new_block; /* entry in the main block table */ + void *addr = NULL; + void *mapped_addr; + ishm_fragment_t *fragment = NULL; + + new_block = &ishm_tbl->block[block_index]; + + /* + * Creates a file to /tmp/odp-<pid>-<sequence> (for normal pages) + * or /mnt/huge/odp-<pid>-<sequence> (for huge pages) + * unless a fd was already given + */ + if (*fd < 0) { + *fd = create_file(block_index, huge, len); + if (*fd < 0) + return NULL; + } else { + new_block->filename[0] = 0; + } + + /* allocate an address range in the prebooked VA area if needed */ + if (flags & _ODP_ISHM_SINGLE_VA) { + addr = alloc_fragment(len, block_index, align, &fragment); + if (!addr) { + ODP_ERR("alloc_fragment failed.\n"); + if (new_block->filename[0]) { + close(*fd); + *fd = -1; + unlink(new_block->filename); + } + return NULL; + } + ishm_tbl->block[block_index].fragment = fragment; + } + + /* try to mmap: */ + mapped_addr = _odp_ishmphy_map(*fd, addr, len, flags); + if (mapped_addr == NULL) { + if (flags & _ODP_ISHM_SINGLE_VA) + free_fragment(fragment); + if (new_block->filename[0]) { + close(*fd); + *fd = -1; + unlink(new_block->filename); + } + return NULL; + } + + new_block->huge = huge; + + return mapped_addr; +} + +/* + * Performs an extra mapping (for a process trying to see an existing block + * i.e. performing a lookup). + * Mutex must be assured by the caller. + */ +static void *do_remap(int block_index, int fd) +{ + void *mapped_addr; + ishm_fragment_t *fragment; + uint64_t len; + uint32_t flags; + + len = ishm_tbl->block[block_index].len; + flags = ishm_tbl->block[block_index].flags; + + if (flags & _ODP_ISHM_SINGLE_VA) { + fragment = ishm_tbl->block[block_index].fragment; + if (!fragment) { + ODP_ERR("invalid fragment failure.\n"); + return NULL; + } + + /* try to mmap: */ + mapped_addr = _odp_ishmphy_map(fd, fragment->start, len, flags); + if (mapped_addr == NULL) + return NULL; + return mapped_addr; + } + + /* try to mmap: */ + mapped_addr = _odp_ishmphy_map(fd, NULL, len, flags); + if (mapped_addr == NULL) + return NULL; + + return mapped_addr; +} + +/* + * Performs unmapping, possibly freeing a prereserved VA space fragment, + * if the _ODP_ISHM_SINGLE_VA flag was set at alloc time + * Mutex must be assured by the caller. + */ +static int do_unmap(void *start, uint64_t size, uint32_t flags, + int block_index) +{ + int ret; + + if (start) + ret = _odp_ishmphy_unmap(start, size, flags); + else + ret = 0; + + if ((block_index >= 0) && (flags & _ODP_ISHM_SINGLE_VA)) { + /* mark reserved address space as free */ + free_fragment(ishm_tbl->block[block_index].fragment); + } + + return ret; +} + +/* + * Search for a given used and allocated block name. + * (search is performed in the global ishm table) + * Returns the index of the found block (if any) or -1 if none. + * Mutex must be assured by the caller. + */ +static int find_block_by_name(const char *name) +{ + int i; + + if (name == NULL || name[0] == 0) + return -1; + + for (i = 0; i < ISHM_MAX_NB_BLOCKS; i++) { + if ((ishm_tbl->block[i].len) && + (strcmp(name, ishm_tbl->block[i].name) == 0)) + return i; + } + + return -1; +} + +/* + * Search for a block by address (only works when flag _ODP_ISHM_SINGLE_VA + * was set at reserve() time, or if the block is already known by this + * process). + * Search is performed in the process table and in the global ishm table. + * The provided address does not have to be at start: any address + * within the fragment is OK. + * Returns the index to the found block (if any) or -1 if none. + * Mutex must be assured by the caller. + */ +static int find_block_by_address(void *addr) +{ + int block_index; + int i; + ishm_fragment_t *fragmnt; + + /* + * first check if there is already a process known block for this + * address + */ + for (i = 0; i < ishm_proctable->nb_entries; i++) { + block_index = ishm_proctable->entry[i].block_index; + if ((addr > ishm_proctable->entry[i].start) && + ((char *)addr < ((char *)ishm_proctable->entry[i].start + + ishm_tbl->block[block_index].len))) + return block_index; + } + + /* + * then check if there is a existing single VA block known by some other + * process and containing the given address + */ + for (i = 0; i < ISHM_MAX_NB_BLOCKS; i++) { + if ((!ishm_tbl->block[i].len) || + (!(ishm_tbl->block[i].flags & _ODP_ISHM_SINGLE_VA))) + continue; + fragmnt = ishm_tbl->block[i].fragment; + if (!fragmnt) { + ODP_ERR("find_fragment: invalid NULL fragment\n"); + return -1; + } + if ((addr >= fragmnt->start) && + ((char *)addr < ((char *)fragmnt->start + fragmnt->len))) + return i; + } + + /* address does not belong to any accessible block: */ + return -1; +} + +/* + * Search a given ishm block in the process local table. Return its index + * in the process table or -1 if not found (meaning that the ishm table + * block index was not referenced in the process local table, i.e. the + * block is known by some other process, but not by the current process). + * Caller must assure mutex. + */ +static int procfind_block(int block_index) +{ + int i; + + for (i = 0; i < ishm_proctable->nb_entries; i++) { + if (ishm_proctable->entry[i].block_index == block_index) + return i; + } + return -1; +} + +/* + * Release the physical memory mapping for blocks which have been freed + * by other processes. Caller must ensure mutex. + * Mutex must be assured by the caller. + */ +static void procsync(void) +{ + int i = 0; + int last; + ishm_block_t *block; + + last = ishm_proctable->nb_entries; + while (i < last) { + /* if the procecess sequence number doesn't match the main + * table seq number, this entry is obsolete + */ + block = &ishm_tbl->block[ishm_proctable->entry[i].block_index]; + if (ishm_proctable->entry[i].seq != block->seq) { + /* obsolete entry: free memory and remove proc entry */ + close(ishm_proctable->entry[i].fd); + _odp_ishmphy_unmap(ishm_proctable->entry[i].start, + ishm_proctable->entry[i].len, + ishm_proctable->entry[i].flags); + ishm_proctable->entry[i] = + ishm_proctable->entry[--last]; + } else { + i++; + } + } + ishm_proctable->nb_entries = last; +} + +/* + * Allocate and map internal shared memory, or other objects: + * If a name is given, check that this name is not already in use. + * If ok, allocate a new shared memory block and map the + * provided fd in it (if fd >=0 was given). + * If no fd is provided, a shared memory file desc named + * /tmp/odp-<pid>-ishm-<name_or_sequence> is created and mapped. + * (the name is different for huge page file as they must be on hugepagefs) + * The function returns the index of the newly created block in the + * main block table (>=0) or -1 on error. + */ +int _odp_ishm_reserve(const char *name, uint64_t size, int fd, + uint32_t align, uint32_t flags, uint32_t user_flags) +{ + int new_index; /* index in the main block table*/ + ishm_block_t *new_block; /* entry in the main block table*/ + uint64_t page_sz; /* normal page size. usually 4K*/ + uint64_t alloc_size; /* includes extra for alignement*/ + uint64_t page_hp_size; /* huge page size */ + uint64_t alloc_hp_size; /* includes extra for alignement*/ + uint32_t hp_align; + uint64_t len; /* mapped length */ + void *addr = NULL; /* mapping address */ + int new_proc_entry; + + page_sz = odp_sys_page_size(); + + odp_spinlock_lock(&ishm_tbl->lock); + + /* update this process view... */ + procsync(); + + /* roundup to page size */ + alloc_size = (size + (page_sz - 1)) & (-page_sz); + + page_hp_size = odp_sys_huge_page_size(); + /* roundup to page size */ + alloc_hp_size = (size + (page_hp_size - 1)) & (-page_hp_size); + + /* check if name already exists */ + if (name && (find_block_by_name(name) >= 0)) { + /* Found a block with the same name */ + odp_spinlock_unlock(&ishm_tbl->lock); + ODP_ERR("name "%s" already used.\n", name); + return -1; + } + + /* grab a new entry: */ + for (new_index = 0; new_index < ISHM_MAX_NB_BLOCKS; new_index++) { + if (ishm_tbl->block[new_index].len == 0) { + /* Found free block */ + break; + } + } + + /* check if we have reached the maximum number of allocation: */ + if (new_index >= ISHM_MAX_NB_BLOCKS) { + odp_spinlock_unlock(&ishm_tbl->lock); + ODP_ERR("ISHM_MAX_NB_BLOCKS limit reached!\n"); + return -1; + } + + new_block = &ishm_tbl->block[new_index]; + + /* save block name (if any given): */ + if (name) + strncpy(new_block->name, name, ISHM_NAME_MAXLEN - 1); + else + new_block->name[0] = 0; + + /* Try first huge pages when possible and needed: */ + if (page_hp_size && (alloc_size > page_sz)) { + /* at least, alignment in VA should match page size, but user + * can request more: If the user requirement exceeds the page + * size then we have to make sure the block will be mapped at + * the same address every where, otherwise alignment may be + * be wrong for some process */ + hp_align = align; + if (hp_align < odp_sys_huge_page_size()) + hp_align = odp_sys_huge_page_size(); + else + flags |= _ODP_ISHM_SINGLE_VA; + len = alloc_hp_size; + addr = do_map(new_index, len, hp_align, flags, 1, &fd); + + if (addr == NULL) + ODP_DBG("No huge pages, fall back to normal pages, " + "check: /proc/sys/vm/nr_hugepages.\n"); + else + new_block->huge = 1; + } + + /* try normal pages if huge pages failed */ + if (addr == NULL) { + /* at least, alignment in VA should match page size, but user + * can request more: If the user requirement exceeds the page + * size then we have to make sure the block will be mapped at + * the same address every where, otherwise alignment may be + * be wrong for some process */ + if (align < odp_sys_page_size()) + align = odp_sys_page_size(); + else + flags |= _ODP_ISHM_SINGLE_VA; + + len = alloc_size; + addr = do_map(new_index, len, align, flags, 0, &fd); + new_block->huge = 0; + } + + /* if neither huge pages or normal pages works, we cannot proceed: */ + if ((addr == NULL) || (len == 0)) { + if ((new_block->filename[0]) && (fd >= 0)) + close(fd); + odp_spinlock_unlock(&ishm_tbl->lock); + ODP_ERR("_ishm_reserve failed.\n"); + return -1; + } + + /* remember block data and increment block seq number to mark change */ + new_block->len = len; + new_block->user_len = size; + new_block->flags = flags; + new_block->user_flags = user_flags; + new_block->seq++; + new_block->refcnt = 1; + new_block->main_odpthread = odp_thread_id(); + new_block->start = addr; /* only for SINGLE_VA*/ + + /* the allocation succeeded: update the process local view */ + new_proc_entry = ishm_proctable->nb_entries++; + ishm_proctable->entry[new_proc_entry].block_index = new_index; + ishm_proctable->entry[new_proc_entry].flags = flags; + ishm_proctable->entry[new_proc_entry].seq = new_block->seq; + ishm_proctable->entry[new_proc_entry].start = addr; + ishm_proctable->entry[new_proc_entry].len = len; + ishm_proctable->entry[new_proc_entry].fd = fd; + + /* register the file descriptor to the file descriptor server. */ + _odp_fdserver_register_fd(FD_SRV_CTX_ISHM, new_index, fd); + + odp_spinlock_unlock(&ishm_tbl->lock); + return new_index; +} + +/* + * Free and unmap internal shared memory: + * The file descriptor is closed and the .../odp-* file deleted, + * unless fd was externally provided at reserve() time. + * return 0 if OK, and -1 on error. + * Mutex must be assured by the caller. + */ +static int block_free(int block_index) +{ + int proc_index; + ishm_block_t *block; /* entry in the main block table*/ + int last; + + if ((block_index < 0) || + (block_index >= ISHM_MAX_NB_BLOCKS) || + (ishm_tbl->block[block_index].len == 0)) { + ODP_ERR("Request to free an invalid block\n"); + return -1; + } + + block = &ishm_tbl->block[block_index]; + + proc_index = procfind_block(block_index); + if (proc_index >= 0) { + /* close the fd, unless if it was externaly provided */ + if ((block->filename[0] != 0) || + (odp_thread_id() != block->main_odpthread)) + close(ishm_proctable->entry[proc_index].fd); + + /* remove the mapping and possible fragment */ + do_unmap(ishm_proctable->entry[proc_index].start, + block->len, + ishm_proctable->entry[proc_index].flags, + block_index); + + /* remove entry from process local table: */ + last = ishm_proctable->nb_entries - 1; + ishm_proctable->entry[proc_index] = + ishm_proctable->entry[last]; + ishm_proctable->nb_entries = last; + } else { + /* just possibly free the fragment as no mapping exist here: */ + do_unmap(NULL, 0, block->flags, block_index); + } + + /* remove the .../odp-* file, unless fd was external: */ + if (block->filename[0] != 0) + unlink(block->filename); + + /* deregister the file descriptor from the file descriptor server. */ + _odp_fdserver_deregister_fd(FD_SRV_CTX_ISHM, block_index); + + /* mark the block as free in the main block table: */ + block->len = 0; + + /* mark the change so other processes see this entry as obsolete: */ + block->seq++; + + return 0; +} + +/* + * Free and unmap internal shared memory, intentified by its block number: + * return -1 on error. 0 if OK. + */ +int _odp_ishm_free_by_index(int block_index) +{ + int ret; + + odp_spinlock_lock(&ishm_tbl->lock); + procsync(); + + ret = block_free(block_index); + odp_spinlock_unlock(&ishm_tbl->lock); + return ret; +} + +/* + * free and unmap internal shared memory, intentified by its block name: + * return -1 on error. 0 if OK. + */ +int _odp_ishm_free_by_name(const char *name) +{ + int block_index; + int ret; + + odp_spinlock_lock(&ishm_tbl->lock); + procsync(); + + /* search the block in main ishm table */ + block_index = find_block_by_name(name); + if (block_index < 0) { + ODP_ERR("Request to free an non existing block..." + " (double free?)\n"); + odp_spinlock_unlock(&ishm_tbl->lock); + return -1; + } + + ret = block_free(block_index); + odp_spinlock_unlock(&ishm_tbl->lock); + return ret; +} + +/* + * Free and unmap internal shared memory identified by address: + * return -1 on error. 0 if OK. + */ +int _odp_ishm_free_by_address(void *addr) +{ + int block_index; + int ret; + + odp_spinlock_lock(&ishm_tbl->lock); + procsync(); + + /* search the block in main ishm table */ + block_index = find_block_by_address(addr); + if (block_index < 0) { + ODP_ERR("Request to free an non existing block..." + " (double free?)\n"); + odp_spinlock_unlock(&ishm_tbl->lock); + return -1; + } + + ret = block_free(block_index); + + odp_spinlock_unlock(&ishm_tbl->lock); + return ret; +} + +/* + * Lookup for an ishm shared memory, identified by its block index + * in the main ishm block table. + * Map this ishm area in the process VA (if not already present). + * Returns the block user address or NULL on error. + * Mutex must be assured by the caller. + */ +static void *block_lookup(int block_index) +{ + int proc_index; + int fd = -1; + ishm_block_t *block; + void *mapped_addr; + int new_entry; + + if ((block_index < 0) || + (block_index >= ISHM_MAX_NB_BLOCKS) || + (ishm_tbl->block[block_index].len == 0)) { + ODP_ERR("Request to lookup an invalid block\n"); + return NULL; + } + + /* search it in process table: if there, this process knows it already*/ + proc_index = procfind_block(block_index); + if (proc_index >= 0) + return ishm_proctable->entry[proc_index].start; + + /* this ishm is not known by this process, yet: we create the mapping.*/ + fd = _odp_fdserver_lookup_fd(FD_SRV_CTX_ISHM, block_index); + if (fd < 0) { + ODP_ERR("Could not find ishm file descriptor (BUG!)\n"); + return NULL; + } + + /* perform the mapping */ + block = &ishm_tbl->block[block_index]; + + mapped_addr = do_remap(block_index, fd); + if (mapped_addr == NULL) { + ODP_ERR(" lookup: Could not map existing shared memory!\n"); + return NULL; + } + + /* the mapping succeeded: update the process local view */ + new_entry = ishm_proctable->nb_entries++; + ishm_proctable->entry[new_entry].block_index = block_index; + ishm_proctable->entry[new_entry].flags = block->flags; + ishm_proctable->entry[new_entry].seq = block->seq; + ishm_proctable->entry[new_entry].start = mapped_addr; + ishm_proctable->entry[new_entry].len = block->len; + ishm_proctable->entry[new_entry].fd = fd; + block->refcnt++; + + return mapped_addr; +} + +/* + * Lookup for an ishm shared memory, identified by its block_index. + * Maps this ishmem area in the process VA (if not already present). + * Returns the block user address, or NULL if the index + * does not match any known ishm blocks. + */ +void *_odp_ishm_lookup_by_index(int block_index) +{ + void *ret; + + odp_spinlock_lock(&ishm_tbl->lock); + procsync(); + + ret = block_lookup(block_index); + odp_spinlock_unlock(&ishm_tbl->lock); + return ret; +} + +/* + * Lookup for an ishm shared memory, identified by its block name. + * Map this ishm area in the process VA (if not already present). + * Return the block index, or -1 if the index + * does not match any known ishm blocks. + */ +int _odp_ishm_lookup_by_name(const char *name) +{ + int block_index; + + odp_spinlock_lock(&ishm_tbl->lock); + procsync(); + + /* search the block in main ishm table: return -1 if not found: */ + block_index = find_block_by_name(name); + if ((block_index < 0) || (!block_lookup(block_index))) { + odp_spinlock_unlock(&ishm_tbl->lock); + return -1; + } + + odp_spinlock_unlock(&ishm_tbl->lock); + return block_index; +} + +/* + * Lookup for an ishm shared memory block, identified by its VA address. + * This works only if the block has already been looked-up (mapped) by the + * current process or it it was created with the _ODP_ISHM_SINGLE_VA flag. + * Map this ishm area in the process VA (if not already present). + * Return the block index, or -1 if the address + * does not match any known ishm blocks. + */ +int _odp_ishm_lookup_by_address(void *addr) +{ + int block_index; + + odp_spinlock_lock(&ishm_tbl->lock); + procsync(); + + /* search the block in main ishm table: return -1 if not found: */ + block_index = find_block_by_address(addr); + if ((block_index < 0) || (!block_lookup(block_index))) { + odp_spinlock_unlock(&ishm_tbl->lock); + return -1; + } + + odp_spinlock_unlock(&ishm_tbl->lock); + return block_index; +} + +/* + * Returns the VA address of a given block (which has to be known in the current + * process). Returns NULL if the block is unknown. + */ +void *_odp_ishm_address(int block_index) +{ + int proc_index; + void *addr; + + odp_spinlock_lock(&ishm_tbl->lock); + procsync(); + + if ((block_index < 0) || + (block_index >= ISHM_MAX_NB_BLOCKS) || + (ishm_tbl->block[block_index].len == 0)) { + ODP_ERR("Request for address on an invalid block\n"); + odp_spinlock_unlock(&ishm_tbl->lock); + return NULL; + } + + proc_index = procfind_block(block_index); + if (proc_index < 0) { + odp_spinlock_unlock(&ishm_tbl->lock); + return NULL; + } + + addr = ishm_proctable->entry[proc_index].start; + odp_spinlock_unlock(&ishm_tbl->lock); + return addr; +} + +int _odp_ishm_info(int block_index, _odp_ishm_info_t *info) +{ + int proc_index; + + odp_spinlock_lock(&ishm_tbl->lock); + procsync(); + + if ((block_index < 0) || + (block_index >= ISHM_MAX_NB_BLOCKS) || + (ishm_tbl->block[block_index].len == 0)) { + odp_spinlock_unlock(&ishm_tbl->lock); + ODP_ERR("Request for info on an invalid block\n"); + return -1; + } + + /* search it in process table: if not there, need to map*/ + proc_index = procfind_block(block_index); + if (proc_index < 0) { + odp_spinlock_unlock(&ishm_tbl->lock); + return -1; + } + + info->name = ishm_tbl->block[block_index].name; + info->addr = ishm_proctable->entry[proc_index].start; + info->size = ishm_tbl->block[block_index].user_len; + info->page_size = ishm_tbl->block[block_index].huge ? + odp_sys_huge_page_size() : odp_sys_page_size(); + info->flags = ishm_tbl->block[block_index].flags; + info->user_flags = ishm_tbl->block[block_index].user_flags; + + odp_spinlock_unlock(&ishm_tbl->lock); + return 0; +} + +int _odp_ishm_init_global(void) +{ + void *addr; + void *spce_addr; + int i; + + if (!odp_global_data.hugepage_info.default_huge_page_dir) + ODP_DBG("NOTE: No support for huge pages\n"); + else + ODP_DBG("Huge pages mount point is: %s\n", + odp_global_data.hugepage_info.default_huge_page_dir); + + /* allocate space for the internal shared mem block table: */ + addr = mmap(NULL, sizeof(ishm_table_t), + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (addr == MAP_FAILED) { + ODP_ERR("unable to mmap the main block table\n."); + goto init_glob_err1; + } + ishm_tbl = addr; + memset(ishm_tbl, 0, sizeof(ishm_table_t)); + ishm_tbl->dev_seq = 0; + odp_spinlock_init(&ishm_tbl->lock); + + /* allocate space for the internal shared mem fragment table: */ + addr = mmap(NULL, sizeof(ishm_ftable_t), + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (addr == MAP_FAILED) { + ODP_ERR("unable to mmap the main fragment table\n."); + goto init_glob_err2; + } + ishm_ftbl = addr; + memset(ishm_ftbl, 0, sizeof(ishm_ftable_t)); + + /* + *reserve the address space for _ODP_ISHM_SINGLE_VA reserved blocks, + * only address space! + */ + spce_addr = _odp_ishmphy_book_va(ODP_CONFIG_ISHM_VA_PREALLOC_SZ, + odp_sys_huge_page_size()); + if (!spce_addr) { + ODP_ERR("unable to reserve virtual space\n."); + goto init_glob_err3; + } + + /* use the first fragment descriptor to describe to whole VA space: */ + ishm_ftbl->fragment[0].block_index = -1; + ishm_ftbl->fragment[0].start = spce_addr; + ishm_ftbl->fragment[0].len = ODP_CONFIG_ISHM_VA_PREALLOC_SZ; + ishm_ftbl->fragment[0].prev = NULL; + ishm_ftbl->fragment[0].next = NULL; + ishm_ftbl->used_fragmnts = &ishm_ftbl->fragment[0]; + + /* and put all other fragment descriptors in the unused list: */ + for (i = 1; i < ISHM_NB_FRAGMNTS - 1; i++) { + ishm_ftbl->fragment[i].prev = NULL; + ishm_ftbl->fragment[i].next = &ishm_ftbl->fragment[i + 1]; + } + ishm_ftbl->fragment[ISHM_NB_FRAGMNTS - 1].prev = NULL; + ishm_ftbl->fragment[ISHM_NB_FRAGMNTS - 1].next = NULL; + ishm_ftbl->unused_fragmnts = &ishm_ftbl->fragment[1]; + + return 0; + +init_glob_err3: + if (munmap(ishm_ftbl, sizeof(ishm_ftable_t)) < 0) + ODP_ERR("unable to munmap main fragment table\n."); +init_glob_err2: + if (munmap(ishm_tbl, sizeof(ishm_table_t)) < 0) + ODP_ERR("unable to munmap main block table\n."); +init_glob_err1: + return -1; +} + +int _odp_ishm_init_local(void) +{ + int i; + int block_index; + + /* + * the ishm_process table is local to each linux process + * Check that no other linux threads (of same or ancestor processes) + * have already created the table, and create it if needed. + * We protect this with the general ishm lock to avoid + * init race condition of different running threads. + */ + odp_spinlock_lock(&ishm_tbl->lock); + if (!ishm_proctable) { + ishm_proctable = malloc(sizeof(ishm_proctable_t)); + if (!ishm_proctable) { + odp_spinlock_unlock(&ishm_tbl->lock); + return -1; + } + memset(ishm_proctable, 0, sizeof(ishm_proctable_t)); + } + if (syscall(SYS_gettid) != getpid()) + ishm_proctable->thrd_refcnt++; /* new linux thread */ + else + ishm_proctable->thrd_refcnt = 1;/* new linux process */ + + /* + * if this ODP thread is actually a new linux process, (as opposed + * to a pthread), i.e, we just forked, then all shmem blocks + * of the parent process are mapped into this child by inheritance. + * (The process local table is inherited as well). We hence have to + * increase the process refcount for each of the inherited mappings: + */ + if (syscall(SYS_gettid) == getpid()) { + for (i = 0; i < ishm_proctable->nb_entries; i++) { + block_index = ishm_proctable->entry[i].block_index; + ishm_tbl->block[block_index].refcnt++; + } + } + + odp_spinlock_unlock(&ishm_tbl->lock); + return 0; +} + +int _odp_ishm_term_global(void) +{ + int ret = 0; + + /* free the fragment table */ + if (munmap(ishm_ftbl, sizeof(ishm_ftable_t)) < 0) { + ret = -1; + ODP_ERR("unable to munmap fragment table\n."); + } + /* free the block table */ + if (munmap(ishm_tbl, sizeof(ishm_table_t)) < 0) { + ret = -1; + ODP_ERR("unable to munmap main table\n."); + } + + /* free the reserved VA space */ + if (_odp_ishmphy_unbook_va()) + ret = -1; + + return ret; +} + +int _odp_ishm_term_local(void) +{ + int i; + int proc_table_refcnt = 0; + int block_index; + ishm_block_t *block; + + odp_spinlock_lock(&ishm_tbl->lock); + procsync(); + + /* + * The ishm_process table is local to each linux process + * Check that no other linux threads (of this linux process) + * still needs the table, and free it if so. + * We protect this with the general ishm lock to avoid + * term race condition of different running threads. + */ + proc_table_refcnt = --ishm_proctable->thrd_refcnt; + if (!proc_table_refcnt) { + /* + * this is the last thread of this process... + * All mappings for this process are about to be lost... + * Go through the table of visible blocks for this process, + * decreasing the refcnt of each visible blocks, and issuing + * warning for those no longer referenced by any process. + * Note that non-referenced blocks are nor freeed: this is + * deliberate as this would imply that the sementic of the + * freeing function would differ depending on whether we run + * with odp_thread as processes or pthreads. With this approach, + * the user should always free the blocks manually, which is + * more consistent + */ + for (i = 0; i < ishm_proctable->nb_entries; i++) { + block_index = ishm_proctable->entry[i].block_index; + block = &ishm_tbl->block[block_index]; + if ((--block->refcnt) <= 0) { + block->refcnt = 0; + ODP_DBG("Warning: block %d: name:%s " + "no longer referenced\n", + i, + ishm_tbl->block[i].name[0] ? + ishm_tbl->block[i].name : "<no name>"); + } + } + + free(ishm_proctable); + ishm_proctable = NULL; + } + + odp_spinlock_unlock(&ishm_tbl->lock); + return 0; +} diff --git a/platform/linux-generic/_ishmphy.c b/platform/linux-generic/_ishmphy.c new file mode 100644 index 0000000..2b2d100 --- /dev/null +++ b/platform/linux-generic/_ishmphy.c @@ -0,0 +1,185 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * This file handles the lower end of the ishm memory allocator: + * It performs the physical mappings. + */ +#include <odp_posix_extensions.h> +#include <odp_config_internal.h> +#include <odp_internal.h> +#include <odp/api/align.h> +#include <odp/api/system_info.h> +#include <odp/api/debug.h> +#include <odp_debug_internal.h> +#include <odp_align_internal.h> +#include <_ishm_internal.h> +#include <_ishmphy_internal.h> + +#include <stdlib.h> +#include <stdio.h> +#include <unistd.h> +#include <string.h> +#include <errno.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <_ishmphy_internal.h> + +static void *common_va_address; +static uint64_t common_va_len; + +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif + +/* Book some virtual address space + * This function is called at odp_init_global() time to pre-book some + * virtual address space inherited by all odpthreads (i.e. descendant + * processes and threads) and later used to guarantee the unicity the + * the mapping VA address when memory is reserver with the _ODP_ISHM_SINGLE_VA + * flag. + * returns the address of the mapping or NULL on error. + */ +void *_odp_ishmphy_book_va(uintptr_t len, intptr_t align) +{ + void *addr; + + addr = mmap(NULL, len + align, PROT_NONE, + MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + if (addr == MAP_FAILED) { + ODP_ERR("_ishmphy_book_va failure\n"); + return NULL; + } + + if (mprotect(addr, len, PROT_NONE)) + ODP_ERR("failure for protect\n"); + + ODP_DBG("VA Reserved: %p, len=%p\n", addr, len + align); + + common_va_address = addr; + common_va_len = len; + + /* return the nearest aligned address: */ + return (void *)(((uintptr_t)addr + align - 1) & (-align)); +} + +/* Un-book some virtual address space + * This function is called at odp_term_global() time to unbook + * the virtual address space booked by _ishmphy_book_va() + */ +int _odp_ishmphy_unbook_va(void) +{ + int ret; + + ret = munmap(common_va_address, common_va_len); + if (ret) + ODP_ERR("_unishmphy_book_va failure\n"); + return ret; +} + +/* + * do a mapping: + * Performs a mapping of the provided file descriptor to the process VA + * space. If the _ODP_ISHM_SINGLE_VA flag is set, 'start' is assumed to be + * the VA address where the mapping is to be done. + * If the flag is not set, a new VA address is taken. + * returns the address of the mapping or NULL on error. + */ +void *_odp_ishmphy_map(int fd, void *start, uint64_t size, + int flags) +{ + void *mapped_addr; + int mmap_flags = 0; + + if (flags & _ODP_ISHM_SINGLE_VA) { + if (!start) { + ODP_ERR("failure: missing address\n"); + return NULL; + } + /* maps over fragment of reserved VA: */ + mapped_addr = mmap(start, size, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED | mmap_flags, fd, 0); + /* if mapping fails, re-block the space we tried to take + * as it seems a mapping failure still affect what was there??*/ + if (mapped_addr == MAP_FAILED) { + mmap_flags = MAP_SHARED | MAP_FIXED | + MAP_ANONYMOUS | MAP_NORESERVE; + mmap(start, size, PROT_NONE, mmap_flags, -1, 0); + mprotect(start, size, PROT_NONE); + } + } else { + /* just do a new mapping in the VA space: */ + mapped_addr = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_SHARED | mmap_flags, fd, 0); + if ((mapped_addr >= common_va_address) && + ((char *)mapped_addr < + (char *)common_va_address + common_va_len)) { + ODP_ERR("VA SPACE OVERLAP!\n"); + } + } + + if (mapped_addr == MAP_FAILED) { + ODP_ERR("mmap failed:%s\n", strerror(errno)); + return NULL; + } + + /* if locking is requested, lock it...*/ + if (flags & _ODP_ISHM_LOCK) { + if (mlock(mapped_addr, size)) { + if (munmap(mapped_addr, size)) + ODP_ERR("munmap failed:%s\n", strerror(errno)); + ODP_ERR("mlock failed:%s\n", strerror(errno)); + return NULL; + } + } + return mapped_addr; +} + +/* free a mapping: + * If the _ODP_ISHM_SINGLE_VA flag was given at creation time the virtual + * address range must be returned to the preoallocated "pool". this is + * done by mapping non accessibly memory there (hence blocking the VA but + * releasing the physical memory). + * If the _ODP_ISHM_SINGLE_VA flag was not given, both physical memory and + * virtual address space are realeased by calling the normal munmap. + * return 0 on success or -1 on error. + */ +int _odp_ishmphy_unmap(void *start, uint64_t len, int flags) +{ + void *addr; + int ret; + int mmap_flgs; + + mmap_flgs = MAP_SHARED | MAP_FIXED | MAP_ANONYMOUS | MAP_NORESERVE; + + /* if locking was requested, unlock...*/ + if (flags & _ODP_ISHM_LOCK) + munlock(start, len); + + if (flags & _ODP_ISHM_SINGLE_VA) { + /* map unnaccessible memory overwrites previous mapping + * and free the physical memory, but guarantees to block + * the VA range from other mappings + */ + addr = mmap(start, len, PROT_NONE, mmap_flgs, -1, 0); + if (addr == MAP_FAILED) { + ODP_ERR("_ishmphy_free failure for ISHM_SINGLE_VA\n"); + return -1; + } + if (mprotect(start, len, PROT_NONE)) + ODP_ERR("_ishmphy_free failure for protect\n"); + return 0; + } + + /* just release the mapping */ + ret = munmap(start, len); + if (ret) + ODP_ERR("_ishmphy_free failure: %s\n", strerror(errno)); + return ret; +} diff --git a/platform/linux-generic/arch/arm/odp/api/cpu_arch.h b/platform/linux-generic/arch/arm/odp/api/cpu_arch.h deleted file mode 120000 index e86e132..0000000 --- a/platform/linux-generic/arch/arm/odp/api/cpu_arch.h +++ /dev/null @@ -1 +0,0 @@ -../../../default/odp/api/cpu_arch.h \ No newline at end of file diff --git a/platform/linux-generic/arch/arm/odp/api/cpu_arch.h b/platform/linux-generic/arch/arm/odp/api/cpu_arch.h new file mode 100644 index 0000000..22b1da2 --- /dev/null +++ b/platform/linux-generic/arch/arm/odp/api/cpu_arch.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_PLAT_CPU_ARCH_H_ +#define ODP_PLAT_CPU_ARCH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define _ODP_CACHE_LINE_SIZE 64 + +static inline void odp_cpu_pause(void) +{ +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/arch/arm/odp_cpu_arch.c b/platform/linux-generic/arch/arm/odp_cpu_arch.c deleted file mode 120000 index deebc47..0000000 --- a/platform/linux-generic/arch/arm/odp_cpu_arch.c +++ /dev/null @@ -1 +0,0 @@ -../default/odp_cpu_arch.c \ No newline at end of file diff --git a/platform/linux-generic/arch/arm/odp_cpu_arch.c b/platform/linux-generic/arch/arm/odp_cpu_arch.c new file mode 100644 index 0000000..2ac223e --- /dev/null +++ b/platform/linux-generic/arch/arm/odp_cpu_arch.c @@ -0,0 +1,48 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_posix_extensions.h> + +#include <stdlib.h> +#include <time.h> + +#include <odp/api/cpu.h> +#include <odp/api/hints.h> +#include <odp/api/system_info.h> +#include <odp_debug_internal.h> + +#define GIGA 1000000000 + +uint64_t odp_cpu_cycles(void) +{ + struct timespec time; + uint64_t sec, ns, hz, cycles; + int ret; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, &time); + + if (ret != 0) + ODP_ABORT("clock_gettime failed\n"); + + hz = odp_cpu_hz_max(); + sec = (uint64_t)time.tv_sec; + ns = (uint64_t)time.tv_nsec; + + cycles = sec * hz; + cycles += (ns * hz) / GIGA; + + return cycles; +} + +uint64_t odp_cpu_cycles_max(void) +{ + return UINT64_MAX; +} + +uint64_t odp_cpu_cycles_resolution(void) +{ + return 1; +} diff --git a/platform/linux-generic/arch/arm/odp_sysinfo_parse.c b/platform/linux-generic/arch/arm/odp_sysinfo_parse.c deleted file mode 120000 index 39962b8..0000000 --- a/platform/linux-generic/arch/arm/odp_sysinfo_parse.c +++ /dev/null @@ -1 +0,0 @@ -../default/odp_sysinfo_parse.c \ No newline at end of file diff --git a/platform/linux-generic/arch/arm/odp_sysinfo_parse.c b/platform/linux-generic/arch/arm/odp_sysinfo_parse.c new file mode 100644 index 0000000..53e2aae --- /dev/null +++ b/platform/linux-generic/arch/arm/odp_sysinfo_parse.c @@ -0,0 +1,27 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_internal.h> +#include <odp_debug_internal.h> +#include <string.h> + +int cpuinfo_parser(FILE *file ODP_UNUSED, system_info_t *sysinfo) +{ + int i; + + ODP_DBG("Warning: use dummy values for freq and model string\n"); + for (i = 0; i < MAX_CPU_NUMBER; i++) { + sysinfo->cpu_hz_max[i] = 1400000000; + strcpy(sysinfo->model_str[i], "UNKNOWN"); + } + + return 0; +} + +uint64_t odp_cpu_hz_current(int id ODP_UNUSED) +{ + return 0; +} diff --git a/platform/linux-generic/arch/powerpc/odp_cpu_arch.c b/platform/linux-generic/arch/powerpc/odp_cpu_arch.c deleted file mode 120000 index deebc47..0000000 --- a/platform/linux-generic/arch/powerpc/odp_cpu_arch.c +++ /dev/null @@ -1 +0,0 @@ -../default/odp_cpu_arch.c \ No newline at end of file diff --git a/platform/linux-generic/arch/powerpc/odp_cpu_arch.c b/platform/linux-generic/arch/powerpc/odp_cpu_arch.c new file mode 100644 index 0000000..2ac223e --- /dev/null +++ b/platform/linux-generic/arch/powerpc/odp_cpu_arch.c @@ -0,0 +1,48 @@ +/* Copyright (c) 2015, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp_posix_extensions.h> + +#include <stdlib.h> +#include <time.h> + +#include <odp/api/cpu.h> +#include <odp/api/hints.h> +#include <odp/api/system_info.h> +#include <odp_debug_internal.h> + +#define GIGA 1000000000 + +uint64_t odp_cpu_cycles(void) +{ + struct timespec time; + uint64_t sec, ns, hz, cycles; + int ret; + + ret = clock_gettime(CLOCK_MONOTONIC_RAW, &time); + + if (ret != 0) + ODP_ABORT("clock_gettime failed\n"); + + hz = odp_cpu_hz_max(); + sec = (uint64_t)time.tv_sec; + ns = (uint64_t)time.tv_nsec; + + cycles = sec * hz; + cycles += (ns * hz) / GIGA; + + return cycles; +} + +uint64_t odp_cpu_cycles_max(void) +{ + return UINT64_MAX; +} + +uint64_t odp_cpu_cycles_resolution(void) +{ + return 1; +} diff --git a/platform/linux-generic/include/_fdserver_internal.h b/platform/linux-generic/include/_fdserver_internal.h index 480ac02..22b2802 100644 --- a/platform/linux-generic/include/_fdserver_internal.h +++ b/platform/linux-generic/include/_fdserver_internal.h @@ -23,6 +23,7 @@ extern "C" { */ typedef enum fd_server_context { FD_SRV_CTX_NA, /* Not Applicable */ + FD_SRV_CTX_ISHM, FD_SRV_CTX_END, /* upper enum limit */ } fd_server_context_e;

diff --git a/platform/linux-generic/include/_ishm_internal.h b/platform/linux-generic/include/_ishm_internal.h new file mode 100644 index 0000000..7d27477 --- /dev/null +++ b/platform/linux-generic/include/_ishm_internal.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_ISHM_INTERNAL_H_ +#define ODP_ISHM_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* flags available at ishm_reserve: */ +#define _ODP_ISHM_SINGLE_VA 1 +#define _ODP_ISHM_LOCK 2 + +/** + * Shared memory block info + */ +typedef struct _odp_ishm_info_t { + const char *name; /**< Block name */ + void *addr; /**< Block address */ + uint64_t size; /**< Block size in bytes */ + uint64_t page_size; /**< Memory page size */ + uint32_t flags; /**< _ODP_ISHM_* flags */ + uint32_t user_flags;/**< user specific flags */ +} _odp_ishm_info_t; + +int _odp_ishm_reserve(const char *name, uint64_t size, int fd, uint32_t align, + uint32_t flags, uint32_t user_flags); +int _odp_ishm_free_by_index(int block_index); +int _odp_ishm_free_by_name(const char *name); +int _odp_ishm_free_by_address(void *addr); +void *_odp_ishm_lookup_by_index(int block_index); +int _odp_ishm_lookup_by_name(const char *name); +int _odp_ishm_lookup_by_address(void *addr); +void *_odp_ishm_address(int block_index); +int _odp_ishm_info(int block_index, _odp_ishm_info_t *info); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/_ishmphy_internal.h b/platform/linux-generic/include/_ishmphy_internal.h new file mode 100644 index 0000000..4fe560f --- /dev/null +++ b/platform/linux-generic/include/_ishmphy_internal.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _ISHMPHY_INTERNAL_H +#define _ISHMPHY_INTERNAL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdint.h> + +void *_odp_ishmphy_book_va(uintptr_t len, intptr_t align); +int _odp_ishmphy_unbook_va(void); +void *_odp_ishmphy_map(int fd, void *start, uint64_t size, int flags); +int _odp_ishmphy_unmap(void *start, uint64_t len, int flags); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/ishmphy_internal.h b/platform/linux-generic/include/ishmphy_internal.h new file mode 100644 index 0000000..0bc4207 --- /dev/null +++ b/platform/linux-generic/include/ishmphy_internal.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _ISHMPHY_INTERNAL_H_ +#define _ISHMPHY_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +void *_ishmphy_book_va(uint64_t len); +int _ishmphy_unbook_va(void); +void *_ishmphy_map(int fd, void *start, uint64_t size, + int flags, int mmap_flags); +int _ishmphy_unmap(void *start, uint64_t len, int flags); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 989ea08..3eabc7b 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -110,6 +110,16 @@ extern "C" { */ #define ODP_CONFIG_SHM_BLOCKS (ODP_CONFIG_POOLS + 48)

+/* + * Size of the virtual address space pre-reserver for ISHM + * + * This is just virtual space preallocation size, not memory allocation. + * This address space is used by ISHM to map things at a common address in + * all ODP threads (when the _ODP_ISHM_SINGLE_VA flag is used). + * In bytes. + */ +#define ODP_CONFIG_ISHM_VA_PREALLOC_SZ (536870912L) + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index 2b21777..6ea8cf0 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -60,6 +60,7 @@ enum init_stage { TIME_INIT, SYSINFO_INIT, FDSERVER_INIT, + ISHM_INIT, SHM_INIT, THREAD_INIT, POOL_INIT, @@ -128,6 +129,11 @@ int _odp_int_name_tbl_term_global(void); int _odp_fdserver_init_global(void); int _odp_fdserver_term_global(void);

+int _odp_ishm_init_global(void); +int _odp_ishm_init_local(void); +int _odp_ishm_term_global(void); +int _odp_ishm_term_local(void); + int cpuinfo_parser(FILE *file, system_info_t *sysinfo); uint64_t odp_cpu_hz_current(int id);

diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index ec84fd0..d33a3ae 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -57,6 +57,12 @@ int odp_init_global(odp_instance_t *instance, } stage = FDSERVER_INIT;

+ if (_odp_ishm_init_global()) { + ODP_ERR("ODP ishm init failed.\n"); + goto init_failed; + } + stage = ISHM_INIT; + if (odp_shm_init_global()) { ODP_ERR("ODP shm init failed.\n"); goto init_failed; @@ -224,6 +230,13 @@ int _odp_term_global(enum init_stage stage) } /* Fall through */

+ case ISHM_INIT: + if (_odp_ishm_term_global()) { + ODP_ERR("ODP ishm term failed.\n"); + rc = -1; + } + /* Fall through */ + case FDSERVER_INIT: if (_odp_fdserver_term_global()) { ODP_ERR("ODP fdserver term failed.\n"); @@ -268,6 +281,12 @@ int odp_init_local(odp_instance_t instance, odp_thread_type_t thr_type) goto init_fail; }

+ if (_odp_ishm_init_local()) { + ODP_ERR("ODP ishm local init failed.\n"); + goto init_fail; + } + stage = ISHM_INIT; + if (odp_shm_init_local()) { ODP_ERR("ODP shm local init failed.\n"); goto init_fail; @@ -343,6 +362,13 @@ int _odp_term_local(enum init_stage stage) } /* Fall through */

+ case ISHM_INIT: + if (_odp_ishm_term_local()) { + ODP_ERR("ODP ishm local term failed.\n"); + rc = -1; + } + /* Fall through */ + default: break; }

commit 442b93274d8f24f31557fe678245f65a2c499f97 Author: Christophe Milard christophe.milard@linaro.org Date: Sat Aug 20 09:45:57 2016 +0200

linux-generic: system_info: adding huge page dir

The Huge page information is separated and a function to get the huge page mount directory is added. This function is called at init so the information is available later on.

Signed-off-by: Christophe Milard christophe.milard@linaro.org Reviewed-and-tested-by: Bill Fischofer bill.fischofer@linaro.org Reviewed-by: Brian Brooks brian.brooks@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index fd770b5..2b21777 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -30,7 +30,6 @@ extern __thread int __odp_errno;

typedef struct { uint64_t cpu_hz_max[MAX_CPU_NUMBER]; - uint64_t default_huge_page_size; uint64_t page_size; int cache_line_size; int cpu_count; @@ -38,11 +37,17 @@ typedef struct { char model_str[MAX_CPU_NUMBER][128]; } system_info_t;

+typedef struct { + uint64_t default_huge_page_size; + char *default_huge_page_dir; +} hugepage_info_t; + struct odp_global_data_s { pid_t main_pid; odp_log_func_t log_fn; odp_abort_func_t abort_fn; system_info_t system_info; + hugepage_info_t hugepage_info; odp_cpumask_t control_cpus; odp_cpumask_t worker_cpus; int num_cpus_installed; diff --git a/platform/linux-generic/odp_system_info.c b/platform/linux-generic/odp_system_info.c index bbe5358..18c61db 100644 --- a/platform/linux-generic/odp_system_info.c +++ b/platform/linux-generic/odp_system_info.c @@ -4,6 +4,13 @@ * SPDX-License-Identifier: BSD-3-Clause */

+/* + * BSD LICENSE + * + * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. + * All rights reserved. + */ + #include <odp_posix_extensions.h>

#include <odp/api/system_info.h> @@ -11,11 +18,13 @@ #include <odp_debug_internal.h> #include <odp/api/align.h> #include <odp/api/cpu.h> +#include <errno.h> #include <pthread.h> #include <sched.h> #include <string.h> #include <stdio.h> #include <inttypes.h> +#include <ctype.h>

/* sysconf */ #include <unistd.h> @@ -97,6 +106,158 @@ static uint64_t default_huge_page_size(void) }

/* + * split string into tokens. largely "inspired" by dpdk: + * lib/librte_eal/common/eal_common_string_fns.c: rte_strsplit + */ +static int strsplit(char *string, int stringlen, + char **tokens, int maxtokens, char delim) +{ + int i, tok = 0; + int tokstart = 1; /* first token is right at start of string */ + + if (string == NULL || tokens == NULL) + return -1; + + for (i = 0; i < stringlen; i++) { + if (string[i] == '\0' || tok >= maxtokens) + break; + if (tokstart) { + tokstart = 0; + tokens[tok++] = &string[i]; + } + if (string[i] == delim) { + string[i] = '\0'; + tokstart = 1; + } + } + return tok; +} + +/* + * Converts a numeric string to the equivalent uint64_t value. + * As well as straight number conversion, also recognises the suffixes + * k, m and g for kilobytes, megabytes and gigabytes respectively. + * + * If a negative number is passed in i.e. a string with the first non-black + * character being "-", zero is returned. Zero is also returned in the case of + * an error with the strtoull call in the function. + * largely "inspired" by dpdk: + * lib/librte_eal/common/include/rte_common.h: rte_str_to_size + * + * param str + * String containing number to convert. + * return + * Number. + */ +static inline uint64_t str_to_size(const char *str) +{ + char *endptr; + unsigned long long size; + + while (isspace((int)*str)) + str++; + if (*str == '-') + return 0; + + errno = 0; + size = strtoull(str, &endptr, 0); + if (errno) + return 0; + + if (*endptr == ' ') + endptr++; /* allow 1 space gap */ + + switch (*endptr) { + case 'G': + case 'g': + size *= 1024; /* fall-through */ + case 'M': + case 'm': + size *= 1024; /* fall-through */ + case 'K': + case 'k': + size *= 1024; /* fall-through */ + default: + break; + } + return size; +} + +/* + * returns a malloced string containing the name of the directory for + * huge pages of a given size (0 for default) + * largely "inspired" by dpdk: + * lib/librte_eal/linuxapp/eal/eal_hugepage_info.c: get_hugepage_dir + * + * Analysis of /proc/mounts + */ +static char *get_hugepage_dir(uint64_t hugepage_sz) +{ + enum proc_mount_fieldnames { + DEVICE = 0, + MOUNTPT, + FSTYPE, + OPTIONS, + _FIELDNAME_MAX + }; + static uint64_t default_size; + const char proc_mounts[] = "/proc/mounts"; + const char hugetlbfs_str[] = "hugetlbfs"; + const size_t htlbfs_str_len = sizeof(hugetlbfs_str) - 1; + const char pagesize_opt[] = "pagesize="; + const size_t pagesize_opt_len = sizeof(pagesize_opt) - 1; + const char split_tok = ' '; + char *tokens[_FIELDNAME_MAX]; + char buf[BUFSIZ]; + char *retval = NULL; + const char *pagesz_str; + uint64_t pagesz; + FILE *fd = fopen(proc_mounts, "r"); + + if (fd == NULL) + return NULL; + + if (default_size == 0) + default_size = default_huge_page_size(); + + if (hugepage_sz == 0) + hugepage_sz = default_size; + + while (fgets(buf, sizeof(buf), fd)) { + if (strsplit(buf, sizeof(buf), tokens, + _FIELDNAME_MAX, split_tok) != _FIELDNAME_MAX) { + ODP_ERR("Error parsing %s\n", proc_mounts); + break; /* return NULL */ + } + + /* is this hugetlbfs? */ + if (!strncmp(tokens[FSTYPE], hugetlbfs_str, htlbfs_str_len)) { + pagesz_str = strstr(tokens[OPTIONS], pagesize_opt); + + /* No explicit size, default page size is compared */ + if (pagesz_str == NULL) { + if (hugepage_sz == default_size) { + retval = strdup(tokens[MOUNTPT]); + break; + } + } + /* there is an explicit page size, so check it */ + else { + pagesz = + str_to_size(&pagesz_str[pagesize_opt_len]); + if (pagesz == hugepage_sz) { + retval = strdup(tokens[MOUNTPT]); + break; + } + } + } /* end if strncmp hugetlbfs */ + } /* end while fgets */ + + fclose(fd); + return retval; +} + +/* * Analysis of /sys/devices/system/cpu/ files */ static int systemcpu(system_info_t *sysinfo) @@ -125,11 +286,21 @@ static int systemcpu(system_info_t *sysinfo) return -1; }

- sysinfo->default_huge_page_size = default_huge_page_size(); - return 0; }

+/* + * Huge page information + */ +static int system_hp(hugepage_info_t *hugeinfo) +{ + hugeinfo->default_huge_page_size = default_huge_page_size(); + + /* default_huge_page_dir may be NULL if no huge page support */ + hugeinfo->default_huge_page_dir = get_hugepage_dir(0); + + return 0; +}

/* * System info initialisation @@ -157,6 +328,8 @@ int odp_system_info_init(void) return -1; }

+ system_hp(&odp_global_data.hugepage_info); + return 0; }

@@ -165,6 +338,8 @@ int odp_system_info_init(void) */ int odp_system_info_term(void) { + free(odp_global_data.hugepage_info.default_huge_page_dir); + return 0; }

@@ -200,7 +375,7 @@ uint64_t odp_cpu_hz_max_id(int id)

uint64_t odp_sys_huge_page_size(void) { - return odp_global_data.system_info.default_huge_page_size; + return odp_global_data.hugepage_info.default_huge_page_size; }

uint64_t odp_sys_page_size(void)

commit a4dd0bade1ca02eb91178834a159c9d26f140c38 Author: Christophe Milard christophe.milard@linaro.org Date: Sat Aug 20 09:45:56 2016 +0200

linux-gen: fdserver: new fdserver added

A fdserver is added and started at init time. The role of the fdserver (file descriptor server) is to enable sharing of file descriptors between unrelated processes: processes which wish to share a file descriptor may register their files descriptor to the server and processes wishing to use the shared file descriptors can do a lookup. When registration occurs, a triple {context, key, fd} is provided to the server. The context identifies the client and scope (i.e. shmem). The key is implemented as a long, and can be whatever. The server won't care as long as keys are unique. The file descriptor can be retrieved by another process providing the same context and key. Files descriptors passed this way are converted on the fly during the unix domain socket communication that occurs between the server and its clients. This is done by using the ancillary control data part of the msg.

Signed-off-by: Christophe Milard christophe.milard@linaro.org Reviewed-and-tested-by: Bill Fischofer bill.fischofer@linaro.org Reviewed-by: Brian Brooks brian.brooks@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 271f5b5..cb7c8e9 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -112,6 +112,7 @@ odpdrvplatinclude_HEADERS = \ $(srcdir)/include/odp/drv/plat/strong_types.h

noinst_HEADERS = \ + ${srcdir}/include/_fdserver_internal.h \ ${srcdir}/include/odp_align_internal.h \ ${srcdir}/include/odp_atomic_internal.h \ ${srcdir}/include/odp_buffer_inlines.h \ @@ -154,6 +155,7 @@ noinst_HEADERS = \ ${srcdir}/Makefile.inc

__LIB__libodp_linux_la_SOURCES = \ + _fdserver.c \ odp_atomic.c \ odp_barrier.c \ odp_buffer.c \ diff --git a/platform/linux-generic/_fdserver.c b/platform/linux-generic/_fdserver.c new file mode 100644 index 0000000..bf36eb2 --- /dev/null +++ b/platform/linux-generic/_fdserver.c @@ -0,0 +1,655 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/* + * This file implements a file descriptor sharing server enabling + * sharing of file descriptors between processes, regardless of fork time. + * + * File descriptors are process scoped, but they can be "sent and converted + * on the fly" between processes using special unix domain socket ancillary + * data. + * The receiving process gets a file descriptor "pointing" to the same thing + * as the one sent (but the value of the file descriptor itself may be different + * from the one sent). + * Because ODP applications are responsible for creating ODP threads (i.e. + * pthreads or linux processes), ODP has no control on the order things happen: + * Nothing prevent a thread A to fork B and C, and then C creating a pktio + * which will be used by A and B to send/receive packets. + * Assuming this pktio uses a file descriptor, the latter will need to be + * shared between the processes, despite the "non convenient" fork time. + * The shared memory allocator is likely to use this as well to be able to + * share memory regardless of fork() time. + * This server handles a table of {(context,key)<-> fd} pair, and is + * interfaced by the following functions: + * + * _odp_fdserver_register_fd(context, key, fd_to_send); + * _odp_fdserver_deregister_fd(context, key); + * _odp_fdserver_lookup_fd(context, key); + * + * which are used to register/deregister or querry for file descriptor based + * on a context and key value couple, which has to be unique. + * + * Note again that the file descriptors stored here are local to this server + * process and get converted both when registered or looked up. + */ + +#include <odp_posix_extensions.h> +#include <odp/api/spinlock.h> +#include <odp_internal.h> +#include <odp_debug_internal.h> +#include <_fdserver_internal.h> + +#include <stdio.h> +#include <stdlib.h> +#include <errno.h> +#include <string.h> +#include <sys/types.h> +#include <signal.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <unistd.h> +#include <inttypes.h> +#include <sys/mman.h> +#include <sys/wait.h> + +#define FDSERVER_SOCKPATH_MAXLEN 32 +#define FDSERVER_SOCKPATH_FORMAT "/tmp/odp-%d-fdserver" +#define FDSERVER_BACKLOG 5 + +#ifndef MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif + +/* when accessing the client functions, clients should be mutexed: */ +odp_spinlock_t *client_lock; + +/* define the tables of file descriptors handled by this server: */ +#define FDSERVER_MAX_ENTRIES 256 +typedef struct fdentry_s { + fd_server_context_e context; + uint64_t key; + int fd; +} fdentry_t; +static fdentry_t fd_table[FDSERVER_MAX_ENTRIES]; +static int fd_table_nb_entries; + +/* + * define the message struct used for communication between client and server + * (this single message is used in both direction) + * The file descriptors are sent out of band as ancillary data for conversion. + */ +typedef struct fd_server_msg { + int command; + fd_server_context_e context; + uint64_t key; +} fdserver_msg_t; +/* possible commands are: */ +#define FD_REGISTER_REQ 1 /* client -> server */ +#define FD_REGISTER_ACK 2 /* server -> client */ +#define FD_REGISTER_NACK 3 /* server -> client */ +#define FD_LOOKUP_REQ 4 /* client -> server */ +#define FD_LOOKUP_ACK 5 /* server -> client */ +#define FD_LOOKUP_NACK 6 /* server -> client */ +#define FD_DEREGISTER_REQ 7 /* client -> server */ +#define FD_DEREGISTER_ACK 8 /* server -> client */ +#define FD_DEREGISTER_NACK 9 /* server -> client */ +#define FD_SERVERSTOP_REQ 10 /* client -> server (stops) */ + +/* + * Client and server function: + * Send a fdserver_msg, possibly including a file descriptor, on the socket + * This function is used both by: + * -the client (sending a FD_REGISTER_REQ with a file descriptor to be shared, + * or FD_LOOKUP_REQ/FD_DEREGISTER_REQ without a file descirptor) + * -the server (sending FD_REGISTER_ACK/NACK, FD_LOOKUP_NACK, + * FD_DEREGISTER_ACK/NACK... without a fd or a + * FD_LOOKUP_ACK with a fd) + * This function make use of the ancillary data (control data) to pass and + * convert file descriptors over UNIX sockets + * Return -1 on error, 0 on success. + */ +static int send_fdserver_msg(int sock, int command, + fd_server_context_e context, uint64_t key, + int fd_to_send) +{ + struct msghdr socket_message; + struct iovec io_vector[1]; /* one msg frgmt only */ + struct cmsghdr *control_message = NULL; + int *fd_location; + fdserver_msg_t msg; + int res; + + char ancillary_data[CMSG_SPACE(sizeof(int))]; + + /* prepare the register request body (single framgent): */ + msg.command = command; + msg.context = context; + msg.key = key; + io_vector[0].iov_base = &msg; + io_vector[0].iov_len = sizeof(fdserver_msg_t); + + /* initialize socket message */ + memset(&socket_message, 0, sizeof(struct msghdr)); + socket_message.msg_iov = io_vector; + socket_message.msg_iovlen = 1; + + if (fd_to_send >= 0) { + /* provide space for the ancillary data */ + memset(ancillary_data, 0, CMSG_SPACE(sizeof(int))); + socket_message.msg_control = ancillary_data; + socket_message.msg_controllen = CMSG_SPACE(sizeof(int)); + + /* initialize a single ancillary data element for fd passing */ + control_message = CMSG_FIRSTHDR(&socket_message); + control_message->cmsg_level = SOL_SOCKET; + control_message->cmsg_type = SCM_RIGHTS; + control_message->cmsg_len = CMSG_LEN(sizeof(int)); + fd_location = (int *)(void *)CMSG_DATA(control_message); + *fd_location = fd_to_send; + } + res = sendmsg(sock, &socket_message, 0); + if (res < 0) { + ODP_ERR("send_fdserver_msg: %s\n", strerror(errno)); + return(-1); + } + + return 0; +} + +/* + * Client and server function + * Receive a fdserver_msg, possibly including a file descriptor, on the + * given socket. + * This function is used both by: + * -the server (receiving a FD_REGISTER_REQ with a file descriptor to be shared, + * or FD_LOOKUP_REQ, FD_DEREGISTER_REQ without a file descirptor) + * -the client (receiving FD_REGISTER_ACK...without a fd or a FD_LOOKUP_ACK with + * a fd) + * This function make use of the ancillary data (control data) to pass and + * convert file descriptors over UNIX sockets. + * Return -1 on error, 0 on success. + */ +static int recv_fdserver_msg(int sock, int *command, + fd_server_context_e *context, uint64_t *key, + int *recvd_fd) +{ + struct msghdr socket_message; + struct iovec io_vector[1]; /* one msg frgmt only */ + struct cmsghdr *control_message = NULL; + int *fd_location; + fdserver_msg_t msg; + char ancillary_data[CMSG_SPACE(sizeof(int))]; + + memset(&socket_message, 0, sizeof(struct msghdr)); + memset(ancillary_data, 0, CMSG_SPACE(sizeof(int))); + + /* setup a place to fill in message contents */ + io_vector[0].iov_base = &msg; + io_vector[0].iov_len = sizeof(fdserver_msg_t); + socket_message.msg_iov = io_vector; + socket_message.msg_iovlen = 1; + + /* provide space for the ancillary data */ + socket_message.msg_control = ancillary_data; + socket_message.msg_controllen = CMSG_SPACE(sizeof(int)); + + /* receive the message */ + if (recvmsg(sock, &socket_message, MSG_CMSG_CLOEXEC) < 0) { + ODP_ERR("recv_fdserver_msg: %s\n", strerror(errno)); + return(-1); + } + + *command = msg.command; + *context = msg.context; + *key = msg.key; + + /* grab the converted file descriptor (if any) */ + *recvd_fd = -1; + + if ((socket_message.msg_flags & MSG_CTRUNC) == MSG_CTRUNC) + return 0; + + /* iterate ancillary elements to find the file descriptor: */ + for (control_message = CMSG_FIRSTHDR(&socket_message); + control_message != NULL; + control_message = CMSG_NXTHDR(&socket_message, control_message)) { + if ((control_message->cmsg_level == SOL_SOCKET) && + (control_message->cmsg_type == SCM_RIGHTS)) { + fd_location = (int *)(void *)CMSG_DATA(control_message); + *recvd_fd = *fd_location; + break; + } + } + + return 0; +} + +/* opens and returns a connected socket to the server */ +static int get_socket(void) +{ + char sockpath[FDSERVER_SOCKPATH_MAXLEN]; + int s_sock; /* server socket */ + struct sockaddr_un remote; + int len; + + /* construct the named socket path: */ + snprintf(sockpath, FDSERVER_SOCKPATH_MAXLEN, FDSERVER_SOCKPATH_FORMAT, + odp_global_data.main_pid); + + s_sock = socket(AF_UNIX, SOCK_STREAM, 0); + if (s_sock == -1) { + ODP_ERR("cannot connect to server: %s\n", strerror(errno)); + return(-1); + } + + remote.sun_family = AF_UNIX; + strcpy(remote.sun_path, sockpath); + len = strlen(remote.sun_path) + sizeof(remote.sun_family); + if (connect(s_sock, (struct sockaddr *)&remote, len) == -1) { + ODP_ERR("cannot connect to server: %s\n", strerror(errno)); + close(s_sock); + return(-1); + } + + return s_sock; +} + +/* + * Client function: + * Register a file descriptor to the server. Return -1 on error. + */ +int _odp_fdserver_register_fd(fd_server_context_e context, uint64_t key, + int fd_to_send) +{ + int s_sock; /* server socket */ + int res; + int command; + int fd; + + odp_spinlock_lock(client_lock); + + ODP_DBG("FD client register: pid=%d key=%" PRIu64 ", fd=%d\n", + getpid(), key, fd_to_send); + + s_sock = get_socket(); + if (s_sock < 0) { + odp_spinlock_unlock(client_lock); + return(-1); + } + + res = send_fdserver_msg(s_sock, FD_REGISTER_REQ, context, key, + fd_to_send); + if (res < 0) { + ODP_ERR("fd registration failure\n"); + close(s_sock); + odp_spinlock_unlock(client_lock); + return -1; + } + + res = recv_fdserver_msg(s_sock, &command, &context, &key, &fd); + + if ((res < 0) || (command != FD_REGISTER_ACK)) { + ODP_ERR("fd registration failure\n"); + close(s_sock); + odp_spinlock_unlock(client_lock); + return -1; + } + + close(s_sock); + + odp_spinlock_unlock(client_lock); + return 0; +} + +/* + * Client function: + * Deregister a file descriptor from the server. Return -1 on error. + */ +int _odp_fdserver_deregister_fd(fd_server_context_e context, uint64_t key) +{ + int s_sock; /* server socket */ + int res; + int command; + int fd; + + odp_spinlock_lock(client_lock); + + ODP_DBG("FD client deregister: pid=%d key=%" PRIu64 "\n", + getpid(), key); + + s_sock = get_socket(); + if (s_sock < 0) { + odp_spinlock_unlock(client_lock); + return(-1); + } + + res = send_fdserver_msg(s_sock, FD_DEREGISTER_REQ, context, key, -1); + if (res < 0) { + ODP_ERR("fd de-registration failure\n"); + close(s_sock); + odp_spinlock_unlock(client_lock); + return -1; + } + + res = recv_fdserver_msg(s_sock, &command, &context, &key, &fd); + + if ((res < 0) || (command != FD_DEREGISTER_ACK)) { + ODP_ERR("fd de-registration failure\n"); + close(s_sock); + odp_spinlock_unlock(client_lock); + return -1; + } + + close(s_sock); + + odp_spinlock_unlock(client_lock); + return 0; +} + +/* + * client function: + * lookup a file descriptor from the server. return -1 on error, + * or the file descriptor on success (>=0). + */ +int _odp_fdserver_lookup_fd(fd_server_context_e context, uint64_t key) +{ + int s_sock; /* server socket */ + int res; + int command; + int fd; + + odp_spinlock_lock(client_lock); + + s_sock = get_socket(); + if (s_sock < 0) { + odp_spinlock_unlock(client_lock); + return(-1); + } + + res = send_fdserver_msg(s_sock, FD_LOOKUP_REQ, context, key, -1); + if (res < 0) { + ODP_ERR("fd lookup failure\n"); + close(s_sock); + odp_spinlock_unlock(client_lock); + return -1; + } + + res = recv_fdserver_msg(s_sock, &command, &context, &key, &fd); + + if ((res < 0) || (command != FD_LOOKUP_ACK)) { + ODP_ERR("fd lookup failure\n"); + close(s_sock); + odp_spinlock_unlock(client_lock); + return -1; + } + + close(s_sock); + ODP_DBG("FD client lookup: pid=%d, key=%" PRIu64 ", fd=%d\n", + getpid(), key, fd); + + odp_spinlock_unlock(client_lock); + return fd; +} + +/* + * request server terminaison: + */ +static int stop_server(void) +{ + int s_sock; /* server socket */ + int res; + + odp_spinlock_lock(client_lock); + + ODP_DBG("FD sending server stop request\n"); + + s_sock = get_socket(); + if (s_sock < 0) { + odp_spinlock_unlock(client_lock); + return(-1); + } + + res = send_fdserver_msg(s_sock, FD_SERVERSTOP_REQ, 0, 0, -1); + if (res < 0) { + ODP_ERR("fd stop request failure\n"); + close(s_sock); + odp_spinlock_unlock(client_lock); + return -1; + } + + close(s_sock); + + odp_spinlock_unlock(client_lock); + return 0; +} + +/* + * server function + * receive a client request and handle it. + * Always returns 0 unless a stop request is received. + */ +static int handle_request(int client_sock) +{ + int command; + fd_server_context_e context; + uint64_t key; + int fd; + int i; + + /* get a client request: */ + recv_fdserver_msg(client_sock, &command, &context, &key, &fd); + switch (command) { + case FD_REGISTER_REQ: + if ((fd < 0) || (context >= FD_SRV_CTX_END)) { + ODP_ERR("Invalid register fd or context\n"); + send_fdserver_msg(client_sock, FD_REGISTER_NACK, + FD_SRV_CTX_NA, 0, -1); + return 0; + } + + /* store the file descriptor in table: */ + if (fd_table_nb_entries < FDSERVER_MAX_ENTRIES) { + fd_table[fd_table_nb_entries].context = context; + fd_table[fd_table_nb_entries].key = key; + fd_table[fd_table_nb_entries++].fd = fd; + ODP_DBG("storing {ctx=%d, key=%" PRIu64 "}->fd=%d\n", + context, key, fd); + } else { + ODP_ERR("FD table full\n"); + send_fdserver_msg(client_sock, FD_REGISTER_NACK, + FD_SRV_CTX_NA, 0, -1); + return 0; + } + + send_fdserver_msg(client_sock, FD_REGISTER_ACK, + FD_SRV_CTX_NA, 0, -1); + break; + + case FD_LOOKUP_REQ: + if (context >= FD_SRV_CTX_END) { + ODP_ERR("invalid lookup context\n"); + send_fdserver_msg(client_sock, FD_LOOKUP_NACK, + FD_SRV_CTX_NA, 0, -1); + return 0; + } + + /* search key in table and sent reply: */ + for (i = 0; i < fd_table_nb_entries; i++) { + if ((fd_table[i].context == context) && + (fd_table[i].key == key)) { + fd = fd_table[i].fd; + ODP_DBG("lookup {ctx=%d," + " key=%" PRIu64 "}->fd=%d\n", + context, key, fd); + send_fdserver_msg(client_sock, + FD_LOOKUP_ACK, context, key, + fd); + return 0; + } + } + + /* context+key not found... send nack */ + send_fdserver_msg(client_sock, FD_LOOKUP_NACK, context, key, + -1); + break; + + case FD_DEREGISTER_REQ: + if (context >= FD_SRV_CTX_END) { + ODP_ERR("invalid deregister context\n"); + send_fdserver_msg(client_sock, FD_DEREGISTER_NACK, + FD_SRV_CTX_NA, 0, -1); + return 0; + } + + /* search key in table and remove it if found, and reply: */ + for (i = 0; i < fd_table_nb_entries; i++) { + if ((fd_table[i].context == context) && + (fd_table[i].key == key)) { + ODP_DBG("drop {ctx=%d," + " key=%" PRIu64 "}->fd=%d\n", + context, key, fd_table[i].fd); + close(fd_table[i].fd); + fd_table[i] = fd_table[--fd_table_nb_entries]; + send_fdserver_msg(client_sock, + FD_DEREGISTER_ACK, + context, key, -1); + return 0; + } + } + + /* key not found... send nack */ + send_fdserver_msg(client_sock, FD_DEREGISTER_NACK, + context, key, -1); + break; + + case FD_SERVERSTOP_REQ: + ODP_DBG("Stoping FD server\n"); + return 1; + + default: + ODP_ERR("Unexpected request\n"); + break; + } + return 0; +} + +/* + * server function + * loop forever, handling client requests one by one + */ +static void wait_requests(int sock) +{ + int c_socket; /* client connection */ + unsigned int addr_sz; + struct sockaddr_un remote; + + for (;;) { + addr_sz = sizeof(remote); + c_socket = accept(sock, (struct sockaddr *)&remote, &addr_sz); + if (c_socket == -1) { + ODP_ERR("wait_requests: %s\n", strerror(errno)); + return; + } + + if (handle_request(c_socket)) + break; + close(c_socket); + } + close(c_socket); +} + +/* + * Create a unix domain socket and fork a process to listen to incoming + * requests. + */ +int _odp_fdserver_init_global(void) +{ + char sockpath[FDSERVER_SOCKPATH_MAXLEN]; + int sock; + struct sockaddr_un local; + pid_t server_pid; + int res; + + /* create the client spinlock that any client can see: */ + client_lock = mmap(NULL, sizeof(odp_spinlock_t), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + + odp_spinlock_init(client_lock); + + /* construct the server named socket path: */ + snprintf(sockpath, FDSERVER_SOCKPATH_MAXLEN, FDSERVER_SOCKPATH_FORMAT, + odp_global_data.main_pid); + + /* create UNIX domain socket: */ + sock = socket(AF_UNIX, SOCK_STREAM, 0); + if (sock == -1) { + ODP_ERR("_odp_fdserver_init_global: %s\n", strerror(errno)); + return(-1); + } + + /* remove previous named socket if it already exists: */ + unlink(sockpath); + + /* bind to new named socket: */ + local.sun_family = AF_UNIX; + strncpy(local.sun_path, sockpath, sizeof(local.sun_path)); + res = bind(sock, (struct sockaddr *)&local, sizeof(struct sockaddr_un)); + if (res == -1) { + ODP_ERR("_odp_fdserver_init_global: %s\n", strerror(errno)); + close(sock); + return(-1); + } + + /* listen for incoming conections: */ + if (listen(sock, FDSERVER_BACKLOG) == -1) { + ODP_ERR("_odp_fdserver_init_global: %s\n", strerror(errno)); + close(sock); + return(-1); + } + + /* fork a server process: */ + server_pid = fork(); + if (server_pid == -1) { + ODP_ERR("Could not fork!\n"); + close(sock); + return(-1); + } + + if (server_pid == 0) { /*child */ + /* TODO: pin the server on appropriate service cpu mask */ + /* when (if) we can agree on the usage of service mask */ + + wait_requests(sock); /* Returns when server is stopped */ + close(sock); + exit(0); + } + + /* parent */ + close(sock); + return 0; +} + +/* + * Terminate the server + */ +int _odp_fdserver_term_global(void) +{ + int status; + char sockpath[FDSERVER_SOCKPATH_MAXLEN]; + + /* close the server and wait for child terminaison*/ + stop_server(); + wait(&status); + + /* construct the server named socket path: */ + snprintf(sockpath, FDSERVER_SOCKPATH_MAXLEN, FDSERVER_SOCKPATH_FORMAT, + odp_global_data.main_pid); + + /* delete the UNIX domain socket: */ + unlink(sockpath); + + return 0; +} diff --git a/platform/linux-generic/include/_fdserver_internal.h b/platform/linux-generic/include/_fdserver_internal.h new file mode 100644 index 0000000..480ac02 --- /dev/null +++ b/platform/linux-generic/include/_fdserver_internal.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _FD_SERVER_INTERNAL_H +#define _FD_SERVER_INTERNAL_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * the following enum defines the different contextes by which the + * FD server may be used: In the FD server, the keys used to store/retrieve + * a file descriptor are actually context based: + * Both the context and the key are stored at fd registration time, + * and both the context and the key are used to retrieve a fd. + * In other words a context identifies a FD server usage, so that different + * unrelated fd server users do not have to guarantee key unicity between + * them. + */ +typedef enum fd_server_context { + FD_SRV_CTX_NA, /* Not Applicable */ + FD_SRV_CTX_END, /* upper enum limit */ +} fd_server_context_e; + +int _odp_fdserver_register_fd(fd_server_context_e context, uint64_t key, + int fd); +int _odp_fdserver_deregister_fd(fd_server_context_e context, uint64_t key); +int _odp_fdserver_lookup_fd(fd_server_context_e context, uint64_t key); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index 8bad450..fd770b5 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -54,6 +54,7 @@ enum init_stage { CPUMASK_INIT, TIME_INIT, SYSINFO_INIT, + FDSERVER_INIT, SHM_INIT, THREAD_INIT, POOL_INIT, @@ -119,6 +120,9 @@ int odp_tm_term_global(void); int _odp_int_name_tbl_init_global(void); int _odp_int_name_tbl_term_global(void);

+int _odp_fdserver_init_global(void); +int _odp_fdserver_term_global(void); + int cpuinfo_parser(FILE *file, system_info_t *sysinfo); uint64_t odp_cpu_hz_current(int id);

diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index f534759..ec84fd0 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -51,6 +51,12 @@ int odp_init_global(odp_instance_t *instance, } stage = SYSINFO_INIT;

+ if (_odp_fdserver_init_global()) { + ODP_ERR("ODP fdserver init failed.\n"); + goto init_failed; + } + stage = FDSERVER_INIT; + if (odp_shm_init_global()) { ODP_ERR("ODP shm init failed.\n"); goto init_failed; @@ -218,6 +224,13 @@ int _odp_term_global(enum init_stage stage) } /* Fall through */

+ case FDSERVER_INIT: + if (_odp_fdserver_term_global()) { + ODP_ERR("ODP fdserver term failed.\n"); + rc = -1; + } + /* Fall through */ + case SYSINFO_INIT: if (odp_system_info_term()) { ODP_ERR("ODP system info term failed.\n");

commit 4871852cc509ca998802616fadfbb5619e3324da Author: Christophe Milard christophe.milard@linaro.org Date: Sat Aug 20 09:45:55 2016 +0200

drv: adding shared memory

The public definition of the shared memory allocator for the driver interface is added.

Signed-off-by: Christophe Milard christophe.milard@linaro.org Reviewed-and-tested-by: Bill Fischofer bill.fischofer@linaro.org Reviewed-by: Brian Brooks brian.brooks@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/include/odp/drv/spec/shm.h b/include/odp/drv/spec/shm.h new file mode 100644 index 0000000..ef64f5d --- /dev/null +++ b/include/odp/drv/spec/shm.h @@ -0,0 +1,231 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODPRDV shared memory (shm) + */ + +#ifndef ODPDRV_SHM_H_ +#define ODPDRV_SHM_H_ +#include <odp/visibility_begin.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** @addtogroup odpdrv_shm ODPDRV SHARED MEMORY + * Operations on driver shared memory. + * @{ + */ + +/** + * @typedef odpdrv_shm_t + * odpdrv shared memory block + */ + +/** + * @def ODPDRV_SHM_INVALID + * Invalid odpdrv shared memory block + */ + +/** Maximum shared memory block name length in chars */ +#define ODPDRV_SHM_NAME_LEN 32 + +/* + * Shared memory flags + */ +#define ODPDRV_SHM_SINGLE_VA 0x01 /**< Memory shall be mapped at same VA */ +#define ODPDRV_SHM_LOCK 0x02 /**< Memory shall be locked (no swap) */ + +/** + * Shared memory block info + */ +typedef struct odpdrv_shm_info_t { + const char *name; /**< Block name */ + void *addr; /**< Block address */ + uint64_t size; /**< Block size in bytes */ + uint64_t page_size; /**< Memory page size */ + uint32_t flags; /**< ODPDRV_SHM_* flags */ +} odpdrv_shm_info_t; + +/** + * Shared memory capabilities + */ +typedef struct odpdrv_shm_capability_t { + /** Maximum number of shared memory blocks + * + * This number of separate shared memory blocks can be + * reserved concurrently. */ + unsigned max_blocks; + + /** Maximum memory block size in bytes + * + * The value of zero means that size is limited only by the available + * memory size. */ + uint64_t max_size; + + /** Maximum memory block alignment in bytes + * + * The value of zero means that alignment is limited only by the + * available memory size. */ + uint64_t max_align; + +} odpdrv_shm_capability_t; + +/** + * Query shared memory capabilities + * + * Outputs shared memory capabilities on success. + * + * @param[out] capa Pointer to capability structure for output + * + * @retval 0 on success + * @retval <0 on failure + */ +int odpdrv_shm_capability(odpdrv_shm_capability_t *capa); + +/** + * Reserve a contiguous block of shared memory + * + * @param[in] name Name of the block (maximum ODPDRV_SHM_NAME_LEN - 1 chars) + * @param[in] size Block size in bytes + * @param[in] align Block alignment in bytes + * @param[in] flags Shared memory parameter flags (ODPDRV_SHM_*). + * Default value is 0. + * + * @return Handle of the reserved block + * @retval ODPDRV_SHM_INVALID on failure + */ +odpdrv_shm_t odpdrv_shm_reserve(const char *name, uint64_t size, uint64_t align, + uint32_t flags); + +/** + * Free a contiguous block of shared memory + * + * Frees a previously reserved block of shared memory (found by its handle). + * @note Freeing memory that is in use will result in UNDEFINED behavior + * + * @param[in] shm odpdrv_shm Block handle + * + * @retval 0 on success + * @retval <0 on failure + */ +int odpdrv_shm_free_by_handle(odpdrv_shm_t shm); + +/** + * Free a contiguous block of shared memory (found from its name) + * + * Frees a previously reserved block of shared memory. + * @note Freeing memory that is in use will result in UNDEFINED behavior + * + * @param[in] name odpdrv_shm Block name + * + * @retval 0 on success + * @retval <0 on failure + */ +int odpdrv_shm_free_by_name(const char *name); + +/** + * Free a contiguous block of shared memory (found from its address) + * + * Frees a previously reserved block of shared memory. + * @note Freeing memory that is in use will result in UNDEFINED behavior + * + * @param[in] address odpdrv_shm Block address + * + * @retval 0 on success + * @retval <0 on failure + */ +int odpdrv_shm_free_by_address(void *address); + +/** + * Lookup and map a block of shared memory (identified by its handle) + * + * @param[in] shm odpdrv_shm Block handle + * + * @return The address of the newly mapped block. + * @retval NULL on failure + */ +void *odpdrv_shm_lookup_by_handle(odpdrv_shm_t shm); + +/** + * Lookup and map a block of shared memory (identified by its name) + * + * @param[in] name odpdrv_shm Block name + * + * @return The handle of the newly mapped block. + * @retval ODPDRV_SHM_INVALID on failure + */ +odpdrv_shm_t odpdrv_shm_lookup_by_name(const char *name); + +/** + * Lookup and map a block of shared memory (identified by its address) + * + * @note This only works when the flag ODPDRV_SHM_SINGLE_VA was set, + * as otherwise addresses are odp-thread local and hence meaningless to + * identify the block between odp-threads. + * + * @param[in] address odpdrv_shm Block address + * + * @return The handle of the newly mapped block. + * @retval ODPDRV_SHM_INVALID on failure + */ +odpdrv_shm_t odpdrv_shm_lookup_by_address(void *address); + +/** + * Get a Shared memory block address + * + * @param[in] shm odpdrv_shm Block handle + * + * @return Memory block address + * @retval NULL on failure + */ +void *odpdrv_shm_addr(odpdrv_shm_t shm); + +/** + * Shared memory block info + * + * @param[in] shm Odpdrv_shm block handle + * @param[out] info Block info pointer for output + * + * @retval 0 on success + * @retval <0 on failure + */ +int odpdrv_shm_info(odpdrv_shm_t shm, odpdrv_shm_info_t *info); + +/** + * Print all shared memory blocks and returns the number of allocated blocks. + * This function is meant for debug. + * @param title A string to be printed before the shared memory status + * @return The total number of allocated blocks + */ +int odpdrv_shm_print_all(const char *title); + +/** + * Get printable value for an odpdrv_shm_t + * + * @param hdl odpdrv_shm_t handle to be printed + * @return uint64_t value that can be used to print/display this + * handle + * + * @note This routine is intended to be used for diagnostic purposes + * to enable applications to generate a printable value that represents + * an odpdrv_shm_t handle. + */ +uint64_t odpdrv_shm_to_u64(odpdrv_shm_t hdl); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#include <odp/visibility_end.h> +#endif diff --git a/platform/Makefile.inc b/platform/Makefile.inc index 6ade6df..7c53d1d 100644 --- a/platform/Makefile.inc +++ b/platform/Makefile.inc @@ -68,6 +68,7 @@ odpdrvspecinclude_HEADERS = \ $(top_srcdir)/include/odp/drv/spec/barrier.h \ $(top_srcdir)/include/odp/drv/spec/byteorder.h \ $(top_srcdir)/include/odp/drv/spec/compiler.h \ + $(top_srcdir)/include/odp/drv/spec/shm.h \ $(top_srcdir)/include/odp/drv/spec/spinlock.h \ $(top_srcdir)/include/odp/drv/spec/std_types.h \ $(top_srcdir)/include/odp/drv/spec/sync.h

commit 48ae68961873bb818a75aa8aa1b93d697fe39517 Author: Christophe Milard christophe.milard@linaro.org Date: Sat Aug 20 09:45:54 2016 +0200

linux-generic: drv: strong typing

Mirrored from its couterpart on the api interface, the drv interface gets its strong typing file.

Signed-off-by: Christophe Milard christophe.milard@linaro.org Reviewed-and-tested-by: Bill Fischofer bill.fischofer@linaro.org Reviewed-by: Brian Brooks brian.brooks@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 42e9996..271f5b5 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -108,7 +108,8 @@ odpdrvplatinclude_HEADERS = \ $(srcdir)/include/odp/drv/plat/atomic_types.h \ $(srcdir)/include/odp/drv/plat/barrier_types.h \ $(srcdir)/include/odp/drv/plat/byteorder_types.h \ - $(srcdir)/include/odp/drv/plat/spinlock_types.h + $(srcdir)/include/odp/drv/plat/spinlock_types.h \ + $(srcdir)/include/odp/drv/plat/strong_types.h

noinst_HEADERS = \ ${srcdir}/include/odp_align_internal.h \ diff --git a/platform/linux-generic/include/odp/drv/plat/strong_types.h b/platform/linux-generic/include/odp/drv/plat/strong_types.h new file mode 100644 index 0000000..a9c48ef --- /dev/null +++ b/platform/linux-generic/include/odp/drv/plat/strong_types.h @@ -0,0 +1,35 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODPDRV Strong Types. Common macros for implementing strong typing + * for ODPDRV abstract data types + */ + +#ifndef ODPDRV_STRONG_TYPES_H_ +#define ODPDRV_STRONG_TYPES_H_ + +/** Use strong typing for ODPDRV types */ +#ifdef __cplusplus +#define ODPDRV_HANDLE_T(type) struct _##type { uint8_t unused_dummy_var; } *type +#else +#define odpdrv_handle_t struct { uint8_t unused_dummy_var; } * +/** C/C++ helper macro for strong typing */ +#define ODPDRV_HANDLE_T(type) odpdrv_handle_t type +#endif + +/** Internal macro to get value of an ODPDRV handle */ +#define _odpdrv_typeval(handle) ((uint32_t)(uintptr_t)(handle)) + +/** Internal macro to get printable value of an ODPDRV handle */ +#define _odpdrv_pri(handle) ((uint64_t)_odpdrv_typeval(handle)) + +/** Internal macro to convert a scalar to a typed handle */ +#define _odpdrv_cast_scalar(type, val) ((type)(uintptr_t)(val)) + +#endif

commit 85c4f0bd3f18900e484b0e71670fc18c555e75b1 Author: Christophe Milard christophe.milard@linaro.org Date: Sat Aug 20 09:45:53 2016 +0200

linux-gen: drv: adding barrier

Based on API interface files.

Signed-off-by: Christophe Milard christophe.milard@linaro.org Reviewed-and-tested-by: Bill Fischofer bill.fischofer@linaro.org Reviewed-by: Brian Brooks brian.brooks@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/include/odp_drv.h b/include/odp_drv.h index 776cf12..31c620e 100644 --- a/include/odp_drv.h +++ b/include/odp_drv.h @@ -20,6 +20,7 @@ extern C {

#include <odp/drv/align.h> #include <odp/drv/atomic.h> +#include <odp/drv/barrier.h> #include <odp/drv/byteorder.h> #include <odp/drv/compiler.h> #include <odp/drv/spinlock.h> diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 747e96f..42e9996 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -96,6 +96,7 @@ odpdrvincludedir = $(includedir)/odp/drv odpdrvinclude_HEADERS = \ $(srcdir)/include/odp/drv/align.h \ $(srcdir)/include/odp/drv/atomic.h \ + $(srcdir)/include/odp/drv/barrier.h \ $(srcdir)/include/odp/drv/byteorder.h \ $(srcdir)/include/odp/drv/compiler.h \ $(srcdir)/include/odp/drv/spinlock.h \ @@ -105,6 +106,7 @@ odpdrvinclude_HEADERS = \ odpdrvplatincludedir = $(includedir)/odp/drv/plat odpdrvplatinclude_HEADERS = \ $(srcdir)/include/odp/drv/plat/atomic_types.h \ + $(srcdir)/include/odp/drv/plat/barrier_types.h \ $(srcdir)/include/odp/drv/plat/byteorder_types.h \ $(srcdir)/include/odp/drv/plat/spinlock_types.h

@@ -204,6 +206,7 @@ __LIB__libodp_linux_la_SOURCES = \ odp_version.c \ odp_weak.c \ drv_atomic.c \ + drv_barrier.c \ drv_spinlock.c \ arch/@ARCH_DIR@/odp_cpu_arch.c \ arch/@ARCH_DIR@/odp_sysinfo_parse.c diff --git a/platform/linux-generic/drv_barrier.c b/platform/linux-generic/drv_barrier.c new file mode 100644 index 0000000..7a83981 --- /dev/null +++ b/platform/linux-generic/drv_barrier.c @@ -0,0 +1,50 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/drv/barrier.h> +#include <odp/drv/sync.h> +#include <odp/api/cpu.h> +#include <odp/drv/atomic.h> + +void odpdrv_barrier_init(odpdrv_barrier_t *barrier, int count) +{ + barrier->count = (uint32_t)count; + odpdrv_atomic_init_u32(&barrier->bar, 0); +} + +/* + * Efficient barrier_sync - + * + * Barriers are initialized with a count of the number of callers + * that must sync on the barrier before any may proceed. + * + * To avoid race conditions and to permit the barrier to be fully + * reusable, the barrier value cycles between 0..2*count-1. When + * synchronizing the wasless variable simply tracks which half of + * the cycle the barrier was in upon entry. Exit is when the + * barrier crosses to the other half of the cycle. + */ +void odpdrv_barrier_wait(odpdrv_barrier_t *barrier) +{ + uint32_t count; + int wasless; + + odpdrv_mb_full(); + + count = odpdrv_atomic_fetch_inc_u32(&barrier->bar); + wasless = count < barrier->count; + + if (count == 2 * barrier->count - 1) { + /* Wrap around *atomically* */ + odpdrv_atomic_sub_u32(&barrier->bar, 2 * barrier->count); + } else { + while ((odpdrv_atomic_load_u32(&barrier->bar) < barrier->count) + == wasless) + odp_cpu_pause(); + } + + odpdrv_mb_full(); +} diff --git a/platform/linux-generic/include/odp/drv/barrier.h b/platform/linux-generic/include/odp/drv/barrier.h new file mode 100644 index 0000000..877cacf --- /dev/null +++ b/platform/linux-generic/include/odp/drv/barrier.h @@ -0,0 +1,30 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODPDRV execution barriers + */ + +#ifndef ODPDRV_PLAT_BARRIER_H_ +#define ODPDRV_PLAT_BARRIER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/drv/std_types.h> +#include <odp/drv/atomic.h> +#include <odp/drv/plat/barrier_types.h> + +#include <odp/drv/spec/barrier.h> + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp/drv/plat/barrier_types.h b/platform/linux-generic/include/odp/drv/plat/barrier_types.h new file mode 100644 index 0000000..32264a1 --- /dev/null +++ b/platform/linux-generic/include/odp/drv/plat/barrier_types.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODPDRV barrier + */ + +#ifndef ODPDRV_BARRIER_TYPES_H_ +#define ODPDRV_BARRIER_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/drv/std_types.h> +#include <odp/drv/atomic.h> + +/** + * @internal + * ODPDRV thread synchronization barrier + */ +struct odpdrv_barrier_s { + uint32_t count; /**< Thread count */ + odpdrv_atomic_u32_t bar; /**< Barrier counter */ +}; + +typedef struct odpdrv_barrier_s odpdrv_barrier_t; + +#ifdef __cplusplus +} +#endif + +#endif

commit 57962f1b9c3a5a41a7dd6146f9666da18e62ec9c Author: Christophe Milard christophe.milard@linaro.org Date: Sat Aug 20 09:45:52 2016 +0200

drv: adding barrier.h

Based on API interface file.

Signed-off-by: Christophe Milard christophe.milard@linaro.org Reviewed-and-tested-by: Bill Fischofer bill.fischofer@linaro.org Reviewed-by: Brian Brooks brian.brooks@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/include/odp/drv/spec/barrier.h b/include/odp/drv/spec/barrier.h new file mode 100644 index 0000000..5ac65a3 --- /dev/null +++ b/include/odp/drv/spec/barrier.h @@ -0,0 +1,66 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODPDRV execution barriers + */ + +#ifndef ODPDRV_API_BARRIER_H_ +#define ODPDRV_API_BARRIER_H_ +#include <odp/visibility_begin.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @defgroup odpdrv_barrier ODPDRV BARRIER + * Thread execution and memory ordering barriers. + * + * @details + * <b> Thread execution barrier (odpdrv_barrier_t) </b> + * + * Thread execution barrier synchronizes a group of threads to wait on the + * barrier until the entire group has reached the barrier. + * @{ + */ + +/** + * @typedef odpdrv_barrier_t + * ODPDRV thread synchronization barrier + */ + +/** + * Initialize barrier with thread count. + * + * @param barr Pointer to a barrier variable + * @param count Thread count + */ +void odpdrv_barrier_init(odpdrv_barrier_t *barr, int count); + +/** + * Synchronize thread execution on barrier. + * Wait for all threads to arrive at the barrier until they are let loose again. + * Threads will block (spin) until the last thread has arrived at the barrier. + * All memory operations before the odpdrv_barrier_wait() call will be visible + * to all threads when they leave the barrier. + * + * @param barr Pointer to a barrier variable + */ +void odpdrv_barrier_wait(odpdrv_barrier_t *barr); + +/** + * @} + */ + +#ifdef __cplusplus +} +#endif + +#include <odp/visibility_end.h> +#endif diff --git a/platform/Makefile.inc b/platform/Makefile.inc index 079f34d..6ade6df 100644 --- a/platform/Makefile.inc +++ b/platform/Makefile.inc @@ -65,6 +65,7 @@ odpdrvspecincludedir= $(includedir)/odp/drv/spec odpdrvspecinclude_HEADERS = \ $(top_srcdir)/include/odp/drv/spec/align.h \ $(top_srcdir)/include/odp/drv/spec/atomic.h \ + $(top_srcdir)/include/odp/drv/spec/barrier.h \ $(top_srcdir)/include/odp/drv/spec/byteorder.h \ $(top_srcdir)/include/odp/drv/spec/compiler.h \ $(top_srcdir)/include/odp/drv/spec/spinlock.h \

commit 9a45802c94f58d9dd443c10e6dc204a58fa0f9ce Author: Christophe Milard christophe.milard@linaro.org Date: Sat Aug 20 09:45:51 2016 +0200

linux-gen: cosmetic changes on barrier

To please check-patch before the copy to the drv interface.

Signed-off-by: Christophe Milard christophe.milard@linaro.org Reviewed-and-tested-by: Bill Fischofer bill.fischofer@linaro.org Reviewed-by: Brian Brooks brian.brooks@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/include/odp/api/spec/barrier.h b/include/odp/api/spec/barrier.h index 678d39a..6de683c 100644 --- a/include/odp/api/spec/barrier.h +++ b/include/odp/api/spec/barrier.h @@ -4,7 +4,6 @@ * SPDX-License-Identifier: BSD-3-Clause */

- /** * @file * @@ -44,7 +43,6 @@ extern "C" { */ void odp_barrier_init(odp_barrier_t *barr, int count);

- /** * Synchronize thread execution on barrier. * Wait for all threads to arrive at the barrier until they are let loose again. diff --git a/platform/linux-generic/include/odp/api/plat/barrier_types.h b/platform/linux-generic/include/odp/api/plat/barrier_types.h index 440275e..00b383c 100644 --- a/platform/linux-generic/include/odp/api/plat/barrier_types.h +++ b/platform/linux-generic/include/odp/api/plat/barrier_types.h @@ -4,7 +4,6 @@ * SPDX-License-Identifier: BSD-3-Clause */

- /** * @file * diff --git a/platform/linux-generic/odp_barrier.c b/platform/linux-generic/odp_barrier.c index ef10f29..a2c6267 100644 --- a/platform/linux-generic/odp_barrier.c +++ b/platform/linux-generic/odp_barrier.c @@ -37,7 +37,7 @@ void odp_barrier_wait(odp_barrier_t *barrier) count = odp_atomic_fetch_inc_u32(&barrier->bar); wasless = count < barrier->count;

- if (count == 2*barrier->count-1) { + if (count == 2 * barrier->count - 1) { /* Wrap around *atomically* */ odp_atomic_sub_u32(&barrier->bar, 2 * barrier->count); } else {

-----------------------------------------------------------------------

Summary of changes: include/odp/api/spec/barrier.h | 2 - include/odp/{api => drv}/spec/barrier.h | 24 +- include/odp/drv/spec/shm.h | 231 ++++ include/odp_drv.h | 2 + platform/Makefile.inc | 2 + platform/linux-generic/Makefile.am | 15 +- platform/linux-generic/_fdserver.c | 655 +++++++++ platform/linux-generic/_ishm.c | 1460 ++++++++++++++++++++ platform/linux-generic/_ishmphy.c | 185 +++ platform/linux-generic/arch/arm/odp/api/cpu_arch.h | 25 +- platform/linux-generic/arch/arm/odp_cpu_arch.c | 49 +- .../linux-generic/arch/arm/odp_sysinfo_parse.c | 28 +- platform/linux-generic/arch/powerpc/odp_cpu_arch.c | 49 +- .../linux-generic/{odp_barrier.c => drv_barrier.c} | 26 +- platform/linux-generic/drv_shm.c | 102 ++ .../linux-generic/include/_fdserver_internal.h | 39 + platform/linux-generic/include/_ishm_internal.h | 46 + platform/linux-generic/include/_ishmphy_internal.h | 25 + platform/linux-generic/include/ishmphy_internal.h | 24 + .../include/odp/api/plat/barrier_types.h | 1 - platform/linux-generic/include/odp/drv/barrier.h | 30 + .../include/odp/drv/plat/barrier_types.h | 38 + .../linux-generic/include/odp/drv/plat/shm_types.h | 46 + .../include/odp/drv/plat/strong_types.h | 35 + .../include/odp/drv/{compiler.h => shm.h} | 12 +- .../linux-generic/include/odp_config_internal.h | 16 + platform/linux-generic/include/odp_internal.h | 17 +- platform/linux-generic/odp_barrier.c | 2 +- platform/linux-generic/odp_init.c | 39 + platform/linux-generic/odp_system_info.c | 181 ++- test/common_plat/m4/configure.m4 | 3 +- test/common_plat/validation/drv/Makefile.am | 3 +- .../common_plat/validation/drv/drvshmem/.gitignore | 1 + .../validation/drv/drvshmem/Makefile.am | 10 + .../common_plat/validation/drv/drvshmem/drvshmem.c | 794 +++++++++++ .../common_plat/validation/drv/drvshmem/drvshmem.h | 27 + .../drvatomic_main.c => drvshmem/drvshmem_main.c} | 4 +- test/linux-generic/Makefile.am | 1 + 38 files changed, 4201 insertions(+), 48 deletions(-) copy include/odp/{api => drv}/spec/barrier.h (65%) create mode 100644 include/odp/drv/spec/shm.h create mode 100644 platform/linux-generic/_fdserver.c create mode 100644 platform/linux-generic/_ishm.c create mode 100644 platform/linux-generic/_ishmphy.c mode change 120000 => 100644 platform/linux-generic/arch/arm/odp/api/cpu_arch.h mode change 120000 => 100644 platform/linux-generic/arch/arm/odp_cpu_arch.c mode change 120000 => 100644 platform/linux-generic/arch/arm/odp_sysinfo_parse.c mode change 120000 => 100644 platform/linux-generic/arch/powerpc/odp_cpu_arch.c copy platform/linux-generic/{odp_barrier.c => drv_barrier.c} (59%) create mode 100644 platform/linux-generic/drv_shm.c create mode 100644 platform/linux-generic/include/_fdserver_internal.h create mode 100644 platform/linux-generic/include/_ishm_internal.h create mode 100644 platform/linux-generic/include/_ishmphy_internal.h create mode 100644 platform/linux-generic/include/ishmphy_internal.h create mode 100644 platform/linux-generic/include/odp/drv/barrier.h create mode 100644 platform/linux-generic/include/odp/drv/plat/barrier_types.h create mode 100644 platform/linux-generic/include/odp/drv/plat/shm_types.h create mode 100644 platform/linux-generic/include/odp/drv/plat/strong_types.h copy platform/linux-generic/include/odp/drv/{compiler.h => shm.h} (56%) create mode 100644 test/common_plat/validation/drv/drvshmem/.gitignore create mode 100644 test/common_plat/validation/drv/drvshmem/Makefile.am create mode 100644 test/common_plat/validation/drv/drvshmem/drvshmem.c create mode 100644 test/common_plat/validation/drv/drvshmem/drvshmem.h copy test/common_plat/validation/drv/{drvatomic/drvatomic_main.c => drvshmem/drvshmem_main.c} (72%)

hooks/post-receive