This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "".
The branch, master has been updated via 939166d4414609717264e6f51f51d4b22cdd022a (commit) via 5fd772d236db2059935d12ab27fcbf1f50d10591 (commit) via 0b9d1e85d1f4d9b98c88c4d34fb26bc801556210 (commit) from 12eaa3d2ce51cc5cd48e54deded2740161162022 (commit)
Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below.
- Log ----------------------------------------------------------------- commit 939166d4414609717264e6f51f51d4b22cdd022a Author: Matias Elo matias.elo@nokia.com Date: Wed Apr 10 08:45:32 2019 +0300
linux-gen: pool: warn about entire pool fitting into thread local caches
Add debug print when entire pool fits into thread local caches. Pool starvation may occur if the pool is used by multiple threads.
Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Petri Savolainen petri.savolainen@nokia.com Suggested-by: Mikko Kurikka mikko.kurikka@nokia.com
diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index e3f53cfaa..832991d32 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -681,12 +681,22 @@ error: static int check_params(odp_pool_param_t *params) { odp_pool_capability_t capa; + odp_bool_t cache_warning = false; + uint32_t cache_size = pool_tbl->config.local_cache_size; + int num_threads = odp_global_ro.init_param.num_control + + odp_global_ro.init_param.num_worker;
if (!params || odp_pool_capability(&capa) < 0) return -1;
+ if (num_threads) + cache_size = num_threads * pool_tbl->config.local_cache_size; + switch (params->type) { case ODP_POOL_BUFFER: + if (params->buf.num <= cache_size) + cache_warning = true; + if (params->buf.num > capa.buf.max_num) { ODP_ERR("buf.num too large %u\n", params->buf.num); return -1; @@ -705,6 +715,9 @@ static int check_params(odp_pool_param_t *params) break;
case ODP_POOL_PACKET: + if (params->pkt.num <= cache_size) + cache_warning = true; + if (params->pkt.num > capa.pkt.max_num) { ODP_ERR("pkt.num too large %u\n", params->pkt.num); return -1; @@ -748,6 +761,9 @@ static int check_params(odp_pool_param_t *params) break;
case ODP_POOL_TIMEOUT: + if (params->tmo.num <= cache_size) + cache_warning = true; + if (params->tmo.num > capa.tmo.max_num) { ODP_ERR("tmo.num too large %u\n", params->tmo.num); return -1; @@ -759,6 +775,11 @@ static int check_params(odp_pool_param_t *params) return -1; }
+ if (cache_warning) + ODP_DBG("Entire pool fits into thread local caches. Pool " + "starvation may occur if the pool is used by multiple " + "threads.\n"); + return 0; }
commit 5fd772d236db2059935d12ab27fcbf1f50d10591 Author: Matias Elo matias.elo@nokia.com Date: Tue Apr 9 11:22:57 2019 +0300
linux-gen: pool: enable setting local cache and burst size using config
Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Petri Savolainen petri.savolainen@nokia.com
diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index e00e4a518..fcc5d75aa 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -16,7 +16,7 @@
# Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.6" +config_file_version = "0.1.7"
# Shared memory options shm: { @@ -45,6 +45,16 @@ shm: {
# Pool options pool: { + # Thread local cache size. Value must be a multiple of burst_size + # (min 2 x burst_size). + # + # The total maximum number of cached events is the number of threads + # using the pool multiplied with local_cache_size. + local_cache_size = 256 + + # Transfer size between local cache and global pool. + burst_size = 32 + # Packet pool options pkt: { # Maximum number of packets per pool. Power of two minus one diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 810576b9a..27f1db599 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -142,7 +142,7 @@ extern "C" { /* * Maximum number of events in a thread local pool cache */ -#define CONFIG_POOL_CACHE_SIZE 256 +#define CONFIG_POOL_CACHE_MAX_SIZE 256
#ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index a5242d60e..67d7e7b7b 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -27,8 +27,11 @@ extern "C" { #include <odp/api/plat/strong_types.h>
typedef struct ODP_ALIGNED_CACHE pool_cache_t { + uint32_t size; /* Size of cache */ + uint32_t burst_size; /* Cache burst size */ uint32_t num; /* Number of buffers in cache */ - odp_buffer_hdr_t *buf_hdr[CONFIG_POOL_CACHE_SIZE]; /* Cached buffers */ + /* Cached buffers */ + odp_buffer_hdr_t *buf_hdr[CONFIG_POOL_CACHE_MAX_SIZE];
} pool_cache_t;
@@ -92,6 +95,8 @@ typedef struct pool_table_t {
struct { uint32_t pkt_max_num; + uint32_t local_cache_size; + uint32_t burst_size; } config;
} pool_table_t; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 2e0898461..e3f53cfaa 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -36,8 +36,7 @@ #define UNLOCK(a) odp_ticketlock_unlock(a) #define LOCK_INIT(a) odp_ticketlock_init(a)
-#define CACHE_BURST 32 -#define RING_SIZE_MIN (2 * CACHE_BURST) +#define RING_SIZE_MIN 64 #define POOL_MAX_NUM_MIN RING_SIZE_MIN
/* Make sure packet buffers don't cross huge page boundaries starting from this @@ -51,9 +50,6 @@ * rounded up to this value. */ #define BUFFER_ALIGN_MIN ODP_CACHE_LINE_SIZE
-ODP_STATIC_ASSERT(CONFIG_POOL_CACHE_SIZE > (2 * CACHE_BURST), - "cache_burst_size_too_large_compared_to_cache_size"); - ODP_STATIC_ASSERT(CONFIG_PACKET_SEG_LEN_MIN >= 256, "ODP Segment size must be a minimum of 256 bytes");
@@ -94,6 +90,9 @@ static inline pool_t *pool_from_buf(odp_buffer_t buf) static inline void cache_init(pool_cache_t *cache) { memset(cache, 0, sizeof(pool_cache_t)); + + cache->size = pool_tbl->config.local_cache_size; + cache->burst_size = pool_tbl->config.burst_size; }
static inline uint32_t cache_pop(pool_cache_t *cache, @@ -150,6 +149,48 @@ static int read_config_file(pool_table_t *pool_tbl)
ODP_PRINT("Pool config:\n");
+ str = "pool.local_cache_size"; + if (!_odp_libconfig_lookup_int(str, &val)) { + ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + + if (val > CONFIG_POOL_CACHE_MAX_SIZE || val < 0) { + ODP_ERR("Bad value %s = %u, max %d\n", str, val, + CONFIG_POOL_CACHE_MAX_SIZE); + return -1; + } + + pool_tbl->config.local_cache_size = val; + ODP_PRINT(" %s: %i\n", str, val); + + str = "pool.burst_size"; + if (!_odp_libconfig_lookup_int(str, &val)) { + ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + + if (val <= 0) { + ODP_ERR("Bad value %s = %u\n", str, val); + return -1; + } + + pool_tbl->config.burst_size = val; + ODP_PRINT(" %s: %i\n", str, val); + + /* Check local cache size and burst size relation */ + if (pool_tbl->config.local_cache_size % pool_tbl->config.burst_size) { + ODP_ERR("Pool cache size not multiple of burst size\n"); + return -1; + } + + if (pool_tbl->config.local_cache_size && + (pool_tbl->config.local_cache_size / + pool_tbl->config.burst_size < 2)) { + ODP_ERR("Cache burst size too large compared to cache size\n"); + return -1; + } + str = "pool.pkt.max_num"; if (!_odp_libconfig_lookup_int(str, &val)) { ODP_ERR("Config option '%s' not found.\n", str); @@ -832,17 +873,18 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_num) odp_buffer_hdr_t *hdr; uint32_t mask, num_ch, i; uint32_t num_deq = 0; + uint32_t burst_size = cache->burst_size;
/* First pull packets from local cache */ num_ch = cache_pop(cache, buf_hdr, max_num);
/* If needed, get more from the global pool */ if (odp_unlikely(num_ch != (uint32_t)max_num)) { - uint32_t burst = CACHE_BURST; + uint32_t burst = burst_size; uint32_t cache_num;
num_deq = max_num - num_ch; - if (odp_unlikely(num_deq > CACHE_BURST)) + if (odp_unlikely(num_deq > burst_size)) burst = num_deq;
uint32_t data[burst]; @@ -891,10 +933,11 @@ static inline void buffer_free_to_pool(pool_t *pool, ring_t *ring; int i; uint32_t cache_num, mask; + uint32_t cache_size = cache->size;
/* Special case of a very large free. Move directly to * the global pool. */ - if (odp_unlikely(num > CONFIG_POOL_CACHE_SIZE)) { + if (odp_unlikely(num > (int)cache_size)) { uint32_t buf_index[num];
ring = &pool->ring->hdr; @@ -911,13 +954,13 @@ static inline void buffer_free_to_pool(pool_t *pool, * transfer. */ cache_num = cache->num;
- if (odp_unlikely((int)(CONFIG_POOL_CACHE_SIZE - cache_num) < num)) { - int burst = CACHE_BURST; + if (odp_unlikely((int)(cache_size - cache_num) < num)) { + int burst = cache->burst_size;
ring = &pool->ring->hdr; mask = pool->ring_mask;
- if (odp_unlikely(num > CACHE_BURST)) + if (odp_unlikely(num > burst)) burst = num; if (odp_unlikely((uint32_t)num > cache_num)) burst = cache_num; diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf index 6cae241da..36bc7f323 100644 --- a/platform/linux-generic/test/inline-timer.conf +++ b/platform/linux-generic/test/inline-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.6" +config_file_version = "0.1.7"
timer: { # Enable inline timer implementation diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf index fc8974944..12c79617f 100644 --- a/platform/linux-generic/test/process-mode.conf +++ b/platform/linux-generic/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.6" +config_file_version = "0.1.7"
# Shared memory options shm: {
commit 0b9d1e85d1f4d9b98c88c4d34fb26bc801556210 Author: Matias Elo matias.elo@nokia.com Date: Mon Apr 8 13:48:54 2019 +0300
linux-gen: pool: optimize buffer caching
Refactor local buffer caching into separate functions and optimize implementation by caching buffer header pointers instead of indices.
Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Petri Savolainen petri.savolainen@nokia.com
diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index cfd0872af..a5242d60e 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -1,4 +1,5 @@ -/* Copyright (c) 2013-2018, Linaro Limited +/* Copyright (c) 2019, Nokia + * Copyright (c) 2013-2018, Linaro Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -26,8 +27,8 @@ extern "C" { #include <odp/api/plat/strong_types.h>
typedef struct ODP_ALIGNED_CACHE pool_cache_t { - uint32_t num; - uint32_t buf_index[CONFIG_POOL_CACHE_SIZE]; + uint32_t num; /* Number of buffers in cache */ + odp_buffer_hdr_t *buf_hdr[CONFIG_POOL_CACHE_SIZE]; /* Cached buffers */
} pool_cache_t;
diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index feb25cf39..2e0898461 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -91,6 +91,58 @@ static inline pool_t *pool_from_buf(odp_buffer_t buf) return buf_hdr->pool_ptr; }
+static inline void cache_init(pool_cache_t *cache) +{ + memset(cache, 0, sizeof(pool_cache_t)); +} + +static inline uint32_t cache_pop(pool_cache_t *cache, + odp_buffer_hdr_t *buf_hdr[], int max_num) +{ + uint32_t cache_num = cache->num; + uint32_t num_ch = max_num; + uint32_t cache_begin; + uint32_t i; + + /* Cache does not have enough buffers */ + if (odp_unlikely(cache_num < (uint32_t)max_num)) + num_ch = cache_num; + + /* Get buffers from the cache */ + cache_begin = cache_num - num_ch; + for (i = 0; i < num_ch; i++) + buf_hdr[i] = cache->buf_hdr[cache_begin + i]; + + cache->num = cache_num - num_ch; + + return num_ch; +} + +static inline void cache_push(pool_cache_t *cache, odp_buffer_hdr_t *buf_hdr[], + uint32_t num) +{ + uint32_t cache_num = cache->num; + uint32_t i; + + for (i = 0; i < num; i++) + cache->buf_hdr[cache_num + i] = buf_hdr[i]; + + cache->num = cache_num + num; +} + +static void cache_flush(pool_cache_t *cache, pool_t *pool) +{ + odp_buffer_hdr_t *buf_hdr; + ring_t *ring; + uint32_t mask; + + ring = &pool->ring->hdr; + mask = pool->ring_mask; + + while (cache_pop(cache, &buf_hdr, 1)) + ring_enq(ring, mask, buf_hdr->index.buffer); +} + static int read_config_file(pool_table_t *pool_tbl) { const char *str; @@ -193,29 +245,13 @@ int odp_pool_init_local(void) for (i = 0; i < ODP_CONFIG_POOLS; i++) { pool = pool_entry(i); local.cache[i] = &pool->local_cache[thr_id]; - local.cache[i]->num = 0; + cache_init(local.cache[i]); }
local.thr_id = thr_id; return 0; }
-static void flush_cache(pool_cache_t *cache, pool_t *pool) -{ - ring_t *ring; - uint32_t mask; - uint32_t cache_num, i; - - ring = &pool->ring->hdr; - mask = pool->ring_mask; - cache_num = cache->num; - - for (i = 0; i < cache_num; i++) - ring_enq(ring, mask, cache->buf_index[i]); - - cache->num = 0; -} - int odp_pool_term_local(void) { int i; @@ -223,7 +259,7 @@ int odp_pool_term_local(void) for (i = 0; i < ODP_CONFIG_POOLS; i++) { pool_t *pool = pool_entry(i);
- flush_cache(local.cache[i], pool); + cache_flush(local.cache[i], pool); }
return 0; @@ -725,7 +761,7 @@ int odp_pool_destroy(odp_pool_t pool_hdl)
/* Make sure local caches are empty */ for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) - flush_cache(&pool->local_cache[i], pool); + cache_flush(&pool->local_cache[i], pool);
odp_shm_free(pool->shm);
@@ -791,40 +827,26 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info)
int buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_num) { + pool_cache_t *cache = local.cache[pool->pool_idx]; ring_t *ring; - uint32_t mask, i; - pool_cache_t *cache; - uint32_t cache_num, num_ch, num_deq, burst; odp_buffer_hdr_t *hdr; + uint32_t mask, num_ch, i; + uint32_t num_deq = 0;
- cache = local.cache[pool->pool_idx]; - - cache_num = cache->num; - num_ch = max_num; - num_deq = 0; - burst = CACHE_BURST; + /* First pull packets from local cache */ + num_ch = cache_pop(cache, buf_hdr, max_num);
- if (odp_unlikely(cache_num < (uint32_t)max_num)) { - /* Cache does not have enough buffers */ - num_ch = cache_num; - num_deq = max_num - cache_num; + /* If needed, get more from the global pool */ + if (odp_unlikely(num_ch != (uint32_t)max_num)) { + uint32_t burst = CACHE_BURST; + uint32_t cache_num;
+ num_deq = max_num - num_ch; if (odp_unlikely(num_deq > CACHE_BURST)) burst = num_deq; - } - - /* Get buffers from the cache */ - for (i = 0; i < num_ch; i++) { - uint32_t j = cache_num - num_ch + i; - - buf_hdr[i] = buf_hdr_from_index(pool, cache->buf_index[j]); - }
- /* Declare variable here to fix clang compilation bug */ - uint32_t data[burst]; + uint32_t data[burst];
- /* If needed, get more from the global pool */ - if (odp_unlikely(num_deq)) { /* Temporary copy to data[] needed since odp_buffer_t is * uintptr_t and not uint32_t. */ ring = &pool->ring->hdr; @@ -845,13 +867,18 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_num) buf_hdr[idx] = hdr; }
- /* Cache extra buffers. Cache is currently empty. */ - for (i = 0; i < cache_num; i++) - cache->buf_index[i] = data[num_deq + i]; + /* Cache possible extra buffers. Cache is currently empty. */ + if (cache_num) { + odp_buffer_hdr_t *buf_hdr[cache_num];
- cache->num = cache_num; - } else { - cache->num = cache_num - num_ch; + for (i = 0; i < cache_num; i++) { + uint32_t idx = num_deq + i; + + buf_hdr[i] = buf_hdr_from_index(pool, + data[idx]); + } + cache_push(cache, buf_hdr, cache_num); + } }
return num_ch + num_deq; @@ -860,13 +887,10 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int max_num) static inline void buffer_free_to_pool(pool_t *pool, odp_buffer_hdr_t *buf_hdr[], int num) { - int i; + pool_cache_t *cache = local.cache[pool->pool_idx]; ring_t *ring; - uint32_t mask; - pool_cache_t *cache; - uint32_t cache_num; - - cache = local.cache[pool->pool_idx]; + int i; + uint32_t cache_num, mask;
/* Special case of a very large free. Move directly to * the global pool. */ @@ -888,7 +912,6 @@ static inline void buffer_free_to_pool(pool_t *pool, cache_num = cache->num;
if (odp_unlikely((int)(CONFIG_POOL_CACHE_SIZE - cache_num) < num)) { - uint32_t index; int burst = CACHE_BURST;
ring = &pool->ring->hdr; @@ -899,26 +922,20 @@ static inline void buffer_free_to_pool(pool_t *pool, if (odp_unlikely((uint32_t)num > cache_num)) burst = cache_num;
- { - /* Temporary copy needed since odp_buffer_t is - * uintptr_t and not uint32_t. */ - uint32_t data[burst]; - - index = cache_num - burst; + /* Temporary copy needed since odp_buffer_t is + * uintptr_t and not uint32_t. */ + odp_buffer_hdr_t *buf_hdr[burst]; + uint32_t data[burst];
- for (i = 0; i < burst; i++) - data[i] = cache->buf_index[index + i]; + cache_pop(cache, buf_hdr, burst);
- ring_enq_multi(ring, mask, data, burst); - } + for (i = 0; i < burst; i++) + data[i] = buf_hdr[i]->index.buffer;
- cache_num -= burst; + ring_enq_multi(ring, mask, data, burst); }
- for (i = 0; i < num; i++) - cache->buf_index[cache_num + i] = buf_hdr[i]->index.buffer; - - cache->num = cache_num + num; + cache_push(cache, buf_hdr, num); }
void buffer_free_multi(odp_buffer_hdr_t *buf_hdr[], int num_total)
-----------------------------------------------------------------------
Summary of changes: config/odp-linux-generic.conf | 12 +- .../linux-generic/include/odp_config_internal.h | 2 +- platform/linux-generic/include/odp_pool_internal.h | 12 +- platform/linux-generic/odp_pool.c | 245 ++++++++++++++------- platform/linux-generic/test/inline-timer.conf | 2 +- platform/linux-generic/test/process-mode.conf | 2 +- 6 files changed, 186 insertions(+), 89 deletions(-)
hooks/post-receive