git-lng-odp

git-lng-odp@lists.linaro.org

1 participants
1207 discussions

[COMMIT] branch api-next updated. v1.10.1.0-114-g414c4c7

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via 414c4c75b0beb8cd4e4d2abe090921bcf058ec69 (commit) via 2b5e5bdfc8b650feba79ccd23c175fad16fd2b33 (commit) from 2c494b4a67123c838b4b6915679cca06f9e0bdd2 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 414c4c75b0beb8cd4e4d2abe090921bcf058ec69 Merge: 2c494b4 2b5e5bd Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Wed Jul 27 19:21:41 2016 +0300 Merge branch 'master' into api-next Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> ----------------------------------------------------------------------- Summary of changes: platform/linux-generic/arch/default/odp/api/cpu_arch.h | 10 +--------- platform/linux-generic/arch/mips64/odp/api/cpu_arch.h | 12 +++--------- platform/linux-generic/arch/powerpc/odp/api/cpu_arch.h | 10 +--------- platform/linux-generic/arch/x86/odp/api/cpu_arch.h | 10 +--------- platform/linux-generic/include/odp/api/align.h | 11 ++++------- 5 files changed, 10 insertions(+), 43 deletions(-) hooks/post-receive --

9 years, 7 months

[COMMIT] branch master updated. v1.10.1.0-68-g2b5e5bd

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via 2b5e5bdfc8b650feba79ccd23c175fad16fd2b33 (commit) from e6fd300ada7529c9ae454c2ce537844a1c822d21 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 2b5e5bdfc8b650feba79ccd23c175fad16fd2b33 Author: Brian Brooks <brian.brooks(a)linaro.org> Date: Tue Jul 26 11:38:27 2016 -0500 linux-generic: internal cache line size Define the ODP API for cache line size to the cache line size defined in the internal architecture directories. Prefix internal cache line size identifier with '_odp'. Signed-off-by: Brian Brooks <brian.brooks(a)linaro.org> Reviewed-by: Christophe Milard <christophe.milard(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/arch/default/odp/api/cpu_arch.h b/platform/linux-generic/arch/default/odp/api/cpu_arch.h index 29f8889..22b1da2 100644 --- a/platform/linux-generic/arch/default/odp/api/cpu_arch.h +++ b/platform/linux-generic/arch/default/odp/api/cpu_arch.h @@ -11,15 +11,7 @@ extern "C" { #endif -/** @ingroup odp_compiler_optim - * @{ - */ - -#define ODP_CACHE_LINE_SIZE 64 - -/** - * @} - */ +#define _ODP_CACHE_LINE_SIZE 64 static inline void odp_cpu_pause(void) { diff --git a/platform/linux-generic/arch/mips64/odp/api/cpu_arch.h b/platform/linux-generic/arch/mips64/odp/api/cpu_arch.h index 7b5bfd2..3582b12 100644 --- a/platform/linux-generic/arch/mips64/odp/api/cpu_arch.h +++ b/platform/linux-generic/arch/mips64/odp/api/cpu_arch.h @@ -11,18 +11,12 @@ extern "C" { #endif -/** @ingroup odp_compiler_optim - * @{ - */ - #if defined __OCTEON__ -#define ODP_CACHE_LINE_SIZE 128 +#define _ODP_CACHE_LINE_SIZE 128 +#else +#error Please add support for your arch in cpu_arch.h #endif -/** - * @} - */ - static inline void odp_cpu_pause(void) { __asm__ __volatile__ ("nop"); diff --git a/platform/linux-generic/arch/powerpc/odp/api/cpu_arch.h b/platform/linux-generic/arch/powerpc/odp/api/cpu_arch.h index 29f8889..22b1da2 100644 --- a/platform/linux-generic/arch/powerpc/odp/api/cpu_arch.h +++ b/platform/linux-generic/arch/powerpc/odp/api/cpu_arch.h @@ -11,15 +11,7 @@ extern "C" { #endif -/** @ingroup odp_compiler_optim - * @{ - */ - -#define ODP_CACHE_LINE_SIZE 64 - -/** - * @} - */ +#define _ODP_CACHE_LINE_SIZE 64 static inline void odp_cpu_pause(void) { diff --git a/platform/linux-generic/arch/x86/odp/api/cpu_arch.h b/platform/linux-generic/arch/x86/odp/api/cpu_arch.h index 3a16fa6..44e6b30 100644 --- a/platform/linux-generic/arch/x86/odp/api/cpu_arch.h +++ b/platform/linux-generic/arch/x86/odp/api/cpu_arch.h @@ -11,15 +11,7 @@ extern "C" { #endif -/** @ingroup odp_compiler_optim - * @{ - */ - -#define ODP_CACHE_LINE_SIZE 64 - -/** - * @} - */ +#define _ODP_CACHE_LINE_SIZE 64 static inline void odp_cpu_pause(void) { diff --git a/platform/linux-generic/include/odp/api/align.h b/platform/linux-generic/include/odp/api/align.h index d8bc653..c238b80 100644 --- a/platform/linux-generic/include/odp/api/align.h +++ b/platform/linux-generic/include/odp/api/align.h @@ -17,6 +17,8 @@ extern "C" { #endif +#include <odp/api/cpu_arch.h> + /** @ingroup odp_compiler_optim * @{ */ @@ -31,16 +33,12 @@ extern "C" { #define ODP_FIELD_SIZEOF(type, member) sizeof(((type *)0)->member) -#if defined __arm__ || defined __aarch64__ - -#define ODP_CACHE_LINE_SIZE 64 - -#endif - #else #error Non-gcc compatible compiler #endif +#define ODP_CACHE_LINE_SIZE _ODP_CACHE_LINE_SIZE + #define ODP_PAGE_SIZE 4096 #define ODP_ALIGNED_CACHE ODP_ALIGNED(ODP_CACHE_LINE_SIZE) @@ -52,7 +50,6 @@ extern "C" { */ #include <odp/api/spec/align.h> -#include <odp/api/cpu_arch.h> #ifdef __cplusplus } ----------------------------------------------------------------------- Summary of changes: platform/linux-generic/arch/default/odp/api/cpu_arch.h | 10 +--------- platform/linux-generic/arch/mips64/odp/api/cpu_arch.h | 12 +++--------- platform/linux-generic/arch/powerpc/odp/api/cpu_arch.h | 10 +--------- platform/linux-generic/arch/x86/odp/api/cpu_arch.h | 10 +--------- platform/linux-generic/include/odp/api/align.h | 11 ++++------- 5 files changed, 10 insertions(+), 43 deletions(-) hooks/post-receive --

9 years, 7 months

[COMMIT] branch api-next updated. v1.10.1.0-112-g2c494b4

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via 2c494b4a67123c838b4b6915679cca06f9e0bdd2 (commit) via e6fd300ada7529c9ae454c2ce537844a1c822d21 (commit) from bbb10c23626657723466a42b94612b554858ce1d (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 2c494b4a67123c838b4b6915679cca06f9e0bdd2 Merge: bbb10c2 e6fd300 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Tue Jul 26 19:03:43 2016 +0300 Merge branch 'master' into api-next Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> ----------------------------------------------------------------------- Summary of changes: platform/linux-generic/odp_pool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) hooks/post-receive --

9 years, 7 months

[COMMIT] branch master updated. v1.10.1.0-67-ge6fd300

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via e6fd300ada7529c9ae454c2ce537844a1c822d21 (commit) from 870ef79d2dae0ad94a5079d0ca1e222493937aef (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit e6fd300ada7529c9ae454c2ce537844a1c822d21 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Tue Jul 26 18:05:00 2016 +0300 linux-gen: fix odp_pool_print commit: ec0d570 linux-gen: pool: optimize thread local buffer cache removed elements for prints but did not remove that from format string. Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 0a427ed..415c9fa 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -1103,7 +1103,7 @@ void odp_pool_print(odp_pool_t pool_hdl) odp_shm_to_u64(pool->s.pool_shm)); ODP_DBG(" pool status %s\n", pool->s.quiesced ? "quiesced" : "active"); - ODP_DBG(" pool opts %s, %s, %s\n", + ODP_DBG(" pool opts %s, %s\n", pool->s.flags.unsegmented ? "unsegmented" : "segmented", pool->s.flags.predefined ? "predefined" : "created"); ODP_DBG(" pool base %p\n", pool->s.pool_base_addr); ----------------------------------------------------------------------- Summary of changes: platform/linux-generic/odp_pool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) hooks/post-receive --

9 years, 7 months

[COMMIT] branch api-next updated. v1.10.1.0-110-gbbb10c2

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via bbb10c23626657723466a42b94612b554858ce1d (commit) via 5c4d38e42aa63c3d8b30b243f0b58f2cab04a466 (commit) via 870ef79d2dae0ad94a5079d0ca1e222493937aef (commit) via 4329099f32dc7e097eac6ec263264bc0f31d042c (commit) via b5354ade4337447d2914e5ffa4bdeb36dd689113 (commit) via ec0d570b8f762ea79960d54cf8c4f36efdea2528 (commit) from ca7d1dd9372957732fc1dc95d0ffe36cf34c23ba (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit bbb10c23626657723466a42b94612b554858ce1d Author: Christophe Milard <christophe.milard(a)linaro.org> Date: Fri Jul 22 15:19:17 2016 +0200 helper: test: gitignore add iplookuptable Obviously a miss from commit: c4aefb88d31452b3add8cf16f9eef152525c3e93 Signed-off-by: Christophe Milard <christophe.milard(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/helper/test/.gitignore b/helper/test/.gitignore index 482fdb5..e5b6a0f 100644 --- a/helper/test/.gitignore +++ b/helper/test/.gitignore @@ -2,6 +2,7 @@ *.log chksum cuckootable +iplookuptable odpthreads parse process commit 5c4d38e42aa63c3d8b30b243f0b58f2cab04a466 Merge: ca7d1dd 870ef79 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Tue Jul 26 17:21:24 2016 +0300 Merge branch 'master' into api-next Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> ----------------------------------------------------------------------- Summary of changes: helper/test/.gitignore | 1 + .../linux-generic/include/odp_buffer_inlines.h | 26 +- .../linux-generic/include/odp_buffer_internal.h | 5 +- platform/linux-generic/include/odp_internal.h | 2 - .../linux-generic/include/odp_packet_internal.h | 4 +- platform/linux-generic/include/odp_packet_netmap.h | 6 + platform/linux-generic/include/odp_packet_socket.h | 1 + platform/linux-generic/include/odp_pool_internal.h | 143 +------ platform/linux-generic/odp_buffer.c | 3 - platform/linux-generic/odp_packet.c | 68 +-- platform/linux-generic/odp_pool.c | 465 ++++++++++++++++++--- platform/linux-generic/pktio/dpdk.c | 24 +- platform/linux-generic/pktio/netmap.c | 127 +++--- platform/linux-generic/pktio/pcap.c | 26 +- platform/linux-generic/pktio/socket.c | 21 +- platform/linux-generic/pktio/socket_mmap.c | 7 +- platform/linux-generic/pktio/tap.c | 7 +- test/common_plat/performance/odp_scheduling.c | 50 ++- 18 files changed, 622 insertions(+), 364 deletions(-) hooks/post-receive --

9 years, 7 months

[COMMIT] branch master updated. v1.10.1.0-66-g870ef79

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via 870ef79d2dae0ad94a5079d0ca1e222493937aef (commit) via 4329099f32dc7e097eac6ec263264bc0f31d042c (commit) via b5354ade4337447d2914e5ffa4bdeb36dd689113 (commit) via ec0d570b8f762ea79960d54cf8c4f36efdea2528 (commit) from 20f14c1e726c97cb675c26627c073b730bcb58e6 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 870ef79d2dae0ad94a5079d0ca1e222493937aef Author: Matias Elo <matias.elo(a)nokia.com> Date: Fri Jul 22 13:04:57 2016 +0300 performance: odp_scheduling: use multi alloc/free buffer functions Use odp_buffer_alloc_multi() and odp_buffer_free_multi() functions where appropriate for improved performance. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-and-tested-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/common_plat/performance/odp_scheduling.c b/test/common_plat/performance/odp_scheduling.c index 5a2997f..1d1bf01 100644 --- a/test/common_plat/performance/odp_scheduling.c +++ b/test/common_plat/performance/odp_scheduling.c @@ -29,7 +29,7 @@ #include <getopt.h> #define MSG_POOL_SIZE (4 * 1024 * 1024) /**< Message pool size */ -#define MAX_ALLOCS 35 /**< Alloc burst size */ +#define MAX_ALLOCS 32 /**< Alloc burst size */ #define QUEUES_PER_PRIO 64 /**< Queue per priority */ #define NUM_PRIOS 2 /**< Number of tested priorities */ #define QUEUE_ROUNDS (512 * 1024) /**< Queue test rounds */ @@ -131,9 +131,10 @@ static void clear_sched_queues(void) static int enqueue_events(int thr, int prio, int num_queues, int num_events, test_globals_t *globals) { - odp_buffer_t buf; + odp_buffer_t buf[num_events]; + odp_event_t ev[num_events]; odp_queue_t queue; - int i, j, k; + int i, j, k, ret; if (prio == ODP_SCHED_PRIO_HIGHEST) i = 0; @@ -144,19 +145,28 @@ static int enqueue_events(int thr, int prio, int num_queues, int num_events, for (j = 0; j < num_queues; j++) { queue = globals->queue[i][j]; + ret = odp_buffer_alloc_multi(globals->pool, buf, num_events); + if (ret != num_events) { + LOG_ERR(" [%i] buffer alloc failed\n", thr); + ret = ret < 0 ? 0 : ret; + odp_buffer_free_multi(buf, ret); + return -1; + } for (k = 0; k < num_events; k++) { - buf = odp_buffer_alloc(globals->pool); - - if (!odp_buffer_is_valid(buf)) { + if (!odp_buffer_is_valid(buf[k])) { LOG_ERR(" [%i] buffer alloc failed\n", thr); + odp_buffer_free_multi(buf, num_events); return -1; } + ev[k] = odp_buffer_to_event(buf[k]); + } - if (odp_queue_enq(queue, odp_buffer_to_event(buf))) { - LOG_ERR(" [%i] Queue enqueue failed.\n", thr); - odp_buffer_free(buf); - return -1; - } + ret = odp_queue_enq_multi(queue, ev, num_events); + if (ret != num_events) { + LOG_ERR(" [%i] Queue enqueue failed.\n", thr); + ret = ret < 0 ? 0 : ret; + odp_buffer_free_multi(&buf[ret], num_events - ret); + return -1; } } @@ -210,24 +220,30 @@ static int test_alloc_single(int thr, test_globals_t *globals) */ static int test_alloc_multi(int thr, test_globals_t *globals) { - int i, j; + int i, j, ret; odp_buffer_t temp_buf[MAX_ALLOCS]; uint64_t c1, c2, cycles; c1 = odp_cpu_cycles(); for (i = 0; i < ALLOC_ROUNDS; i++) { - for (j = 0; j < MAX_ALLOCS; j++) { - temp_buf[j] = odp_buffer_alloc(globals->pool); + ret = odp_buffer_alloc_multi(globals->pool, temp_buf, + MAX_ALLOCS); + if (ret != MAX_ALLOCS) { + LOG_ERR(" [%i] buffer alloc failed\n", thr); + ret = ret < 0 ? 0 : ret; + odp_buffer_free_multi(temp_buf, ret); + return -1; + } + for (j = 0; j < MAX_ALLOCS; j++) { if (!odp_buffer_is_valid(temp_buf[j])) { LOG_ERR(" [%i] alloc_multi failed\n", thr); + odp_buffer_free_multi(temp_buf, MAX_ALLOCS); return -1; } } - - for (; j > 0; j--) - odp_buffer_free(temp_buf[j - 1]); + odp_buffer_free_multi(temp_buf, MAX_ALLOCS); } c2 = odp_cpu_cycles(); commit 4329099f32dc7e097eac6ec263264bc0f31d042c Author: Matias Elo <matias.elo(a)nokia.com> Date: Fri Jul 22 13:04:56 2016 +0300 linux-gen: netmap: use multi alloc/free Modify netmap pktio to take advantage of multi-alloc/free. Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-and-tested-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_packet_netmap.h b/platform/linux-generic/include/odp_packet_netmap.h index 0e6c0dc..2aadbed 100644 --- a/platform/linux-generic/include/odp_packet_netmap.h +++ b/platform/linux-generic/include/odp_packet_netmap.h @@ -35,6 +35,12 @@ typedef union { uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct netmap_ring_t))]; } netmap_ring_t ODP_ALIGNED_CACHE; +/** Netmap ring slot */ +typedef struct { + char *buf; /**< Slot buffer pointer */ + uint16_t len; /**< Slot length */ +} netmap_slot_t; + /** Packet socket using netmap mmaped rings for both Rx and Tx */ typedef struct { odp_pool_t pool; /**< pool to alloc packets from */ diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 6498429..412beec 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -580,67 +580,85 @@ static int netmap_stop(pktio_entry_t *pktio_entry ODP_UNUSED) } /** - * Create ODP packet from netmap packet + * Create ODP packets from netmap packets * * @param pktio_entry Packet IO entry - * @param pkt_out Storage for new ODP packet handle - * @param buf Netmap buffer address - * @param len Netmap buffer length + * @param pkt_tbl Array for new ODP packet handles + * @param slot_tbl Array of netmap ring slots + * @param slot_num Number of netmap ring slots * @param ts Pointer to pktin timestamp * * @retval Number of created packets - * @retval <0 on failure */ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, - odp_packet_t *pkt_out, const char *buf, - uint16_t len, odp_time_t *ts) + odp_packet_t pkt_tbl[], + netmap_slot_t slot_tbl[], int16_t slot_num, + odp_time_t *ts) { odp_packet_t pkt; odp_pool_t pool = pktio_entry->s.pkt_nm.pool; odp_packet_hdr_t *pkt_hdr; odp_packet_hdr_t parsed_hdr; + int i; int num; + int alloc_len; - if (odp_unlikely(len > pktio_entry->s.pkt_nm.max_frame_len)) { - ODP_ERR("RX: frame too big %" PRIu16 " %zu!\n", len, - pktio_entry->s.pkt_nm.max_frame_len); - return -1; - } + /* Allocate maximum sized packets */ + alloc_len = pktio_entry->s.pkt_nm.mtu; - if (odp_unlikely(len < _ODP_ETH_LEN_MIN)) { - ODP_ERR("RX: Frame truncated: %" PRIu16 "\n", len); - return -1; - } + num = packet_alloc_multi(pool, alloc_len, pkt_tbl, slot_num); - if (pktio_cls_enabled(pktio_entry)) { - if (cls_classify_packet(pktio_entry, (const uint8_t *)buf, len, - len, &pool, &parsed_hdr)) - return -1; - } - num = packet_alloc_multi(pool, len, &pkt, 1); - if (num != 1) - return -1; + for (i = 0; i < num; i++) { + netmap_slot_t slot; + uint16_t len; - pkt_hdr = odp_packet_hdr(pkt); + slot = slot_tbl[i]; + len = slot.len; - /* For now copy the data in the mbuf, - worry about zero-copy later */ - if (odp_packet_copy_from_mem(pkt, 0, len, buf) != 0) { - odp_packet_free(pkt); - return -1; - } - pkt_hdr->input = pktio_entry->s.handle; + odp_prefetch(slot.buf); - if (pktio_cls_enabled(pktio_entry)) - copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); - else - packet_parse_l2(&pkt_hdr->p, len); + if (odp_unlikely(len > pktio_entry->s.pkt_nm.max_frame_len)) { + ODP_ERR("RX: frame too big %" PRIu16 " %zu!\n", len, + pktio_entry->s.pkt_nm.max_frame_len); + goto fail; + } + + if (odp_unlikely(len < _ODP_ETH_LEN_MIN)) { + ODP_ERR("RX: Frame truncated: %" PRIu16 "\n", len); + goto fail; + } + + if (pktio_cls_enabled(pktio_entry)) { + if (cls_classify_packet(pktio_entry, + (const uint8_t *)slot.buf, len, + len, &pool, &parsed_hdr)) + goto fail; + } + + pkt = pkt_tbl[i]; + pkt_hdr = odp_packet_hdr(pkt); + pull_tail(pkt_hdr, alloc_len - len); - packet_set_ts(pkt_hdr, ts); + /* For now copy the data in the mbuf, + worry about zero-copy later */ + if (odp_packet_copy_from_mem(pkt, 0, len, slot.buf) != 0) + goto fail; - *pkt_out = pkt; + pkt_hdr->input = pktio_entry->s.handle; - return 1; + if (pktio_cls_enabled(pktio_entry)) + copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); + else + packet_parse_l2(&pkt_hdr->p, len); + + packet_set_ts(pkt_hdr, ts); + } + + return i; + +fail: + odp_packet_free_multi(&pkt_tbl[i], num - i); + return i; } static inline int netmap_recv_desc(pktio_entry_t *pktio_entry, @@ -650,10 +668,10 @@ static inline int netmap_recv_desc(pktio_entry_t *pktio_entry, struct netmap_ring *ring; odp_time_t ts_val; odp_time_t *ts = NULL; + netmap_slot_t slot_tbl[num]; char *buf; uint32_t slot_id; int i; - int ret; int ring_id = desc->cur_rx_ring; int num_rx = 0; int num_rings = desc->last_rx_ring - desc->first_rx_ring + 1; @@ -668,29 +686,28 @@ static inline int netmap_recv_desc(pktio_entry_t *pktio_entry, ring = NETMAP_RXRING(desc->nifp, ring_id); - /* Take timestamp beforehand per ring to improve performance */ - if (ts != NULL) - ts_val = odp_time_global(); - while (!nm_ring_empty(ring) && num_rx != num) { slot_id = ring->cur; buf = NETMAP_BUF(ring, ring->slot[slot_id].buf_idx); - odp_prefetch(buf); - - ret = netmap_pkt_to_odp(pktio_entry, &pkt_table[num_rx], - buf, ring->slot[slot_id].len, - ts); - if (ret > 0) - num_rx += ret; + slot_tbl[num_rx].buf = buf; + slot_tbl[num_rx].len = ring->slot[slot_id].len; + num_rx++; ring->cur = nm_ring_next(ring, slot_id); - ring->head = ring->cur; } + ring->head = ring->cur; ring_id++; } desc->cur_rx_ring = ring_id; - return num_rx; + + if (num_rx) { + if (ts != NULL) + ts_val = odp_time_global(); + return netmap_pkt_to_odp(pktio_entry, pkt_table, slot_tbl, + num_rx, ts); + } + return 0; } static int netmap_recv(pktio_entry_t *pktio_entry, int index, @@ -806,7 +823,6 @@ static int netmap_send(pktio_entry_t *pktio_entry, int index, } if (i == NM_INJECT_RETRIES) break; - odp_packet_free(pkt); } /* Send pending packets */ poll(&polld, 1, 0); @@ -814,6 +830,8 @@ static int netmap_send(pktio_entry_t *pktio_entry, int index, if (!pkt_nm->lockless_tx) odp_ticketlock_unlock(&pkt_nm->tx_desc_ring[index].s.lock); + odp_packet_free_multi(pkt_table, nb_tx); + if (odp_unlikely(nb_tx == 0 && __odp_errno != 0)) return -1; commit b5354ade4337447d2914e5ffa4bdeb36dd689113 Author: Matias Elo <matias.elo(a)nokia.com> Date: Fri Jul 22 13:04:55 2016 +0300 linux-gen: pktio: use multi alloc Modify dpdk pktio to take advantage of multi-alloc/free. Others pktios do alloc/free still one packet at a time. For example, l2fwd test application packet throughput is increased about 10% (with dpdk pktio). Signed-off-by: Petri Savolainen <petri.savolainen(a)nokia.com> Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-and-tested-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 4c4e36c..392d670 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -306,7 +306,9 @@ static inline int packet_parse_not_complete(odp_packet_hdr_t *pkt_hdr) /* Forward declarations */ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt); -odp_packet_t packet_alloc(odp_pool_t pool_hdl, uint32_t len, int parse); +/* Packet alloc of pktios */ +int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, + odp_packet_t pkt[], int max_num); /* Fill in parser metadata for L2 */ void packet_parse_l2(packet_parser_t *prs, uint32_t frame_len); diff --git a/platform/linux-generic/include/odp_packet_socket.h b/platform/linux-generic/include/odp_packet_socket.h index ccff69a..dbfc9f1 100644 --- a/platform/linux-generic/include/odp_packet_socket.h +++ b/platform/linux-generic/include/odp_packet_socket.h @@ -45,6 +45,7 @@ typedef struct { int sockfd; /**< socket descriptor */ odp_pool_t pool; /**< pool to alloc packets from */ + uint32_t mtu; /**< maximum transmission unit */ unsigned char if_mac[ETH_ALEN]; /**< IF eth mac addr */ uint8_t *cache_ptr[ODP_PACKET_SOCKET_MAX_BURST_RX]; odp_shm_t shm; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 474fa81..c4cf324 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -76,35 +76,46 @@ static void packet_init(pool_entry_t *pool, odp_packet_hdr_t *pkt_hdr, pkt_hdr->input = ODP_PKTIO_INVALID; } -odp_packet_t packet_alloc(odp_pool_t pool_hdl, uint32_t len, int parse) +int packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, + odp_packet_t pkt[], int max_num) { - odp_packet_t pkt; odp_packet_hdr_t *pkt_hdr; pool_entry_t *pool = odp_pool_to_entry(pool_hdl); + int num, i; - if (pool->s.params.type != ODP_POOL_PACKET) - return ODP_PACKET_INVALID; - - /* Handle special case for zero-length packets */ - if (len == 0) { - len = pool->s.params.buf.size; + num = buffer_alloc_multi(pool_hdl, len, (odp_buffer_t *)pkt, max_num); - pkt = (odp_packet_t)buffer_alloc(pool_hdl, len); + for (i = 0; i < num; i++) { + pkt_hdr = odp_packet_hdr(pkt[i]); + packet_init(pool, pkt_hdr, len, 1 /* do parse */); - if (pkt == ODP_PACKET_INVALID) - return ODP_PACKET_INVALID; + if (pkt_hdr->tailroom >= pkt_hdr->buf_hdr.segsize) + pull_tail_seg(pkt_hdr); + } - pull_tail(odp_packet_hdr(pkt), len); + return num; +} - } else { - pkt = (odp_packet_t)buffer_alloc(pool_hdl, len); +odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) +{ + pool_entry_t *pool = odp_pool_to_entry(pool_hdl); + size_t pkt_size = len ? len : pool->s.params.buf.size; + odp_packet_t pkt; + odp_packet_hdr_t *pkt_hdr; - if (pkt == ODP_PACKET_INVALID) - return ODP_PACKET_INVALID; + if (pool->s.params.type != ODP_POOL_PACKET) { + __odp_errno = EINVAL; + return ODP_PACKET_INVALID; } + pkt = (odp_packet_t)buffer_alloc(pool_hdl, pkt_size); + if (pkt == ODP_PACKET_INVALID) + return ODP_PACKET_INVALID; + pkt_hdr = odp_packet_hdr(pkt); - packet_init(pool, pkt_hdr, len, parse); + packet_init(pool, pkt_hdr, pkt_size, 0 /* do not parse */); + if (len == 0) + pull_tail(pkt_hdr, pkt_size); if (pkt_hdr->tailroom >= pkt_hdr->buf_hdr.segsize) pull_tail_seg(pkt_hdr); @@ -112,11 +123,6 @@ odp_packet_t packet_alloc(odp_pool_t pool_hdl, uint32_t len, int parse) return pkt; } -odp_packet_t odp_packet_alloc(odp_pool_t pool_hdl, uint32_t len) -{ - return packet_alloc(pool_hdl, len, 0); -} - int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, odp_packet_t pkt[], int num) { @@ -135,9 +141,12 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, for (i = 0; i < count; ++i) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt[i]); - packet_init(pool, pkt_hdr, pkt_size, 0); + packet_init(pool, pkt_hdr, pkt_size, 0 /* do not parse */); if (len == 0) pull_tail(pkt_hdr, pkt_size); + + if (pkt_hdr->tailroom >= pkt_hdr->buf_hdr.segsize) + pull_tail_seg(pkt_hdr); } return count; @@ -145,12 +154,16 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, void odp_packet_free(odp_packet_t pkt) { - odp_buffer_free((odp_buffer_t)pkt); + uint32_t pool_id = pool_id_from_buf((odp_buffer_t)pkt); + + buffer_free(pool_id, (odp_buffer_t)pkt); } void odp_packet_free_multi(const odp_packet_t pkt[], int num) { - odp_buffer_free_multi((const odp_buffer_t *)pkt, num); + uint32_t pool_id = pool_id_from_buf((odp_buffer_t)pkt[0]); + + buffer_free_multi(pool_id, (const odp_buffer_t * const)pkt, num); } int odp_packet_reset(odp_packet_t pkt, uint32_t len) diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index a5934a5..1fad8a9 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -706,7 +706,7 @@ static int dpdk_stop(pktio_entry_t *pktio_entry) static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], struct rte_mbuf *mbuf_table[], - uint16_t num, odp_time_t *ts) + uint16_t mbuf_num, odp_time_t *ts) { odp_packet_t pkt; odp_packet_hdr_t *pkt_hdr; @@ -715,9 +715,15 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, void *buf; int i, j; int nb_pkts = 0; + int alloc_len, num; + odp_pool_t pool = pktio_entry->s.pkt_dpdk.pool; + + /* Allocate maximum sized packets */ + alloc_len = pktio_entry->s.pkt_dpdk.data_room; + + num = packet_alloc_multi(pool, alloc_len, pkt_table, mbuf_num); for (i = 0; i < num; i++) { - odp_pool_t pool = pktio_entry->s.pkt_dpdk.pool; odp_packet_hdr_t parsed_hdr; mbuf = mbuf_table[i]; @@ -738,18 +744,16 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, &parsed_hdr)) goto fail; } - pkt = packet_alloc(pool, pkt_len, 1); - if (pkt == ODP_PACKET_INVALID) - goto fail; + pkt = pkt_table[i]; pkt_hdr = odp_packet_hdr(pkt); + pull_tail(pkt_hdr, alloc_len - pkt_len); /* For now copy the data in the mbuf, worry about zero-copy later */ - if (odp_packet_copy_from_mem(pkt, 0, pkt_len, buf) != 0) { - odp_packet_free(pkt); + if (odp_packet_copy_from_mem(pkt, 0, pkt_len, buf) != 0) goto fail; - } + pkt_hdr->input = pktio_entry->s.handle; if (pktio_cls_enabled(pktio_entry)) @@ -770,7 +774,9 @@ static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, return nb_pkts; fail: - for (j = i; j < num; j++) + odp_packet_free_multi(&pkt_table[i], mbuf_num - i); + + for (j = i; j < mbuf_num; j++) rte_pktmbuf_free(mbuf_table[j]); return (i > 0 ? i : -1); diff --git a/platform/linux-generic/pktio/netmap.c b/platform/linux-generic/pktio/netmap.c index 3017e40..6498429 100644 --- a/platform/linux-generic/pktio/netmap.c +++ b/platform/linux-generic/pktio/netmap.c @@ -599,6 +599,7 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, odp_pool_t pool = pktio_entry->s.pkt_nm.pool; odp_packet_hdr_t *pkt_hdr; odp_packet_hdr_t parsed_hdr; + int num; if (odp_unlikely(len > pktio_entry->s.pkt_nm.max_frame_len)) { ODP_ERR("RX: frame too big %" PRIu16 " %zu!\n", len, @@ -616,8 +617,8 @@ static inline int netmap_pkt_to_odp(pktio_entry_t *pktio_entry, len, &pool, &parsed_hdr)) return -1; } - pkt = packet_alloc(pool, len, 1); - if (pkt == ODP_PACKET_INVALID) + num = packet_alloc_multi(pool, len, &pkt, 1); + if (num != 1) return -1; pkt_hdr = odp_packet_hdr(pkt); diff --git a/platform/linux-generic/pktio/pcap.c b/platform/linux-generic/pktio/pcap.c index e501858..e54a56f 100644 --- a/platform/linux-generic/pktio/pcap.c +++ b/platform/linux-generic/pktio/pcap.c @@ -224,19 +224,9 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pktio_entry->s.config.pktin.bit.ts_ptp) ts = &ts_val; - pkt = ODP_PACKET_INVALID; - pkt_len = 0; - for (i = 0; i < len; ) { int ret; - if (pkt == ODP_PACKET_INVALID) { - pkt = packet_alloc(pcap->pool, 0 /*default len*/, 1); - if (odp_unlikely(pkt == ODP_PACKET_INVALID)) - break; - pkt_len = odp_packet_len(pkt); - } - ret = pcap_next_ex(pcap->rx, &hdr, &data); /* end of file, attempt to reopen if within loop limit */ @@ -246,17 +236,17 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, if (ret != 1) break; + pkt_len = hdr->caplen; + + ret = packet_alloc_multi(pcap->pool, pkt_len, &pkt, 1); + if (odp_unlikely(ret != 1)) + break; + if (ts != NULL) ts_val = odp_time_global(); pkt_hdr = odp_packet_hdr(pkt); - if (!odp_packet_pull_tail(pkt, pkt_len - hdr->caplen)) { - ODP_ERR("failed to pull tail: pkt_len: %d caplen: %d\n", - pkt_len, hdr->caplen); - break; - } - if (odp_packet_copy_from_mem(pkt, 0, hdr->caplen, data) != 0) { ODP_ERR("failed to copy packet data\n"); break; @@ -269,7 +259,6 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt_hdr->input = pktio_entry->s.handle; pkts[i] = pkt; - pkt = ODP_PACKET_INVALID; i++; } @@ -277,9 +266,6 @@ static int pcapif_recv_pkt(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_ticketlock_unlock(&pktio_entry->s.rxl); - if (pkt != ODP_PACKET_INVALID) - odp_packet_free(pkt); - return i; } diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index 0309b0d..e01b0a5 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -527,6 +527,10 @@ static int sock_setup_pkt(pktio_entry_t *pktio_entry, const char *netdev, if (err != 0) goto error; + pkt_sock->mtu = mtu_get_fd(sockfd, netdev); + if (!pkt_sock->mtu) + goto error; + /* bind socket to if */ memset(&sa_ll, 0, sizeof(sa_ll)); sa_ll.sll_family = AF_PACKET; @@ -659,6 +663,7 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, void *base = msgvec[i].msg_hdr.msg_iov->iov_base; struct ethhdr *eth_hdr = base; uint16_t pkt_len = msgvec[i].msg_len; + int num; /* Don't receive packets sent by ourselves */ if (odp_unlikely(ethaddrs_equal(pkt_sock->if_mac, @@ -668,8 +673,8 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, if (cls_classify_packet(pktio_entry, base, pkt_len, pkt_len, &pool, &parsed_hdr)) continue; - pkt = packet_alloc(pool, pkt_len, 1); - if (pkt == ODP_PACKET_INVALID) + num = packet_alloc_multi(pool, pkt_len, &pkt, 1); + if (num != 1) continue; pkt_hdr = odp_packet_hdr(pkt); @@ -690,10 +695,14 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, [ODP_BUFFER_MAX_SEG]; for (i = 0; i < (int)len; i++) { - pkt_table[i] = packet_alloc(pkt_sock->pool, - 0 /*default*/, 1); - if (odp_unlikely(pkt_table[i] == ODP_PACKET_INVALID)) + int num; + + num = packet_alloc_multi(pkt_sock->pool, pkt_sock->mtu, + &pkt_table[i], 1); + if (odp_unlikely(num != 1)) { + pkt_table[i] = ODP_PACKET_INVALID; break; + } msgvec[i].msg_hdr.msg_iovlen = _rx_pkt_to_iovec(pkt_table[i], iovecs[i]); @@ -818,7 +827,7 @@ static int sock_mmsg_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, */ static uint32_t sock_mtu_get(pktio_entry_t *pktio_entry) { - return mtu_get_fd(pktio_entry->s.pkt_sock.sockfd, pktio_entry->s.name); + return pktio_entry->s.pkt_sock.mtu; } /* diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index b6be81a..8b24c99 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -171,6 +171,7 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, odp_packet_hdr_t *hdr; odp_packet_hdr_t parsed_hdr; odp_pool_t pool = pkt_sock->pool; + int num; if (!mmap_rx_kernel_ready(ring->rd[frame_num].iov_base)) break; @@ -208,8 +209,10 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, } } - pkt_table[nb_rx] = packet_alloc(pool, pkt_len, 1); - if (odp_unlikely(pkt_table[nb_rx] == ODP_PACKET_INVALID)) { + num = packet_alloc_multi(pool, pkt_len, &pkt_table[nb_rx], 1); + + if (odp_unlikely(num != 1)) { + pkt_table[nb_rx] = ODP_PACKET_INVALID; mmap_rx_user_ready(ppd.raw); /* drop */ frame_num = next_frame_num; continue; diff --git a/platform/linux-generic/pktio/tap.c b/platform/linux-generic/pktio/tap.c index a9a8886..d758a39 100644 --- a/platform/linux-generic/pktio/tap.c +++ b/platform/linux-generic/pktio/tap.c @@ -185,11 +185,12 @@ static odp_packet_t pack_odp_pkt(pktio_entry_t *pktio_entry, const void *data, { odp_packet_t pkt; odp_packet_hdr_t *pkt_hdr; + int num; - pkt = packet_alloc(pktio_entry->s.pkt_tap.pool, len, 1); + num = packet_alloc_multi(pktio_entry->s.pkt_tap.pool, len, &pkt, 1); - if (pkt == ODP_PACKET_INVALID) - return pkt; + if (num != 1) + return ODP_PACKET_INVALID; if (odp_packet_copy_from_mem(pkt, 0, len, data) < 0) { ODP_ERR("failed to copy packet data\n"); commit ec0d570b8f762ea79960d54cf8c4f36efdea2528 Author: Matias Elo <matias.elo(a)nokia.com> Date: Fri Jul 22 13:04:54 2016 +0300 linux-gen: pool: optimize thread local buffer cache Optimize local buffer cache performance which is critical to many use cases - including packet IO. Main parts of the optimization are: * Local cache implemented as an array of buf_hdr pointers, instead of a linked list (which causes a lot of cache misses) * Alloc and free N buffers per operation All above steps are needed to demonstrate the performance upgrade. Some related pool functions (get_buf(), ret_buf(), etc) were moved from pool header to c source file, since those were actual local to the c source file. Also some unused pool variables are removed also. Signed-off-by: Petri Savolainen <petri.savolainen(a)nokia.com> Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-and-tested-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_buffer_inlines.h b/platform/linux-generic/include/odp_buffer_inlines.h index 3f4d9fd..2b1ab42 100644 --- a/platform/linux-generic/include/odp_buffer_inlines.h +++ b/platform/linux-generic/include/odp_buffer_inlines.h @@ -56,30 +56,12 @@ static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf) (pool->pool_mdata_addr + (index * ODP_CACHE_LINE_SIZE)); } -static inline uint32_t odp_buffer_refcount(odp_buffer_hdr_t *buf) +static inline uint32_t pool_id_from_buf(odp_buffer_t buf) { - return odp_atomic_load_u32(&buf->ref_count); -} - -static inline uint32_t odp_buffer_incr_refcount(odp_buffer_hdr_t *buf, - uint32_t val) -{ - return odp_atomic_fetch_add_u32(&buf->ref_count, val) + val; -} - -static inline uint32_t odp_buffer_decr_refcount(odp_buffer_hdr_t *buf, - uint32_t val) -{ - uint32_t tmp; - - tmp = odp_atomic_fetch_sub_u32(&buf->ref_count, val); + odp_buffer_bits_t handle; - if (tmp < val) { - odp_atomic_fetch_add_u32(&buf->ref_count, val - tmp); - return 0; - } else { - return tmp - val; - } + handle.handle = buf; + return handle.pool_id; } static inline odp_buffer_hdr_t *validate_buf(odp_buffer_t buf) diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index f21364c..7b0ef8b 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -114,7 +114,6 @@ struct odp_buffer_hdr_t { union { uint32_t all; struct { - uint32_t zeroized:1; /* Zeroize buf data on free */ uint32_t hdrdata:1; /* Data is in buffer hdr */ uint32_t sustain:1; /* Sustain order */ }; @@ -123,7 +122,6 @@ struct odp_buffer_hdr_t { int8_t type; /* buffer type */ odp_event_type_t event_type; /* for reuse as event */ uint32_t size; /* max data size */ - odp_atomic_u32_t ref_count; /* reference count */ odp_pool_t pool_hdl; /* buffer pool handle */ union { uint64_t buf_u64; /* user u64 */ @@ -174,6 +172,9 @@ typedef struct { odp_buffer_t buffer_alloc(odp_pool_t pool, size_t size); int buffer_alloc_multi(odp_pool_t pool_hdl, size_t size, odp_buffer_t buf[], int num); +void buffer_free(uint32_t pool_id, const odp_buffer_t buf); +void buffer_free_multi(uint32_t pool_id, + const odp_buffer_t buf[], int num_free); int seg_alloc_head(odp_buffer_hdr_t *buf_hdr, int segcount); void seg_free_head(odp_buffer_hdr_t *buf_hdr, int segcount); int seg_alloc_tail(odp_buffer_hdr_t *buf_hdr, int segcount); diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index d12f850..8bad450 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -119,8 +119,6 @@ int odp_tm_term_global(void); int _odp_int_name_tbl_init_global(void); int _odp_int_name_tbl_term_global(void); -void _odp_flush_caches(void); - int cpuinfo_parser(FILE *file, system_info_t *sysinfo); uint64_t odp_cpu_hz_current(int id); diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index 3317bd0..d6717ff 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -51,15 +51,25 @@ typedef struct _odp_buffer_pool_init_t { void *buf_init_arg; /**< Argument to be passed to buf_init() */ } _odp_buffer_pool_init_t; /**< Type of buffer initialization struct */ +#define POOL_MAX_LOCAL_CHUNKS 4 +#define POOL_CHUNK_SIZE 32 +#define POOL_MAX_LOCAL_BUFS (POOL_MAX_LOCAL_CHUNKS * POOL_CHUNK_SIZE) + +struct local_cache_s { + uint64_t bufallocs; /* Local buffer alloc count */ + uint64_t buffrees; /* Local buffer free count */ + + uint32_t num_buf; + odp_buffer_hdr_t *buf[POOL_MAX_LOCAL_BUFS]; +}; + /* Local cache for buffer alloc/free acceleration */ typedef struct local_cache_t { union { - struct { - odp_buffer_hdr_t *buf_freelist; /* The local cache */ - uint64_t bufallocs; /* Local buffer alloc count */ - uint64_t buffrees; /* Local buffer free count */ - }; - uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(uint64_t))]; + struct local_cache_s s; + + uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP( + sizeof(struct local_cache_s))]; }; } local_cache_t; @@ -214,127 +224,6 @@ static inline void ret_blk(struct pool_entry_s *pool, void *block) odp_atomic_inc_u64(&pool->poolstats.blkfrees); } -static inline odp_buffer_hdr_t *get_buf(struct pool_entry_s *pool) -{ - odp_buffer_hdr_t *myhead; - POOL_LOCK(&pool->buf_lock); - - myhead = pool->buf_freelist; - - if (odp_unlikely(myhead == NULL)) { - POOL_UNLOCK(&pool->buf_lock); - odp_atomic_inc_u64(&pool->poolstats.bufempty); - } else { - pool->buf_freelist = myhead->next; - POOL_UNLOCK(&pool->buf_lock); - uint64_t bufcount = - odp_atomic_fetch_sub_u32(&pool->bufcount, 1) - 1; - - /* Check for low watermark condition */ - if (bufcount == pool->buf_low_wm && !pool->buf_low_wm_assert) { - pool->buf_low_wm_assert = 1; - odp_atomic_inc_u64(&pool->poolstats.buf_low_wm_count); - } - - odp_atomic_inc_u64(&pool->poolstats.bufallocs); - } - - return (void *)myhead; -} - -static inline void ret_buf(struct pool_entry_s *pool, odp_buffer_hdr_t *buf) -{ - if (!buf->flags.hdrdata && buf->type != ODP_EVENT_BUFFER) { - while (buf->segcount > 0) { - if (buffer_is_secure(buf) || pool_is_secure(pool)) - memset(buf->addr[buf->segcount - 1], - 0, buf->segsize); - ret_blk(pool, buf->addr[--buf->segcount]); - } - buf->size = 0; - } - - buf->allocator = ODP_FREEBUF; /* Mark buffer free */ - POOL_LOCK(&pool->buf_lock); - buf->next = pool->buf_freelist; - pool->buf_freelist = buf; - POOL_UNLOCK(&pool->buf_lock); - - uint64_t bufcount = odp_atomic_fetch_add_u32(&pool->bufcount, 1) + 1; - - /* Check if low watermark condition should be deasserted */ - if (bufcount == pool->buf_high_wm && pool->buf_low_wm_assert) { - pool->buf_low_wm_assert = 0; - odp_atomic_inc_u64(&pool->poolstats.buf_high_wm_count); - } - - odp_atomic_inc_u64(&pool->poolstats.buffrees); -} - -static inline void *get_local_buf(local_cache_t *buf_cache, - struct pool_entry_s *pool, - size_t totsize) -{ - odp_buffer_hdr_t *buf = buf_cache->buf_freelist; - - if (odp_likely(buf != NULL)) { - buf_cache->buf_freelist = buf->next; - - if (odp_unlikely(buf->size < totsize)) { - intmax_t needed = totsize - buf->size; - - do { - void *blk = get_blk(pool); - if (odp_unlikely(blk == NULL)) { - ret_buf(pool, buf); - buf_cache->buffrees--; - return NULL; - } - buf->addr[buf->segcount++] = blk; - needed -= pool->seg_size; - } while (needed > 0); - - buf->size = buf->segcount * pool->seg_size; - } - - buf_cache->bufallocs++; - } - - return buf; -} - -static inline void ret_local_buf(local_cache_t *buf_cache, - odp_buffer_hdr_t *buf) -{ - buf->allocator = ODP_FREEBUF; - buf->next = buf_cache->buf_freelist; - buf_cache->buf_freelist = buf; - - buf_cache->buffrees++; -} - -static inline void flush_cache(local_cache_t *buf_cache, - struct pool_entry_s *pool) -{ - odp_buffer_hdr_t *buf = buf_cache->buf_freelist; - uint32_t flush_count = 0; - - while (buf != NULL) { - odp_buffer_hdr_t *next = buf->next; - ret_buf(pool, buf); - buf = next; - flush_count++; - } - - odp_atomic_add_u64(&pool->poolstats.bufallocs, buf_cache->bufallocs); - odp_atomic_add_u64(&pool->poolstats.buffrees, - buf_cache->buffrees - flush_count); - - buf_cache->buf_freelist = NULL; - buf_cache->bufallocs = 0; - buf_cache->buffrees = 0; -} - static inline odp_pool_t pool_index_to_handle(uint32_t pool_id) { return _odp_cast_scalar(odp_pool_t, pool_id); diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c index e7e4d58..ce2fdba 100644 --- a/platform/linux-generic/odp_buffer.c +++ b/platform/linux-generic/odp_buffer.c @@ -67,9 +67,6 @@ int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf) len += snprintf(&str[len], n-len, " size %" PRIu32 "\n", hdr->size); len += snprintf(&str[len], n-len, - " ref_count %" PRIu32 "\n", - odp_atomic_load_u32(&hdr->ref_count)); - len += snprintf(&str[len], n-len, " type %i\n", hdr->type); return len; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 0e319d2..474fa81 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -972,10 +972,7 @@ int _odp_packet_copy_md_to_packet(odp_packet_t srcpkt, odp_packet_t dstpkt) srchdr->buf_hdr.uarea_size ? dsthdr->buf_hdr.uarea_size : srchdr->buf_hdr.uarea_size); - odp_atomic_store_u32( - &dsthdr->buf_hdr.ref_count, - odp_atomic_load_u32( - &srchdr->buf_hdr.ref_count)); + copy_packet_parser_metadata(srchdr, dsthdr); /* Metadata copied, but return indication of whether the packet diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 419f03f..0a427ed 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -57,8 +57,15 @@ static const char SHM_DEFAULT_NAME[] = "odp_buffer_pools"; /* Pool entry pointers (for inlining) */ void *pool_entry_ptr[ODP_CONFIG_POOLS]; -/* Cache thread id locally for local cache performance */ -static __thread int local_id; +/* Thread local variables */ +typedef struct pool_local_t { + local_cache_t *cache[ODP_CONFIG_POOLS]; + int thr_id; +} pool_local_t; + +static __thread pool_local_t local; + +static void flush_cache(local_cache_t *buf_cache, struct pool_entry_s *pool); int odp_pool_init_global(void) { @@ -111,7 +118,19 @@ int odp_pool_init_global(void) int odp_pool_init_local(void) { - local_id = odp_thread_id(); + pool_entry_t *pool; + int i; + int thr_id = odp_thread_id(); + + memset(&local, 0, sizeof(pool_local_t)); + + for (i = 0; i < ODP_CONFIG_POOLS; i++) { + pool = get_pool_entry(i); + local.cache[i] = &pool->s.local_cache[thr_id]; + local.cache[i]->s.num_buf = 0; + } + + local.thr_id = thr_id; return 0; } @@ -144,7 +163,14 @@ int odp_pool_term_global(void) int odp_pool_term_local(void) { - _odp_flush_caches(); + int i; + + for (i = 0; i < ODP_CONFIG_POOLS; i++) { + pool_entry_t *pool = get_pool_entry(i); + + flush_cache(local.cache[i], &pool->s); + } + return 0; } @@ -179,10 +205,53 @@ int odp_pool_capability(odp_pool_capability_t *capa) return 0; } -/** +static inline odp_buffer_hdr_t *get_buf(struct pool_entry_s *pool) +{ + odp_buffer_hdr_t *myhead; + + POOL_LOCK(&pool->buf_lock); + + myhead = pool->buf_freelist; + + if (odp_unlikely(myhead == NULL)) { + POOL_UNLOCK(&pool->buf_lock); + odp_atomic_inc_u64(&pool->poolstats.bufempty); + } else { + pool->buf_freelist = myhead->next; + POOL_UNLOCK(&pool->buf_lock); + + odp_atomic_fetch_sub_u32(&pool->bufcount, 1); + odp_atomic_inc_u64(&pool->poolstats.bufallocs); + } + + return (void *)myhead; +} + +static inline void ret_buf(struct pool_entry_s *pool, odp_buffer_hdr_t *buf) +{ + if (!buf->flags.hdrdata && buf->type != ODP_EVENT_BUFFER) { + while (buf->segcount > 0) { + if (buffer_is_secure(buf) || pool_is_secure(pool)) + memset(buf->addr[buf->segcount - 1], + 0, buf->segsize); + ret_blk(pool, buf->addr[--buf->segcount]); + } + buf->size = 0; + } + + buf->allocator = ODP_FREEBUF; /* Mark buffer free */ + POOL_LOCK(&pool->buf_lock); + buf->next = pool->buf_freelist; + pool->buf_freelist = buf; + POOL_UNLOCK(&pool->buf_lock); + + odp_atomic_fetch_add_u32(&pool->bufcount, 1); + odp_atomic_inc_u64(&pool->poolstats.buffrees); +} + +/* * Pool creation */ - odp_pool_t _pool_create(const char *name, odp_pool_param_t *params, uint32_t shmflags) @@ -208,9 +277,6 @@ odp_pool_t _pool_create(const char *name, /* Restriction for v1.0: All non-packet buffers are unsegmented */ int unseg = 1; - /* Restriction for v1.0: No zeroization support */ - const int zeroized = 0; - uint32_t blk_size, buf_stride, buf_num, blk_num, seg_len = 0; uint32_t buf_align = params->type == ODP_POOL_BUFFER ? params->buf.align : 0; @@ -350,7 +416,6 @@ odp_pool_t _pool_create(const char *name, POOL_UNLOCK(&pool->s.lock); pool->s.flags.unsegmented = unseg; - pool->s.flags.zeroized = zeroized; pool->s.seg_size = unseg ? blk_size : seg_len; pool->s.blk_size = blk_size; @@ -383,9 +448,7 @@ odp_pool_t _pool_create(const char *name, /* Iniitalize buffer metadata */ tmp->allocator = ODP_FREEBUF; tmp->flags.all = 0; - tmp->flags.zeroized = zeroized; tmp->size = 0; - odp_atomic_init_u32(&tmp->ref_count, 0); tmp->type = params->type; tmp->event_type = params->type; tmp->pool_hdl = pool->s.pool_hdl; @@ -503,6 +566,41 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info) return 0; } +static inline void get_local_cache_bufs(local_cache_t *buf_cache, uint32_t idx, + odp_buffer_hdr_t *buf_hdr[], + uint32_t num) +{ + uint32_t i; + + for (i = 0; i < num; i++) { + buf_hdr[i] = buf_cache->s.buf[idx + i]; + odp_prefetch(buf_hdr[i]); + odp_prefetch_store(buf_hdr[i]); + } +} + +static void flush_cache(local_cache_t *buf_cache, struct pool_entry_s *pool) +{ + uint32_t flush_count = 0; + uint32_t num; + + while ((num = buf_cache->s.num_buf)) { + odp_buffer_hdr_t *buf; + + buf = buf_cache->s.buf[num - 1]; + ret_buf(pool, buf); + flush_count++; + buf_cache->s.num_buf--; + } + + odp_atomic_add_u64(&pool->poolstats.bufallocs, buf_cache->s.bufallocs); + odp_atomic_add_u64(&pool->poolstats.buffrees, + buf_cache->s.buffrees - flush_count); + + buf_cache->s.bufallocs = 0; + buf_cache->s.buffrees = 0; +} + int odp_pool_destroy(odp_pool_t pool_hdl) { uint32_t pool_id = pool_handle_to_index(pool_hdl); @@ -621,71 +719,207 @@ void seg_free_tail(odp_buffer_hdr_t *buf_hdr, int segcount) buf_hdr->size = buf_hdr->segcount * pool->s.seg_size; } -odp_buffer_t buffer_alloc(odp_pool_t pool_hdl, size_t size) +static inline int get_local_bufs(local_cache_t *buf_cache, + odp_buffer_hdr_t *buf_hdr[], uint32_t max_num) +{ + uint32_t num_buf = buf_cache->s.num_buf; + uint32_t num = num_buf; + + if (odp_unlikely(num_buf == 0)) + return 0; + + if (odp_likely(max_num < num)) + num = max_num; + + get_local_cache_bufs(buf_cache, num_buf - num, buf_hdr, num); + buf_cache->s.num_buf -= num; + buf_cache->s.bufallocs += num; + + return num; +} + +static inline void ret_local_buf(local_cache_t *buf_cache, uint32_t idx, + odp_buffer_hdr_t *buf) +{ + buf_cache->s.buf[idx] = buf; + buf_cache->s.num_buf++; + buf_cache->s.buffrees++; +} + +static inline void ret_local_bufs(local_cache_t *buf_cache, uint32_t idx, + odp_buffer_hdr_t *buf[], int num_buf) +{ + int i; + + for (i = 0; i < num_buf; i++) + buf_cache->s.buf[idx + i] = buf[i]; + + buf_cache->s.num_buf += num_buf; + buf_cache->s.buffrees += num_buf; +} + +int buffer_alloc_multi(odp_pool_t pool_hdl, size_t size, + odp_buffer_t buf[], int max_num) { uint32_t pool_id = pool_handle_to_index(pool_hdl); pool_entry_t *pool = get_pool_entry(pool_id); uintmax_t totsize = pool->s.headroom + size + pool->s.tailroom; - odp_anybuf_t *buf; + odp_buffer_hdr_t *buf_tbl[max_num]; + odp_buffer_hdr_t *buf_hdr; + int num, i; + intmax_t needed; + void *blk; /* Reject oversized allocation requests */ if ((pool->s.flags.unsegmented && totsize > pool->s.seg_size) || (!pool->s.flags.unsegmented && totsize > pool->s.seg_size * ODP_BUFFER_MAX_SEG)) - return ODP_BUFFER_INVALID; + return 0; /* Try to satisfy request from the local cache */ - buf = (odp_anybuf_t *) - (void *)get_local_buf(&pool->s.local_cache[local_id], - &pool->s, totsize); + num = get_local_bufs(local.cache[pool_id], buf_tbl, max_num); /* If cache is empty, satisfy request from the pool */ - if (odp_unlikely(buf == NULL)) { - buf = (odp_anybuf_t *)(void *)get_buf(&pool->s); + if (odp_unlikely(num < max_num)) { + for (; num < max_num; num++) { + buf_hdr = get_buf(&pool->s); + + if (odp_unlikely(buf_hdr == NULL)) + goto pool_empty; + + /* Get blocks for this buffer, if pool uses + * application data */ + if (buf_hdr->size < totsize) { + uint32_t segcount; + + needed = totsize - buf_hdr->size; + do { + blk = get_blk(&pool->s); + if (odp_unlikely(blk == NULL)) { + ret_buf(&pool->s, buf_hdr); + goto pool_empty; + } + + segcount = buf_hdr->segcount++; + buf_hdr->addr[segcount] = blk; + needed -= pool->s.seg_size; + } while (needed > 0); + buf_hdr->size = buf_hdr->segcount * + pool->s.seg_size; + } - if (odp_unlikely(buf == NULL)) - return ODP_BUFFER_INVALID; + buf_tbl[num] = buf_hdr; + } + } - /* Get blocks for this buffer, if pool uses application data */ - if (buf->buf.size < totsize) { - intmax_t needed = totsize - buf->buf.size; +pool_empty: + for (i = 0; i < num; i++) { + buf_hdr = buf_tbl[i]; + + /* Mark buffer as allocated */ + buf_hdr->allocator = local.thr_id; + + /* By default, buffers are not associated with + * an ordered queue */ + buf_hdr->origin_qe = NULL; + + buf[i] = odp_hdr_to_buf(buf_hdr); + + /* Add more segments if buffer from local cache is too small */ + if (odp_unlikely(buf_hdr->size < totsize)) { + needed = totsize - buf_hdr->size; do { - uint8_t *blk = get_blk(&pool->s); - if (blk == NULL) { - ret_buf(&pool->s, &buf->buf); - return ODP_BUFFER_INVALID; + blk = get_blk(&pool->s); + if (odp_unlikely(blk == NULL)) { + int j; + + ret_buf(&pool->s, buf_hdr); + buf_hdr = NULL; + local.cache[pool_id]->s.buffrees--; + + /* move remaining bufs up one step + * and update loop counters */ + num--; + for (j = i; j < num; j++) + buf_tbl[j] = buf_tbl[j + 1]; + + i--; + break; } - buf->buf.addr[buf->buf.segcount++] = blk; needed -= pool->s.seg_size; + buf_hdr->addr[buf_hdr->segcount++] = blk; + buf_hdr->size = buf_hdr->segcount * + pool->s.seg_size; } while (needed > 0); - buf->buf.size = buf->buf.segcount * pool->s.seg_size; } } - /* Mark buffer as allocated */ - buf->buf.allocator = local_id; + return num; +} - /* By default, buffers inherit their pool's zeroization setting */ - buf->buf.flags.zeroized = pool->s.flags.zeroized; +odp_buffer_t buffer_alloc(odp_pool_t pool_hdl, size_t size) +{ + uint32_t pool_id = pool_handle_to_index(pool_hdl); + pool_entry_t *pool = get_pool_entry(pool_id); + uintmax_t totsize = pool->s.headroom + size + pool->s.tailroom; + odp_buffer_hdr_t *buf_hdr; + intmax_t needed; + void *blk; - /* By default, buffers are not associated with an ordered queue */ - buf->buf.origin_qe = NULL; + /* Reject oversized allocation requests */ + if ((pool->s.flags.unsegmented && totsize > pool->s.seg_size) || + (!pool->s.flags.unsegmented && + totsize > pool->s.seg_size * ODP_BUFFER_MAX_SEG)) + return 0; - return odp_hdr_to_buf(&buf->buf); -} + /* Try to satisfy request from the local cache. If cache is empty, + * satisfy request from the pool */ + if (odp_unlikely(!get_local_bufs(local.cache[pool_id], &buf_hdr, 1))) { + buf_hdr = get_buf(&pool->s); -int buffer_alloc_multi(odp_pool_t pool_hdl, size_t size, - odp_buffer_t buf[], int num) -{ - int count; + if (odp_unlikely(buf_hdr == NULL)) + return ODP_BUFFER_INVALID; + + /* Get blocks for this buffer, if pool uses application data */ + if (buf_hdr->size < totsize) { + needed = totsize - buf_hdr->size; + do { + blk = get_blk(&pool->s); + if (odp_unlikely(blk == NULL)) { + ret_buf(&pool->s, buf_hdr); + return ODP_BUFFER_INVALID; + } + buf_hdr->addr[buf_hdr->segcount++] = blk; + needed -= pool->s.seg_size; + } while (needed > 0); + buf_hdr->size = buf_hdr->segcount * pool->s.seg_size; + } + } + /* Mark buffer as allocated */ + buf_hdr->allocator = local.thr_id; - for (count = 0; count < num; ++count) { - buf[count] = buffer_alloc(pool_hdl, size); - if (buf[count] == ODP_BUFFER_INVALID) - break; + /* By default, buffers are not associated with + * an ordered queue */ + buf_hdr->origin_qe = NULL; + + /* Add more segments if buffer from local cache is too small */ + if (odp_unlikely(buf_hdr->size < totsize)) { + needed = totsize - buf_hdr->size; + do { + blk = get_blk(&pool->s); + if (odp_unlikely(blk == NULL)) { + ret_buf(&pool->s, buf_hdr); + buf_hdr = NULL; + local.cache[pool_id]->s.buffrees--; + return ODP_BUFFER_INVALID; + } + buf_hdr->addr[buf_hdr->segcount++] = blk; + needed -= pool->s.seg_size; + } while (needed > 0); + buf_hdr->size = buf_hdr->segcount * pool->s.seg_size; } - return count; + return odp_hdr_to_buf(buf_hdr); } odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) @@ -701,35 +935,132 @@ int odp_buffer_alloc_multi(odp_pool_t pool_hdl, odp_buffer_t buf[], int num) return buffer_alloc_multi(pool_hdl, buf_size, buf, num); } -void odp_buffer_free(odp_buffer_t buf) +static void multi_pool_free(odp_buffer_hdr_t *buf_hdr[], int num_buf) { - odp_buffer_hdr_t *buf_hdr = odp_buf_to_hdr(buf); - pool_entry_t *pool = odp_buf_to_pool(buf_hdr); + uint32_t pool_id, num; + local_cache_t *buf_cache; + pool_entry_t *pool; + int i, j, idx; - ODP_ASSERT(buf_hdr->allocator != ODP_FREEBUF); + for (i = 0; i < num_buf; i++) { + pool_id = pool_handle_to_index(buf_hdr[i]->pool_hdl); + buf_cache = local.cache[pool_id]; + num = buf_cache->s.num_buf; + + if (num < POOL_MAX_LOCAL_BUFS) { + ret_local_buf(buf_cache, num, buf_hdr[i]); + continue; + } + + idx = POOL_MAX_LOCAL_BUFS - POOL_CHUNK_SIZE; + pool = get_pool_entry(pool_id); + + /* local cache full, return a chunk */ + for (j = 0; j < POOL_CHUNK_SIZE; j++) { + odp_buffer_hdr_t *tmp; + + tmp = buf_cache->s.buf[idx + i]; + ret_buf(&pool->s, tmp); + } - if (odp_unlikely(pool->s.buf_low_wm_assert || pool->s.blk_low_wm_assert)) - ret_buf(&pool->s, buf_hdr); - else - ret_local_buf(&pool->s.local_cache[local_id], buf_hdr); + num = POOL_MAX_LOCAL_BUFS - POOL_CHUNK_SIZE; + buf_cache->s.num_buf = num; + ret_local_buf(buf_cache, num, buf_hdr[i]); + } } -void odp_buffer_free_multi(const odp_buffer_t buf[], int num) +void buffer_free_multi(uint32_t pool_id, + const odp_buffer_t buf[], int num_free) { - int i; + local_cache_t *buf_cache = local.cache[pool_id]; + uint32_t num; + int i, idx; + pool_entry_t *pool; + odp_buffer_hdr_t *buf_hdr[num_free]; + int multi_pool = 0; + + for (i = 0; i < num_free; i++) { + uint32_t id; + + buf_hdr[i] = odp_buf_to_hdr(buf[i]); + ODP_ASSERT(buf_hdr[i]->allocator != ODP_FREEBUF); + buf_hdr[i]->allocator = ODP_FREEBUF; + id = pool_handle_to_index(buf_hdr[i]->pool_hdl); + multi_pool |= (pool_id != id); + } + + if (odp_unlikely(multi_pool)) { + multi_pool_free(buf_hdr, num_free); + return; + } + + num = buf_cache->s.num_buf; - for (i = 0; i < num; ++i) - odp_buffer_free(buf[i]); + if (odp_likely((num + num_free) < POOL_MAX_LOCAL_BUFS)) { + ret_local_bufs(buf_cache, num, buf_hdr, num_free); + return; + } + + pool = get_pool_entry(pool_id); + + /* Return at least one chunk into the global pool */ + if (odp_unlikely(num_free > POOL_CHUNK_SIZE)) { + for (i = 0; i < num_free; i++) + ret_buf(&pool->s, buf_hdr[i]); + + return; + } + + idx = num - POOL_CHUNK_SIZE; + for (i = 0; i < POOL_CHUNK_SIZE; i++) + ret_buf(&pool->s, buf_cache->s.buf[idx + i]); + + num -= POOL_CHUNK_SIZE; + buf_cache->s.num_buf = num; + ret_local_bufs(buf_cache, num, buf_hdr, num_free); } -void _odp_flush_caches(void) +void buffer_free(uint32_t pool_id, const odp_buffer_t buf) { + local_cache_t *buf_cache = local.cache[pool_id]; + uint32_t num; int i; + pool_entry_t *pool; + odp_buffer_hdr_t *buf_hdr; - for (i = 0; i < ODP_CONFIG_POOLS; i++) { - pool_entry_t *pool = get_pool_entry(i); - flush_cache(&pool->s.local_cache[local_id], &pool->s); + buf_hdr = odp_buf_to_hdr(buf); + ODP_ASSERT(buf_hdr->allocator != ODP_FREEBUF); + buf_hdr->allocator = ODP_FREEBUF; + + num = buf_cache->s.num_buf; + + if (odp_likely((num + 1) < POOL_MAX_LOCAL_BUFS)) { + ret_local_bufs(buf_cache, num, &buf_hdr, 1); + return; } + + pool = get_pool_entry(pool_id); + + num -= POOL_CHUNK_SIZE; + for (i = 0; i < POOL_CHUNK_SIZE; i++) + ret_buf(&pool->s, buf_cache->s.buf[num + i]); + + buf_cache->s.num_buf = num; + ret_local_bufs(buf_cache, num, &buf_hdr, 1); +} + +void odp_buffer_free(odp_buffer_t buf) +{ + uint32_t pool_id = pool_id_from_buf(buf); + + buffer_free(pool_id, buf); +} + +void odp_buffer_free_multi(const odp_buffer_t buf[], int num) +{ + uint32_t pool_id = pool_id_from_buf(buf[0]); + + buffer_free_multi(pool_id, buf, num); } void odp_pool_print(odp_pool_t pool_hdl) @@ -774,7 +1105,6 @@ void odp_pool_print(odp_pool_t pool_hdl) pool->s.quiesced ? "quiesced" : "active"); ODP_DBG(" pool opts %s, %s, %s\n", pool->s.flags.unsegmented ? "unsegmented" : "segmented", - pool->s.flags.zeroized ? "zeroized" : "non-zeroized", pool->s.flags.predefined ? "predefined" : "created"); ODP_DBG(" pool base %p\n", pool->s.pool_base_addr); ODP_DBG(" pool size %zu (%zu pages)\n", @@ -817,10 +1147,11 @@ void odp_pool_print(odp_pool_t pool_hdl) ODP_DBG(" blk low wm count %lu\n", blklowmct); } - odp_pool_t odp_buffer_pool(odp_buffer_t buf) { - return odp_buf_to_hdr(buf)->pool_hdl; + uint32_t pool_id = pool_id_from_buf(buf); + + return pool_index_to_handle(pool_id); } void odp_pool_param_init(odp_pool_param_t *params) ----------------------------------------------------------------------- Summary of changes: .../linux-generic/include/odp_buffer_inlines.h | 26 +- .../linux-generic/include/odp_buffer_internal.h | 5 +- platform/linux-generic/include/odp_internal.h | 2 - .../linux-generic/include/odp_packet_internal.h | 4 +- platform/linux-generic/include/odp_packet_netmap.h | 6 + platform/linux-generic/include/odp_packet_socket.h | 1 + platform/linux-generic/include/odp_pool_internal.h | 143 +------ platform/linux-generic/odp_buffer.c | 3 - platform/linux-generic/odp_packet.c | 68 +-- platform/linux-generic/odp_pool.c | 465 ++++++++++++++++++--- platform/linux-generic/pktio/dpdk.c | 24 +- platform/linux-generic/pktio/netmap.c | 127 +++--- platform/linux-generic/pktio/pcap.c | 26 +- platform/linux-generic/pktio/socket.c | 21 +- platform/linux-generic/pktio/socket_mmap.c | 7 +- platform/linux-generic/pktio/tap.c | 7 +- test/common_plat/performance/odp_scheduling.c | 50 ++- 17 files changed, 621 insertions(+), 364 deletions(-) hooks/post-receive --

9 years, 7 months

[COMMIT] branch api-next updated. v1.10.1.0-104-gca7d1dd

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via ca7d1dd9372957732fc1dc95d0ffe36cf34c23ba (commit) from 064cdece468e83bb72ef9ece63e8f0e05856eeb7 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit ca7d1dd9372957732fc1dc95d0ffe36cf34c23ba Author: Mike Holmes <mike.holmes(a)linaro.org> Date: Mon Jul 11 12:46:50 2016 -0400 doc: driver-guide: initial revision Add an initial driver interface document structure for the existing driver framework. Signed-off-by: Mike Holmes <mike.holmes(a)linaro.org> Reviewed-and-tested-by: Bill Fischofer <bill.fischofer(a)linaro.org> Reviewed-by: Christophe Milard <christophe.milard(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/configure.ac b/configure.ac index c0f0f21..3f2209b 100644 --- a/configure.ac +++ b/configure.ac @@ -188,7 +188,9 @@ DX_INIT_DOXYGEN($PACKAGE_NAME, ${srcdir}/doc/helper-guide/Doxyfile, ${builddir}/doc/helper-guide/output, ${srcdir}/doc/platform-api-guide/Doxyfile, - ${builddir}/doc/platform-api-guide/output) + ${builddir}/doc/platform-api-guide/output, + ${srcdir}/doc/driver-api-guide/Doxyfile, + ${builddir}/doc/driver-api-guide/output) ########################################################################## # Enable/disable ODP_DEBUG_PRINT diff --git a/doc/Makefile.am b/doc/Makefile.am index d49d84b..59d6a6c 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -1,4 +1,8 @@ -SUBDIRS = application-api-guide helper-guide platform-api-guide +SUBDIRS = \ + application-api-guide \ + helper-guide \ + platform-api-guide \ + driver-api-guide if user_guide SUBDIRS += implementers-guide users-guide process-guide diff --git a/doc/driver-api-guide/.gitignore b/doc/driver-api-guide/.gitignore new file mode 100644 index 0000000..53752db --- /dev/null +++ b/doc/driver-api-guide/.gitignore @@ -0,0 +1 @@ +output diff --git a/doc/driver-api-guide/Doxyfile b/doc/driver-api-guide/Doxyfile new file mode 100644 index 0000000..680d1d4 --- /dev/null +++ b/doc/driver-api-guide/Doxyfile @@ -0,0 +1,14 @@ +@INCLUDE = $(SRCDIR)/doc/Doxyfile_common + +PROJECT_NAME = "Driver Interface (drv) Reference Manual" +PROJECT_NUMBER = $(VERSION) +PROJECT_LOGO = $(SRCDIR)/doc/images/ODP-Logo-HQ.svg +INPUT = $(SRCDIR)/doc/driver-api-guide \ + $(SRCDIR)/include/odp/drv \ + $(SRCDIR)/include/odp_drv.h +EXCLUDE_PATTERNS = drv* odp_drv.h +EXAMPLE_PATH = $(SRCDIR)/example $(SRCDIR) +PREDEFINED = __GNUC__ \ + "ODP_HANDLE_T(type)=odp_handle_t type" \ + odpdrv_bool_t=int +WARNINGS = NO diff --git a/doc/driver-api-guide/Makefile.am b/doc/driver-api-guide/Makefile.am new file mode 100644 index 0000000..4fc4755 --- /dev/null +++ b/doc/driver-api-guide/Makefile.am @@ -0,0 +1,5 @@ +EXTRA_DIST = \ + odp.dox + +clean-local: + rm -rf output diff --git a/doc/driver-api-guide/odp.dox b/doc/driver-api-guide/odp.dox new file mode 100644 index 0000000..687a79e --- /dev/null +++ b/doc/driver-api-guide/odp.dox @@ -0,0 +1,20 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @mainpage + * + * @section sec_1 Introduction + * + * OpenDataPlane (ODP) provides a driver interface + + * + * @section contact Contact Details + * - The main web site is http://www.opendataplane.org/ + * - The git repo is https://git.linaro.org/lng/odp.git + * - Bug tracking is https://bugs.linaro.org/buglist.cgi?product=OpenDataPlane + * + */ diff --git a/doc/m4/configure.m4 b/doc/m4/configure.m4 index ed9451d..6e02f76 100644 --- a/doc/m4/configure.m4 +++ b/doc/m4/configure.m4 @@ -42,4 +42,5 @@ AC_CONFIG_FILES([doc/application-api-guide/Makefile doc/Makefile doc/platform-api-guide/Makefile doc/process-guide/Makefile - doc/users-guide/Makefile]) + doc/users-guide/Makefile + doc/driver-api-guide/Makefile]) ----------------------------------------------------------------------- Summary of changes: configure.ac | 4 +++- doc/Makefile.am | 6 +++++- doc/{application-api-guide => driver-api-guide}/.gitignore | 0 doc/{application-api-guide => driver-api-guide}/Doxyfile | 10 ++++++---- doc/{platform-api-guide => driver-api-guide}/Makefile.am | 2 +- doc/{helper-guide => driver-api-guide}/odp.dox | 10 ++-------- doc/m4/configure.m4 | 3 ++- 7 files changed, 19 insertions(+), 16 deletions(-) copy doc/{application-api-guide => driver-api-guide}/.gitignore (100%) copy doc/{application-api-guide => driver-api-guide}/Doxyfile (51%) copy doc/{platform-api-guide => driver-api-guide}/Makefile.am (74%) copy doc/{helper-guide => driver-api-guide}/odp.dox (55%) hooks/post-receive --

9 years, 7 months

[COMMIT] branch api-next updated. v1.10.1.0-103-g064cdec

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via 064cdece468e83bb72ef9ece63e8f0e05856eeb7 (commit) via 20f14c1e726c97cb675c26627c073b730bcb58e6 (commit) via d440fa6b463d9dad5e852d7970c5f77493f190f0 (commit) from 8fede35ce4e41dc8a5de3164be40e2e630cd0812 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 064cdece468e83bb72ef9ece63e8f0e05856eeb7 Merge: 8fede35 20f14c1 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Fri Jul 22 15:38:17 2016 +0300 Merge branch 'master' into api-next Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --cc platform/Makefile.inc index 404c798,a44f88f..079f34d --- a/platform/Makefile.inc +++ b/platform/Makefile.inc @@@ -61,12 -61,19 +61,29 @@@ odpapispecinclude_HEADERS = $(top_srcdir)/include/odp/api/spec/traffic_mngr.h \ $(top_srcdir)/include/odp/api/spec/version.h +odpdrvspecincludedir= $(includedir)/odp/drv/spec +odpdrvspecinclude_HEADERS = \ + $(top_srcdir)/include/odp/drv/spec/align.h \ + $(top_srcdir)/include/odp/drv/spec/atomic.h \ + $(top_srcdir)/include/odp/drv/spec/byteorder.h \ + $(top_srcdir)/include/odp/drv/spec/compiler.h \ + $(top_srcdir)/include/odp/drv/spec/spinlock.h \ + $(top_srcdir)/include/odp/drv/spec/std_types.h \ + $(top_srcdir)/include/odp/drv/spec/sync.h ++ + EXTRA_DIST = \ + arch/arm/odp/api/cpu_arch.h \ + arch/arm/odp_cpu_arch.c \ + arch/arm/odp_sysinfo_parse.c \ + arch/default/odp/api/cpu_arch.h \ + arch/default/odp_cpu_arch.c \ + arch/default/odp_sysinfo_parse.c \ + arch/mips64/odp/api/cpu_arch.h \ + arch/mips64/odp_cpu_arch.c \ + arch/mips64/odp_sysinfo_parse.c \ + arch/powerpc/odp/api/cpu_arch.h \ + arch/powerpc/odp_cpu_arch.c \ + arch/powerpc/odp_sysinfo_parse.c \ + arch/x86/odp/api/cpu_arch.h \ + arch/x86/odp_cpu_arch.c \ + arch/x86/odp_sysinfo_parse.c ----------------------------------------------------------------------- Summary of changes: platform/Makefile.inc | 17 +++++++++++++++++ platform/linux-generic/Makefile.am | 17 ----------------- platform/linux-generic/include/odp_config_internal.h | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) hooks/post-receive --

9 years, 7 months

[COMMIT] branch master updated. v1.10.1.0-62-g20f14c1

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via 20f14c1e726c97cb675c26627c073b730bcb58e6 (commit) from d440fa6b463d9dad5e852d7970c5f77493f190f0 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 20f14c1e726c97cb675c26627c073b730bcb58e6 Author: Anders Roxell <anders.roxell(a)linaro.org> Date: Fri Jul 22 12:52:22 2016 +0200 linux-gen: Makefile: Move EXTRA_DIST to to platform/Makefile.inc Make it easier for other platforms like odp-dpdk to reuse arch specific files in EXTRA_DIST for "free". Reported-by: Fathi Boudra <fathi.boudra(a)linaro.org> Signed-off-by: Anders Roxell <anders.roxell(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/Makefile.inc b/platform/Makefile.inc index 053bd12..a44f88f 100644 --- a/platform/Makefile.inc +++ b/platform/Makefile.inc @@ -60,3 +60,20 @@ odpapispecinclude_HEADERS = \ $(top_srcdir)/include/odp/api/spec/timer.h \ $(top_srcdir)/include/odp/api/spec/traffic_mngr.h \ $(top_srcdir)/include/odp/api/spec/version.h + +EXTRA_DIST = \ + arch/arm/odp/api/cpu_arch.h \ + arch/arm/odp_cpu_arch.c \ + arch/arm/odp_sysinfo_parse.c \ + arch/default/odp/api/cpu_arch.h \ + arch/default/odp_cpu_arch.c \ + arch/default/odp_sysinfo_parse.c \ + arch/mips64/odp/api/cpu_arch.h \ + arch/mips64/odp_cpu_arch.c \ + arch/mips64/odp_sysinfo_parse.c \ + arch/powerpc/odp/api/cpu_arch.h \ + arch/powerpc/odp_cpu_arch.c \ + arch/powerpc/odp_sysinfo_parse.c \ + arch/x86/odp/api/cpu_arch.h \ + arch/x86/odp_cpu_arch.c \ + arch/x86/odp_sysinfo_parse.c diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index c8fd8cb..0cfd0fe 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -186,23 +186,6 @@ __LIB__libodp_linux_la_SOURCES = \ arch/@ARCH_DIR@/odp_cpu_arch.c \ arch/@ARCH_DIR@/odp_sysinfo_parse.c -EXTRA_DIST = \ - arch/default/odp/api/cpu_arch.h \ - arch/default/odp_cpu_arch.c \ - arch/default/odp_sysinfo_parse.c \ - arch/mips64/odp/api/cpu_arch.h \ - arch/mips64/odp_cpu_arch.c \ - arch/mips64/odp_sysinfo_parse.c \ - arch/powerpc/odp/api/cpu_arch.h \ - arch/powerpc/odp_cpu_arch.c \ - arch/powerpc/odp_sysinfo_parse.c \ - arch/x86/odp/api/cpu_arch.h \ - arch/x86/odp_cpu_arch.c \ - arch/x86/odp_sysinfo_parse.c \ - arch/arm/odp/api/cpu_arch.h \ - arch/arm/odp_cpu_arch.c \ - arch/arm/odp_sysinfo_parse.c - if HAVE_PCAP __LIB__libodp_linux_la_SOURCES += pktio/pcap.c endif ----------------------------------------------------------------------- Summary of changes: platform/Makefile.inc | 17 +++++++++++++++++ platform/linux-generic/Makefile.am | 17 ----------------- 2 files changed, 17 insertions(+), 17 deletions(-) hooks/post-receive --

9 years, 7 months

[COMMIT] branch master updated. v1.10.1.0-61-gd440fa6

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via d440fa6b463d9dad5e852d7970c5f77493f190f0 (commit) from 5d5304117434ab340cdd138066561e00a6b0642d (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit d440fa6b463d9dad5e852d7970c5f77493f190f0 Author: Oriol Arcas <oriol(a)starflownetworks.com> Date: Tue Jul 19 11:16:43 2016 +0200 linux-generic: pool: set maximum number of pools to 64 Signed-off-by: Oriol Arcas <oriol(a)starflownetworks.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index b108c07..989ea08 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -14,7 +14,7 @@ extern "C" { /* * Maximum number of pools */ -#define ODP_CONFIG_POOLS 16 +#define ODP_CONFIG_POOLS 64 /* * Maximum number of queues ----------------------------------------------------------------------- Summary of changes: platform/linux-generic/include/odp_config_internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) hooks/post-receive --

9 years, 7 months

Jump to page:

2026

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

git-lng-odp