This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "".

The branch, master has been updated via 9bb623dfe97590991dfe3ab04e1dcfb9ff745169 (commit) via 8b1dd78079de09d5f51c61b5b2ede2766ac74685 (commit) via 920857807c4d7831ada68ebed4ffa542564ee3a5 (commit) via 43bfbbf66db3d8fd6a22762d7ac346461857f653 (commit) via a959a61e3d00965af8b93536e3fbc3bc09cad1e9 (commit) via 9e4826c45af05eb339d129140ac88bc9d4e5a2fd (commit) via fc539b10fff8b92c234f422905eec2266ec13b26 (commit) via 04e5b3dd319d1affb61e0f6860e255082a6c15df (commit) via c65cfd73c6103d02f33a62099391a98cf60d098f (commit) from 490f4bf22129638899ce71c99a8847e8ba849692 (commit)

Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below.

- Log ----------------------------------------------------------------- commit 9bb623dfe97590991dfe3ab04e1dcfb9ff745169 Author: Dmitry Eremin-Solenikov dmitry.ereminsolenikov@linaro.org Date: Fri Jul 14 03:17:05 2017 +0300

example: ipsec: fix build with OpenSSL located in non-standard path

IPsec example uses OpenSSL directly to generate packages stream. Thus use $(OPENSSL_CPPFLAGS) and $(OPENSSL_LIBS) to let compiler find OpenSSL if it is present in non-standard location.

Signed-off-by: Dmitry Eremin-Solenikov dmitry.ereminsolenikov@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/example/ipsec/Makefile.am b/example/ipsec/Makefile.am index 01ccd426..fd9b3c72 100644 --- a/example/ipsec/Makefile.am +++ b/example/ipsec/Makefile.am @@ -1,5 +1,7 @@ include $(top_srcdir)/example/Makefile.inc

+AM_CPPFLAGS = $(OPENSSL_CPPFLAGS) + bin_PROGRAMS = odp_ipsec$(EXEEXT) odp_ipsec_LDFLAGS = $(AM_LDFLAGS) -static odp_ipsec_CFLAGS = $(AM_CFLAGS) -I${top_srcdir}/example

commit 8b1dd78079de09d5f51c61b5b2ede2766ac74685 Author: Matias Elo matias.elo@nokia.com Date: Mon Jul 10 16:32:05 2017 +0300

travis: add zero-copy dpdk pktio to CI

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/.travis.yml b/.travis.yml index 665d70f8..a1306206 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,6 +64,7 @@ env: - CONF="--disable-abi-compat" - CONF="--enable-schedule-sp" - CONF="--enable-schedule-iquery" + - CONF="--enable-dpdk-zero-copy"

before_install: # Install cunit for the validation tests because distro version is too old and fails C99 compile

commit 920857807c4d7831ada68ebed4ffa542564ee3a5 Author: Matias Elo matias.elo@nokia.com Date: Mon Jun 26 16:31:41 2017 +0300

linux-gen: dpdk: enable zero-copy operation

Implements experimental zero-copy mode for DPDK pktio. This can be enabled with additional '--enable-dpdk-zero-copy' configure flag.

This feature has been put behind an extra configure flag as it doesn't entirely adhere to the DPDK API and may behave unexpectedly with untested DPDK NIC drivers. Zero-copy operation has been tested with pcap, ixgbe, and i40e drivers.

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 1e8a6486..20127004 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -100,7 +100,7 @@ struct odp_buffer_hdr_t {

/* Data or next header */ uint8_t data[0]; -}; +} ODP_ALIGNED_CACHE;

ODP_STATIC_ASSERT(CONFIG_PACKET_MAX_SEGS < 256, "CONFIG_PACKET_MAX_SEGS_TOO_LARGE"); diff --git a/platform/linux-generic/include/odp_packet_dpdk.h b/platform/linux-generic/include/odp_packet_dpdk.h index 4d7e0fc4..5d80d84a 100644 --- a/platform/linux-generic/include/odp_packet_dpdk.h +++ b/platform/linux-generic/include/odp_packet_dpdk.h @@ -52,8 +52,6 @@ typedef struct { odp_pktio_capability_t capa; /**< interface capabilities */ uint32_t data_room; /**< maximum packet length */ uint16_t mtu; /**< maximum transmission unit */ - /** DPDK packet pool name (pktpool_<ifname>) */ - char pool_name[IF_NAMESIZE + 8]; /** Use system call to get/set vdev promisc mode */ odp_bool_t vdev_sysc_promisc; uint8_t port_id; /**< DPDK port identifier */ diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index b4bec35b..f77987c8 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -93,6 +93,12 @@ typedef struct { uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ } packet_parser_t;

+/* Packet extra data length */ +#define PKT_EXTRA_LEN 128 + +/* Packet extra data types */ +#define PKT_EXTRA_TYPE_DPDK 1 + /** * Internal Packet header * @@ -132,6 +138,13 @@ typedef struct { /* Result for crypto */ odp_crypto_generic_op_result_t op_result;

+#ifdef ODP_PKTIO_DPDK + /* Type of extra data */ + uint8_t extra_type; + /* Extra space for packet descriptors. E.g. DPDK mbuf */ + uint8_t extra[PKT_EXTRA_LEN] ODP_ALIGNED_CACHE; +#endif + /* Packet data storage */ uint8_t data[0]; } odp_packet_hdr_t; diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index a4c0e532..f2d2e2ca 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -42,6 +42,9 @@ typedef struct {

} pool_ring_t ODP_ALIGNED_CACHE;

+/* Callback function for pool destroy */ +typedef void (*pool_destroy_cb_fn)(void *pool); + typedef struct pool_t { odp_ticketlock_t lock ODP_ALIGNED_CACHE;

@@ -67,6 +70,10 @@ typedef struct pool_t { uint8_t *base_addr; uint8_t *uarea_base_addr;

+ /* Used by DPDK zero-copy pktio */ + pool_destroy_cb_fn ext_destroy; + void *ext_desc; + pool_cache_t local_cache[ODP_THREAD_COUNT_MAX];

odp_shm_t ring_shm; diff --git a/platform/linux-generic/m4/odp_dpdk.m4 b/platform/linux-generic/m4/odp_dpdk.m4 index 1995e0fe..cebf1028 100644 --- a/platform/linux-generic/m4/odp_dpdk.m4 +++ b/platform/linux-generic/m4/odp_dpdk.m4 @@ -9,6 +9,16 @@ AC_HELP_STRING([--with-dpdk-path=DIR path to dpdk build directory]), pktio_dpdk_support=yes],[])

########################################################################## +# Enable zero-copy DPDK pktio +########################################################################## +zero_copy=0 +AC_ARG_ENABLE([dpdk-zero-copy], + [ --enable-dpdk-zero-copy enable experimental zero-copy DPDK pktio mode], + [if test x$enableval = xyes; then + zero_copy=1 + fi]) + +########################################################################## # Save and set temporary compilation flags ########################################################################## OLD_CPPFLAGS="$CPPFLAGS" @@ -37,8 +47,8 @@ then done AS_VAR_APPEND([DPDK_PMDS], [--no-whole-archive])

- ODP_CFLAGS="$ODP_CFLAGS -DODP_PKTIO_DPDK" - DPDK_LIBS="-L$DPDK_PATH/lib -ldpdk -lpthread -ldl -lpcap" + ODP_CFLAGS="$ODP_CFLAGS -DODP_PKTIO_DPDK -DODP_DPDK_ZERO_COPY=$zero_copy" + DPDK_LIBS="-L$DPDK_PATH/lib -ldpdk -lpthread -ldl -lpcap -lm" AC_SUBST([DPDK_CPPFLAGS]) AC_SUBST([DPDK_LIBS]) AC_SUBST([DPDK_PMDS]) diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index bbd56b06..f2410b6e 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -393,6 +393,8 @@ static odp_pool_t pool_create(const char *name, odp_pool_param_t *params, pool->uarea_size = uarea_size; pool->shm_size = num * block_size; pool->uarea_shm_size = num * uarea_size; + pool->ext_desc = NULL; + pool->ext_destroy = NULL;

shm = odp_shm_reserve(pool->name, pool->shm_size, ODP_PAGE_SIZE, shmflags); @@ -537,6 +539,13 @@ int odp_pool_destroy(odp_pool_t pool_hdl) return -1; }

+ /* Destroy external DPDK mempool */ + if (pool->ext_destroy) { + pool->ext_destroy(pool->ext_desc); + pool->ext_destroy = NULL; + pool->ext_desc = NULL; + } + /* Make sure local caches are empty */ for (i = 0; i < ODP_THREAD_COUNT_MAX; i++) flush_cache(&pool->local_cache[i], pool); diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index c52cd09d..9e3e583d 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -11,6 +11,7 @@ #include <sched.h> #include <ctype.h> #include <unistd.h> +#include <math.h>

#include <odp/api/cpumask.h>

@@ -25,10 +26,19 @@ #include <protocols/eth.h>

#include <rte_config.h> +#include <rte_malloc.h> #include <rte_mbuf.h> +#include <rte_mempool.h> #include <rte_ethdev.h> #include <rte_string_fns.h>

+#if ODP_DPDK_ZERO_COPY +ODP_STATIC_ASSERT(CONFIG_PACKET_HEADROOM == RTE_PKTMBUF_HEADROOM, + "ODP and DPDK headroom sizes not matching!"); +ODP_STATIC_ASSERT(PKT_EXTRA_LEN >= sizeof(struct rte_mbuf), + "DPDK rte_mbuf won't fit in odp_packet_hdr_t.extra!"); +#endif + static int disable_pktio; /** !0 this pktio disabled, 0 enabled */

/* Has dpdk_pktio_init() been called */ @@ -58,6 +68,464 @@ void refer_constructors(void) mp_hdlr_init_ops_stack(); }

+/** + * Calculate valid cache size for DPDK packet pool + */ +static unsigned cache_size(uint32_t num) +{ + unsigned size = 0; + unsigned i; + + if (!RTE_MEMPOOL_CACHE_MAX_SIZE) + return 0; + + i = ceil((double)num / RTE_MEMPOOL_CACHE_MAX_SIZE); + i = RTE_MAX(i, 2UL); + for (; i <= (num / 2); ++i) + if ((num % i) == 0) { + size = num / i; + break; + } + if (odp_unlikely(size > RTE_MEMPOOL_CACHE_MAX_SIZE || + (uint32_t)size * 1.5 > num)) { + ODP_ERR("Cache size calc failure: %d\n", size); + size = 0; + } + + return size; +} + +static inline uint16_t mbuf_data_off(struct rte_mbuf *mbuf, + odp_packet_hdr_t *pkt_hdr) +{ + return (uint64_t)pkt_hdr->buf_hdr.seg[0].data - + (uint64_t)mbuf->buf_addr; +} + +/** + * Update mbuf + * + * Called always before rte_mbuf is passed to DPDK. + */ +static inline void mbuf_update(struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr, + uint16_t pkt_len) +{ + mbuf->data_len = pkt_len; + mbuf->pkt_len = pkt_len; + mbuf->refcnt = 1; + + if (odp_unlikely(pkt_hdr->buf_hdr.base_data != + pkt_hdr->buf_hdr.seg[0].data)) + mbuf->data_off = mbuf_data_off(mbuf, pkt_hdr); +} + +/** + * Initialize mbuf + * + * Called once per ODP packet. + */ +static void mbuf_init(struct rte_mempool *mp, struct rte_mbuf *mbuf, + odp_packet_hdr_t *pkt_hdr) +{ + void *buf_addr = pkt_hdr->buf_hdr.base_data - RTE_PKTMBUF_HEADROOM; + + rte_mem_lock_page(buf_addr); + + memset(mbuf, 0, sizeof(struct rte_mbuf)); + + mbuf->priv_size = 0; + mbuf->buf_addr = buf_addr; + mbuf->buf_physaddr = rte_mem_virt2phy(buf_addr); + if (odp_unlikely(mbuf->buf_physaddr == RTE_BAD_PHYS_ADDR)) + ODP_ABORT("Failed to map virt addr to phy"); + + mbuf->buf_len = (uint16_t)rte_pktmbuf_data_room_size(mp); + mbuf->data_off = RTE_PKTMBUF_HEADROOM; + mbuf->pool = mp; + mbuf->refcnt = 1; + mbuf->nb_segs = 1; + mbuf->port = 0xff; + + /* Store ODP packet handle inside rte_mbuf */ + mbuf->userdata = packet_handle(pkt_hdr); + pkt_hdr->extra_type = PKT_EXTRA_TYPE_DPDK; +} + +/** + * Create custom DPDK packet pool + */ +static struct rte_mempool *mbuf_pool_create(const char *name, + pool_t *pool_entry) +{ + struct rte_mempool *mp; + struct rte_pktmbuf_pool_private mbp_priv; + unsigned elt_size; + unsigned num; + uint16_t data_room_size; + + num = pool_entry->num; + data_room_size = pool_entry->max_seg_len + CONFIG_PACKET_HEADROOM; + elt_size = sizeof(struct rte_mbuf) + (unsigned)data_room_size; + mbp_priv.mbuf_data_room_size = data_room_size; + mbp_priv.mbuf_priv_size = 0; + + mp = rte_mempool_create_empty(name, num, elt_size, cache_size(num), + sizeof(struct rte_pktmbuf_pool_private), + rte_socket_id(), 0); + if (mp == NULL) { + ODP_ERR("Failed to create empty DPDK packet pool\n"); + return NULL; + } + + if (rte_mempool_set_ops_byname(mp, "odp_pool", pool_entry)) { + ODP_ERR("Failed setting mempool operations\n"); + return NULL; + } + + rte_pktmbuf_pool_init(mp, &mbp_priv); + + if (rte_mempool_ops_alloc(mp)) { + ODP_ERR("Failed allocating mempool\n"); + return NULL; + } + + return mp; +} + +/* DPDK external memory pool operations */ + +static int pool_enqueue(struct rte_mempool *mp ODP_UNUSED, + void * const *obj_table, unsigned num) +{ + odp_packet_t pkt_tbl[num]; + unsigned i; + + if (odp_unlikely(num == 0)) + return 0; + + for (i = 0; i < num; i++) + pkt_tbl[i] = (odp_packet_t)((struct rte_mbuf *) + obj_table[i])->userdata; + + odp_packet_free_multi(pkt_tbl, num); + + return 0; +} + +static int pool_dequeue_bulk(struct rte_mempool *mp, void **obj_table, + unsigned num) +{ + odp_pool_t pool = (odp_pool_t)mp->pool_data; + pool_t *pool_entry = (pool_t *)mp->pool_config; + odp_packet_t packet_tbl[num]; + int pkts; + int i; + + pkts = packet_alloc_multi(pool, pool_entry->max_seg_len, packet_tbl, + num); + + if (odp_unlikely(pkts != (int)num)) { + if (pkts > 0) + odp_packet_free_multi(packet_tbl, pkts); + return -ENOENT; + } + + for (i = 0; i < pkts; i++) { + odp_packet_t pkt = packet_tbl[i]; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + struct rte_mbuf *mbuf = (struct rte_mbuf *) + (uintptr_t)pkt_hdr->extra; + if (pkt_hdr->extra_type != PKT_EXTRA_TYPE_DPDK) + mbuf_init(mp, mbuf, pkt_hdr); + obj_table[i] = mbuf; + } + + return 0; +} + +static int pool_alloc(struct rte_mempool *mp) +{ + pool_t *pool_entry = (pool_t *)mp->pool_config; + + mp->pool_data = pool_entry->pool_hdl; + mp->flags |= MEMPOOL_F_POOL_CREATED; + + return 0; +} + +static unsigned pool_get_count(const struct rte_mempool *mp) +{ + odp_pool_t pool = (odp_pool_t)mp->pool_data; + odp_pool_info_t info; + + if (odp_pool_info(pool, &info)) { + ODP_ERR("Failed to read pool info\n"); + return 0; + } + return info.params.pkt.num; +} + +static void pool_free(struct rte_mempool *mp) +{ + unsigned lcore_id; + + RTE_LCORE_FOREACH(lcore_id) { + struct rte_mempool_cache *cache; + + cache = rte_mempool_default_cache(mp, lcore_id); + if (cache != NULL) + rte_mempool_cache_flush(cache, mp); + } +} + +static void pool_destroy(void *pool) +{ + struct rte_mempool *mp = (struct rte_mempool *)pool; + + if (mp != NULL) + rte_mempool_free(mp); +} + +static struct rte_mempool_ops ops_stack = { + .name = "odp_pool", + .alloc = pool_alloc, + .free = pool_free, + .enqueue = pool_enqueue, + .dequeue = pool_dequeue_bulk, + .get_count = pool_get_count +}; + +MEMPOOL_REGISTER_OPS(ops_stack); + +static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, + odp_packet_t pkt_table[], + struct rte_mbuf *mbuf_table[], + uint16_t mbuf_num, odp_time_t *ts) +{ + odp_packet_t pkt; + odp_packet_hdr_t *pkt_hdr; + uint16_t pkt_len; + struct rte_mbuf *mbuf; + void *data; + int i, j; + int nb_pkts = 0; + int alloc_len, num; + odp_pool_t pool = pktio_entry->s.pkt_dpdk.pool; + + /* Allocate maximum sized packets */ + alloc_len = pktio_entry->s.pkt_dpdk.data_room; + + num = packet_alloc_multi(pool, alloc_len, pkt_table, mbuf_num); + if (num != mbuf_num) { + ODP_DBG("packet_alloc_multi() unable to allocate all packets: " + "%d/%" PRIu16 " allocated\n", num, mbuf_num); + for (i = num; i < mbuf_num; i++) + rte_pktmbuf_free(mbuf_table[i]); + } + + for (i = 0; i < num; i++) { + odp_packet_hdr_t parsed_hdr; + + mbuf = mbuf_table[i]; + if (odp_unlikely(mbuf->nb_segs != 1)) { + ODP_ERR("Segmented buffers not supported\n"); + goto fail; + } + + data = rte_pktmbuf_mtod(mbuf, char *); + odp_prefetch(data); + + pkt_len = rte_pktmbuf_pkt_len(mbuf); + + if (pktio_cls_enabled(pktio_entry)) { + if (cls_classify_packet(pktio_entry, + (const uint8_t *)data, + pkt_len, pkt_len, &pool, + &parsed_hdr)) + goto fail; + } + + pkt = pkt_table[i]; + pkt_hdr = odp_packet_hdr(pkt); + pull_tail(pkt_hdr, alloc_len - pkt_len); + + if (odp_packet_copy_from_mem(pkt, 0, pkt_len, data) != 0) + goto fail; + + pkt_hdr->input = pktio_entry->s.handle; + + if (pktio_cls_enabled(pktio_entry)) + copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); + else + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); + + if (mbuf->ol_flags & PKT_RX_RSS_HASH) + odp_packet_flow_hash_set(pkt, mbuf->hash.rss); + + packet_set_ts(pkt_hdr, ts); + + pkt_table[nb_pkts++] = pkt; + + rte_pktmbuf_free(mbuf); + } + + return nb_pkts; + +fail: + odp_packet_free_multi(&pkt_table[i], num - i); + + for (j = i; j < num; j++) + rte_pktmbuf_free(mbuf_table[j]); + + return (i > 0 ? i : -1); +} + +static inline int pkt_to_mbuf(pktio_entry_t *pktio_entry, + struct rte_mbuf *mbuf_table[], + const odp_packet_t pkt_table[], uint16_t num) +{ + pkt_dpdk_t *pkt_dpdk = &pktio_entry->s.pkt_dpdk; + int i, j; + char *data; + uint16_t pkt_len; + + if (odp_unlikely((rte_pktmbuf_alloc_bulk(pkt_dpdk->pkt_pool, + mbuf_table, num)))) { + ODP_ERR("Failed to alloc mbuf\n"); + return 0; + } + for (i = 0; i < num; i++) { + pkt_len = _odp_packet_len(pkt_table[i]); + + if (pkt_len > pkt_dpdk->mtu) { + if (i == 0) + __odp_errno = EMSGSIZE; + goto fail; + } + + /* Packet always fits in mbuf */ + data = rte_pktmbuf_append(mbuf_table[i], pkt_len); + + odp_packet_copy_to_mem(pkt_table[i], 0, pkt_len, data); + } + return i; + +fail: + for (j = i; j < num; j++) + rte_pktmbuf_free(mbuf_table[j]); + + return i; +} + +static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, + odp_packet_t pkt_table[], + struct rte_mbuf *mbuf_table[], + uint16_t mbuf_num, odp_time_t *ts) +{ + odp_packet_t pkt; + odp_packet_hdr_t *pkt_hdr; + uint16_t pkt_len; + struct rte_mbuf *mbuf; + void *data; + int i; + int nb_pkts = 0; + odp_pool_t pool = pktio_entry->s.pkt_dpdk.pool; + + for (i = 0; i < mbuf_num; i++) { + odp_packet_hdr_t parsed_hdr; + + mbuf = mbuf_table[i]; + if (odp_unlikely(mbuf->nb_segs != 1)) { + ODP_ERR("Segmented buffers not supported\n"); + rte_pktmbuf_free(mbuf); + continue; + } + + data = rte_pktmbuf_mtod(mbuf, char *); + pkt_len = rte_pktmbuf_pkt_len(mbuf); + + pkt = (odp_packet_t)mbuf->userdata; + pkt_hdr = odp_packet_hdr(pkt); + + if (pktio_cls_enabled(pktio_entry)) { + if (cls_classify_packet(pktio_entry, + (const uint8_t *)data, + pkt_len, pkt_len, &pool, + &parsed_hdr)) + ODP_ERR("Unable to classify packet\n"); + rte_pktmbuf_free(mbuf); + continue; + } + + /* Init buffer segments. Currently, only single segment packets + * are supported. */ + pkt_hdr->buf_hdr.seg[0].data = data; + + packet_init(pkt_hdr, pkt_len); + pkt_hdr->input = pktio_entry->s.handle; + + if (pktio_cls_enabled(pktio_entry)) + copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); + else + packet_parse_layer(pkt_hdr, + pktio_entry->s.config.parser.layer); + + if (mbuf->ol_flags & PKT_RX_RSS_HASH) + odp_packet_flow_hash_set(pkt, mbuf->hash.rss); + + packet_set_ts(pkt_hdr, ts); + + pkt_table[nb_pkts++] = pkt; + } + + return nb_pkts; +} + +static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry, + struct rte_mbuf *mbuf_table[], + const odp_packet_t pkt_table[], uint16_t num, + uint16_t *seg_count) +{ + pkt_dpdk_t *pkt_dpdk = &pktio_entry->s.pkt_dpdk; + int i; + + *seg_count = 0; + + for (i = 0; i < num; i++) { + odp_packet_t pkt = pkt_table[i]; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + struct rte_mbuf *mbuf = (struct rte_mbuf *) + (uintptr_t)pkt_hdr->extra; + uint16_t pkt_len = odp_packet_len(pkt); + + if (odp_unlikely(pkt_len > pkt_dpdk->mtu)) + goto fail; + + if (odp_likely(pkt_hdr->buf_hdr.segcount == 1)) { + if (odp_unlikely(pkt_hdr->extra_type != + PKT_EXTRA_TYPE_DPDK)) + mbuf_init(pkt_dpdk->pkt_pool, mbuf, pkt_hdr); + + mbuf_update(mbuf, pkt_hdr, pkt_len); + } else { + /* Fall back to packet copy */ + if (odp_unlikely(pkt_to_mbuf(pktio_entry, &mbuf, + &pkt, 1) != 1)) + goto fail; + (*seg_count)++; + } + + mbuf_table[i] = mbuf; + } + return i; + +fail: + if (i == 0) + __odp_errno = EMSGSIZE; + return i; +} + /* Test if s has only digits or not. Dpdk pktio uses only digits.*/ static int dpdk_netdev_is_valid(const char *s) { @@ -95,7 +563,7 @@ static uint32_t dpdk_vdev_mtu_get(uint8_t port_id) static uint32_t dpdk_mtu_get(pktio_entry_t *pktio_entry) { pkt_dpdk_t *pkt_dpdk = &pktio_entry->s.pkt_dpdk; - uint32_t mtu; + uint32_t mtu = 0;

if (rte_eth_dev_get_mtu(pkt_dpdk->port_id, (uint16_t *)&mtu)) return 0; @@ -201,13 +669,12 @@ static int dpdk_setup_port(pktio_entry_t *pktio_entry) struct rte_eth_conf port_conf = { .rxmode = { .mq_mode = ETH_MQ_RX_RSS, - .max_rx_pkt_len = pkt_dpdk->data_room, .split_hdr_size = 0, .header_split = 0, .hw_ip_checksum = 0, .hw_vlan_filter = 0, - .jumbo_frame = 1, .hw_strip_crc = 0, + .enable_scatter = 0, }, .rx_adv_conf = { .rss_conf = rss_conf, @@ -245,7 +712,8 @@ static int dpdk_close(pktio_entry_t *pktio_entry) if (pktio_entry->s.state != PKTIO_STATE_OPENED) rte_eth_dev_close(pkt_dpdk->port_id);

- rte_mempool_free(pkt_dpdk->pkt_pool); + if (!ODP_DPDK_ZERO_COPY) + rte_mempool_free(pkt_dpdk->pkt_pool);

return 0; } @@ -444,16 +912,18 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, pkt_dpdk_t *pkt_dpdk = &pktio_entry->s.pkt_dpdk; struct rte_eth_dev_info dev_info; struct rte_mempool *pkt_pool; - odp_pool_info_t pool_info; + char pool_name[RTE_MEMPOOL_NAMESIZE]; uint16_t data_room; uint32_t mtu; int i; + pool_t *pool_entry;

if (disable_pktio) return -1;

if (pool == ODP_POOL_INVALID) return -1; + pool_entry = pool_entry_from_hdl(pool);

if (!dpdk_netdev_is_valid(netdev)) { ODP_ERR("Invalid dpdk netdev: %s\n", netdev); @@ -473,19 +943,11 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, pkt_dpdk->pool = pool; pkt_dpdk->port_id = atoi(netdev);

- snprintf(pkt_dpdk->pool_name, sizeof(pkt_dpdk->pool_name), "pktpool_%s", - netdev); - if (rte_eth_dev_count() == 0) { ODP_ERR("No DPDK ports found\n"); return -1; }

- if (odp_pool_info(pool, &pool_info) < 0) { - ODP_ERR("Failed to read pool info\n"); - return -1; - } - dpdk_init_capability(pktio_entry, &dev_info);

mtu = dpdk_mtu_get(pktio_entry); @@ -506,19 +968,35 @@ static int dpdk_open(odp_pktio_t id ODP_UNUSED, pkt_dpdk->min_rx_burst = DPDK_IXGBE_MIN_RX_BURST; else pkt_dpdk->min_rx_burst = 0; - - pkt_pool = rte_pktmbuf_pool_create(pkt_dpdk->pool_name, DPDK_NB_MBUF, - DPDK_MEMPOOL_CACHE_SIZE, 0, - DPDK_MBUF_BUF_SIZE, rte_socket_id()); + if (ODP_DPDK_ZERO_COPY) { + if (pool_entry->ext_desc != NULL) { + pkt_pool = (struct rte_mempool *)pool_entry->ext_desc; + } else { + snprintf(pool_name, sizeof(pool_name), + "pktpool_%" PRIu32 "", pool_entry->pool_idx); + pkt_pool = mbuf_pool_create(pool_name, + pool_entry); + pool_entry->ext_destroy = pool_destroy; + pool_entry->ext_desc = pkt_pool; + } + } else { + snprintf(pool_name, sizeof(pool_name), "pktpool_%s", netdev); + pkt_pool = rte_pktmbuf_pool_create(pool_name, + DPDK_NB_MBUF, + DPDK_MEMPOOL_CACHE_SIZE, 0, + DPDK_MBUF_BUF_SIZE, + rte_socket_id()); + } if (pkt_pool == NULL) { ODP_ERR("Cannot init mbuf packet pool\n"); return -1; } + pkt_dpdk->pkt_pool = pkt_pool;

data_room = rte_pktmbuf_data_room_size(pkt_dpdk->pkt_pool) - RTE_PKTMBUF_HEADROOM; - pkt_dpdk->data_room = RTE_MIN(pool_info.params.pkt.len, data_room); + pkt_dpdk->data_room = RTE_MIN(pool_entry->max_seg_len, data_room);

/* Mbuf chaining not yet supported */ pkt_dpdk->mtu = RTE_MIN(pkt_dpdk->mtu, pkt_dpdk->data_room); @@ -591,129 +1069,6 @@ static int dpdk_stop(pktio_entry_t *pktio_entry) return 0; }

-static inline int mbuf_to_pkt(pktio_entry_t *pktio_entry, - odp_packet_t pkt_table[], - struct rte_mbuf *mbuf_table[], - uint16_t mbuf_num, odp_time_t *ts) -{ - odp_packet_t pkt; - odp_packet_hdr_t *pkt_hdr; - uint16_t pkt_len; - struct rte_mbuf *mbuf; - void *buf; - int i, j; - int nb_pkts = 0; - int alloc_len, num; - odp_pool_t pool = pktio_entry->s.pkt_dpdk.pool; - - /* Allocate maximum sized packets */ - alloc_len = pktio_entry->s.pkt_dpdk.data_room; - - num = packet_alloc_multi(pool, alloc_len, pkt_table, mbuf_num); - if (num != mbuf_num) { - ODP_DBG("packet_alloc_multi() unable to allocate all packets: " - "%d/%" PRIu16 " allocated\n", num, mbuf_num); - for (i = num; i < mbuf_num; i++) - rte_pktmbuf_free(mbuf_table[i]); - } - - for (i = 0; i < num; i++) { - odp_packet_hdr_t parsed_hdr; - - mbuf = mbuf_table[i]; - if (odp_unlikely(mbuf->nb_segs != 1)) { - ODP_ERR("Segmented buffers not supported\n"); - goto fail; - } - - buf = rte_pktmbuf_mtod(mbuf, char *); - odp_prefetch(buf); - - pkt_len = rte_pktmbuf_pkt_len(mbuf); - - if (pktio_cls_enabled(pktio_entry)) { - if (cls_classify_packet(pktio_entry, - (const uint8_t *)buf, - pkt_len, pkt_len, &pool, - &parsed_hdr)) - goto fail; - } - - pkt = pkt_table[i]; - pkt_hdr = odp_packet_hdr(pkt); - pull_tail(pkt_hdr, alloc_len - pkt_len); - - /* For now copy the data in the mbuf, - worry about zero-copy later */ - if (odp_packet_copy_from_mem(pkt, 0, pkt_len, buf) != 0) - goto fail; - - pkt_hdr->input = pktio_entry->s.handle; - - if (pktio_cls_enabled(pktio_entry)) - copy_packet_cls_metadata(&parsed_hdr, pkt_hdr); - else - packet_parse_layer(pkt_hdr, - pktio_entry->s.config.parser.layer); - - if (mbuf->ol_flags & PKT_RX_RSS_HASH) - odp_packet_flow_hash_set(pkt, mbuf->hash.rss); - - packet_set_ts(pkt_hdr, ts); - - pkt_table[nb_pkts++] = pkt; - - rte_pktmbuf_free(mbuf); - } - - return nb_pkts; - -fail: - odp_packet_free_multi(&pkt_table[i], num - i); - - for (j = i; j < num; j++) - rte_pktmbuf_free(mbuf_table[j]); - - return (i > 0 ? i : -1); -} - -static inline int pkt_to_mbuf(pktio_entry_t *pktio_entry, - struct rte_mbuf *mbuf_table[], - const odp_packet_t pkt_table[], uint16_t num) -{ - pkt_dpdk_t *pkt_dpdk = &pktio_entry->s.pkt_dpdk; - int i, j; - char *data; - uint16_t pkt_len; - - if (odp_unlikely((rte_pktmbuf_alloc_bulk(pkt_dpdk->pkt_pool, - mbuf_table, num)))) { - ODP_ERR("Failed to alloc mbuf\n"); - return 0; - } - for (i = 0; i < num; i++) { - pkt_len = _odp_packet_len(pkt_table[i]); - - if (pkt_len > pkt_dpdk->mtu) { - if (i == 0) - __odp_errno = EMSGSIZE; - goto fail; - } - - /* Packet always fits in mbuf */ - data = rte_pktmbuf_append(mbuf_table[i], pkt_len); - - odp_packet_copy_to_mem(pkt_table[i], 0, pkt_len, data); - } - return i; - -fail: - for (j = i; j < num; j++) - rte_pktmbuf_free(mbuf_table[j]); - - return i; -} - static int dpdk_recv(pktio_entry_t *pktio_entry, int index, odp_packet_t pkt_table[], int num) { @@ -777,8 +1132,12 @@ static int dpdk_recv(pktio_entry_t *pktio_entry, int index, ts_val = odp_time_global(); ts = &ts_val; } - nb_rx = mbuf_to_pkt(pktio_entry, pkt_table, rx_mbufs, nb_rx, - ts); + if (ODP_DPDK_ZERO_COPY) + nb_rx = mbuf_to_pkt_zero(pktio_entry, pkt_table, + rx_mbufs, nb_rx, ts); + else + nb_rx = mbuf_to_pkt(pktio_entry, pkt_table, rx_mbufs, + nb_rx, ts); }

return nb_rx; @@ -789,6 +1148,7 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, { struct rte_mbuf *tx_mbufs[num]; pkt_dpdk_t *pkt_dpdk = &pktio_entry->s.pkt_dpdk; + uint16_t seg_count = 0; int tx_pkts; int i; int mbufs; @@ -796,7 +1156,11 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, if (odp_unlikely(pktio_entry->s.state != PKTIO_STATE_STARTED)) return 0;

- mbufs = pkt_to_mbuf(pktio_entry, tx_mbufs, pkt_table, num); + if (ODP_DPDK_ZERO_COPY) + mbufs = pkt_to_mbuf_zero(pktio_entry, tx_mbufs, pkt_table, num, + &seg_count); + else + mbufs = pkt_to_mbuf(pktio_entry, tx_mbufs, pkt_table, num);

if (!pkt_dpdk->lockless_tx) odp_ticketlock_lock(&pkt_dpdk->tx_lock[index]); @@ -807,16 +1171,38 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, if (!pkt_dpdk->lockless_tx) odp_ticketlock_unlock(&pkt_dpdk->tx_lock[index]);

- if (odp_unlikely(tx_pkts < num)) { - for (i = tx_pkts; i < mbufs; i++) - rte_pktmbuf_free(tx_mbufs[i]); - } - - if (odp_unlikely(tx_pkts == 0)) { - if (__odp_errno != 0) - return -1; + if (ODP_DPDK_ZERO_COPY) { + /* Free copied segmented packets */ + if (odp_unlikely(seg_count)) { + uint16_t freed = 0; + + for (i = 0; i < mbufs && freed != seg_count; i++) { + odp_packet_t pkt = pkt_table[i]; + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + + if (pkt_hdr->buf_hdr.segcount > 1) { + if (odp_likely(i < tx_pkts)) + odp_packet_free(pkt); + else + rte_pktmbuf_free(tx_mbufs[i]); + freed++; + } + } + } + if (odp_unlikely(tx_pkts == 0 && __odp_errno != 0)) + return -1; } else { - odp_packet_free_multi(pkt_table, tx_pkts); + if (odp_unlikely(tx_pkts < mbufs)) { + for (i = tx_pkts; i < mbufs; i++) + rte_pktmbuf_free(tx_mbufs[i]); + } + + if (odp_unlikely(tx_pkts == 0)) { + if (__odp_errno != 0) + return -1; + } else { + odp_packet_free_multi(pkt_table, tx_pkts); + } }

return tx_pkts;

commit 43bfbbf66db3d8fd6a22762d7ac346461857f653 Author: Matias Elo matias.elo@nokia.com Date: Mon Jul 3 12:27:34 2017 +0300

linux-gen: packet: increase default headroom

The packet default headroom is modified to be the same as in DPDK to enable interoperability between ODP and DPDK packets.

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 292be356..a798851f 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -56,10 +56,10 @@ extern "C" { * size e.g. due to HW or a protocol specific alignment requirement. * * @internal In odp-linux implementation: - * The default value (66) allows a 1500-byte packet to be received into a single - * segment with Ethernet offset alignment and room for some header expansion. + * The default value (128) allows a 1500-byte packet to be received into a + * single segment with room for some header expansion. */ -#define CONFIG_PACKET_HEADROOM 66 +#define CONFIG_PACKET_HEADROOM 128

/* * Default packet tailroom

commit a959a61e3d00965af8b93536e3fbc3bc09cad1e9 Author: Matias Elo matias.elo@nokia.com Date: Thu Jun 29 15:06:59 2017 +0300

linux-gen: pool: fix segfault condition in buffer_free_to_pool()

One can remove max 'cache_num' objects from the local cache.

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index e08c8985..bbd56b06 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -703,6 +703,8 @@ static inline void buffer_free_to_pool(pool_t *pool,

if (odp_unlikely(num > CACHE_BURST)) burst = num; + if (odp_unlikely((uint32_t)num > cache_num)) + burst = cache_num;

{ /* Temporary copy needed since odp_buffer_t is

commit 9e4826c45af05eb339d129140ac88bc9d4e5a2fd Author: Matias Elo matias.elo@nokia.com Date: Tue Jun 27 10:53:35 2017 +0300

linux-gen: pktio: add missing global locks to odp_pktio_close()

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/odp_packet_io.c b/platform/linux-generic/odp_packet_io.c index 6a181f53..64c6e452 100644 --- a/platform/linux-generic/odp_packet_io.c +++ b/platform/linux-generic/odp_packet_io.c @@ -376,7 +376,9 @@ int odp_pktio_close(odp_pktio_t hdl) entry->s.num_in_queue = 0; entry->s.num_out_queue = 0;

+ odp_spinlock_lock(&pktio_tbl->lock); res = _pktio_close(entry); + odp_spinlock_unlock(&pktio_tbl->lock); if (res) ODP_ABORT("unable to close pktio\n");

commit fc539b10fff8b92c234f422905eec2266ec13b26 Author: Matias Elo matias.elo@nokia.com Date: Mon Jun 26 16:03:58 2017 +0300

linux-gen: packet: move packet_init() function to header file

Move packet_init() inline function to odp_packet_internal.h. This way e.g. pktio devices can use it directly.

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index d38f831f..b4bec35b 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -159,6 +159,45 @@ static inline odp_packet_t packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) return (odp_packet_t)(odp_packet_hdr_t *)buf_hdr; }

+/** + * Initialize packet + */ +static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) +{ + uint32_t seg_len; + int num = pkt_hdr->buf_hdr.segcount; + + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num == 1)) { + seg_len = len; + pkt_hdr->buf_hdr.seg[0].len = len; + } else { + seg_len = len - ((num - 1) * CONFIG_PACKET_MAX_SEG_LEN); + + /* Last segment data length */ + pkt_hdr->buf_hdr.seg[num - 1].len = seg_len; + } + + pkt_hdr->p.input_flags.all = 0; + pkt_hdr->p.output_flags.all = 0; + pkt_hdr->p.error_flags.all = 0; + + pkt_hdr->p.l2_offset = 0; + pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; + pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; + + /* + * Packet headroom is set from the pool's headroom + * Packet tailroom is rounded up to fill the last + * segment occupied by the allocated length. + */ + pkt_hdr->frame_len = len; + pkt_hdr->headroom = CONFIG_PACKET_HEADROOM; + pkt_hdr->tailroom = CONFIG_PACKET_MAX_SEG_LEN - seg_len + + CONFIG_PACKET_TAILROOM; + + pkt_hdr->input = ODP_PKTIO_INVALID; +} + static inline void copy_packet_parser_metadata(odp_packet_hdr_t *src_hdr, odp_packet_hdr_t *dst_hdr) { diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index ab325692..dbde7280 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -241,45 +241,6 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; }

-/** - * Initialize packet - */ -static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) -{ - uint32_t seg_len; - int num = pkt_hdr->buf_hdr.segcount; - - if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num == 1)) { - seg_len = len; - pkt_hdr->buf_hdr.seg[0].len = len; - } else { - seg_len = len - ((num - 1) * CONFIG_PACKET_MAX_SEG_LEN); - - /* Last segment data length */ - pkt_hdr->buf_hdr.seg[num - 1].len = seg_len; - } - - pkt_hdr->p.input_flags.all = 0; - pkt_hdr->p.output_flags.all = 0; - pkt_hdr->p.error_flags.all = 0; - - pkt_hdr->p.l2_offset = 0; - pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; - pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; - - /* - * Packet headroom is set from the pool's headroom - * Packet tailroom is rounded up to fill the last - * segment occupied by the allocated length. - */ - pkt_hdr->frame_len = len; - pkt_hdr->headroom = CONFIG_PACKET_HEADROOM; - pkt_hdr->tailroom = CONFIG_PACKET_MAX_SEG_LEN - seg_len + - CONFIG_PACKET_TAILROOM; - - pkt_hdr->input = ODP_PKTIO_INVALID; -} - static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) { odp_packet_hdr_t *hdr;

commit 04e5b3dd319d1affb61e0f6860e255082a6c15df Author: Matias Elo matias.elo@nokia.com Date: Thu Jun 22 16:32:36 2017 +0300

linux-gen: socket: fix ring frame size

Frame size should include also packet headroom and tailroom.

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 2dba7b08..6fc4b4cc 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -355,9 +355,9 @@ static void mmap_fill_ring(struct ring *ring, odp_pool_t pool_hdl, int fanout) pool = pool_entry_from_hdl(pool_hdl);

/* Frame has to capture full packet which can fit to the pool block.*/ - ring->req.tp_frame_size = (pool->data_size + - TPACKET_HDRLEN + TPACKET_ALIGNMENT + - + (pz - 1)) & (-pz); + ring->req.tp_frame_size = (pool->headroom + pool->data_size + + pool->tailroom + TPACKET_HDRLEN + + TPACKET_ALIGNMENT + + (pz - 1)) & (-pz);

/* Calculate how many pages do we need to hold all pool packets * and align size to page boundary.

commit c65cfd73c6103d02f33a62099391a98cf60d098f Author: Matias Elo matias.elo@nokia.com Date: Thu Jun 22 14:52:27 2017 +0300

linux-gen: packet: fix odp_packet_reset() implementation

Follow the API definition and use total buffer length instead of single segment length when checking 'len' argument validity.

Reset also packet segment data pointers and lengths.

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 3cde2086..ab325692 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -308,6 +308,20 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) } }

+static inline void reset_seg(odp_packet_hdr_t *pkt_hdr, int first, int num) +{ + odp_buffer_hdr_t *hdr; + void *base; + int i; + + for (i = first; i < first + num; i++) { + hdr = pkt_hdr->buf_hdr.seg[i].hdr; + base = hdr->base_data; + pkt_hdr->buf_hdr.seg[i].len = BASE_LEN; + pkt_hdr->buf_hdr.seg[i].data = base; + } +} + /* Calculate the number of segments */ static inline int num_segments(uint32_t len) { @@ -627,10 +641,13 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *const pkt_hdr = packet_hdr(pkt); pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; + int num = pkt_hdr->buf_hdr.segcount;

- if (len > pool->headroom + pool->data_size + pool->tailroom) + if (odp_unlikely(len > (pool->max_seg_len * num))) return -1;

+ reset_seg(pkt_hdr, 0, num); + packet_init(pkt_hdr, len);

return 0; @@ -844,20 +861,6 @@ static inline int move_data_to_tail(odp_packet_hdr_t *pkt_hdr, int segs) return dst_seg; }

-static inline void reset_seg(odp_packet_hdr_t *pkt_hdr, int first, int num) -{ - odp_buffer_hdr_t *hdr; - void *base; - int i; - - for (i = first; i < first + num; i++) { - hdr = pkt_hdr->buf_hdr.seg[i].hdr; - base = hdr->base_data; - pkt_hdr->buf_hdr.seg[i].len = BASE_LEN; - pkt_hdr->buf_hdr.seg[i].data = base; - } -} - int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len) {

-----------------------------------------------------------------------

Summary of changes: .travis.yml | 1 + example/ipsec/Makefile.am | 2 + .../linux-generic/include/odp_buffer_internal.h | 2 +- .../linux-generic/include/odp_config_internal.h | 6 +- platform/linux-generic/include/odp_packet_dpdk.h | 2 - .../linux-generic/include/odp_packet_internal.h | 52 ++ platform/linux-generic/include/odp_pool_internal.h | 7 + platform/linux-generic/m4/odp_dpdk.m4 | 14 +- platform/linux-generic/odp_packet.c | 72 +-- platform/linux-generic/odp_packet_io.c | 2 + platform/linux-generic/odp_pool.c | 11 + platform/linux-generic/pktio/dpdk.c | 692 ++++++++++++++++----- platform/linux-generic/pktio/socket_mmap.c | 6 +- 13 files changed, 651 insertions(+), 218 deletions(-)

hooks/post-receive