This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "".

The branch, master has been updated via 86745e999aa2bf644fd72775df55042289e07b2b (commit) via b11ca4c8889eef7444253003eec531377c7167f6 (commit) via f4b03adaf872f8c2e593d94a8619aa23bfecd51f (commit) via c723055094546df3890efb0f6a5f02b59cb4cb34 (commit) via 7610aab45d2a307d883c62ee56fbc0430fabf2b2 (commit) via 11353856ea2c1add12ec941bdb2a2e5854bdeb6d (commit) via 77dbff3c7054c607ff0c3371f62b226742f08aec (commit) via cc4e3434ca806736fd0c586ed36bc34f3d3b2bd4 (commit) via 916bfae5098242dd7e5121c789b1dbe20c8453ea (commit) via 65e5ee3c37c28a9365385516e5279f917a6ff41a (commit) from f0f83828a483f90edcf320ec2cf0107cb1e9362e (commit)

Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below.

- Log ----------------------------------------------------------------- commit 86745e999aa2bf644fd72775df55042289e07b2b Author: Matias Elo matias.elo@nokia.com Date: Wed Oct 9 10:38:15 2019 +0300

linux-gen: packet: clean up internal naming

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Petri Savolainen petri.savolainen@nokia.com

diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index adaf877ca..e491d2512 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -383,20 +383,18 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) link_segments(pkt_hdr, num); }

-static inline void reset_seg(odp_packet_hdr_t *pkt_hdr, int num) +static inline void reset_segments(odp_packet_hdr_t *pkt_hdr) { - odp_packet_hdr_t *hdr = pkt_hdr; void *base; - int i; - uint32_t seg_len = ((pool_t *)(hdr->buf_hdr.pool_ptr))->seg_len; + uint32_t seg_len = ((pool_t *)(pkt_hdr->buf_hdr.pool_ptr))->seg_len;

- for (i = 0; i < num; i++) { - base = hdr->buf_hdr.base_data; + while (pkt_hdr != NULL) { + base = pkt_hdr->buf_hdr.base_data;

- hdr->seg_len = seg_len; - hdr->seg_data = base; + pkt_hdr->seg_len = seg_len; + pkt_hdr->seg_data = base;

- hdr = hdr->seg_next; + pkt_hdr = pkt_hdr->seg_next; } }

@@ -559,11 +557,11 @@ static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) { int i; odp_buffer_hdr_t *buf_hdr[num]; - odp_packet_hdr_t *seg = pkt_hdr; + odp_packet_hdr_t *seg_hdr = pkt_hdr;

for (i = 0; i < num; i++) { - buf_hdr[i] = &seg->buf_hdr; - seg = seg->seg_next; + buf_hdr[i] = &seg_hdr->buf_hdr; + seg_hdr = seg_hdr->seg_next; }

packet_free_multi(buf_hdr, num); @@ -573,7 +571,7 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, int num, uint32_t free_len, uint32_t pull_len, int head) { - odp_packet_hdr_t *seg; + odp_packet_hdr_t *seg_hdr; int i; int num_remain = pkt_hdr->seg_count - num; odp_packet_hdr_t *hdr = pkt_hdr; @@ -584,8 +582,8 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, odp_packet_hdr_t *new_hdr;

for (i = 0; i < num; i++) { - seg = packet_seg_step(&hdr); - buf_hdr[i] = &seg->buf_hdr; + seg_hdr = packet_seg_step(&hdr); + buf_hdr[i] = &seg_hdr->buf_hdr; }

/* The first remaining header is the new packet descriptor. @@ -619,8 +617,8 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, packet_seg_step(&hdr);

for (i = 0; i < num; i++) { - seg = packet_seg_step(&hdr); - buf_hdr[i] = &seg->buf_hdr; + seg_hdr = packet_seg_step(&hdr); + buf_hdr[i] = &seg_hdr->buf_hdr; }

packet_free_multi(buf_hdr, num); @@ -770,9 +768,7 @@ void odp_packet_free(odp_packet_t pkt) void odp_packet_free_multi(const odp_packet_t pkt[], int num) { odp_buffer_hdr_t *buf_hdr[num]; - odp_buffer_hdr_t *buf_hdr2[num]; int i; - int links = 0; int num_freed = 0;

for (i = 0; i < num; i++) { @@ -790,9 +786,6 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) buf_hdr[i - num_freed] = &pkt_hdr->buf_hdr; }

- if (odp_unlikely(links)) - packet_free_multi(buf_hdr2, links); - if (odp_likely(num - num_freed)) packet_free_multi(buf_hdr, num - num_freed); } @@ -816,7 +809,7 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) num_req = num_segments(len, pool->seg_len); if (odp_unlikely(num_req < num)) free_segments(pkt_hdr, num - num_req, 0, 0, 0); - reset_seg(pkt_hdr, num_req); + reset_segments(pkt_hdr);

packet_init(pkt_hdr, len);

commit b11ca4c8889eef7444253003eec531377c7167f6 Author: Matias Elo matias.elo@nokia.com Date: Wed Oct 9 10:11:05 2019 +0300

linux-gen: packet: make sure max segment count is not exceeded

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Petri Savolainen petri.savolainen@nokia.com

diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index bdff0875b..adaf877ca 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -894,6 +894,9 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, return -1;

num = num_segments(len - headroom, pool->seg_len); + if (odp_unlikely(pkt_hdr->seg_count + num > PKT_MAX_SEGS)) + return -1; + push_head(pkt_hdr, headroom); ptr = add_segments(pkt_hdr, pool, len - headroom, num, 1);

@@ -1000,6 +1003,9 @@ int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, return -1;

num = num_segments(len - tailroom, pool->seg_len); + if (odp_unlikely(pkt_hdr->seg_count + num > PKT_MAX_SEGS)) + return -1; + push_tail(pkt_hdr, tailroom); ptr = add_segments(pkt_hdr, pool, len - tailroom, num, 0);

@@ -1356,6 +1362,10 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) return -1; }

+ if (odp_unlikely(dst_hdr->seg_count + src_hdr->seg_count > + PKT_MAX_SEGS)) + return -1; + add_all_segs(dst_hdr, src_hdr);

dst_hdr->frame_len = dst_len + src_len;

commit f4b03adaf872f8c2e593d94a8619aa23bfecd51f Author: Matias Elo matias.elo@nokia.com Date: Wed Oct 9 09:00:27 2019 +0300

linux-gen: dpdk: optimize performance

Minor performance tweaks.

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Petri Savolainen petri.savolainen@nokia.com

diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index 93cec349d..039c38d36 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -804,7 +804,7 @@ static inline void prefetch_pkt(struct rte_mbuf *mbuf) void *data = rte_pktmbuf_mtod(mbuf, char *);

odp_prefetch(pkt_hdr); - odp_prefetch(&pkt_hdr->p); + odp_prefetch_store((uint8_t *)pkt_hdr + ODP_CACHE_LINE_SIZE); odp_prefetch(data); }

@@ -920,6 +920,7 @@ static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry, odp_pktout_config_opt_t *pktout_cfg = &pktio_entry->s.config.pktout; odp_pktout_config_opt_t *pktout_capa = &pktio_entry->s.capa.config.pktout; + uint16_t mtu = pkt_dpdk->mtu; int i; *copy_count = 0;

@@ -929,7 +930,7 @@ static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry, struct rte_mbuf *mbuf = mbuf_from_pkt_hdr(pkt_hdr); uint16_t pkt_len = odp_packet_len(pkt);

- if (odp_unlikely(pkt_len > pkt_dpdk->mtu)) + if (odp_unlikely(pkt_len > mtu)) goto fail;

if (odp_likely(pkt_hdr->seg_count == 1)) {

commit c723055094546df3890efb0f6a5f02b59cb4cb34 Author: Matias Elo matias.elo@nokia.com Date: Tue Oct 8 13:38:21 2019 +0300

linux-gen: dpdk: make storing packet flow hash configurable

Make saving RX RSS hash as ODP flow hash optional (disabled by default). This improves performance by removing a write operation to an additional cache line.

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Petri Savolainen petri.savolainen@nokia.com

diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index 4fec05047..4e24ef82d 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -16,7 +16,7 @@

# Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.8" +config_file_version = "0.1.9"

# Shared memory options shm: { @@ -70,6 +70,9 @@ pktio_dpdk: { num_tx_desc = 512 rx_drop_en = 0

+ # Store RX RSS hash result as ODP flow hash + set_flow_hash = 0 + # Driver specific options (use PMD names from DPDK) net_ixgbe: { rx_drop_en = 1 diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index cc0a4368c..93cec349d 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -92,7 +92,8 @@ ODP_STATIC_ASSERT((DPDK_NB_MBUF % DPDK_MEMPOOL_CACHE_SIZE == 0) && typedef struct { int num_rx_desc; int num_tx_desc; - int rx_drop_en; + uint8_t rx_drop_en; + uint8_t set_flow_hash; } dpdk_opt_t;

struct pkt_cache_t { @@ -169,6 +170,7 @@ static int init_options(pktio_entry_t *pktio_entry, const struct rte_eth_dev_info *dev_info) { dpdk_opt_t *opt = &pkt_priv(pktio_entry)->opt; + int val;

if (!lookup_opt("num_rx_desc", dev_info->driver_name, &opt->num_rx_desc)) @@ -190,10 +192,13 @@ static int init_options(pktio_entry_t *pktio_entry, return -1; }

- if (!lookup_opt("rx_drop_en", dev_info->driver_name, - &opt->rx_drop_en)) + if (!lookup_opt("rx_drop_en", dev_info->driver_name, &val)) return -1; - opt->rx_drop_en = !!opt->rx_drop_en; + opt->rx_drop_en = !!val; + + if (!lookup_opt("set_flow_hash", NULL, &val)) + return -1; + opt->set_flow_hash = !!val;

ODP_PRINT("DPDK interface (%s): %" PRIu16 "\n", dev_info->driver_name, pkt_priv(pktio_entry)->port_id); @@ -810,6 +815,7 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, { odp_packet_hdr_t *pkt_hdr; uint16_t pkt_len; + uint8_t set_flow_hash; struct rte_mbuf *mbuf; void *data; int i, nb_pkts; @@ -824,6 +830,7 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, pkt_dpdk = pkt_priv(pktio_entry); nb_pkts = 0; pool = pkt_dpdk->pool; + set_flow_hash = pkt_dpdk->opt.set_flow_hash; pktin_cfg = pktio_entry->s.config.pktin; parse_layer = pktio_entry->s.config.parser.layer; input = pktio_entry->s.handle; @@ -893,8 +900,7 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry, continue; } } - - if (mbuf->ol_flags & PKT_RX_RSS_HASH) + if (set_flow_hash && (mbuf->ol_flags & PKT_RX_RSS_HASH)) packet_set_flow_hash(pkt_hdr, mbuf->hash.rss);

packet_set_ts(pkt_hdr, ts); diff --git a/platform/linux-generic/test/inline-timer.conf b/platform/linux-generic/test/inline-timer.conf index 03998f5f5..435f51e6d 100644 --- a/platform/linux-generic/test/inline-timer.conf +++ b/platform/linux-generic/test/inline-timer.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.8" +config_file_version = "0.1.9"

timer: { # Enable inline timer implementation diff --git a/platform/linux-generic/test/process-mode.conf b/platform/linux-generic/test/process-mode.conf index dbf2acb81..708b9b4a8 100644 --- a/platform/linux-generic/test/process-mode.conf +++ b/platform/linux-generic/test/process-mode.conf @@ -1,6 +1,6 @@ # Mandatory fields odp_implementation = "linux-generic" -config_file_version = "0.1.8" +config_file_version = "0.1.9"

# Shared memory options shm: {

commit 7610aab45d2a307d883c62ee56fbc0430fabf2b2 Author: Matias Elo matias.elo@nokia.com Date: Tue Oct 1 14:40:36 2019 +0300

linux-gen: packet: prefetch packet metadata in packet_alloc()

Improve packet alloc performance by prefetching required metadata.

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Petri Savolainen petri.savolainen@nokia.com

diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 8d06e8fcc..bdff0875b 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -646,6 +646,8 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, int num = max_pkt; int max_buf = max_pkt * num_seg; odp_packet_hdr_t *pkt_hdr[max_buf]; + odp_packet_hdr_t *hdr_next; + odp_packet_hdr_t *hdr;

num_buf = buffer_alloc_multi(pool, (odp_buffer_hdr_t **)pkt_hdr, max_buf); @@ -668,17 +670,29 @@ static inline int packet_alloc(pool_t *pool, uint32_t len, int max_pkt, return 0; }

- for (i = 0; i < num; i++) { - odp_packet_hdr_t *hdr; + hdr_next = pkt_hdr[0]; + odp_prefetch(hdr_next); + odp_prefetch((uint8_t *)hdr_next + ODP_CACHE_LINE_SIZE); + + for (i = 0; i < num - 1; i++) { + hdr = hdr_next; + hdr_next = pkt_hdr[(i + 1) * num_seg]; + + odp_prefetch(hdr_next); + odp_prefetch((uint8_t *)hdr_next + ODP_CACHE_LINE_SIZE);

/* First buffer is the packet descriptor */ - hdr = pkt_hdr[i * num_seg]; pkt[i] = packet_handle(hdr); init_segments(&pkt_hdr[i * num_seg], num_seg);

packet_init(hdr, len); }

+ /* Last packet */ + pkt[i] = packet_handle(hdr_next); + init_segments(&pkt_hdr[i * num_seg], num_seg); + packet_init(hdr_next, len); + return num; }

commit 11353856ea2c1add12ec941bdb2a2e5854bdeb6d Author: Matias Elo matias.elo@nokia.com Date: Fri Sep 27 12:24:54 2019 +0300

linux-gen: packet: inline packet segment functions

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Petri Savolainen petri.savolainen@nokia.com

diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index 0f9dea38b..fd18488fd 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -31,6 +31,7 @@ extern "C" { typedef struct _odp_packet_inline_offset_t { uint16_t seg_data; uint16_t seg_len; + uint16_t seg_next; uint16_t frame_len; uint16_t headroom; uint16_t tailroom; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 5232fb1e3..08048a993 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -57,6 +57,9 @@ #define odp_packet_head __odp_packet_head #define odp_packet_is_segmented __odp_packet_is_segmented #define odp_packet_first_seg __odp_packet_first_seg + #define odp_packet_seg_data __odp_packet_seg_data + #define odp_packet_seg_data_len __odp_packet_seg_data_len + #define odp_packet_next_seg __odp_packet_next_seg #define odp_packet_prefetch __odp_packet_prefetch #define odp_packet_copy_from_mem __odp_packet_copy_from_mem #define odp_packet_copy_to_mem __odp_packet_copy_to_mem @@ -256,6 +259,29 @@ _ODP_INLINE odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) return (odp_packet_seg_t)pkt; }

+_ODP_INLINE void *odp_packet_seg_data(odp_packet_t pkt ODP_UNUSED, + odp_packet_seg_t seg) +{ + return _odp_pkt_get((odp_packet_t)seg, void *, seg_data); +} + +_ODP_INLINE uint32_t odp_packet_seg_data_len(odp_packet_t pkt ODP_UNUSED, + odp_packet_seg_t seg) +{ + return _odp_pkt_get((odp_packet_t)seg, uint32_t, seg_len); +} + +_ODP_INLINE odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt ODP_UNUSED, + odp_packet_seg_t seg) +{ + void *next_seg = _odp_pkt_get((odp_packet_t)seg, void *, seg_next); + + if (odp_unlikely(next_seg == NULL)) + return ODP_PACKET_SEG_INVALID; + + return (odp_packet_seg_t)next_seg; +} + _ODP_INLINE void odp_packet_prefetch(odp_packet_t pkt, uint32_t offset, uint32_t len) { diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 4b0a4d8d3..8d06e8fcc 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -38,6 +38,7 @@ const _odp_packet_inline_offset_t ODP_ALIGNED_CACHE _odp_packet_inline = { .seg_data = offsetof(odp_packet_hdr_t, seg_data), .seg_len = offsetof(odp_packet_hdr_t, seg_len), + .seg_next = offsetof(odp_packet_hdr_t, seg_next), .frame_len = offsetof(odp_packet_hdr_t, frame_len), .headroom = offsetof(odp_packet_hdr_t, headroom), .tailroom = offsetof(odp_packet_hdr_t, tailroom), @@ -1194,37 +1195,11 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) * */

-void *odp_packet_seg_data(odp_packet_t pkt ODP_UNUSED, odp_packet_seg_t seg) -{ - odp_packet_hdr_t *seg_hdr = packet_seg_to_hdr(seg); - - return seg_hdr->seg_data; -} - -uint32_t odp_packet_seg_data_len(odp_packet_t pkt ODP_UNUSED, - odp_packet_seg_t seg) -{ - odp_packet_hdr_t *seg_hdr = packet_seg_to_hdr(seg); - - return seg_hdr->seg_len; -} - odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) { return (odp_packet_seg_t)packet_last_seg(packet_hdr(pkt)); }

-odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt ODP_UNUSED, - odp_packet_seg_t seg) -{ - odp_packet_hdr_t *pkt_hdr = packet_seg_to_hdr(seg); - - if (odp_unlikely(pkt_hdr->seg_next == NULL)) - return ODP_PACKET_SEG_INVALID; - - return (odp_packet_seg_t)pkt_hdr->seg_next; -} - /* * * Manipulation

commit 77dbff3c7054c607ff0c3371f62b226742f08aec Author: Matias Elo matias.elo@nokia.com Date: Thu Sep 26 16:30:58 2019 +0300

linux-gen: packet: free possible extra segments in odp_packet_reset()

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Petri Savolainen petri.savolainen@nokia.com

diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index e2d768a23..4b0a4d8d3 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -792,11 +792,16 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) odp_packet_hdr_t *const pkt_hdr = packet_hdr(pkt); pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; int num = pkt_hdr->seg_count; + int num_req;

if (odp_unlikely(len > (pool->seg_len * num))) return -1;

- reset_seg(pkt_hdr, num); + /* Free possible extra segments */ + num_req = num_segments(len, pool->seg_len); + if (odp_unlikely(num_req < num)) + free_segments(pkt_hdr, num - num_req, 0, 0, 0); + reset_seg(pkt_hdr, num_req);

packet_init(pkt_hdr, len);

commit cc4e3434ca806736fd0c586ed36bc34f3d3b2bd4 Author: Matias Elo matias.elo@nokia.com Date: Tue Sep 17 14:58:07 2019 +0300

linux-gen: packet: reimplement segmentation using linked list

Packet segmentation has been reimplemented using a linked list. The new implementation is more simple and has the added benefit of reducing packet and buffer header sizes: odp_packet_hdr_t: 384B -> 256B odp_buffer_hdr_t: 256B -> 64B

The dynamic packet reference implementation has been temporarily downgraded to a simple packet copy.

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Petri Savolainen petri.savolainen@nokia.com

diff --git a/platform/linux-generic/include-abi/odp/api/abi/packet.h b/platform/linux-generic/include-abi/odp/api/abi/packet.h index e11aed978..0967f1ec6 100644 --- a/platform/linux-generic/include-abi/odp/api/abi/packet.h +++ b/platform/linux-generic/include-abi/odp/api/abi/packet.h @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2019, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -30,22 +31,11 @@ typedef ODP_HANDLE_T(odp_packet_t);

#define ODP_PACKET_OFFSET_INVALID 0xffff

-typedef uint8_t odp_packet_seg_t; +typedef ODP_HANDLE_T(odp_packet_seg_t);

-/* or it will be provided by packet_inlines.h */ -#define _ODP_HAVE_PACKET_SEG_NDX 1 +#define ODP_PACKET_SEG_INVALID _odp_cast_scalar(odp_packet_seg_t, 0)

-static inline uint8_t _odp_packet_seg_to_ndx(odp_packet_seg_t seg) -{ - return (uint8_t)seg; -} - -static inline odp_packet_seg_t _odp_packet_seg_from_ndx(uint8_t ndx) -{ - return (odp_packet_seg_t)ndx; -} - -#define ODP_PACKET_SEG_INVALID ((odp_packet_seg_t)-1) +#define ODP_PACKET_OFFSET_INVALID 0xffff

typedef uint8_t odp_proto_l2_type_t;

diff --git a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h index 71065172d..0f9dea38b 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inline_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inline_types.h @@ -1,4 +1,5 @@ /* Copyright (c) 2015-2018, Linaro Limited + * Copyright (c) 2019, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -28,14 +29,14 @@ extern "C" {

/* Packet header field offsets for inline functions */ typedef struct _odp_packet_inline_offset_t { - uint16_t data; + uint16_t seg_data; uint16_t seg_len; uint16_t frame_len; uint16_t headroom; uint16_t tailroom; uint16_t pool; uint16_t input; - uint16_t segcount; + uint16_t seg_count; uint16_t user_ptr; uint16_t user_area; uint16_t l2_offset; diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 3851af4c7..5232fb1e3 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -1,4 +1,5 @@ /* Copyright (c) 2017-2018, Linaro Limited + * Copyright (c) 2019, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -56,8 +57,6 @@ #define odp_packet_head __odp_packet_head #define odp_packet_is_segmented __odp_packet_is_segmented #define odp_packet_first_seg __odp_packet_first_seg - #define odp_packet_last_seg __odp_packet_last_seg - #define odp_packet_next_seg __odp_packet_next_seg #define odp_packet_prefetch __odp_packet_prefetch #define odp_packet_copy_from_mem __odp_packet_copy_from_mem #define odp_packet_copy_to_mem __odp_packet_copy_to_mem @@ -72,7 +71,7 @@ #endif

void *_odp_packet_map(void *pkt_ptr, uint32_t offset, uint32_t *seg_len, - int *seg_idx); + odp_packet_seg_t *seg);

int _odp_packet_copy_from_mem_seg(odp_packet_t pkt, uint32_t offset, uint32_t len, const void *src); @@ -83,22 +82,9 @@ int _odp_packet_copy_to_mem_seg(odp_packet_t pkt, uint32_t offset, extern const _odp_packet_inline_offset_t _odp_packet_inline; extern const _odp_pool_inline_offset_t _odp_pool_inline;

-#ifndef _ODP_HAVE_PACKET_SEG_NDX -#include <odp/api/plat/strong_types.h> -static inline uint32_t _odp_packet_seg_to_ndx(odp_packet_seg_t seg) -{ - return _odp_typeval(seg) - 1; -} - -static inline odp_packet_seg_t _odp_packet_seg_from_ndx(uint32_t ndx) -{ - return _odp_cast_scalar(odp_packet_seg_t, ndx + 1); -} -#endif - _ODP_INLINE void *odp_packet_data(odp_packet_t pkt) { - return _odp_pkt_get(pkt, void *, data); + return _odp_pkt_get(pkt, void *, seg_data); }

_ODP_INLINE uint32_t odp_packet_seg_len(odp_packet_t pkt) @@ -149,7 +135,7 @@ _ODP_INLINE int odp_packet_input_index(odp_packet_t pkt)

_ODP_INLINE int odp_packet_num_segs(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint8_t, segcount); + return _odp_pkt_get(pkt, uint8_t, seg_count); }

_ODP_INLINE void *odp_packet_user_ptr(odp_packet_t pkt) @@ -262,29 +248,12 @@ _ODP_INLINE void *odp_packet_head(odp_packet_t pkt)

_ODP_INLINE int odp_packet_is_segmented(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint8_t, segcount) > 1; + return _odp_pkt_get(pkt, uint8_t, seg_count) > 1; }

_ODP_INLINE odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) { - (void)pkt; - - return _odp_packet_seg_from_ndx(0); -} - -_ODP_INLINE odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) -{ - return _odp_packet_seg_from_ndx(odp_packet_num_segs(pkt) - 1); -} - -_ODP_INLINE odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, - odp_packet_seg_t seg) -{ - if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= - _odp_packet_seg_to_ndx(odp_packet_last_seg(pkt)))) - return ODP_PACKET_SEG_INVALID; - - return seg + 1; + return (odp_packet_seg_t)pkt; }

_ODP_INLINE void odp_packet_prefetch(odp_packet_t pkt, uint32_t offset, diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index f759aac35..1c52b028f 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2019, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -32,12 +33,6 @@ extern "C" { #include <odp_schedule_if.h> #include <stddef.h>

-typedef struct seg_entry_t { - void *hdr; - uint8_t *data; - uint32_t len; -} seg_entry_t; - typedef union buffer_index_t { uint32_t u32;

@@ -58,47 +53,15 @@ ODP_STATIC_ASSERT(CONFIG_POOL_MAX_NUM <= (0xFFFFFF + 1), "TOO_LARGE_POOL");

/* Common buffer header */ struct ODP_ALIGNED_CACHE odp_buffer_hdr_t { - /* Combined pool and buffer index */ - buffer_index_t index; - - /* Total segment count */ - uint16_t segcount; - - /* Pool type */ - int8_t type; - - /* Number of seg[] entries used */ - uint8_t num_seg; - - /* Next header which continues the segment list */ - void *next_seg; - - /* Last header of the segment list */ - void *last_seg; - /* Initial buffer data pointer */ uint8_t *base_data;

/* Pool pointer */ - void *pool_ptr; - - /* --- 40 bytes --- */ - - /* Segments */ - seg_entry_t seg[CONFIG_PACKET_SEGS_PER_HDR]; + void *pool_ptr;

/* --- Mostly read only data --- */ const void *user_ptr;

- /* Reference count */ - odp_atomic_u32_t ref_cnt; - - /* Event type. Maybe different than pool type (crypto compl event) */ - int8_t event_type; - - /* Event flow id */ - uint8_t flow_id; - /* Initial buffer tail pointer */ uint8_t *buf_end;

@@ -109,13 +72,25 @@ struct ODP_ALIGNED_CACHE odp_buffer_hdr_t { * offset has to be used */ uint64_t ipc_data_offset;

+ /* Combined pool and buffer index */ + buffer_index_t index; + + /* Reference count */ + odp_atomic_u32_t ref_cnt; + + /* Pool type */ + int8_t type; + + /* Event type. Maybe different than pool type (crypto compl event) */ + int8_t event_type; + + /* Event flow id */ + uint8_t flow_id; + /* Data or next header */ uint8_t data[0]; };

-ODP_STATIC_ASSERT(CONFIG_PACKET_SEGS_PER_HDR < 256, - "CONFIG_PACKET_SEGS_PER_HDR_TOO_LARGE"); - odp_event_type_t _odp_buffer_event_type(odp_buffer_t buf); void _odp_buffer_event_type_set(odp_buffer_t buf, int ev); int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf); diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 2e1d61d36..e283472a0 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -1,4 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2019, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -88,21 +89,6 @@ extern "C" { */ #define CONFIG_PACKET_TAILROOM 0

-/* - * Maximum number of segments per packet - */ -#define CONFIG_PACKET_MAX_SEGS 255 - -/* - * Packet segmentation disabled - */ -#define CONFIG_PACKET_SEG_DISABLED (CONFIG_PACKET_MAX_SEGS == 1) - -/* - * Number of segments stored in a packet header - */ -#define CONFIG_PACKET_SEGS_PER_HDR 6 - /* * Maximum packet data length in bytes */ diff --git a/platform/linux-generic/include/odp_packet_dpdk.h b/platform/linux-generic/include/odp_packet_dpdk.h index 2b6b859c4..970b97fca 100644 --- a/platform/linux-generic/include/odp_packet_dpdk.h +++ b/platform/linux-generic/include/odp_packet_dpdk.h @@ -1,4 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2019, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -51,8 +52,8 @@ static inline int _odp_dpdk_packet_parse_layer(odp_packet_hdr_t *pkt_hdr, uint32_t supported_ptypes, odp_pktin_config_opt_t pktin_cfg) { - uint32_t seg_len = pkt_hdr->buf_hdr.seg[0].len; - void *base = pkt_hdr->buf_hdr.seg[0].data; + uint32_t seg_len = pkt_hdr->seg_len; + void *base = pkt_hdr->seg_data;

return _odp_dpdk_packet_parse_common(&pkt_hdr->p, base, pkt_hdr->frame_len, seg_len, mbuf, diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 2e5ef30b8..a0e33222e 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2019, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -30,6 +31,8 @@ extern "C" { #include <odp/api/abi/packet.h> #include <odp_queue_if.h>

+#include <stdint.h> + /** Minimum segment length expected by packet_parse_common() */ #define PACKET_PARSE_SEG_LEN 96

@@ -65,6 +68,11 @@ typedef struct { /* Packet extra data types */ #define PKT_EXTRA_TYPE_DPDK 1

+/* Maximum number of segments per packet */ +#define PKT_MAX_SEGS 255 + +ODP_STATIC_ASSERT(PKT_MAX_SEGS < UINT16_MAX, "PACKET_MAX_SEGS_ERROR"); + /** * Internal Packet header * @@ -72,32 +80,39 @@ typedef struct { * packet_init(). Because of this any new fields added must be reviewed for * initialization requirements. */ -typedef struct { - /* common buffer header */ +typedef struct odp_packet_hdr_t { + /* Common buffer header */ odp_buffer_hdr_t buf_hdr;

- /* - * Following members are initialized by packet_init() - */ + /* --- 64 bytes --- */ + + /* Segment data start */ + uint8_t *seg_data;

packet_parser_t p;

odp_pktio_t input;

+ /* Next header which continues the segment list */ + struct odp_packet_hdr_t *seg_next; + + /* Total packet length */ uint32_t frame_len;

+ /* Segment data length */ + uint32_t seg_len; + + /* Total segment count */ + uint16_t seg_count; + uint16_t headroom; + uint16_t tailroom;

/* Event subtype */ - int8_t subtype; + int8_t subtype;

- /* - * Members below are not initialized by packet_init() - */ - - /* Flow hash value */ - uint32_t flow_hash; + /* --- 128 bytes --- */

/* Timestamp value */ odp_time_t timestamp; @@ -105,6 +120,9 @@ typedef struct { /* Classifier destination queue */ odp_queue_t dst_queue;

+ /* Flow hash value */ + uint32_t flow_hash; + union { struct { /* Result for crypto packet op */ @@ -145,14 +163,12 @@ static inline odp_packet_t packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) return (odp_packet_t)(odp_packet_hdr_t *)buf_hdr; }

-static inline seg_entry_t *seg_entry_last(odp_packet_hdr_t *hdr) +static inline odp_packet_hdr_t *packet_last_seg(odp_packet_hdr_t *hdr) { - odp_packet_hdr_t *last; - uint8_t last_seg; + while (hdr->seg_next != NULL) + hdr = hdr->seg_next;

- last = hdr->buf_hdr.last_seg; - last_seg = last->buf_hdr.num_seg - 1; - return &last->buf_hdr.seg[last_seg]; + return hdr; }

static inline void packet_subtype_set(odp_packet_t pkt, int ev) @@ -167,19 +183,19 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; uint32_t seg_len; - int num = pkt_hdr->buf_hdr.segcount; + int num = pkt_hdr->seg_count;

- if (odp_likely(CONFIG_PACKET_SEG_DISABLED || num == 1)) { + if (odp_likely(num == 1)) { seg_len = len; - pkt_hdr->buf_hdr.seg[0].len = len; + pkt_hdr->seg_len = len; } else { - seg_entry_t *last; + odp_packet_hdr_t *last;

seg_len = len - ((num - 1) * pool->seg_len);

/* Last segment data length */ - last = seg_entry_last(pkt_hdr); - last->len = seg_len; + last = packet_last_seg(pkt_hdr); + last->seg_len = seg_len; }

pkt_hdr->p.input_flags.all = 0; @@ -221,11 +237,11 @@ static inline void copy_packet_cls_metadata(odp_packet_hdr_t *src_hdr,

static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - seg_entry_t *last = seg_entry_last(pkt_hdr); + odp_packet_hdr_t *last = packet_last_seg(pkt_hdr);

pkt_hdr->tailroom += len; pkt_hdr->frame_len -= len; - last->len -= len; + last->seg_len -= len; }

static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c index 67f0d4e26..f39f76a57 100644 --- a/platform/linux-generic/odp_buffer.c +++ b/platform/linux-generic/odp_buffer.c @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2019, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -38,7 +39,7 @@ void *odp_buffer_addr(odp_buffer_t buf) { odp_buffer_hdr_t *hdr = buf_hdl_to_hdr(buf);

- return hdr->seg[0].data; + return hdr->base_data; }

uint32_t odp_buffer_size(odp_buffer_t buf) @@ -69,7 +70,7 @@ int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf) " pool %" PRIu64 "\n", odp_pool_to_u64(pool->pool_hdl)); len += snprintf(&str[len], n - len, - " addr %p\n", hdr->seg[0].data); + " addr %p\n", hdr->base_data); len += snprintf(&str[len], n - len, " size %" PRIu32 "\n", odp_buffer_size(buf)); len += snprintf(&str[len], n - len, diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index a785ecef9..e2d768a23 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1,4 +1,5 @@ /* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2019, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -35,14 +36,14 @@

/* Fill in packet header field offsets for inline functions */ const _odp_packet_inline_offset_t ODP_ALIGNED_CACHE _odp_packet_inline = { - .data = offsetof(odp_packet_hdr_t, buf_hdr.seg[0].data), - .seg_len = offsetof(odp_packet_hdr_t, buf_hdr.seg[0].len), + .seg_data = offsetof(odp_packet_hdr_t, seg_data), + .seg_len = offsetof(odp_packet_hdr_t, seg_len), .frame_len = offsetof(odp_packet_hdr_t, frame_len), .headroom = offsetof(odp_packet_hdr_t, headroom), .tailroom = offsetof(odp_packet_hdr_t, tailroom), .pool = offsetof(odp_packet_hdr_t, buf_hdr.pool_ptr), .input = offsetof(odp_packet_hdr_t, input), - .segcount = offsetof(odp_packet_hdr_t, buf_hdr.segcount), + .seg_count = offsetof(odp_packet_hdr_t, seg_count), .user_ptr = offsetof(odp_packet_hdr_t, buf_hdr.user_ptr), .user_area = offsetof(odp_packet_hdr_t, buf_hdr.uarea_addr), .l2_offset = offsetof(odp_packet_hdr_t, p.l2_offset), @@ -72,155 +73,80 @@ static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) return (odp_buffer_t)pkt; }

-static inline seg_entry_t *seg_entry(odp_packet_hdr_t *hdr, - uint32_t seg_idx) +static inline odp_packet_hdr_t *packet_seg_to_hdr(odp_packet_seg_t seg) { - uint32_t idx = 0; - uint8_t num_seg = hdr->buf_hdr.num_seg; - - while (odp_unlikely(idx + num_seg - 1 < seg_idx)) { - idx += num_seg; - hdr = hdr->buf_hdr.next_seg; - num_seg = hdr->buf_hdr.num_seg; - } - - idx = seg_idx - idx; - - return &hdr->buf_hdr.seg[idx]; + return (odp_packet_hdr_t *)(uintptr_t)seg; }

-static inline void seg_entry_find_idx(odp_packet_hdr_t **p_hdr, - uint8_t *p_idx, - uint32_t find_idx) +static inline odp_packet_seg_t packet_hdr_to_seg(odp_packet_hdr_t *pkt_hdr) { - odp_packet_hdr_t *hdr = *p_hdr; - uint32_t idx = 0; - uint8_t num_seg = hdr->buf_hdr.num_seg; - - while (odp_unlikely(idx + num_seg - 1 < find_idx)) { - idx += num_seg; - hdr = hdr->buf_hdr.next_seg; - num_seg = hdr->buf_hdr.num_seg; - } - - idx = find_idx - idx; - *p_hdr = hdr; - *p_idx = idx; + return (odp_packet_seg_t)pkt_hdr; }

-/* Return pointer to the current segment entry and step cur_hdr / cur_idx - * forward. +/* + * Return pointer to the current segment and step cur_hdr forward. */ -static inline seg_entry_t *seg_entry_next(odp_packet_hdr_t **cur_hdr, - uint8_t *cur_idx) +static inline odp_packet_hdr_t *packet_seg_step(odp_packet_hdr_t **cur_hdr) { odp_packet_hdr_t *hdr = *cur_hdr; - uint8_t idx = *cur_idx; - uint8_t num_seg = hdr->buf_hdr.num_seg;

- if (idx == num_seg - 1) { - *cur_hdr = hdr->buf_hdr.next_seg; - *cur_idx = 0; - } else { - *cur_idx = idx + 1; - } + *cur_hdr = hdr->seg_next;

- return &hdr->buf_hdr.seg[idx]; + return hdr; }

-static inline void seg_entry_find_offset(odp_packet_hdr_t **p_hdr, - uint8_t *p_idx, - uint32_t *seg_offset, - uint32_t *seg_idx, - uint32_t offset) +static inline void packet_seg_find_idx(odp_packet_hdr_t **pkt_hdr, + uint32_t find_idx) { - int i; - odp_packet_hdr_t *hdr, *cur_hdr; - uint8_t idx, cur_idx; - seg_entry_t *seg = NULL; - uint32_t seg_start = 0, seg_end = 0; - int seg_count; - - hdr = *p_hdr; - cur_hdr = hdr; - idx = 0; - cur_idx = 0; - seg_count = hdr->buf_hdr.segcount; - - for (i = 0; i < seg_count; i++) { - cur_hdr = hdr; - cur_idx = idx; - seg = seg_entry_next(&hdr, &idx); - seg_end += seg->len; - - if (odp_likely(offset < seg_end)) - break; + odp_packet_hdr_t *hdr = *pkt_hdr; + uint32_t idx = 0;

- seg_start = seg_end; + while (odp_unlikely(idx < find_idx)) { + idx++; + hdr = hdr->seg_next; }

- *p_hdr = cur_hdr; - *p_idx = cur_idx; - *seg_offset = offset - seg_start; - *seg_idx = i; + *pkt_hdr = hdr; }

static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { - seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); - - return seg->len; -} - -static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) -{ - seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); - - return seg->data; -} + packet_seg_find_idx(&pkt_hdr, seg_idx);

-static inline uint16_t packet_last_seg(odp_packet_hdr_t *pkt_hdr) -{ - if (CONFIG_PACKET_SEG_DISABLED) - return 0; - else - return pkt_hdr->buf_hdr.segcount - 1; + return pkt_hdr->seg_len; }

static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) { - return pkt_hdr->buf_hdr.seg[0].len; + return pkt_hdr->seg_len; }

static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) { - return pkt_hdr->buf_hdr.seg[0].data; + return pkt_hdr->seg_data; }

static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) { - seg_entry_t *last_seg = seg_entry_last(pkt_hdr); + odp_packet_hdr_t *last_seg = packet_last_seg(pkt_hdr);

- return last_seg->data + last_seg->len; + return last_seg->seg_data + last_seg->seg_len; }

-static inline uint32_t seg_headroom(odp_packet_hdr_t *pkt_hdr, int seg_idx) +static inline uint32_t seg_headroom(odp_packet_hdr_t *pkt_seg) { - seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); - odp_buffer_hdr_t *hdr = seg->hdr; + odp_buffer_hdr_t *hdr = &pkt_seg->buf_hdr; uint8_t *base = hdr->base_data; - uint8_t *head = seg->data; + uint8_t *head = pkt_seg->seg_data;

return CONFIG_PACKET_HEADROOM + (head - base); }

-static inline uint32_t seg_tailroom(odp_packet_hdr_t *pkt_hdr, int seg_idx) +static inline uint32_t seg_tailroom(odp_packet_hdr_t *pkt_seg) { - seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); - - odp_buffer_hdr_t *hdr = seg->hdr; - uint8_t *tail = seg->data + seg->len; + odp_buffer_hdr_t *hdr = &pkt_seg->buf_hdr; + uint8_t *tail = pkt_seg->seg_data + pkt_seg->seg_len;

return hdr->buf_end - tail; } @@ -229,25 +155,25 @@ static inline void push_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->headroom -= len; pkt_hdr->frame_len += len; - pkt_hdr->buf_hdr.seg[0].data -= len; - pkt_hdr->buf_hdr.seg[0].len += len; + pkt_hdr->seg_data -= len; + pkt_hdr->seg_len += len; }

static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->headroom += len; pkt_hdr->frame_len -= len; - pkt_hdr->buf_hdr.seg[0].data += len; - pkt_hdr->buf_hdr.seg[0].len -= len; + pkt_hdr->seg_data += len; + pkt_hdr->seg_len -= len; }

static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - seg_entry_t *last_seg = seg_entry_last(pkt_hdr); + odp_packet_hdr_t *last_seg = packet_last_seg(pkt_hdr);

pkt_hdr->tailroom -= len; pkt_hdr->frame_len += len; - last_seg->len += len; + last_seg->seg_len += len; }

/* Copy all metadata for segmentation modification. Segment data and lengths @@ -273,39 +199,34 @@ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr;

/* segmentation data is not copied: - * buf_hdr.seg[] - * buf_hdr.segcount - * buf_hdr.num_seg - * buf_hdr.next_seg - * buf_hdr.last_seg + * seg_next + * seg_data + * seg_len + * seg_count */ }

static inline void *packet_map(void *pkt_ptr, uint32_t offset, - uint32_t *seg_len, int *seg_idx) + uint32_t *seg_len, odp_packet_seg_t *seg) { void *addr; uint32_t len; odp_packet_hdr_t *pkt_hdr = pkt_ptr; - int seg_id = 0; - int seg_count = pkt_hdr->buf_hdr.segcount; + int seg_count = pkt_hdr->seg_count;

if (odp_unlikely(offset >= pkt_hdr->frame_len)) return NULL;

- if (odp_likely(CONFIG_PACKET_SEG_DISABLED || seg_count == 1)) { - addr = pkt_hdr->buf_hdr.seg[0].data + offset; - len = pkt_hdr->buf_hdr.seg[0].len - offset; + if (odp_likely(seg_count == 1)) { + addr = pkt_hdr->seg_data + offset; + len = pkt_hdr->seg_len - offset; } else { - int i; - seg_entry_t *seg = NULL; + odp_packet_hdr_t *next_hdr = pkt_hdr; uint32_t seg_start = 0, seg_end = 0; - odp_packet_hdr_t *hdr = pkt_hdr; - uint8_t idx = 0;

- for (i = 0; i < seg_count; i++) { - seg = seg_entry_next(&hdr, &idx); - seg_end += seg->len; + while (next_hdr != NULL) { + pkt_hdr = packet_seg_step(&next_hdr); + seg_end += pkt_hdr->seg_len;

if (odp_likely(offset < seg_end)) break; @@ -313,16 +234,15 @@ static inline void *packet_map(void *pkt_ptr, uint32_t offset, seg_start = seg_end; }

- addr = seg->data + (offset - seg_start); - len = seg->len - (offset - seg_start); - seg_id = i; + addr = pkt_hdr->seg_data + (offset - seg_start); + len = pkt_hdr->seg_len - (offset - seg_start); }

if (seg_len) *seg_len = len;

- if (seg_idx) - *seg_idx = seg_id; + if (seg) + *seg = packet_hdr_to_seg(pkt_hdr);

return addr; } @@ -332,9 +252,9 @@ static inline void *packet_map(void *pkt_ptr, uint32_t offset, /* This file uses the inlined version directly. Inlined API calls use this when * offset does not point to the first segment. */ void *_odp_packet_map(void *pkt_ptr, uint32_t offset, uint32_t *seg_len, - int *seg_idx) + odp_packet_seg_t *seg) { - return packet_map(pkt_ptr, offset, seg_len, seg_idx); + return packet_map(pkt_ptr, offset, seg_len, seg); }

int _odp_packet_copy_from_mem_seg(odp_packet_t pkt, uint32_t offset, @@ -399,46 +319,38 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr)

static inline void link_segments(odp_packet_hdr_t *pkt_hdr[], int num) { - int cur, i; + int cur = 0; odp_packet_hdr_t *hdr; odp_packet_hdr_t *head = pkt_hdr[0]; uint32_t seg_len = ((pool_t *)(head->buf_hdr.pool_ptr))->seg_len;

- cur = 0; - while (1) { - hdr = pkt_hdr[cur]; + odp_buffer_hdr_t *buf_hdr = &pkt_hdr[cur]->buf_hdr;

- for (i = 0; i < CONFIG_PACKET_SEGS_PER_HDR; i++) { - odp_buffer_hdr_t *buf_hdr; + hdr = pkt_hdr[cur]; + hdr->seg_data = buf_hdr->base_data; + hdr->seg_len = seg_len;

- buf_hdr = &pkt_hdr[cur]->buf_hdr; - hdr->buf_hdr.seg[i].hdr = buf_hdr; - hdr->buf_hdr.seg[i].data = buf_hdr->base_data; - hdr->buf_hdr.seg[i].len = seg_len; + /* init_segments() handles first seg ref_cnt init */ + if (ODP_DEBUG == 1 && cur > 0) { + uint32_t prev_ref; + odp_atomic_u32_t *ref_cnt;

- /* init_segments() handles first seg ref_cnt init */ - if (ODP_DEBUG == 1 && cur > 0) { - uint32_t prev_ref = - odp_atomic_fetch_inc_u32( - &pkt_hdr[cur]->buf_hdr.ref_cnt); + ref_cnt = &pkt_hdr[cur]->buf_hdr.ref_cnt; + prev_ref = odp_atomic_fetch_inc_u32(ref_cnt);

- ODP_ASSERT(prev_ref == 0); - } + ODP_ASSERT(prev_ref == 0); + }

- cur++; + cur++;

- if (cur == num) { - /* Last segment */ - hdr->buf_hdr.num_seg = i + 1; - hdr->buf_hdr.next_seg = NULL; - head->buf_hdr.last_seg = &hdr->buf_hdr; - return; - } + if (cur == num) { + /* Last segment */ + hdr->seg_next = NULL; + return; }

- hdr->buf_hdr.num_seg = CONFIG_PACKET_SEGS_PER_HDR; - hdr->buf_hdr.next_seg = pkt_hdr[cur]; + hdr->seg_next = pkt_hdr[cur]; } }

@@ -452,8 +364,11 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) seg_len = ((pool_t *)(hdr->buf_hdr.pool_ptr))->seg_len;

/* Defaults for single segment packet */ - hdr->buf_hdr.seg[0].data = hdr->buf_hdr.base_data; - hdr->buf_hdr.seg[0].len = seg_len; + hdr->seg_data = hdr->buf_hdr.base_data; + hdr->seg_len = seg_len; + hdr->seg_next = NULL; + + hdr->seg_count = num;

if (ODP_DEBUG == 1) { uint32_t prev_ref = @@ -462,46 +377,32 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) ODP_ASSERT(prev_ref == 0); }

- if (!CONFIG_PACKET_SEG_DISABLED) { - hdr->buf_hdr.segcount = num; - hdr->buf_hdr.num_seg = 1; - hdr->buf_hdr.next_seg = NULL; - hdr->buf_hdr.last_seg = &hdr->buf_hdr; - - /* Link segments */ - if (odp_unlikely(num > 1)) - link_segments(pkt_hdr, num); - } + /* Link segments */ + if (odp_unlikely(num > 1)) + link_segments(pkt_hdr, num); }

-static inline void reset_seg(odp_packet_hdr_t *pkt_hdr, int first, int num) +static inline void reset_seg(odp_packet_hdr_t *pkt_hdr, int num) { odp_packet_hdr_t *hdr = pkt_hdr; void *base; int i; - seg_entry_t *seg; uint32_t seg_len = ((pool_t *)(hdr->buf_hdr.pool_ptr))->seg_len; - uint8_t idx; - - seg_entry_find_idx(&hdr, &idx, first);

for (i = 0; i < num; i++) { base = hdr->buf_hdr.base_data; - seg = seg_entry_next(&hdr, &idx); - seg->len = seg_len; - seg->data = base; + + hdr->seg_len = seg_len; + hdr->seg_data = base; + + hdr = hdr->seg_next; } }

/* Calculate the number of segments */ static inline int num_segments(uint32_t len, uint32_t seg_len) { - int num; - - if (CONFIG_PACKET_SEG_DISABLED) - return 1; - - num = 1; + int num = 1;

if (odp_unlikely(len > seg_len)) { num = len / seg_len; @@ -515,11 +416,10 @@ static inline int num_segments(uint32_t len, uint32_t seg_len)

static inline void add_all_segs(odp_packet_hdr_t *to, odp_packet_hdr_t *from) { - odp_packet_hdr_t *last = to->buf_hdr.last_seg; + odp_packet_hdr_t *last = packet_last_seg(to);

- last->buf_hdr.next_seg = from; - to->buf_hdr.last_seg = from->buf_hdr.last_seg; - to->buf_hdr.segcount += from->buf_hdr.segcount; + last->seg_next = from; + to->seg_count += from->seg_count; }

static inline odp_packet_hdr_t *alloc_segments(pool_t *pool, int num) @@ -561,8 +461,8 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, add_all_segs(new_hdr, pkt_hdr);

/* adjust first segment length */ - new_hdr->buf_hdr.seg[0].data += offset; - new_hdr->buf_hdr.seg[0].len = seg_len; + new_hdr->seg_data += offset; + new_hdr->seg_len = seg_len;

packet_seg_copy_md(new_hdr, pkt_hdr); new_hdr->frame_len = pkt_hdr->frame_len + len; @@ -571,14 +471,14 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr,

pkt_hdr = new_hdr; } else { - seg_entry_t *last_seg; + odp_packet_hdr_t *last_seg;

/* add into the tail */ add_all_segs(pkt_hdr, new_hdr);

/* adjust last segment length */ - last_seg = seg_entry_last(pkt_hdr); - last_seg->len = seg_len; + last_seg = packet_last_seg(pkt_hdr); + last_seg->seg_len = seg_len;

pkt_hdr->frame_len += len; pkt_hdr->tailroom = pool->tailroom + offset; @@ -587,13 +487,6 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, return pkt_hdr; }

-static inline int seg_is_link(void *hdr) -{ - odp_packet_hdr_t *pkt_hdr = hdr; - - return pkt_hdr != pkt_hdr->buf_hdr.seg[0].hdr; -} - static inline void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr) { uint32_t ref_cnt = odp_atomic_load_u32(&buf_hdr->ref_cnt); @@ -622,15 +515,11 @@ static inline int is_multi_ref(uint32_t ref_cnt)

static inline void packet_ref_inc(odp_packet_hdr_t *pkt_hdr) { - seg_entry_t *seg; - int i; - int seg_count = pkt_hdr->buf_hdr.segcount; odp_packet_hdr_t *hdr = pkt_hdr; - uint8_t idx = 0;

- for (i = 0; i < seg_count; i++) { - seg = seg_entry_next(&hdr, &idx); - buffer_ref_inc(seg->hdr); + while (hdr != NULL) { + buffer_ref_inc(&hdr->buf_hdr); + hdr = hdr->seg_next; } }

@@ -653,10 +542,6 @@ static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) } }

- /* Reset link header back to normal header */ - if (odp_unlikely(seg_is_link(hdr[i]))) - hdr[i]->seg[0].hdr = hdr[i]; - /* Skip references and pack to be freed headers to array head */ if (odp_unlikely(num_ref)) hdr[i - num_ref] = hdr[i]; @@ -672,35 +557,12 @@ static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) { int i; - odp_buffer_hdr_t *buf_hdr[num + 1]; - - if (odp_likely(pkt_hdr->buf_hdr.num_seg == num)) { - for (i = 0; i < num; i++) - buf_hdr[i] = pkt_hdr->buf_hdr.seg[i].hdr; - - if (odp_unlikely(seg_is_link(pkt_hdr))) { - buf_hdr[num] = &pkt_hdr->buf_hdr; - num++; - } - } else { - seg_entry_t *seg; - odp_buffer_hdr_t *link_hdr[num]; - uint8_t idx = 0; - int links = 0; - - for (i = 0; i < num; i++) { - /* Free also link headers */ - if (odp_unlikely(idx == 0 && seg_is_link(pkt_hdr))) { - link_hdr[links] = &pkt_hdr->buf_hdr; - links++; - } - - seg = seg_entry_next(&pkt_hdr, &idx); - buf_hdr[i] = seg->hdr; - } + odp_buffer_hdr_t *buf_hdr[num]; + odp_packet_hdr_t *seg = pkt_hdr;

- if (odp_unlikely(links)) - packet_free_multi(link_hdr, links); + for (i = 0; i < num; i++) { + buf_hdr[i] = &seg->buf_hdr; + seg = seg->seg_next; }

packet_free_multi(buf_hdr, num); @@ -710,65 +572,35 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, int num, uint32_t free_len, uint32_t pull_len, int head) { - seg_entry_t *seg; + odp_packet_hdr_t *seg; int i; - int num_remain = pkt_hdr->buf_hdr.segcount - num; + int num_remain = pkt_hdr->seg_count - num; odp_packet_hdr_t *hdr = pkt_hdr; - odp_packet_hdr_t *last_hdr = pkt_hdr->buf_hdr.last_seg; - uint8_t idx; - uint8_t num_seg; + odp_packet_hdr_t *last_hdr = packet_last_seg(pkt_hdr); odp_buffer_hdr_t *buf_hdr[num]; - odp_buffer_hdr_t *link_hdr[num]; - odp_packet_hdr_t *tmp_hdr; - int links = 0;

if (head) { odp_packet_hdr_t *new_hdr;

- idx = 0; for (i = 0; i < num; i++) { - tmp_hdr = hdr; - seg = seg_entry_next(&hdr, &idx); - buf_hdr[i] = seg->hdr; - - /* Free link headers, if those become empty */ - if (odp_unlikely(idx == 0 && seg_is_link(tmp_hdr))) { - link_hdr[links] = &tmp_hdr->buf_hdr; - links++; - } + seg = packet_seg_step(&hdr); + buf_hdr[i] = &seg->buf_hdr; }

/* The first remaining header is the new packet descriptor. * Copy remaining segments from the last to-be-removed header * to the new header. */ - new_hdr = hdr->buf_hdr.seg[idx].hdr; - num_seg = hdr->buf_hdr.num_seg - idx; - - new_hdr->buf_hdr.next_seg = hdr->buf_hdr.next_seg; - - if (hdr == last_hdr) - new_hdr->buf_hdr.last_seg = new_hdr; - else - new_hdr->buf_hdr.last_seg = last_hdr; - - new_hdr->buf_hdr.num_seg = num_seg; - new_hdr->buf_hdr.segcount = num_remain; + new_hdr = hdr;

- for (i = 0; i < num_seg; i++) { - seg = seg_entry_next(&hdr, &idx); - new_hdr->buf_hdr.seg[i] = *seg; - } + new_hdr->seg_next = hdr->seg_next; + new_hdr->seg_count = num_remain;

packet_seg_copy_md(new_hdr, pkt_hdr);

/* Tailroom not changed */ new_hdr->tailroom = pkt_hdr->tailroom;

- /* Link header does not have headroom */ - if (seg_is_link(new_hdr)) - new_hdr->headroom = 0; - else - new_hdr->headroom = seg_headroom(new_hdr, 0); + new_hdr->headroom = seg_headroom(new_hdr);

new_hdr->frame_len = pkt_hdr->frame_len - free_len;

@@ -776,45 +608,29 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr,

pkt_hdr = new_hdr;

- if (odp_unlikely(links)) - packet_free_multi(link_hdr, links); - packet_free_multi(buf_hdr, num); } else { /* Free last 'num' bufs. * First, find the last remaining header. */ - seg_entry_find_idx(&hdr, &idx, num_remain - 1); + packet_seg_find_idx(&hdr, num_remain - 1); last_hdr = hdr; - num_seg = idx + 1;

- seg_entry_next(&hdr, &idx); + packet_seg_step(&hdr);

for (i = 0; i < num; i++) { - tmp_hdr = hdr; - seg = seg_entry_next(&hdr, &idx); - buf_hdr[i] = seg->hdr; - - /* Free link headers, if those become empty */ - if (odp_unlikely(idx == 0 && seg_is_link(tmp_hdr))) { - link_hdr[links] = &tmp_hdr->buf_hdr; - links++; - } + seg = packet_seg_step(&hdr); + buf_hdr[i] = &seg->buf_hdr; }

- if (odp_unlikely(links)) - packet_free_multi(link_hdr, links); - packet_free_multi(buf_hdr, num);

/* Head segment remains, no need to copy or update majority * of the metadata. */ - last_hdr->buf_hdr.num_seg = num_seg; - last_hdr->buf_hdr.next_seg = NULL; + last_hdr->seg_next = NULL;

- pkt_hdr->buf_hdr.last_seg = last_hdr; - pkt_hdr->buf_hdr.segcount = num_remain; + pkt_hdr->seg_count = num_remain; pkt_hdr->frame_len -= free_len; - pkt_hdr->tailroom = seg_tailroom(pkt_hdr, num_remain - 1); + pkt_hdr->tailroom = seg_tailroom(pkt_hdr);

pull_tail(pkt_hdr, pull_len); } @@ -923,22 +739,14 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, void odp_packet_free(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - int num_seg = pkt_hdr->buf_hdr.segcount; + int num_seg = pkt_hdr->seg_count;

ODP_ASSERT(buffer_ref(&pkt_hdr->buf_hdr) > 0);

- if (odp_likely(CONFIG_PACKET_SEG_DISABLED || num_seg == 1)) { - odp_buffer_hdr_t *buf_hdr[2]; - int num = 1; - - buf_hdr[0] = &pkt_hdr->buf_hdr; + if (odp_likely(num_seg == 1)) { + odp_buffer_hdr_t *buf_hdr = &pkt_hdr->buf_hdr;

- if (odp_unlikely(seg_is_link(pkt_hdr))) { - num = 2; - buf_hdr[1] = pkt_hdr->buf_hdr.seg[0].hdr; - } - - packet_free_multi(buf_hdr, num); + packet_free_multi(&buf_hdr, 1); } else { free_all_segments(pkt_hdr, num_seg); } @@ -954,7 +762,7 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num)

for (i = 0; i < num; i++) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); - int num_seg = pkt_hdr->buf_hdr.segcount; + int num_seg = pkt_hdr->seg_count;

ODP_ASSERT(buffer_ref(&pkt_hdr->buf_hdr) > 0);

@@ -964,11 +772,6 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) continue; }

- if (odp_unlikely(seg_is_link(pkt_hdr))) { - buf_hdr2[links] = pkt_hdr->buf_hdr.seg[0].hdr; - links++; - } - buf_hdr[i - num_freed] = &pkt_hdr->buf_hdr; }

@@ -988,12 +791,12 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *const pkt_hdr = packet_hdr(pkt); pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; - int num = pkt_hdr->buf_hdr.segcount; + int num = pkt_hdr->seg_count;

if (odp_unlikely(len > (pool->seg_len * num))) return -1;

- reset_seg(pkt_hdr, 0, num); + reset_seg(pkt_hdr, num);

packet_init(pkt_hdr, len);

@@ -1033,7 +836,7 @@ uint32_t odp_packet_buf_len(odp_packet_t pkt) odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); pool_t *pool = pkt_hdr->buf_hdr.pool_ptr;

- return pool->max_seg_len * pkt_hdr->buf_hdr.segcount; + return pool->max_seg_len * pkt_hdr->seg_count; }

void *odp_packet_tail(odp_packet_t pkt) @@ -1117,7 +920,7 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len,

if (len < seg_len) { pull_head(pkt_hdr, len); - } else if (!CONFIG_PACKET_SEG_DISABLED) { + } else { int num = 0; uint32_t pull_len = 0;

@@ -1198,11 +1001,11 @@ int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - seg_entry_t *last_seg = seg_entry_last(pkt_hdr); + odp_packet_hdr_t *last_seg = packet_last_seg(pkt_hdr);

ODP_ASSERT(odp_packet_has_ref(pkt) == 0);

- if (len > last_seg->len) + if (len > last_seg->seg_len) return NULL;

pull_tail(pkt_hdr, len); @@ -1215,7 +1018,7 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, { int last; uint32_t seg_len; - seg_entry_t *last_seg; + odp_packet_hdr_t *last_seg; odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt);

if (len > pkt_hdr->frame_len) @@ -1223,16 +1026,17 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len,

ODP_ASSERT(odp_packet_has_ref(*pkt) == 0);

- last = packet_last_seg(pkt_hdr); - last_seg = seg_entry_last(pkt_hdr); - seg_len = last_seg->len; + last = pkt_hdr->seg_count - 1; + last_seg = packet_last_seg(pkt_hdr); + seg_len = last_seg->seg_len;

if (len < seg_len) { pull_tail(pkt_hdr, len); - } else if (!CONFIG_PACKET_SEG_DISABLED) { + } else { int num = 0; uint32_t pull_len = 0;

+ /* Reverse order */ while (seg_len <= len) { pull_len = len - seg_len; num++; @@ -1253,12 +1057,9 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len, odp_packet_seg_t *seg) { - int seg_idx; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - void *addr = packet_map(pkt_hdr, offset, len, &seg_idx);

- if (addr != NULL && seg != NULL) - *seg = _odp_packet_seg_from_ndx(seg_idx); + void *addr = packet_map(pkt_hdr, offset, len, seg);

return addr; } @@ -1388,26 +1189,35 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) * */

-void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) +void *odp_packet_seg_data(odp_packet_t pkt ODP_UNUSED, odp_packet_seg_t seg) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_packet_hdr_t *seg_hdr = packet_seg_to_hdr(seg);

- if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= - pkt_hdr->buf_hdr.segcount)) - return NULL; + return seg_hdr->seg_data; +}

- return packet_seg_data(pkt_hdr, _odp_packet_seg_to_ndx(seg)); +uint32_t odp_packet_seg_data_len(odp_packet_t pkt ODP_UNUSED, + odp_packet_seg_t seg) +{ + odp_packet_hdr_t *seg_hdr = packet_seg_to_hdr(seg); + + return seg_hdr->seg_len; }

-uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + return (odp_packet_seg_t)packet_last_seg(packet_hdr(pkt)); +}

- if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= - pkt_hdr->buf_hdr.segcount)) - return 0; +odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt ODP_UNUSED, + odp_packet_seg_t seg) +{ + odp_packet_hdr_t *pkt_hdr = packet_seg_to_hdr(seg); + + if (odp_unlikely(pkt_hdr->seg_next == NULL)) + return ODP_PACKET_SEG_INVALID;

- return packet_seg_len(pkt_hdr, _odp_packet_seg_to_ndx(seg)); + return (odp_packet_seg_t)pkt_hdr->seg_next; }

/* @@ -1778,10 +1588,7 @@ static int packet_print_input_flags(odp_packet_hdr_t *hdr, char *str, int max) void odp_packet_print(odp_packet_t pkt) { odp_packet_seg_t seg; - seg_entry_t *seg_entry; - odp_packet_hdr_t *seg_hdr; - uint8_t idx; - int max_len = 1024; + int max_len = 4096; char str[max_len]; int len = 0; int n = max_len - 1; @@ -1819,32 +1626,27 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " num_segs %i\n", odp_packet_num_segs(pkt));

- seg_hdr = hdr; - idx = 0; seg = odp_packet_first_seg(pkt);

while (seg != ODP_PACKET_SEG_INVALID) { - odp_buffer_hdr_t *buf_hdr; - odp_packet_hdr_t *tmp_hdr; - - tmp_hdr = seg_hdr; - seg_entry = seg_entry_next(&seg_hdr, &idx); - buf_hdr = seg_entry->hdr; - - len += snprintf(&str[len], n - len, - " seg_len %-4" PRIu32 " seg_data %p ", - odp_packet_seg_data_len(pkt, seg), - odp_packet_seg_data(pkt, seg)); - len += snprintf(&str[len], n - len, "ref_cnt %u", - buffer_ref(buf_hdr)); - if (seg_is_link(tmp_hdr)) { - uint32_t ref; - - ref = buffer_ref(&tmp_hdr->buf_hdr); - len += snprintf(&str[len], n - len, "L(%u)\n", ref); - } else { - len += snprintf(&str[len], n - len, "\n"); + odp_packet_hdr_t *seg_hdr = packet_seg_to_hdr(seg); + odp_buffer_hdr_t *buf_hdr = &seg_hdr->buf_hdr; + char seg_str[max_len]; + int str_len; + + str_len = snprintf(&seg_str[0], max_len, + " seg_len %-4" PRIu32 " seg_data %p " + "ref_cnt %u\n", + odp_packet_seg_data_len(pkt, seg), + odp_packet_seg_data(pkt, seg), + buffer_ref(buf_hdr)); + + /* Prevent print buffer overflow */ + if (n - len - str_len < 10) { + len += snprintf(&str[len], n - len, " ...\n"); + break; } + len += snprintf(&str[len], n - len, "%s", seg_str);

seg = odp_packet_next_seg(pkt, seg); } @@ -1872,7 +1674,7 @@ void odp_packet_print_data(odp_packet_t pkt, uint32_t offset, " buf index %" PRIu32 "\n", hdr->buf_hdr.index.buffer); len += snprintf(&str[len], n - len, - " segcount %" PRIu16 "\n", hdr->buf_hdr.segcount); + " seg_count %" PRIu16 "\n", hdr->seg_count); len += snprintf(&str[len], n - len, " data len %" PRIu32 "\n", data_len); len += snprintf(&str[len], n - len, @@ -2953,90 +2755,25 @@ odp_packet_t odp_packet_ref_static(odp_packet_t pkt)

odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) { - odp_packet_t ref; - odp_packet_hdr_t *link_hdr; - odp_packet_hdr_t *next_hdr; - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - odp_packet_hdr_t *hdr = pkt_hdr; - seg_entry_t *seg; - uint32_t seg_idx = 0; - uint8_t idx = 0; - uint32_t seg_offset = 0; - int i, num_copy, segcount; - uint32_t len; + odp_packet_t new; + int ret;

- if (offset >= pkt_hdr->frame_len) { - ODP_DBG("offset too large\n"); - return ODP_PACKET_INVALID; - } + new = odp_packet_copy(pkt, odp_packet_pool(pkt));

- /* Allocate link segment */ - if (packet_alloc(pkt_hdr->buf_hdr.pool_ptr, 0, 1, 1, &ref) != 1) { - ODP_DBG("segment alloc failed\n"); + if (new == ODP_PACKET_INVALID) { + ODP_ERR("copy failed\n"); return ODP_PACKET_INVALID; }

- link_hdr = packet_hdr(ref); - - seg_entry_find_offset(&hdr, &idx, &seg_offset, &seg_idx, offset); - num_copy = hdr->buf_hdr.num_seg - idx; - segcount = pkt_hdr->buf_hdr.segcount; - - /* In addition to segments, update reference count of - * an existing link header. */ - if (seg_is_link(hdr)) - buffer_ref_inc((odp_buffer_hdr_t *)hdr); - - seg = seg_entry_next(&hdr, &idx); - link_hdr->buf_hdr.num_seg = 1; - link_hdr->buf_hdr.seg[0].hdr = seg->hdr; - link_hdr->buf_hdr.seg[0].data = seg->data + seg_offset; - link_hdr->buf_hdr.seg[0].len = seg->len - seg_offset; - buffer_ref_inc(seg->hdr); + ret = odp_packet_trunc_head(&new, offset, NULL, NULL);

- /* The 'CONFIG_PACKET_SEGS_PER_HDR > 1' condition is required to fix an - * invalid error ('array subscript is above array bounds') thrown by - * gcc (5.4.0). */ - for (i = 1; CONFIG_PACKET_SEGS_PER_HDR > 1 && i < num_copy; i++) { - /* Update link header reference count */ - if (idx == 0 && seg_is_link(hdr)) - buffer_ref_inc((odp_buffer_hdr_t *)hdr); - - seg = seg_entry_next(&hdr, &idx); - - link_hdr->buf_hdr.num_seg++; - link_hdr->buf_hdr.seg[i].hdr = seg->hdr; - link_hdr->buf_hdr.seg[i].data = seg->data; - link_hdr->buf_hdr.seg[i].len = seg->len; - buffer_ref_inc(seg->hdr); - } - - next_hdr = hdr; - - /* Increment ref count for remaining segments */ - for (i = seg_idx + num_copy; i < segcount; i++) { - /* Update link header reference count */ - if (idx == 0 && seg_is_link(hdr)) - buffer_ref_inc((odp_buffer_hdr_t *)hdr); - - seg = seg_entry_next(&hdr, &idx); - buffer_ref_inc(seg->hdr); + if (ret < 0) { + ODP_ERR("trunk_head failed\n"); + odp_packet_free(new); + return ODP_PACKET_INVALID; }

- len = pkt_hdr->frame_len - offset; - link_hdr->buf_hdr.next_seg = next_hdr; - link_hdr->buf_hdr.last_seg = pkt_hdr->buf_hdr.last_seg; - link_hdr->buf_hdr.segcount = segcount - seg_idx; - link_hdr->frame_len = len; - link_hdr->tailroom = pkt_hdr->tailroom; - - /* Link header does not have headroom, it just points to other - * buffers. Zero length headroom ensures that head of the other buffer - * is not pushed through a reference. */ - link_hdr->headroom = 0; - - return ref; - + return new; }

odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, @@ -3066,21 +2803,18 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, int odp_packet_has_ref(odp_packet_t pkt) { odp_buffer_hdr_t *buf_hdr; - seg_entry_t *seg; - int i; - uint32_t ref_cnt; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - int seg_count = pkt_hdr->buf_hdr.segcount; - odp_packet_hdr_t *hdr = pkt_hdr; - uint8_t idx = 0; + uint32_t ref_cnt; + + while (pkt_hdr != NULL) { + buf_hdr = &pkt_hdr->buf_hdr;

- for (i = 0; i < seg_count; i++) { - seg = seg_entry_next(&hdr, &idx); - buf_hdr = seg->hdr; ref_cnt = buffer_ref(buf_hdr);

if (is_multi_ref(ref_cnt)) return 1; + + pkt_hdr = pkt_hdr->seg_next; }

return 0; diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 17504a2a7..e98027eb6 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -1,5 +1,5 @@ -/* Copyright (c) 2019, Nokia - * Copyright (c) 2013-2018, Linaro Limited +/* Copyright (c) 2013-2018, Linaro Limited + * Copyright (c) 2019, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -410,20 +410,19 @@ static void init_buffers(pool_t *pool) buf_hdr->event_type = type; buf_hdr->pool_ptr = pool; buf_hdr->uarea_addr = uarea; - buf_hdr->segcount = 1; - buf_hdr->num_seg = 1; - buf_hdr->next_seg = NULL; - buf_hdr->last_seg = buf_hdr;

- /* Pointer to data start (of the first segment) */ - buf_hdr->seg[0].hdr = buf_hdr; - buf_hdr->seg[0].data = &data[offset]; - buf_hdr->seg[0].len = pool->seg_len; + /* Initialize segmentation metadata */ + if (type == ODP_POOL_PACKET) { + pkt_hdr->seg_data = &data[offset]; + pkt_hdr->seg_len = pool->seg_len; + pkt_hdr->seg_count = 1; + pkt_hdr->seg_next = NULL; + }

odp_atomic_init_u32(&buf_hdr->ref_cnt, 0);

/* Store base values for fast init */ - buf_hdr->base_data = buf_hdr->seg[0].data; + buf_hdr->base_data = &data[offset]; buf_hdr->buf_end = &data[offset + pool->seg_len + pool->tailroom];

@@ -512,9 +511,8 @@ static odp_pool_t pool_create(const char *name, odp_pool_param_t *params, * pool. */ if (params->pkt.max_len != 0) max_len = params->pkt.max_len; - if ((max_len + seg_len - 1) / seg_len > CONFIG_PACKET_MAX_SEGS) - seg_len = (max_len + CONFIG_PACKET_MAX_SEGS - 1) / - CONFIG_PACKET_MAX_SEGS; + if ((max_len + seg_len - 1) / seg_len > PKT_MAX_SEGS) + seg_len = (max_len + PKT_MAX_SEGS - 1) / PKT_MAX_SEGS; if (seg_len > CONFIG_PACKET_MAX_SEG_LEN) { ODP_ERR("Pool unable to store 'max_len' packet"); return ODP_POOL_INVALID; @@ -1068,7 +1066,7 @@ int odp_pool_capability(odp_pool_capability_t *capa) capa->pkt.min_headroom = CONFIG_PACKET_HEADROOM; capa->pkt.max_headroom = CONFIG_PACKET_HEADROOM; capa->pkt.min_tailroom = CONFIG_PACKET_TAILROOM; - capa->pkt.max_segs_per_pkt = CONFIG_PACKET_MAX_SEGS; + capa->pkt.max_segs_per_pkt = PKT_MAX_SEGS; capa->pkt.min_seg_len = CONFIG_PACKET_SEG_LEN_MIN; capa->pkt.max_seg_len = max_seg_len; capa->pkt.max_uarea_size = MAX_SIZE; diff --git a/platform/linux-generic/pktio/dpdk.c b/platform/linux-generic/pktio/dpdk.c index c38747d3c..cc0a4368c 100644 --- a/platform/linux-generic/pktio/dpdk.c +++ b/platform/linux-generic/pktio/dpdk.c @@ -1,4 +1,5 @@ /* Copyright (c) 2016-2018, Linaro Limited + * Copyright (c) 2019, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -233,8 +234,7 @@ static unsigned cache_size(uint32_t num) static inline uint16_t mbuf_data_off(struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr) { - return (uintptr_t)pkt_hdr->buf_hdr.seg[0].data - - (uintptr_t)mbuf->buf_addr; + return (uintptr_t)pkt_hdr->seg_data - (uintptr_t)mbuf->buf_addr; }

/** @@ -250,8 +250,7 @@ static inline void mbuf_update(struct rte_mbuf *mbuf, odp_packet_hdr_t *pkt_hdr, mbuf->refcnt = 1; mbuf->ol_flags = 0;

- if (odp_unlikely(pkt_hdr->buf_hdr.base_data != - pkt_hdr->buf_hdr.seg[0].data)) + if (odp_unlikely(pkt_hdr->buf_hdr.base_data != pkt_hdr->seg_data)) mbuf->data_off = mbuf_data_off(mbuf, pkt_hdr); }

@@ -876,7 +875,7 @@ static inline int mbuf_to_pkt_zero(pktio_entry_t *pktio_entry,

/* Init buffer segments. Currently, only single segment packets * are supported. */ - pkt_hdr->buf_hdr.seg[0].data = data; + pkt_hdr->seg_data = data;

packet_init(pkt_hdr, pkt_len); pkt_hdr->input = input; @@ -927,7 +926,7 @@ static inline int pkt_to_mbuf_zero(pktio_entry_t *pktio_entry, if (odp_unlikely(pkt_len > pkt_dpdk->mtu)) goto fail;

- if (odp_likely(pkt_hdr->buf_hdr.segcount == 1)) { + if (odp_likely(pkt_hdr->seg_count == 1)) { mbuf_update(mbuf, pkt_hdr, pkt_len);

if (odp_unlikely(pktio_entry->s.chksum_insert_ena)) @@ -1919,7 +1918,7 @@ static int dpdk_send(pktio_entry_t *pktio_entry, int index, odp_packet_t pkt = pkt_table[i]; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);

- if (pkt_hdr->buf_hdr.segcount > 1) { + if (pkt_hdr->seg_count > 1) { if (odp_likely(i < tx_pkts)) odp_packet_free(pkt); else diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index 45d95bc11..1090d760c 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -685,7 +685,7 @@ static int ipc_pktio_send_lockless(pktio_entry_t *pktio_entry,

offsets[i] = (uint8_t *)pkt_hdr - (uint8_t *)odp_shm_addr(pool->shm); - data_pool_off = (uint8_t *)pkt_hdr->buf_hdr.seg[0].data - + data_pool_off = (uint8_t *)pkt_hdr->seg_data - (uint8_t *)odp_shm_addr(pool->shm);

/* compile all function code even if ipc disabled with config */ diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index a6b399b50..76fee0ad7 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -216,7 +216,7 @@ static int sock_mmsg_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, odp_time_t *ts = NULL; const int sockfd = pkt_sock->sockfd; struct mmsghdr msgvec[num]; - struct iovec iovecs[num][CONFIG_PACKET_MAX_SEGS]; + struct iovec iovecs[num][PKT_MAX_SEGS]; int nb_rx = 0; int nb_pkts; int recv_msgs; @@ -421,7 +421,7 @@ static int sock_mmsg_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, { pkt_sock_t *pkt_sock = pkt_priv(pktio_entry); struct mmsghdr msgvec[num]; - struct iovec iovecs[num][CONFIG_PACKET_MAX_SEGS]; + struct iovec iovecs[num][PKT_MAX_SEGS]; int ret; int sockfd = pkt_sock->sockfd; int i;

commit 916bfae5098242dd7e5121c789b1dbe20c8453ea Author: Matias Elo matias.elo@nokia.com Date: Wed Sep 25 14:57:24 2019 +0300

linux-gen: packet: check resulting packet length in odp_packet_concat()

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Petri Savolainen petri.savolainen@nokia.com

diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 4518706f0..a785ecef9 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1533,6 +1533,11 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src)

ODP_ASSERT(odp_packet_has_ref(*dst) == 0);

+ if (odp_unlikely(dst_len + src_len > dst_pool->max_len)) { + ODP_ERR("concat would result oversized packet\n"); + return -1; + } + /* Do a copy if packets are from different pools. */ if (odp_unlikely(dst_pool != src_pool)) { if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) {

commit 65e5ee3c37c28a9365385516e5279f917a6ff41a Author: Matias Elo matias.elo@nokia.com Date: Wed Sep 25 17:04:06 2019 +0300

validation: packet: initialize output variable of odp_packet_offset()

Fixes invalid variable may be used uninitialized error.

Signed-off-by: Matias Elo matias.elo@nokia.com Reviewed-by: Petri Savolainen petri.savolainen@nokia.com

diff --git a/test/validation/api/packet/packet.c b/test/validation/api/packet/packet.c index a12ad13eb..b3001693c 100644 --- a/test/validation/api/packet/packet.c +++ b/test/validation/api/packet/packet.c @@ -1,4 +1,5 @@ /* Copyright (c) 2014-2018, Linaro Limited + * Copyright (c) 2019, Nokia * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -1383,7 +1384,8 @@ static void packet_test_copy(void) odp_packet_t pkt; odp_packet_t pkt_copy, pkt_part; odp_pool_t pool; - uint32_t i, plen, seg_len, src_offset, dst_offset; + uint32_t i, plen, src_offset, dst_offset; + uint32_t seg_len = 0; void *pkt_data;

pkt = odp_packet_copy(test_packet, packet_pool_no_uarea); @@ -1462,7 +1464,7 @@ static void packet_test_copy(void)

/* Test segment crossing if we support segments */ pkt_data = odp_packet_offset(pkt, 0, &seg_len, NULL); - CU_ASSERT(pkt_data != NULL); + CU_ASSERT_FATAL(pkt_data != NULL);

if (seg_len < plen) { src_offset = seg_len - 15; @@ -2166,7 +2168,8 @@ static void packet_test_extend_ref(void) static void packet_test_align(void) { odp_packet_t pkt; - uint32_t pkt_len, seg_len, offset, aligned_seglen; + uint32_t pkt_len, offset; + uint32_t seg_len = 0, aligned_seglen = 0; void *pkt_data, *aligned_data; const uint32_t max_align = 32;

@@ -2224,7 +2227,8 @@ static void packet_test_align(void) static void packet_test_offset(void) { odp_packet_t pkt = test_packet; - uint32_t seg_len, full_seg_len; + uint32_t seg_len = 0; + uint32_t full_seg_len; uint8_t *ptr, *start_ptr; uint32_t offset; odp_packet_seg_t seg = ODP_PACKET_SEG_INVALID;

-----------------------------------------------------------------------

Summary of changes: config/odp-linux-generic.conf | 5 +- .../linux-generic/include-abi/odp/api/abi/packet.h | 18 +- .../include/odp/api/plat/packet_inline_types.h | 6 +- .../include/odp/api/plat/packet_inlines.h | 49 +- .../linux-generic/include/odp_buffer_internal.h | 59 +- .../linux-generic/include/odp_config_internal.h | 16 +- platform/linux-generic/include/odp_packet_dpdk.h | 5 +- .../linux-generic/include/odp_packet_internal.h | 68 +- platform/linux-generic/odp_buffer.c | 5 +- platform/linux-generic/odp_packet.c | 704 +++++++-------------- platform/linux-generic/odp_pool.c | 28 +- platform/linux-generic/pktio/dpdk.c | 36 +- platform/linux-generic/pktio/ipc.c | 2 +- platform/linux-generic/pktio/socket.c | 4 +- platform/linux-generic/test/inline-timer.conf | 2 +- platform/linux-generic/test/process-mode.conf | 2 +- test/validation/api/packet/packet.c | 12 +- 17 files changed, 367 insertions(+), 654 deletions(-)

hooks/post-receive