This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "".
The branch, master has been updated via 91ef9f7e224056af351cbaf99ccfa98ee815460d (commit) via ba97c3174d0de3e08cffaad414bf2a55de8853df (commit) via 69a1179a05344b59961aaf997a1b406698b6840b (commit) via 6efe7d971a3a4b8f7eaa42660b48eee31493924c (commit) via ef4e4d4c4ef1f9f569c236494cb178feb7c90343 (commit) via e4289ab4f73221a9d20ecfb8eac6b79a26df06a3 (commit) via a787abf96b17d9714a6b892091d19c1be2bc5e6c (commit) via 20c75b764af6fd15e5e1d4df969ac33d62525405 (commit) via 04029613fbdc85221d1a20354a49ada912302fc0 (commit) via 88d36e6811de494708c520cb12e9e5f97628e9e4 (commit) via a820d8a84e132652b4cef295756ccf135e3bd54b (commit) from f7c5f3e973814e0fc9211e008a37080b1c7d4a76 (commit)
Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below.
- Log ----------------------------------------------------------------- commit 91ef9f7e224056af351cbaf99ccfa98ee815460d Author: Petri Savolainen petri.savolainen@linaro.org Date: Tue Sep 5 09:40:59 2017 +0300
linux-gen: packet: optimize header layout
Pack most often used data into the first and the last cache line of buffer header, and the first cache line of packet header. Reduce offsets and head-/tailroom fields to 16 bits, since those are in maximum as large as a segment, usually much less. Pack header fields into correct alignments (when possible), so that holes are avoided.
These changes reduce packet header size by one cache line.
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/include/odp/arch/default/api/abi/packet.h b/include/odp/arch/default/api/abi/packet.h index 4aac75b9..15cf081b 100644 --- a/include/odp/arch/default/api/abi/packet.h +++ b/include/odp/arch/default/api/abi/packet.h @@ -28,7 +28,7 @@ typedef _odp_abi_packet_seg_t *odp_packet_seg_t;
#define ODP_PACKET_INVALID ((odp_packet_t)0xffffffff) #define ODP_PACKET_SEG_INVALID ((odp_packet_seg_t)0xffffffff) -#define ODP_PACKET_OFFSET_INVALID (0x0fffffff) +#define ODP_PACKET_OFFSET_INVALID 0xffff
typedef enum { ODP_PACKET_GREEN = 0, diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index d0cf1390..6874d149 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -56,13 +56,13 @@ static inline uint32_t _odp_packet_len(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_headroom(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint32_t, headroom); + return _odp_pkt_get(pkt, uint16_t, headroom); }
/** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_tailroom(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint32_t, tailroom); + return _odp_pkt_get(pkt, uint16_t, tailroom); }
/** @internal Inline function @param pkt @return */ diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h b/platform/linux-generic/include/odp/api/plat/packet_types.h index 96379ebf..68c66312 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_types.h @@ -36,7 +36,7 @@ typedef ODP_HANDLE_T(odp_packet_t);
#define ODP_PACKET_INVALID _odp_cast_scalar(odp_packet_t, 0)
-#define ODP_PACKET_OFFSET_INVALID (0x0fffffff) +#define ODP_PACKET_OFFSET_INVALID 0xffff
typedef uint8_t odp_packet_seg_t;
diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index cd067a08..5d40303b 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -65,13 +65,10 @@ struct odp_buffer_hdr_t { /* Initial buffer data pointer */ uint8_t *base_data;
- /* Reference count */ - odp_atomic_u32_t ref_cnt; - - /* Event type. Maybe different than pool type (crypto compl event) */ - int8_t event_type; + /* Pool pointer */ + void *pool_ptr;
- /* --- 37 bytes --- */ + /* --- 40 bytes --- */
/* Segments */ seg_entry_t seg[CONFIG_PACKET_MAX_SEGS]; @@ -95,8 +92,11 @@ struct odp_buffer_hdr_t { const void *buf_cctx; /* const alias for ctx */ };
- /* Pool pointer */ - void *pool_ptr; + /* Reference count */ + odp_atomic_u32_t ref_cnt; + + /* Event type. Maybe different than pool type (crypto compl event) */ + int8_t event_type;
/* Initial buffer tail pointer */ uint8_t *buf_end; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index d8d4584a..d76d7bf0 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -88,9 +88,14 @@ typedef struct { error_flags_t error_flags; output_flags_t output_flags;
- uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */ - uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */ - uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ + /* offset to L2 hdr, e.g. Eth */ + uint16_t l2_offset; + + /* offset to L3 hdr, e.g. IPv4, IPv6 */ + uint16_t l3_offset; + + /* offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */ + uint16_t l4_offset; } packet_parser_t;
/* Packet extra data length */ @@ -116,14 +121,14 @@ typedef struct {
packet_parser_t p;
- uint32_t frame_len; - odp_pktio_t input;
- uint32_t headroom; - uint32_t tailroom; + uint32_t frame_len; uint32_t shared_len;
+ uint16_t headroom; + uint16_t tailroom; + /* * Members below are not initialized by packet_init() */ diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 2f65cb20..2a0a35e4 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -41,6 +41,9 @@ ODP_STATIC_ASSERT(CONFIG_POOL_CACHE_SIZE > (2 * CACHE_BURST), ODP_STATIC_ASSERT(CONFIG_PACKET_SEG_LEN_MIN >= 256, "ODP Segment size must be a minimum of 256 bytes");
+ODP_STATIC_ASSERT(CONFIG_PACKET_SEG_SIZE < 0xffff, + "Segment size must be less than 64k (16 bit offsets)"); + /* Thread local variables */ typedef struct pool_local_t { pool_cache_t *cache[ODP_CONFIG_POOLS];
commit ba97c3174d0de3e08cffaad414bf2a55de8853df Author: Petri Savolainen petri.savolainen@linaro.org Date: Thu Aug 31 15:35:12 2017 +0300
linux-gen: packet: implement dynamic references
Use link headers to implement dynamic references. These are segment headers which first segment (seg[0]) does not point to itself but to a referenced segment. Link headers enable long chains of references (new dynamic references from previous references).
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 5990f878..d2a87f0f 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -130,6 +130,43 @@ static inline seg_entry_t *seg_entry_next(odp_packet_hdr_t **cur_hdr, return &hdr->buf_hdr.seg[idx]; }
+static inline void seg_entry_find_offset(odp_packet_hdr_t **p_hdr, + uint8_t *p_idx, + uint32_t *seg_offset, + uint32_t *seg_idx, + uint32_t offset) +{ + int i; + odp_packet_hdr_t *hdr, *cur_hdr; + uint8_t idx, cur_idx; + seg_entry_t *seg = NULL; + uint32_t seg_start = 0, seg_end = 0; + int seg_count; + + hdr = *p_hdr; + cur_hdr = hdr; + idx = 0; + cur_idx = 0; + seg_count = hdr->buf_hdr.segcount; + + for (i = 0; i < seg_count; i++) { + cur_hdr = hdr; + cur_idx = idx; + seg = seg_entry_next(&hdr, &idx); + seg_end += seg->len; + + if (odp_likely(offset < seg_end)) + break; + + seg_start = seg_end; + } + + *p_hdr = cur_hdr; + *p_idx = cur_idx; + *seg_offset = offset - seg_start; + *seg_idx = i; +} + static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { @@ -459,6 +496,7 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, new_hdr->frame_len = pkt_hdr->frame_len + len; new_hdr->headroom = pool->headroom + offset; new_hdr->tailroom = pkt_hdr->tailroom; + new_hdr->shared_len = pkt_hdr->shared_len;
pkt_hdr = new_hdr; } else { @@ -478,20 +516,11 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, return pkt_hdr; }
-static inline void copy_buf_hdr(odp_packet_hdr_t *pkt_hdr, int first, int num, - odp_buffer_hdr_t *buf_hdr[]) +static inline int seg_is_link(void *hdr) { - seg_entry_t *seg; - int i; - uint8_t idx; - odp_packet_hdr_t *hdr = pkt_hdr; - - seg_entry_find_idx(&hdr, &idx, first); + odp_packet_hdr_t *pkt_hdr = hdr;
- for (i = 0; i < num; i++) { - seg = seg_entry_next(&hdr, &idx); - buf_hdr[i] = seg->hdr; - } + return pkt_hdr != pkt_hdr->buf_hdr.seg[0].hdr; }
static inline void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr) @@ -553,9 +582,14 @@ static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) } }
+ /* Reset link header back to normal header */ + if (odp_unlikely(seg_is_link(hdr[i]))) + hdr[i]->seg[0].hdr = hdr[i]; + /* Skip references and pack to be freed headers to array head */ if (odp_unlikely(num_ref)) hdr[i - num_ref] = hdr[i]; + }
num -= num_ref; @@ -566,14 +600,36 @@ static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num)
static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) { - seg_entry_t *seg; int i; - odp_buffer_hdr_t *buf_hdr[num]; - uint8_t idx = 0; + odp_buffer_hdr_t *buf_hdr[num + 1];
- for (i = 0; i < num; i++) { - seg = seg_entry_next(&pkt_hdr, &idx); - buf_hdr[i] = seg->hdr; + if (odp_likely(pkt_hdr->buf_hdr.num_seg == num)) { + for (i = 0; i < num; i++) + buf_hdr[i] = pkt_hdr->buf_hdr.seg[i].hdr; + + if (odp_unlikely(seg_is_link(pkt_hdr))) { + buf_hdr[num] = &pkt_hdr->buf_hdr; + num++; + } + } else { + seg_entry_t *seg; + odp_buffer_hdr_t *link_hdr[num]; + uint8_t idx = 0; + int links = 0; + + for (i = 0; i < num; i++) { + /* Free also link headers */ + if (odp_unlikely(idx == 0 && seg_is_link(pkt_hdr))) { + link_hdr[links] = &pkt_hdr->buf_hdr; + links++; + } + + seg = seg_entry_next(&pkt_hdr, &idx); + buf_hdr[i] = seg->hdr; + } + + if (odp_unlikely(links)) + packet_free_multi(link_hdr, links); }
packet_free_multi(buf_hdr, num); @@ -591,14 +647,24 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, uint8_t idx; uint8_t num_seg; odp_buffer_hdr_t *buf_hdr[num]; + odp_buffer_hdr_t *link_hdr[num]; + odp_packet_hdr_t *tmp_hdr; + int links = 0;
if (head) { odp_packet_hdr_t *new_hdr;
idx = 0; for (i = 0; i < num; i++) { + tmp_hdr = hdr; seg = seg_entry_next(&hdr, &idx); buf_hdr[i] = seg->hdr; + + /* Free link headers, if those become empty */ + if (odp_unlikely(idx == 0 && seg_is_link(tmp_hdr))) { + link_hdr[links] = &tmp_hdr->buf_hdr; + links++; + } }
/* The first remaining header is the new packet descriptor. @@ -626,13 +692,23 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr,
/* Tailroom not changed */ new_hdr->tailroom = pkt_hdr->tailroom; - new_hdr->headroom = seg_headroom(new_hdr, 0); - new_hdr->frame_len = pkt_hdr->frame_len - free_len; + + /* Link header does not have headroom */ + if (seg_is_link(new_hdr)) + new_hdr->headroom = 0; + else + new_hdr->headroom = seg_headroom(new_hdr, 0); + + new_hdr->frame_len = pkt_hdr->frame_len - free_len; + new_hdr->shared_len = pkt_hdr->shared_len;
pull_head(new_hdr, pull_len);
pkt_hdr = new_hdr;
+ if (odp_unlikely(links)) + packet_free_multi(link_hdr, links); + packet_free_multi(buf_hdr, num); } else { /* Free last 'num' bufs. @@ -644,10 +720,20 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, seg_entry_next(&hdr, &idx);
for (i = 0; i < num; i++) { + tmp_hdr = hdr; seg = seg_entry_next(&hdr, &idx); buf_hdr[i] = seg->hdr; + + /* Free link headers, if those become empty */ + if (odp_unlikely(idx == 0 && seg_is_link(tmp_hdr))) { + link_hdr[links] = &tmp_hdr->buf_hdr; + links++; + } }
+ if (odp_unlikely(links)) + packet_free_multi(link_hdr, links); + packet_free_multi(buf_hdr, num);
/* Head segment remains, no need to copy or update majority @@ -767,42 +853,56 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, void odp_packet_free(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - odp_buffer_t hdl = buffer_handle(pkt_hdr); - int num_seg = pkt_hdr->buf_hdr.segcount;
- if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) - packet_free_multi((odp_buffer_hdr_t **)&hdl, 1); - else + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) { + odp_buffer_hdr_t *buf_hdr[2]; + int num = 1; + + buf_hdr[0] = &pkt_hdr->buf_hdr; + + if (odp_unlikely(seg_is_link(pkt_hdr))) { + num = 2; + buf_hdr[1] = pkt_hdr->buf_hdr.seg[0].hdr; + } + + packet_free_multi(buf_hdr, num); + } else { free_all_segments(pkt_hdr, num_seg); + } }
void odp_packet_free_multi(const odp_packet_t pkt[], int num) { - if (CONFIG_PACKET_MAX_SEGS == 1) { - packet_free_multi((odp_buffer_hdr_t **)(uintptr_t)pkt, num); - } else { - odp_buffer_hdr_t *buf_hdr[num * CONFIG_PACKET_MAX_SEGS]; - int i; - int bufs = 0; - - for (i = 0; i < num; i++) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); - int num_seg = pkt_hdr->buf_hdr.segcount; - odp_buffer_hdr_t *hdr = &pkt_hdr->buf_hdr; + odp_buffer_hdr_t *buf_hdr[num]; + odp_buffer_hdr_t *buf_hdr2[num]; + int i; + int links = 0; + int num_freed = 0;
- buf_hdr[bufs] = hdr; - bufs++; + for (i = 0; i < num; i++) { + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); + int num_seg = pkt_hdr->buf_hdr.segcount;
- if (odp_likely(num_seg == 1)) - continue; + if (odp_unlikely(num_seg > 1)) { + free_all_segments(pkt_hdr, num_seg); + num_freed++; + continue; + }
- copy_buf_hdr(pkt_hdr, 1, num_seg - 1, &buf_hdr[bufs]); - bufs += num_seg - 1; + if (odp_unlikely(seg_is_link(pkt_hdr))) { + buf_hdr2[links] = pkt_hdr->buf_hdr.seg[0].hdr; + links++; }
- packet_free_multi(buf_hdr, bufs); + buf_hdr[i - num_freed] = &pkt_hdr->buf_hdr; } + + if (odp_unlikely(links)) + packet_free_multi(buf_hdr2, links); + + if (odp_likely(num - num_freed)) + packet_free_multi(buf_hdr, num - num_freed); }
int odp_packet_reset(odp_packet_t pkt, uint32_t len) @@ -1627,7 +1727,9 @@ void odp_packet_print(odp_packet_t pkt)
while (seg != ODP_PACKET_SEG_INVALID) { odp_buffer_hdr_t *buf_hdr; + odp_packet_hdr_t *tmp_hdr;
+ tmp_hdr = seg_hdr; seg_entry = seg_entry_next(&seg_hdr, &idx); buf_hdr = seg_entry->hdr;
@@ -1635,8 +1737,16 @@ void odp_packet_print(odp_packet_t pkt) " seg_len %-4" PRIu32 " seg_data %p ", odp_packet_seg_data_len(pkt, seg), odp_packet_seg_data(pkt, seg)); - len += snprintf(&str[len], n - len, "ref_cnt %u\n", + len += snprintf(&str[len], n - len, "ref_cnt %u", buffer_ref(buf_hdr)); + if (seg_is_link(tmp_hdr)) { + uint32_t ref; + + ref = buffer_ref(&tmp_hdr->buf_hdr); + len += snprintf(&str[len], n - len, "L(%u)\n", ref); + } else { + len += snprintf(&str[len], n - len, "\n"); + }
seg = odp_packet_next_seg(pkt, seg); } @@ -2034,59 +2144,122 @@ odp_packet_t odp_packet_ref_static(odp_packet_t pkt)
odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) { - odp_packet_t new; - int ret; + odp_packet_t ref; + odp_packet_hdr_t *link_hdr; + odp_packet_hdr_t *next_hdr; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_packet_hdr_t *hdr = pkt_hdr; + seg_entry_t *seg; + uint32_t seg_idx = 0; + uint8_t idx = 0; + uint32_t seg_offset = 0; + int i, num_copy, segcount; + uint32_t len;
- new = odp_packet_copy(pkt, odp_packet_pool(pkt)); + if (offset >= pkt_hdr->frame_len) { + ODP_DBG("offset too large\n"); + return ODP_PACKET_INVALID; + }
- if (new == ODP_PACKET_INVALID) { - ODP_ERR("copy failed\n"); + /* Allocate link segment */ + if (packet_alloc(pkt_hdr->buf_hdr.pool_ptr, 0, 1, 1, &ref) != 1) { + ODP_DBG("segment alloc failed\n"); return ODP_PACKET_INVALID; }
- ret = odp_packet_trunc_head(&new, offset, NULL, NULL); + link_hdr = packet_hdr(ref);
- if (ret < 0) { - ODP_ERR("trunk_head failed\n"); - odp_packet_free(new); - return ODP_PACKET_INVALID; + seg_entry_find_offset(&hdr, &idx, &seg_offset, &seg_idx, offset); + num_copy = hdr->buf_hdr.num_seg - idx; + segcount = pkt_hdr->buf_hdr.segcount; + + /* In addition to segments, update reference count of + * an existing link header. */ + if (seg_is_link(hdr)) + buffer_ref_inc((odp_buffer_hdr_t *)hdr); + + seg = seg_entry_next(&hdr, &idx); + link_hdr->buf_hdr.num_seg = 1; + link_hdr->buf_hdr.seg[0].hdr = seg->hdr; + link_hdr->buf_hdr.seg[0].data = seg->data + seg_offset; + link_hdr->buf_hdr.seg[0].len = seg->len - seg_offset; + buffer_ref_inc(seg->hdr); + + for (i = 1; i < num_copy; i++) { + /* Update link header reference count */ + if (idx == 0 && seg_is_link(hdr)) + buffer_ref_inc((odp_buffer_hdr_t *)hdr); + + seg = seg_entry_next(&hdr, &idx); + + link_hdr->buf_hdr.num_seg++; + link_hdr->buf_hdr.seg[i].hdr = seg->hdr; + link_hdr->buf_hdr.seg[i].data = seg->data; + link_hdr->buf_hdr.seg[i].len = seg->len; + buffer_ref_inc(seg->hdr); + } + + next_hdr = hdr; + + /* Increment ref count for remaining segments */ + for (i = seg_idx + num_copy; i < segcount; i++) { + /* Update link header reference count */ + if (idx == 0 && seg_is_link(hdr)) + buffer_ref_inc((odp_buffer_hdr_t *)hdr); + + seg = seg_entry_next(&hdr, &idx); + buffer_ref_inc(seg->hdr); }
- return new; + len = pkt_hdr->frame_len - offset; + link_hdr->buf_hdr.next_seg = next_hdr; + link_hdr->buf_hdr.last_seg = pkt_hdr->buf_hdr.last_seg; + link_hdr->buf_hdr.segcount = segcount - seg_idx; + link_hdr->frame_len = len; + link_hdr->tailroom = pkt_hdr->tailroom; + link_hdr->shared_len = len; + + /* Link header does not have headroom, it just points to other + * buffers. Zero length headroom ensures that head of the other buffer + * is not pushed through a reference. */ + link_hdr->headroom = 0; + + if (pkt_hdr->shared_len < len) + pkt_hdr->shared_len = len; + + return ref; + }
odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, odp_packet_t hdr) { - odp_packet_t new; + odp_packet_t ref; int ret; + odp_packet_hdr_t *new_hdr; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t len = pkt_hdr->frame_len;
- new = odp_packet_copy(pkt, odp_packet_pool(pkt)); + ref = odp_packet_ref(pkt, offset);
- if (new == ODP_PACKET_INVALID) { - ODP_ERR("copy failed\n"); + if (ref == ODP_PACKET_INVALID) { + ODP_DBG("reference create failed\n"); return ODP_PACKET_INVALID; }
- if (offset) { - ret = odp_packet_trunc_head(&new, offset, NULL, NULL); - - if (ret < 0) { - ODP_ERR("trunk_head failed\n"); - odp_packet_free(new); - return ODP_PACKET_INVALID; - } - } - - ret = odp_packet_concat(&hdr, new); + ret = odp_packet_concat(&hdr, ref);
if (ret < 0) { - ODP_ERR("concat failed\n"); - odp_packet_free(new); + ODP_DBG("concat failed\n"); + odp_packet_free(ref); return ODP_PACKET_INVALID; }
+ new_hdr = packet_hdr(hdr); + new_hdr->shared_len = len - offset; + return hdr; + }
int odp_packet_has_ref(odp_packet_t pkt) @@ -2115,8 +2288,12 @@ int odp_packet_has_ref(odp_packet_t pkt) uint32_t odp_packet_unshared_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t len = pkt_hdr->frame_len; + + if (odp_packet_has_ref(pkt)) + return len - pkt_hdr->shared_len;
- return packet_len(pkt_hdr) - pkt_hdr->shared_len; + return len; }
/* Include non-inlined versions of API functions */
commit 69a1179a05344b59961aaf997a1b406698b6840b Author: Petri Savolainen petri.savolainen@linaro.org Date: Fri Aug 25 14:03:59 2017 +0300
linux-gen: packet: compress packet offset table
Use uint16_t (instead of size_t) for offsets and thus decrease size of the offset table into one fourth.
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h b/platform/linux-generic/include/odp/api/plat/packet_types.h index 41468bec..96379ebf 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_types.h @@ -74,33 +74,33 @@ typedef enum { /** @internal Packet header field offsets for inline functions */ typedef struct _odp_packet_inline_offset_t { /** @internal field offset */ - size_t data; + uint16_t data; /** @internal field offset */ - size_t seg_len; + uint16_t seg_len; /** @internal field offset */ - size_t frame_len; + uint16_t frame_len; /** @internal field offset */ - size_t headroom; + uint16_t headroom; /** @internal field offset */ - size_t tailroom; + uint16_t tailroom; /** @internal field offset */ - size_t pool; + uint16_t pool; /** @internal field offset */ - size_t input; + uint16_t input; /** @internal field offset */ - size_t segcount; + uint16_t segcount; /** @internal field offset */ - size_t user_ptr; + uint16_t user_ptr; /** @internal field offset */ - size_t user_area; + uint16_t user_area; /** @internal field offset */ - size_t user_area_size; + uint16_t user_area_size; /** @internal field offset */ - size_t flow_hash; + uint16_t flow_hash; /** @internal field offset */ - size_t timestamp; + uint16_t timestamp; /** @internal field offset */ - size_t input_flags; + uint16_t input_flags;
} _odp_packet_inline_offset_t;
commit 6efe7d971a3a4b8f7eaa42660b48eee31493924c Author: Petri Savolainen petri.savolainen@linaro.org Date: Thu Aug 24 16:21:51 2017 +0300
linux-gen: packet: enhance segment prints
Print segment addresses and reference counts in addition to segment lengths.
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 1a21ab0c..5990f878 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1579,11 +1579,13 @@ int _odp_packet_cmp_data(odp_packet_t pkt, uint32_t offset, * ******************************************************** * */ - void odp_packet_print(odp_packet_t pkt) { odp_packet_seg_t seg; - int max_len = 512; + seg_entry_t *seg_entry; + odp_packet_hdr_t *seg_hdr; + uint8_t idx; + int max_len = 1024; char str[max_len]; int len = 0; int n = max_len - 1; @@ -1619,12 +1621,22 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " num_segs %i\n", odp_packet_num_segs(pkt));
+ seg_hdr = hdr; + idx = 0; seg = odp_packet_first_seg(pkt);
while (seg != ODP_PACKET_SEG_INVALID) { + odp_buffer_hdr_t *buf_hdr; + + seg_entry = seg_entry_next(&seg_hdr, &idx); + buf_hdr = seg_entry->hdr; + len += snprintf(&str[len], n - len, - " seg_len %" PRIu32 "\n", - odp_packet_seg_data_len(pkt, seg)); + " seg_len %-4" PRIu32 " seg_data %p ", + odp_packet_seg_data_len(pkt, seg), + odp_packet_seg_data(pkt, seg)); + len += snprintf(&str[len], n - len, "ref_cnt %u\n", + buffer_ref(buf_hdr));
seg = odp_packet_next_seg(pkt, seg); }
commit ef4e4d4c4ef1f9f569c236494cb178feb7c90343 Author: Petri Savolainen petri.savolainen@linaro.org Date: Mon Aug 21 13:51:49 2017 +0300
linux-gen: packet: implement static references
Implemented static references with a reference counter. Counter is zero when not used (reference API not used).
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 0c873d2d..cd067a08 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -62,11 +62,16 @@ struct odp_buffer_hdr_t { /* Last header of the segment list */ void *last_seg;
- /* Initial buffer data pointer and length */ + /* Initial buffer data pointer */ uint8_t *base_data; - uint8_t *buf_end;
- /* --- 40 bytes --- */ + /* Reference count */ + odp_atomic_u32_t ref_cnt; + + /* Event type. Maybe different than pool type (crypto compl event) */ + int8_t event_type; + + /* --- 37 bytes --- */
/* Segments */ seg_entry_t seg[CONFIG_PACKET_MAX_SEGS]; @@ -93,14 +98,17 @@ struct odp_buffer_hdr_t { /* Pool pointer */ void *pool_ptr;
+ /* Initial buffer tail pointer */ + uint8_t *buf_end; + /* User area pointer */ void *uarea_addr;
/* User area size */ uint32_t uarea_size;
- /* Event type. Maybe different than pool type (crypto compl event) */ - int8_t event_type; + /* Max data size */ + uint32_t size;
/* ipc mapped process can not walk over pointers, * offset has to be used */ @@ -110,9 +118,6 @@ struct odp_buffer_hdr_t { * inlining */ odp_pool_t pool_hdl;
- /* Max data size */ - uint32_t size; - /* Data or next header */ uint8_t data[0]; } ODP_ALIGNED_CACHE; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index d8e5766c..d8d4584a 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -116,11 +116,13 @@ typedef struct {
packet_parser_t p;
+ uint32_t frame_len; + odp_pktio_t input;
- uint32_t frame_len; uint32_t headroom; uint32_t tailroom; + uint32_t shared_len;
/* * Members below are not initialized by packet_init() @@ -217,6 +219,7 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) * segment occupied by the allocated length. */ pkt_hdr->frame_len = len; + pkt_hdr->shared_len = 0; pkt_hdr->headroom = CONFIG_PACKET_HEADROOM; pkt_hdr->tailroom = CONFIG_PACKET_MAX_SEG_LEN - seg_len + CONFIG_PACKET_TAILROOM; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index a4330737..1a21ab0c 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -494,6 +494,76 @@ static inline void copy_buf_hdr(odp_packet_hdr_t *pkt_hdr, int first, int num, } }
+static inline void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr) +{ + uint32_t ref_cnt = odp_atomic_load_u32(&buf_hdr->ref_cnt); + + /* First count increment after alloc */ + if (odp_likely(ref_cnt) == 0) + odp_atomic_store_u32(&buf_hdr->ref_cnt, 2); + else + odp_atomic_inc_u32(&buf_hdr->ref_cnt); +} + +static inline uint32_t buffer_ref_dec(odp_buffer_hdr_t *buf_hdr) +{ + return odp_atomic_fetch_dec_u32(&buf_hdr->ref_cnt); +} + +static inline uint32_t buffer_ref(odp_buffer_hdr_t *buf_hdr) +{ + return odp_atomic_load_u32(&buf_hdr->ref_cnt); +} + +static inline int is_multi_ref(uint32_t ref_cnt) +{ + return (ref_cnt > 1); +} + +static inline void packet_ref_inc(odp_packet_hdr_t *pkt_hdr) +{ + seg_entry_t *seg; + int i; + int seg_count = pkt_hdr->buf_hdr.segcount; + odp_packet_hdr_t *hdr = pkt_hdr; + uint8_t idx = 0; + + for (i = 0; i < seg_count; i++) { + seg = seg_entry_next(&hdr, &idx); + buffer_ref_inc(seg->hdr); + } +} + +static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num) +{ + int i; + uint32_t ref_cnt; + int num_ref = 0; + + for (i = 0; i < num; i++) { + /* Zero when reference API has not been used */ + ref_cnt = buffer_ref(hdr[i]); + + if (odp_unlikely(ref_cnt)) { + ref_cnt = buffer_ref_dec(hdr[i]); + + if (is_multi_ref(ref_cnt)) { + num_ref++; + continue; + } + } + + /* Skip references and pack to be freed headers to array head */ + if (odp_unlikely(num_ref)) + hdr[i - num_ref] = hdr[i]; + } + + num -= num_ref; + + if (odp_likely(num)) + buffer_free_multi(hdr, num); +} + static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) { seg_entry_t *seg; @@ -506,7 +576,7 @@ static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) buf_hdr[i] = seg->hdr; }
- buffer_free_multi(buf_hdr, num); + packet_free_multi(buf_hdr, num); }
static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, @@ -563,7 +633,7 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr,
pkt_hdr = new_hdr;
- buffer_free_multi(buf_hdr, num); + packet_free_multi(buf_hdr, num); } else { /* Free last 'num' bufs. * First, find the last remaining header. */ @@ -578,7 +648,7 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, buf_hdr[i] = seg->hdr; }
- buffer_free_multi(buf_hdr, num); + packet_free_multi(buf_hdr, num);
/* Head segment remains, no need to copy or update majority * of the metadata. */ @@ -702,7 +772,7 @@ void odp_packet_free(odp_packet_t pkt) int num_seg = pkt_hdr->buf_hdr.segcount;
if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) - buffer_free_multi((odp_buffer_hdr_t **)&hdl, 1); + packet_free_multi((odp_buffer_hdr_t **)&hdl, 1); else free_all_segments(pkt_hdr, num_seg); } @@ -710,7 +780,7 @@ void odp_packet_free(odp_packet_t pkt) void odp_packet_free_multi(const odp_packet_t pkt[], int num) { if (CONFIG_PACKET_MAX_SEGS == 1) { - buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)pkt, num); + packet_free_multi((odp_buffer_hdr_t **)(uintptr_t)pkt, num); } else { odp_buffer_hdr_t *buf_hdr[num * CONFIG_PACKET_MAX_SEGS]; int i; @@ -731,7 +801,7 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) bufs += num_seg - 1; }
- buffer_free_multi(buf_hdr, bufs); + packet_free_multi(buf_hdr, bufs); } }
@@ -1942,7 +2012,12 @@ uint64_t odp_packet_seg_to_u64(odp_packet_seg_t hdl)
odp_packet_t odp_packet_ref_static(odp_packet_t pkt) { - return odp_packet_copy(pkt, odp_packet_pool(pkt)); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + packet_ref_inc(pkt_hdr); + pkt_hdr->shared_len = pkt_hdr->frame_len; + + return pkt; }
odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) @@ -2004,14 +2079,32 @@ odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset,
int odp_packet_has_ref(odp_packet_t pkt) { - (void)pkt; + odp_buffer_hdr_t *buf_hdr; + seg_entry_t *seg; + int i; + uint32_t ref_cnt; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + int seg_count = pkt_hdr->buf_hdr.segcount; + odp_packet_hdr_t *hdr = pkt_hdr; + uint8_t idx = 0; + + for (i = 0; i < seg_count; i++) { + seg = seg_entry_next(&hdr, &idx); + buf_hdr = seg->hdr; + ref_cnt = buffer_ref(buf_hdr); + + if (is_multi_ref(ref_cnt)) + return 1; + }
return 0; }
uint32_t odp_packet_unshared_len(odp_packet_t pkt) { - return odp_packet_len(pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + return packet_len(pkt_hdr) - pkt_hdr->shared_len; }
/* Include non-inlined versions of API functions */ diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 47a39f5b..2f65cb20 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -277,6 +277,8 @@ static void init_buffers(pool_t *pool) buf_hdr->seg[0].data = &data[offset]; buf_hdr->seg[0].len = pool->data_size;
+ odp_atomic_init_u32(&buf_hdr->ref_cnt, 0); + /* Store base values for fast init */ buf_hdr->base_data = buf_hdr->seg[0].data; buf_hdr->buf_end = &data[offset + pool->data_size +
commit e4289ab4f73221a9d20ecfb8eac6b79a26df06a3 Author: Petri Savolainen petri.savolainen@linaro.org Date: Fri Aug 18 10:54:25 2017 +0300
linux-gen: buffer: optimize header layout
Pack most often used buffer header fields (including seg[0]) into the first cache line.
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 30909465..0c873d2d 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -43,41 +43,46 @@ typedef struct seg_entry_t {
/* Common buffer header */ struct odp_buffer_hdr_t { - /* Buffer index in the pool */ - uint32_t index;
- /* Initial buffer data pointer and length */ - uint8_t *base_data; - uint8_t *buf_end; + /* Buffer index in the pool */ + uint32_t index;
- /* Max data size */ - uint32_t size; + /* Total segment count */ + uint16_t segcount;
/* Pool type */ int8_t type;
- /* Burst counts */ - uint8_t burst_num; - uint8_t burst_first; - /* Number of seg[] entries used */ uint8_t num_seg;
- /* Total segment count */ - uint32_t segcount; - /* Next header which continues the segment list */ void *next_seg;
/* Last header of the segment list */ void *last_seg;
+ /* Initial buffer data pointer and length */ + uint8_t *base_data; + uint8_t *buf_end; + + /* --- 40 bytes --- */ + /* Segments */ seg_entry_t seg[CONFIG_PACKET_MAX_SEGS];
+ /* Burst counts */ + uint8_t burst_num; + uint8_t burst_first; + /* Next buf in a list */ struct odp_buffer_hdr_t *next;
+ /* Burst table */ + struct odp_buffer_hdr_t *burst[BUFFER_BURST_SIZE]; + + /* --- Mostly read only data --- */ + /* User context pointer or u64 */ union { uint64_t buf_u64; @@ -85,6 +90,9 @@ struct odp_buffer_hdr_t { const void *buf_cctx; /* const alias for ctx */ };
+ /* Pool pointer */ + void *pool_ptr; + /* User area pointer */ void *uarea_addr;
@@ -94,9 +102,6 @@ struct odp_buffer_hdr_t { /* Event type. Maybe different than pool type (crypto compl event) */ int8_t event_type;
- /* Burst table */ - struct odp_buffer_hdr_t *burst[BUFFER_BURST_SIZE]; - /* ipc mapped process can not walk over pointers, * offset has to be used */ uint64_t ipc_data_offset; @@ -105,8 +110,8 @@ struct odp_buffer_hdr_t { * inlining */ odp_pool_t pool_hdl;
- /* Pool pointer */ - void *pool_ptr; + /* Max data size */ + uint32_t size;
/* Data or next header */ uint8_t data[0];
commit a787abf96b17d9714a6b892091d19c1be2bc5e6c Author: Petri Savolainen petri.savolainen@linaro.org Date: Thu Aug 17 14:09:47 2017 +0300
linux-gen: packet: re-implement packet segmentation
Removed limitation to fixed number of segments (CONFIG_PACKET_MAX_SEGS) per packet. Packet headers carrying segment tables are now linked together with 'next_seg' pointer. Last header containing a segment table is pointed by 'last_seg' pointer. Other than the first segment are not referred directly but with seg_entry_ functions.
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index e20d1c8d..30909465 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -35,6 +35,12 @@ extern "C" {
#define BUFFER_BURST_SIZE CONFIG_BURST_SIZE
+typedef struct seg_entry_t { + void *hdr; + uint8_t *data; + uint32_t len; +} seg_entry_t; + /* Common buffer header */ struct odp_buffer_hdr_t { /* Buffer index in the pool */ @@ -54,15 +60,20 @@ struct odp_buffer_hdr_t { uint8_t burst_num; uint8_t burst_first;
- /* Segment count */ - uint8_t segcount; + /* Number of seg[] entries used */ + uint8_t num_seg; + + /* Total segment count */ + uint32_t segcount; + + /* Next header which continues the segment list */ + void *next_seg; + + /* Last header of the segment list */ + void *last_seg;
/* Segments */ - struct { - void *hdr; - uint8_t *data; - uint32_t len; - } seg[CONFIG_PACKET_MAX_SEGS]; + seg_entry_t seg[CONFIG_PACKET_MAX_SEGS];
/* Next buf in a list */ struct odp_buffer_hdr_t *next; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index f77987c8..d8e5766c 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -172,6 +172,16 @@ static inline odp_packet_t packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) return (odp_packet_t)(odp_packet_hdr_t *)buf_hdr; }
+static inline seg_entry_t *seg_entry_last(odp_packet_hdr_t *hdr) +{ + odp_packet_hdr_t *last; + uint8_t last_seg; + + last = hdr->buf_hdr.last_seg; + last_seg = last->buf_hdr.num_seg - 1; + return &last->buf_hdr.seg[last_seg]; +} + /** * Initialize packet */ @@ -184,10 +194,13 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) seg_len = len; pkt_hdr->buf_hdr.seg[0].len = len; } else { + seg_entry_t *last; + seg_len = len - ((num - 1) * CONFIG_PACKET_MAX_SEG_LEN);
/* Last segment data length */ - pkt_hdr->buf_hdr.seg[num - 1].len = seg_len; + last = seg_entry_last(pkt_hdr); + last->len = seg_len; }
pkt_hdr->p.input_flags.all = 0; @@ -229,11 +242,11 @@ static inline void copy_packet_cls_metadata(odp_packet_hdr_t *src_hdr,
static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - int last = pkt_hdr->buf_hdr.segcount - 1; + seg_entry_t *last = seg_entry_last(pkt_hdr);
pkt_hdr->tailroom += len; pkt_hdr->frame_len -= len; - pkt_hdr->buf_hdr.seg[last].len -= len; + last->len -= len; }
static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 1e5b136a..a4330737 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -74,15 +74,75 @@ static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) return (odp_buffer_t)pkt; }
+static inline seg_entry_t *seg_entry(odp_packet_hdr_t *hdr, + uint32_t seg_idx) +{ + uint32_t idx = 0; + uint8_t num_seg = hdr->buf_hdr.num_seg; + + while (odp_unlikely(idx + num_seg - 1 < seg_idx)) { + idx += num_seg; + hdr = hdr->buf_hdr.next_seg; + num_seg = hdr->buf_hdr.num_seg; + } + + idx = seg_idx - idx; + + return &hdr->buf_hdr.seg[idx]; +} + +static inline void seg_entry_find_idx(odp_packet_hdr_t **p_hdr, + uint8_t *p_idx, + uint32_t find_idx) +{ + odp_packet_hdr_t *hdr = *p_hdr; + uint32_t idx = 0; + uint8_t num_seg = hdr->buf_hdr.num_seg; + + while (odp_unlikely(idx + num_seg - 1 < find_idx)) { + idx += num_seg; + hdr = hdr->buf_hdr.next_seg; + num_seg = hdr->buf_hdr.num_seg; + } + + idx = find_idx - idx; + *p_hdr = hdr; + *p_idx = idx; +} + +/* Return pointer to the current segment entry and step cur_hdr / cur_idx + * forward. + */ +static inline seg_entry_t *seg_entry_next(odp_packet_hdr_t **cur_hdr, + uint8_t *cur_idx) +{ + odp_packet_hdr_t *hdr = *cur_hdr; + uint8_t idx = *cur_idx; + uint8_t num_seg = hdr->buf_hdr.num_seg; + + if (idx == num_seg - 1) { + *cur_hdr = hdr->buf_hdr.next_seg; + *cur_idx = 0; + } else { + *cur_idx = idx + 1; + } + + return &hdr->buf_hdr.seg[idx]; +} + static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { - return pkt_hdr->buf_hdr.seg[seg_idx].len; + seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); + + return seg->len; }
static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { - return pkt_hdr->buf_hdr.seg[seg_idx].data; + seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); + + return seg->data; }
static inline uint16_t packet_last_seg(odp_packet_hdr_t *pkt_hdr) @@ -95,14 +155,7 @@ static inline uint16_t packet_last_seg(odp_packet_hdr_t *pkt_hdr)
static inline uint32_t packet_first_seg_len(odp_packet_hdr_t *pkt_hdr) { - return packet_seg_len(pkt_hdr, 0); -} - -static inline uint32_t packet_last_seg_len(odp_packet_hdr_t *pkt_hdr) -{ - int last = packet_last_seg(pkt_hdr); - - return packet_seg_len(pkt_hdr, last); + return pkt_hdr->buf_hdr.seg[0].len; }
static inline void *packet_data(odp_packet_hdr_t *pkt_hdr) @@ -112,26 +165,27 @@ static inline void *packet_data(odp_packet_hdr_t *pkt_hdr)
static inline void *packet_tail(odp_packet_hdr_t *pkt_hdr) { - int last = packet_last_seg(pkt_hdr); - uint32_t seg_len = pkt_hdr->buf_hdr.seg[last].len; + seg_entry_t *last_seg = seg_entry_last(pkt_hdr);
- return pkt_hdr->buf_hdr.seg[last].data + seg_len; + return last_seg->data + last_seg->len; }
-static inline uint32_t seg_headroom(odp_packet_hdr_t *pkt_hdr, int seg) +static inline uint32_t seg_headroom(odp_packet_hdr_t *pkt_hdr, int seg_idx) { - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; + seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); + odp_buffer_hdr_t *hdr = seg->hdr; uint8_t *base = hdr->base_data; - uint8_t *head = pkt_hdr->buf_hdr.seg[seg].data; + uint8_t *head = seg->data;
return CONFIG_PACKET_HEADROOM + (head - base); }
-static inline uint32_t seg_tailroom(odp_packet_hdr_t *pkt_hdr, int seg) +static inline uint32_t seg_tailroom(odp_packet_hdr_t *pkt_hdr, int seg_idx) { - uint32_t seg_len = pkt_hdr->buf_hdr.seg[seg].len; - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; - uint8_t *tail = pkt_hdr->buf_hdr.seg[seg].data + seg_len; + seg_entry_t *seg = seg_entry(pkt_hdr, seg_idx); + + odp_buffer_hdr_t *hdr = seg->hdr; + uint8_t *tail = seg->data + seg->len;
return hdr->buf_end - tail; } @@ -154,11 +208,11 @@ static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len)
static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - int last = packet_last_seg(pkt_hdr); + seg_entry_t *last_seg = seg_entry_last(pkt_hdr);
pkt_hdr->tailroom -= len; pkt_hdr->frame_len += len; - pkt_hdr->buf_hdr.seg[last].len += len; + last_seg->len += len; }
/* Copy all metadata for segmentation modification. Segment data and lengths @@ -188,6 +242,9 @@ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, /* segmentation data is not copied: * buf_hdr.seg[] * buf_hdr.segcount + * buf_hdr.num_seg + * buf_hdr.next_seg + * buf_hdr.last_seg */ }
@@ -196,7 +253,7 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, { void *addr; uint32_t len; - int seg = 0; + int seg_id = 0; int seg_count = pkt_hdr->buf_hdr.segcount;
if (odp_unlikely(offset >= pkt_hdr->frame_len)) @@ -207,10 +264,14 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, len = pkt_hdr->buf_hdr.seg[0].len - offset; } else { int i; + seg_entry_t *seg = NULL; uint32_t seg_start = 0, seg_end = 0; + odp_packet_hdr_t *hdr = pkt_hdr; + uint8_t idx = 0;
for (i = 0; i < seg_count; i++) { - seg_end += pkt_hdr->buf_hdr.seg[i].len; + seg = seg_entry_next(&hdr, &idx); + seg_end += seg->len;
if (odp_likely(offset < seg_end)) break; @@ -218,16 +279,16 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, seg_start = seg_end; }
- addr = pkt_hdr->buf_hdr.seg[i].data + (offset - seg_start); - len = pkt_hdr->buf_hdr.seg[i].len - (offset - seg_start); - seg = i; + addr = seg->data + (offset - seg_start); + len = seg->len - (offset - seg_start); + seg_id = i; }
if (seg_len) *seg_len = len;
if (seg_idx) - *seg_idx = seg; + *seg_idx = seg_id;
return addr; } @@ -243,10 +304,43 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; }
+static inline void link_segments(odp_packet_hdr_t *pkt_hdr[], int num) +{ + int cur, i; + odp_packet_hdr_t *hdr; + odp_packet_hdr_t *head = pkt_hdr[0]; + + cur = 0; + + while (1) { + hdr = pkt_hdr[cur]; + + for (i = 0; i < CONFIG_PACKET_MAX_SEGS; i++) { + odp_buffer_hdr_t *buf_hdr; + + buf_hdr = &pkt_hdr[cur]->buf_hdr; + hdr->buf_hdr.seg[i].hdr = buf_hdr; + hdr->buf_hdr.seg[i].data = buf_hdr->base_data; + hdr->buf_hdr.seg[i].len = BASE_LEN; + cur++; + + if (cur == num) { + /* Last segment */ + hdr->buf_hdr.num_seg = i + 1; + hdr->buf_hdr.next_seg = NULL; + head->buf_hdr.last_seg = &hdr->buf_hdr; + return; + } + } + + hdr->buf_hdr.num_seg = CONFIG_PACKET_MAX_SEGS; + hdr->buf_hdr.next_seg = pkt_hdr[cur]; + } +} + static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) { odp_packet_hdr_t *hdr; - int i;
/* First segment is the packet descriptor */ hdr = pkt_hdr[0]; @@ -258,30 +352,33 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) if (CONFIG_PACKET_MAX_SEGS != 1) { hdr->buf_hdr.segcount = num;
+ /* Defaults for single segment packet */ + hdr->buf_hdr.num_seg = 1; + hdr->buf_hdr.next_seg = NULL; + hdr->buf_hdr.last_seg = &hdr->buf_hdr; + if (odp_unlikely(num > 1)) { - for (i = 1; i < num; i++) { - odp_buffer_hdr_t *buf_hdr; + link_segments(pkt_hdr, num);
- buf_hdr = &pkt_hdr[i]->buf_hdr; - hdr->buf_hdr.seg[i].hdr = buf_hdr; - hdr->buf_hdr.seg[i].data = buf_hdr->base_data; - hdr->buf_hdr.seg[i].len = BASE_LEN; - } } } }
static inline void reset_seg(odp_packet_hdr_t *pkt_hdr, int first, int num) { - odp_buffer_hdr_t *hdr; + odp_packet_hdr_t *hdr = pkt_hdr; void *base; int i; + seg_entry_t *seg; + uint8_t idx; + + seg_entry_find_idx(&hdr, &idx, first);
- for (i = first; i < first + num; i++) { - hdr = pkt_hdr->buf_hdr.seg[i].hdr; - base = hdr->base_data; - pkt_hdr->buf_hdr.seg[i].len = BASE_LEN; - pkt_hdr->buf_hdr.seg[i].data = base; + for (i = 0; i < num; i++) { + base = hdr->buf_hdr.base_data; + seg = seg_entry_next(&hdr, &idx); + seg->len = BASE_LEN; + seg->data = base; } }
@@ -309,31 +406,11 @@ static inline int num_segments(uint32_t len)
static inline void add_all_segs(odp_packet_hdr_t *to, odp_packet_hdr_t *from) { - int i; - int n = to->buf_hdr.segcount; - int num = from->buf_hdr.segcount; + odp_packet_hdr_t *last = to->buf_hdr.last_seg;
- for (i = 0; i < num; i++) { - to->buf_hdr.seg[n + i].hdr = from->buf_hdr.seg[i].hdr; - to->buf_hdr.seg[n + i].data = from->buf_hdr.seg[i].data; - to->buf_hdr.seg[n + i].len = from->buf_hdr.seg[i].len; - } - - to->buf_hdr.segcount = n + num; -} - -static inline void copy_num_segs(odp_packet_hdr_t *to, odp_packet_hdr_t *from, - int first, int num) -{ - int i; - - for (i = 0; i < num; i++) { - to->buf_hdr.seg[i].hdr = from->buf_hdr.seg[first + i].hdr; - to->buf_hdr.seg[i].data = from->buf_hdr.seg[first + i].data; - to->buf_hdr.seg[i].len = from->buf_hdr.seg[first + i].len; - } - - to->buf_hdr.segcount = num; + last->buf_hdr.next_seg = from; + to->buf_hdr.last_seg = from->buf_hdr.last_seg; + to->buf_hdr.segcount += from->buf_hdr.segcount; }
static inline odp_packet_hdr_t *alloc_segments(pool_t *pool, int num) @@ -385,14 +462,14 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr,
pkt_hdr = new_hdr; } else { - int last; + seg_entry_t *last_seg;
/* add into the tail */ add_all_segs(pkt_hdr, new_hdr);
/* adjust last segment length */ - last = packet_last_seg(pkt_hdr); - pkt_hdr->buf_hdr.seg[last].len = seg_len; + last_seg = seg_entry_last(pkt_hdr); + last_seg->len = seg_len;
pkt_hdr->frame_len += len; pkt_hdr->tailroom = pool->tailroom + offset; @@ -401,13 +478,33 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, return pkt_hdr; }
-static inline void free_bufs(odp_packet_hdr_t *pkt_hdr, int first, int num) +static inline void copy_buf_hdr(odp_packet_hdr_t *pkt_hdr, int first, int num, + odp_buffer_hdr_t *buf_hdr[]) +{ + seg_entry_t *seg; + int i; + uint8_t idx; + odp_packet_hdr_t *hdr = pkt_hdr; + + seg_entry_find_idx(&hdr, &idx, first); + + for (i = 0; i < num; i++) { + seg = seg_entry_next(&hdr, &idx); + buf_hdr[i] = seg->hdr; + } +} + +static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num) { + seg_entry_t *seg; int i; odp_buffer_hdr_t *buf_hdr[num]; + uint8_t idx = 0;
- for (i = 0; i < num; i++) - buf_hdr[i] = pkt_hdr->buf_hdr.seg[first + i].hdr; + for (i = 0; i < num; i++) { + seg = seg_entry_next(&pkt_hdr, &idx); + buf_hdr[i] = seg->hdr; + }
buffer_free_multi(buf_hdr, num); } @@ -416,20 +513,45 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, int num, uint32_t free_len, uint32_t pull_len, int head) { + seg_entry_t *seg; + int i; int num_remain = pkt_hdr->buf_hdr.segcount - num; + odp_packet_hdr_t *hdr = pkt_hdr; + odp_packet_hdr_t *last_hdr = pkt_hdr->buf_hdr.last_seg; + uint8_t idx; + uint8_t num_seg; + odp_buffer_hdr_t *buf_hdr[num];
if (head) { odp_packet_hdr_t *new_hdr; - int i; - odp_buffer_hdr_t *buf_hdr[num];
- for (i = 0; i < num; i++) - buf_hdr[i] = pkt_hdr->buf_hdr.seg[i].hdr; + idx = 0; + for (i = 0; i < num; i++) { + seg = seg_entry_next(&hdr, &idx); + buf_hdr[i] = seg->hdr; + } + + /* The first remaining header is the new packet descriptor. + * Copy remaining segments from the last to-be-removed header + * to the new header. */ + new_hdr = hdr->buf_hdr.seg[idx].hdr; + num_seg = hdr->buf_hdr.num_seg - idx; + + new_hdr->buf_hdr.next_seg = hdr->buf_hdr.next_seg; + + if (hdr == last_hdr) + new_hdr->buf_hdr.last_seg = new_hdr; + else + new_hdr->buf_hdr.last_seg = last_hdr; + + new_hdr->buf_hdr.num_seg = num_seg; + new_hdr->buf_hdr.segcount = num_remain;
- /* First remaining segment is the new packet descriptor */ - new_hdr = pkt_hdr->buf_hdr.seg[num].hdr; + for (i = 0; i < num_seg; i++) { + seg = seg_entry_next(&hdr, &idx); + new_hdr->buf_hdr.seg[i] = *seg; + }
- copy_num_segs(new_hdr, pkt_hdr, num, num_remain); packet_seg_copy_md(new_hdr, pkt_hdr);
/* Tailroom not changed */ @@ -443,11 +565,27 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr,
buffer_free_multi(buf_hdr, num); } else { - /* Free last 'num' bufs */ - free_bufs(pkt_hdr, num_remain, num); + /* Free last 'num' bufs. + * First, find the last remaining header. */ + seg_entry_find_idx(&hdr, &idx, num_remain - 1); + last_hdr = hdr; + num_seg = idx + 1; + + seg_entry_next(&hdr, &idx); + + for (i = 0; i < num; i++) { + seg = seg_entry_next(&hdr, &idx); + buf_hdr[i] = seg->hdr; + } + + buffer_free_multi(buf_hdr, num);
/* Head segment remains, no need to copy or update majority * of the metadata. */ + last_hdr->buf_hdr.num_seg = num_seg; + last_hdr->buf_hdr.next_seg = NULL; + + pkt_hdr->buf_hdr.last_seg = last_hdr; pkt_hdr->buf_hdr.segcount = num_remain; pkt_hdr->frame_len -= free_len; pkt_hdr->tailroom = seg_tailroom(pkt_hdr, num_remain - 1); @@ -566,7 +704,7 @@ void odp_packet_free(odp_packet_t pkt) if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) buffer_free_multi((odp_buffer_hdr_t **)&hdl, 1); else - free_bufs(pkt_hdr, 0, num_seg); + free_all_segments(pkt_hdr, num_seg); }
void odp_packet_free_multi(const odp_packet_t pkt[], int num) @@ -576,7 +714,6 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) } else { odp_buffer_hdr_t *buf_hdr[num * CONFIG_PACKET_MAX_SEGS]; int i; - int j; int bufs = 0;
for (i = 0; i < num; i++) { @@ -590,10 +727,8 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) if (odp_likely(num_seg == 1)) continue;
- for (j = 1; j < num_seg; j++) { - buf_hdr[bufs] = hdr->seg[j].hdr; - bufs++; - } + copy_buf_hdr(pkt_hdr, 1, num_seg - 1, &buf_hdr[bufs]); + bufs += num_seg - 1; }
buffer_free_multi(buf_hdr, bufs); @@ -664,166 +799,6 @@ void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) return packet_data(pkt_hdr); }
-static inline uint32_t pack_seg_head(odp_packet_hdr_t *pkt_hdr, int seg) -{ - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; - uint32_t len = pkt_hdr->buf_hdr.seg[seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[seg].data; - uint8_t *dst = hdr->base_data; - - if (dst != src) { - memmove(dst, src, len); - pkt_hdr->buf_hdr.seg[seg].data = dst; - } - - return len; -} - -static inline uint32_t pack_seg_tail(odp_packet_hdr_t *pkt_hdr, int seg) -{ - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[seg].hdr; - uint32_t len = pkt_hdr->buf_hdr.seg[seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[seg].data; - uint8_t *dst = hdr->base_data + BASE_LEN - len; - - if (dst != src) { - memmove(dst, src, len); - pkt_hdr->buf_hdr.seg[seg].data = dst; - } - - return len; -} - -static inline uint32_t fill_seg_head(odp_packet_hdr_t *pkt_hdr, int dst_seg, - int src_seg, uint32_t max_len) -{ - uint32_t len = pkt_hdr->buf_hdr.seg[src_seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[src_seg].data; - uint32_t offset = pkt_hdr->buf_hdr.seg[dst_seg].len; - uint8_t *dst = pkt_hdr->buf_hdr.seg[dst_seg].data + offset; - - if (len > max_len) - len = max_len; - - memmove(dst, src, len); - - pkt_hdr->buf_hdr.seg[dst_seg].len += len; - pkt_hdr->buf_hdr.seg[src_seg].len -= len; - pkt_hdr->buf_hdr.seg[src_seg].data += len; - - if (pkt_hdr->buf_hdr.seg[src_seg].len == 0) { - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[src_seg].hdr; - - pkt_hdr->buf_hdr.seg[src_seg].data = hdr->base_data; - } - - return len; -} - -static inline uint32_t fill_seg_tail(odp_packet_hdr_t *pkt_hdr, int dst_seg, - int src_seg, uint32_t max_len) -{ - uint32_t src_len = pkt_hdr->buf_hdr.seg[src_seg].len; - uint8_t *src = pkt_hdr->buf_hdr.seg[src_seg].data; - uint8_t *dst = pkt_hdr->buf_hdr.seg[dst_seg].data; - uint32_t len = src_len; - - if (len > max_len) - len = max_len; - - src += src_len - len; - dst -= len; - - memmove(dst, src, len); - - pkt_hdr->buf_hdr.seg[dst_seg].data -= len; - pkt_hdr->buf_hdr.seg[dst_seg].len += len; - pkt_hdr->buf_hdr.seg[src_seg].len -= len; - - if (pkt_hdr->buf_hdr.seg[src_seg].len == 0) { - odp_buffer_hdr_t *hdr = pkt_hdr->buf_hdr.seg[src_seg].hdr; - - pkt_hdr->buf_hdr.seg[src_seg].data = hdr->base_data; - } - - return len; -} - -static inline int move_data_to_head(odp_packet_hdr_t *pkt_hdr, int segs) -{ - int dst_seg, src_seg; - uint32_t len, free_len; - uint32_t moved = 0; - - for (dst_seg = 0; dst_seg < segs; dst_seg++) { - len = pack_seg_head(pkt_hdr, dst_seg); - moved += len; - - if (len == BASE_LEN) - continue; - - free_len = BASE_LEN - len; - - for (src_seg = dst_seg + 1; CONFIG_PACKET_MAX_SEGS > 1 && - src_seg < segs; src_seg++) { - len = fill_seg_head(pkt_hdr, dst_seg, src_seg, - free_len); - moved += len; - - if (len == free_len) { - /* dst seg is full */ - break; - } - - /* src seg is empty */ - free_len -= len; - } - - if (moved == pkt_hdr->frame_len) - break; - } - - /* last segment which have data */ - return dst_seg; -} - -static inline int move_data_to_tail(odp_packet_hdr_t *pkt_hdr, int segs) -{ - int dst_seg, src_seg; - uint32_t len, free_len; - uint32_t moved = 0; - - for (dst_seg = segs - 1; dst_seg >= 0; dst_seg--) { - len = pack_seg_tail(pkt_hdr, dst_seg); - moved += len; - - if (len == BASE_LEN) - continue; - - free_len = BASE_LEN - len; - - for (src_seg = dst_seg - 1; src_seg >= 0; src_seg--) { - len = fill_seg_tail(pkt_hdr, dst_seg, src_seg, - free_len); - moved += len; - - if (len == free_len) { - /* dst seg is full */ - break; - } - - /* src seg is empty */ - free_len -= len; - } - - if (moved == pkt_hdr->frame_len) - break; - } - - /* first segment which have data */ - return dst_seg; -} - int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len) { @@ -835,87 +810,23 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, if (len > headroom) { pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; int num; - int segs; + void *ptr;
if (odp_unlikely((frame_len + len) > pool->max_len)) return -1;
- num = num_segments(len - headroom); - segs = pkt_hdr->buf_hdr.segcount; - - if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { - /* Cannot directly add new segments */ - odp_packet_hdr_t *new_hdr; - int new_segs = 0; - int free_segs = 0; - uint32_t offset; - - num = num_segments(frame_len + len); - - if (num > segs) { - /* Allocate additional segments */ - new_segs = num - segs; - new_hdr = alloc_segments(pool, new_segs); - - if (new_hdr == NULL) - return -1; - - } else if (num < segs) { - free_segs = segs - num; - } - - /* Pack all data to packet tail */ - move_data_to_tail(pkt_hdr, segs); - reset_seg(pkt_hdr, 0, segs); - - if (new_segs) { - add_all_segs(new_hdr, pkt_hdr); - packet_seg_copy_md(new_hdr, pkt_hdr); - segs += new_segs; - - pkt_hdr = new_hdr; - *pkt = packet_handle(pkt_hdr); - } else if (CONFIG_PACKET_MAX_SEGS > 1 && free_segs) { - new_hdr = pkt_hdr->buf_hdr.seg[free_segs].hdr; - packet_seg_copy_md(new_hdr, pkt_hdr); - - /* Free extra segs */ - free_bufs(pkt_hdr, 0, free_segs); - - segs -= free_segs; - pkt_hdr = new_hdr; - *pkt = packet_handle(pkt_hdr); - } - - frame_len += len; - offset = (segs * BASE_LEN) - frame_len; - - pkt_hdr->buf_hdr.seg[0].data += offset; - pkt_hdr->buf_hdr.seg[0].len -= offset; - - pkt_hdr->buf_hdr.segcount = segs; - pkt_hdr->frame_len = frame_len; - pkt_hdr->headroom = offset + pool->headroom; - pkt_hdr->tailroom = pool->tailroom; - - /* Data was moved */ - ret = 1; - } else { - void *ptr; + num = num_segments(len - headroom); + push_head(pkt_hdr, headroom); + ptr = add_segments(pkt_hdr, pool, len - headroom, num, 1);
- push_head(pkt_hdr, headroom); - ptr = add_segments(pkt_hdr, pool, len - headroom, - num, 1); - - if (ptr == NULL) { - /* segment alloc failed, rollback changes */ - pull_head(pkt_hdr, headroom); - return -1; - } - - *pkt = packet_handle(ptr); - pkt_hdr = ptr; + if (ptr == NULL) { + /* segment alloc failed, rollback changes */ + pull_head(pkt_hdr, headroom); + return -1; } + + *pkt = packet_handle(ptr); + pkt_hdr = ptr; } else { push_head(pkt_hdr, len); } @@ -1001,75 +912,19 @@ int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, if (len > tailroom) { pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; int num; - int segs; + void *ptr;
if (odp_unlikely((frame_len + len) > pool->max_len)) return -1;
- num = num_segments(len - tailroom); - segs = pkt_hdr->buf_hdr.segcount; - - if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { - /* Cannot directly add new segments */ - odp_packet_hdr_t *new_hdr; - int new_segs = 0; - int free_segs = 0; - uint32_t offset; - - num = num_segments(frame_len + len); - - if (num > segs) { - /* Allocate additional segments */ - new_segs = num - segs; - new_hdr = alloc_segments(pool, new_segs); - - if (new_hdr == NULL) - return -1; - - } else if (num < segs) { - free_segs = segs - num; - } - - /* Pack all data to packet head */ - move_data_to_head(pkt_hdr, segs); - reset_seg(pkt_hdr, 0, segs); + num = num_segments(len - tailroom); + push_tail(pkt_hdr, tailroom); + ptr = add_segments(pkt_hdr, pool, len - tailroom, num, 0);
- if (new_segs) { - /* Add new segs */ - add_all_segs(pkt_hdr, new_hdr); - segs += new_segs; - } else if (free_segs) { - /* Free extra segs */ - free_bufs(pkt_hdr, segs - free_segs, free_segs); - - segs -= free_segs; - } - - frame_len += len; - offset = (segs * BASE_LEN) - frame_len; - - pkt_hdr->buf_hdr.seg[segs - 1].len -= offset; - - pkt_hdr->buf_hdr.segcount = segs; - pkt_hdr->frame_len = frame_len; - pkt_hdr->headroom = pool->headroom; - pkt_hdr->tailroom = offset + pool->tailroom; - - /* Data was moved */ - ret = 1; - } else { - void *ptr; - - push_tail(pkt_hdr, tailroom); - - ptr = add_segments(pkt_hdr, pool, len - tailroom, - num, 0); - - if (ptr == NULL) { - /* segment alloc failed, rollback changes */ - pull_tail(pkt_hdr, tailroom); - return -1; - } + if (ptr == NULL) { + /* segment alloc failed, rollback changes */ + pull_tail(pkt_hdr, tailroom); + return -1; } } else { push_tail(pkt_hdr, len); @@ -1084,8 +939,9 @@ int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + seg_entry_t *last_seg = seg_entry_last(pkt_hdr);
- if (len > packet_last_seg_len(pkt_hdr)) + if (len > last_seg->len) return NULL;
pull_tail(pkt_hdr, len); @@ -1098,13 +954,15 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, { int last; uint32_t seg_len; + seg_entry_t *last_seg; odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt);
if (len > pkt_hdr->frame_len) return -1;
- last = packet_last_seg(pkt_hdr); - seg_len = packet_seg_len(pkt_hdr, last); + last = packet_last_seg(pkt_hdr); + last_seg = seg_entry_last(pkt_hdr); + seg_len = last_seg->len;
if (len < seg_len) { pull_tail(pkt_hdr, len); @@ -1397,17 +1255,13 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) { odp_packet_hdr_t *dst_hdr = packet_hdr(*dst); odp_packet_hdr_t *src_hdr = packet_hdr(src); - int dst_segs = dst_hdr->buf_hdr.segcount; - int src_segs = src_hdr->buf_hdr.segcount; pool_t *dst_pool = dst_hdr->buf_hdr.pool_ptr; pool_t *src_pool = src_hdr->buf_hdr.pool_ptr; uint32_t dst_len = dst_hdr->frame_len; uint32_t src_len = src_hdr->frame_len;
- /* Do a copy if resulting packet would be out of segments or packets - * are from different pools. */ - if (odp_unlikely((dst_segs + src_segs) > CONFIG_PACKET_MAX_SEGS) || - odp_unlikely(dst_pool != src_pool)) { + /* Do a copy if packets are from different pools. */ + if (odp_unlikely(dst_pool != src_pool)) { if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) { (void)odp_packet_copy_from_pkt(*dst, dst_len, src, 0, src_len); diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 01edff8b..47a39f5b 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -268,6 +268,9 @@ static void init_buffers(pool_t *pool) /* Show user requested size through API */ buf_hdr->uarea_size = pool->params.pkt.uarea_size; buf_hdr->segcount = 1; + buf_hdr->num_seg = 1; + buf_hdr->next_seg = NULL; + buf_hdr->last_seg = buf_hdr;
/* Pointer to data start (of the first segment) */ buf_hdr->seg[0].hdr = buf_hdr;
commit 20c75b764af6fd15e5e1d4df969ac33d62525405 Author: Petri Savolainen petri.savolainen@linaro.org Date: Wed Aug 23 13:30:19 2017 +0300
validation: packet: check reference packet content
Check that reference and base packets contain the same data.
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/test/common_plat/validation/api/packet/packet.c b/test/common_plat/validation/api/packet/packet.c index f291b6ee..10e011df 100644 --- a/test/common_plat/validation/api/packet/packet.c +++ b/test/common_plat/validation/api/packet/packet.c @@ -2072,14 +2072,107 @@ void packet_test_ref(void) { odp_packet_t base_pkt, segmented_base_pkt, hdr_pkt[4], ref_pkt[4], refhdr_pkt[4], hdr_cpy; + odp_packet_t pkt, pkt2, pkt3, ref, ref2; uint32_t pkt_len, segmented_pkt_len, hdr_len[4], offset[4], hr[4], base_hr, ref_len[4]; - int i; + int i, ret; + odp_pool_t pool; + + /* Create references and compare data */ + pool = odp_packet_pool(test_packet); + + pkt = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + ref = odp_packet_ref_static(pkt); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + + pkt = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + ref = odp_packet_ref(pkt, 0); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + + pkt = odp_packet_copy(test_packet, pool); + pkt3 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt3 != ODP_PACKET_INVALID) + ret = odp_packet_concat(&pkt3, pkt); + CU_ASSERT_FATAL(ret >= 0); + + pkt = odp_packet_copy(test_packet, pool); + pkt2 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt2 != ODP_PACKET_INVALID) + ref = odp_packet_ref_pkt(pkt, 0, pkt2); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt3, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + odp_packet_free(pkt3); + + /* Do the same for segmented packets */ + pool = odp_packet_pool(segmented_test_packet); + + pkt = odp_packet_copy(segmented_test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + ref = odp_packet_ref_static(pkt); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + + pkt = odp_packet_copy(segmented_test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + ref = odp_packet_ref(pkt, 0); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + packet_compare_data(pkt, ref); + odp_packet_free(ref); + odp_packet_free(pkt); + + /* Avoid to create too large packets with concat */ + pool = odp_packet_pool(test_packet); + + pkt = odp_packet_copy(test_packet, pool); + pkt2 = odp_packet_copy(test_packet, pool); + pkt3 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt2 != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt3 != ODP_PACKET_INVALID) + ret = odp_packet_concat(&pkt3, pkt2); + CU_ASSERT_FATAL(ret >= 0); + ret = odp_packet_concat(&pkt3, pkt); + CU_ASSERT_FATAL(ret >= 0); + + pkt = odp_packet_copy(test_packet, pool); + pkt2 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt != ODP_PACKET_INVALID) + CU_ASSERT_FATAL(pkt2 != ODP_PACKET_INVALID) + ref = odp_packet_ref_pkt(pkt, 0, pkt2); + CU_ASSERT_FATAL(ref != ODP_PACKET_INVALID) + pkt2 = odp_packet_copy(test_packet, pool); + CU_ASSERT_FATAL(pkt2 != ODP_PACKET_INVALID) + ref2 = odp_packet_ref_pkt(ref, 0, pkt2); + CU_ASSERT_FATAL(ref2 != ODP_PACKET_INVALID) + packet_compare_data(pkt3, ref2); + + /* Try print function on a reference */ + odp_packet_print(ref2); + + odp_packet_free(ref); + odp_packet_free(ref2); + odp_packet_free(pkt); + odp_packet_free(pkt3);
+ /* Test has_ref, unshared_len, lengths, etc */ base_pkt = odp_packet_copy(test_packet, odp_packet_pool(test_packet)); - base_hr = odp_packet_headroom(base_pkt); - pkt_len = odp_packet_len(test_packet); CU_ASSERT_FATAL(base_pkt != ODP_PACKET_INVALID); + base_hr = odp_packet_headroom(base_pkt); + pkt_len = odp_packet_len(test_packet);
segmented_base_pkt = odp_packet_copy(segmented_test_packet,
commit 04029613fbdc85221d1a20354a49ada912302fc0 Author: Petri Savolainen petri.savolainen@linaro.org Date: Mon Aug 21 13:54:23 2017 +0300
validation: packet: unshared len with static reference
When static references are used all data is shared, and thus unshared length is zero. When there's no references, all data is unique to the packet (unshared len == len).
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/test/common_plat/validation/api/packet/packet.c b/test/common_plat/validation/api/packet/packet.c index 898dac87..f291b6ee 100644 --- a/test/common_plat/validation/api/packet/packet.c +++ b/test/common_plat/validation/api/packet/packet.c @@ -2224,8 +2224,14 @@ void packet_test_ref(void) CU_ASSERT(ref_pkt[0] != ODP_PACKET_INVALID);
if (odp_packet_has_ref(base_pkt) == 1) { - /* CU_ASSERT needs braces */ CU_ASSERT(odp_packet_has_ref(ref_pkt[0]) == 1); + CU_ASSERT(odp_packet_unshared_len(base_pkt) == 0); + CU_ASSERT(odp_packet_unshared_len(ref_pkt[0]) == 0); + } else { + CU_ASSERT(odp_packet_unshared_len(base_pkt) == + odp_packet_len(base_pkt)); + CU_ASSERT(odp_packet_unshared_len(ref_pkt[0]) == + odp_packet_len(ref_pkt[0])); }
CU_ASSERT(odp_packet_len(ref_pkt[0]) == odp_packet_len(base_pkt));
commit 88d36e6811de494708c520cb12e9e5f97628e9e4 Author: Petri Savolainen petri.savolainen@linaro.org Date: Fri Aug 18 11:04:49 2017 +0300
validation: packet: relax small concat test
Limit number of consecutive small concat to fraction of the number of packets in the pool. An implementation linking packets together (no copy) cannot be expected to support one byte concats up to max packet size.
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/test/common_plat/validation/api/packet/packet.c b/test/common_plat/validation/api/packet/packet.c index 28bc12b0..898dac87 100644 --- a/test/common_plat/validation/api/packet/packet.c +++ b/test/common_plat/validation/api/packet/packet.c @@ -1376,7 +1376,7 @@ void packet_test_concat_small(void) int ret; uint8_t *data; uint32_t i; - uint32_t len = 32000; + uint32_t len = PACKET_POOL_NUM / 4; uint8_t buf[len];
CU_ASSERT_FATAL(odp_pool_capability(&capa) == 0);
commit a820d8a84e132652b4cef295756ccf135e3bd54b Author: Petri Savolainen petri.savolainen@linaro.org Date: Thu Aug 10 13:43:52 2017 +0300
linux-gen: packet: roll back to copy based references
In order to re-implement segmentation for the new reference implementation roll back to the copy based reference implementation, which have been proved to work OK.
This zero copy reference implementation introduced multiple issues: * segment pointers and length were not set correctly, but were left to NULL * reference counting has a race condition which causes random assert failures also with non-reference packets * there's larger than expected performance drop with non-reference packets * the first segment of a reference was zero bytes long, which is possible by the API but is unusual for users
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 06b049fc..d0cf1390 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -50,19 +50,7 @@ static inline uint32_t _odp_packet_seg_len(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_len(odp_packet_t pkt) { - uint32_t pkt_len = _odp_pkt_get(pkt, uint32_t, frame_len); - void *ref_nxt = _odp_pkt_get(pkt, void *, ref_hdr); - void *ref_pkt = (void *)pkt; - - while (ref_nxt) { - pkt_len += _odp_pkt_get(ref_pkt, uint32_t, ref_len) - - _odp_pkt_get(ref_pkt, uint32_t, ref_offset); - - ref_pkt = ref_nxt; - ref_nxt = _odp_pkt_get(ref_nxt, void *, ref_hdr); - } - - return pkt_len; + return _odp_pkt_get(pkt, uint32_t, frame_len); }
/** @internal Inline function @param pkt @return */ @@ -90,6 +78,12 @@ static inline odp_pktio_t _odp_packet_input(odp_packet_t pkt) }
/** @internal Inline function @param pkt @return */ +static inline int _odp_packet_num_segs(odp_packet_t pkt) +{ + return _odp_pkt_get(pkt, uint8_t, segcount); +} + +/** @internal Inline function @param pkt @return */ static inline void *_odp_packet_user_ptr(odp_packet_t pkt) { return _odp_pkt_get(pkt, void *, user_ptr); @@ -128,8 +122,7 @@ static inline void *_odp_packet_head(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline int _odp_packet_is_segmented(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint8_t, segcount) > 1 || - _odp_pkt_get(pkt, void *, ref_hdr) != NULL; + return _odp_pkt_get(pkt, uint8_t, segcount) > 1; }
/** @internal Inline function @param pkt @return */ @@ -140,6 +133,23 @@ static inline odp_packet_seg_t _odp_packet_first_seg(odp_packet_t pkt) return _odp_packet_seg_from_ndx(0); }
+/** @internal Inline function @param pkt @return */ +static inline odp_packet_seg_t _odp_packet_last_seg(odp_packet_t pkt) +{ + return _odp_packet_seg_from_ndx(_odp_packet_num_segs(pkt) - 1); +} + +/** @internal Inline function @param pkt @param seg @return */ +static inline odp_packet_seg_t _odp_packet_next_seg(odp_packet_t pkt, + odp_packet_seg_t seg) +{ + if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= + _odp_packet_seg_to_ndx(_odp_packet_last_seg(pkt)))) + return ODP_PACKET_SEG_INVALID; + + return seg + 1; +} + /** @internal Inline function @param pkt @param offset @param len */ static inline void _odp_packet_prefetch(odp_packet_t pkt, uint32_t offset, uint32_t len) diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h b/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h index f818f820..233bc876 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h @@ -48,6 +48,11 @@ _ODP_INLINE odp_pktio_t odp_packet_input(odp_packet_t pkt) return _odp_packet_input(pkt); }
+_ODP_INLINE int odp_packet_num_segs(odp_packet_t pkt) +{ + return _odp_packet_num_segs(pkt); +} + _ODP_INLINE void *odp_packet_user_ptr(odp_packet_t pkt) { return _odp_packet_user_ptr(pkt); @@ -88,6 +93,17 @@ _ODP_INLINE odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) return _odp_packet_first_seg(pkt); }
+_ODP_INLINE odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) +{ + return _odp_packet_last_seg(pkt); +} + +_ODP_INLINE odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, + odp_packet_seg_t seg) +{ + return _odp_packet_next_seg(pkt, seg); +} + _ODP_INLINE void odp_packet_prefetch(odp_packet_t pkt, uint32_t offset, uint32_t len) { diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h b/platform/linux-generic/include/odp/api/plat/packet_types.h index 1554c3af..41468bec 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_types.h @@ -84,14 +84,6 @@ typedef struct _odp_packet_inline_offset_t { /** @internal field offset */ size_t tailroom; /** @internal field offset */ - size_t unshared_len; - /** @internal field offset */ - size_t ref_hdr; - /** @internal field offset */ - size_t ref_offset; - /** *internal field offset */ - size_t ref_len; - /** @internal field offset */ size_t pool; /** @internal field offset */ size_t input; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index d923ee23..f77987c8 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -19,7 +19,6 @@ extern "C" {
#include <odp/api/align.h> #include <odp/api/debug.h> -#include <odp_debug_internal.h> #include <odp_buffer_internal.h> #include <odp_pool_internal.h> #include <odp_buffer_inlines.h> @@ -107,7 +106,7 @@ typedef struct { * packet_init(). Because of this any new fields added must be reviewed for * initialization requirements. */ -typedef struct odp_packet_hdr_t { +typedef struct { /* common buffer header */ odp_buffer_hdr_t buf_hdr;
@@ -123,19 +122,6 @@ typedef struct odp_packet_hdr_t { uint32_t headroom; uint32_t tailroom;
- /* Fields used to support packet references */ - uint32_t unshared_len; - /* Next pkt_hdr in reference chain */ - struct odp_packet_hdr_t *ref_hdr; - /* Offset into next pkt_hdr that ref was created at */ - uint32_t ref_offset; - /* frame_len in next pkt_hdr at time ref was created. This - * allows original offset to be maintained when base pkt len - * is changed */ - uint32_t ref_len; - /* Incremented on refs, decremented on frees. */ - odp_atomic_u32_t ref_count; - /* * Members below are not initialized by packet_init() */ @@ -171,50 +157,6 @@ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) return (odp_packet_hdr_t *)(uintptr_t)pkt; }
-static inline odp_packet_hdr_t *packet_last_hdr(odp_packet_t pkt, - uint32_t *offset) -{ - odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - odp_packet_hdr_t *prev_hdr = pkt_hdr; - uint32_t ref_offset = 0; - - while (pkt_hdr->ref_hdr) { - ref_offset = pkt_hdr->ref_offset; - prev_hdr = pkt_hdr; - pkt_hdr = pkt_hdr->ref_hdr; - } - - if (offset) { - if (prev_hdr != pkt_hdr) - ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; - *offset = ref_offset; - } - - return pkt_hdr; -} - -static inline odp_packet_hdr_t *packet_prev_hdr(odp_packet_hdr_t *pkt_hdr, - odp_packet_hdr_t *cur_hdr, - uint32_t *offset) -{ - uint32_t ref_offset = 0; - odp_packet_hdr_t *prev_hdr = pkt_hdr; - - while (pkt_hdr->ref_hdr != cur_hdr) { - ref_offset = pkt_hdr->ref_offset; - prev_hdr = pkt_hdr; - pkt_hdr = pkt_hdr->ref_hdr; - } - - if (offset) { - if (prev_hdr != pkt_hdr) - ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; - *offset = ref_offset; - } - - return pkt_hdr; -} - static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr) { return (odp_packet_t)pkt_hdr; @@ -267,10 +209,6 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len) CONFIG_PACKET_TAILROOM;
pkt_hdr->input = ODP_PKTIO_INVALID; - - /* By default packet has no references */ - pkt_hdr->unshared_len = len; - pkt_hdr->ref_hdr = NULL; }
static inline void copy_packet_parser_metadata(odp_packet_hdr_t *src_hdr, @@ -295,49 +233,17 @@ static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len)
pkt_hdr->tailroom += len; pkt_hdr->frame_len -= len; - pkt_hdr->unshared_len -= len; pkt_hdr->buf_hdr.seg[last].len -= len; }
static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) { - uint32_t pkt_len = pkt_hdr->frame_len; - odp_packet_hdr_t *ref_hdr = pkt_hdr->ref_hdr; - - while (ref_hdr) { - pkt_len += (pkt_hdr->ref_len - pkt_hdr->ref_offset); - pkt_hdr = ref_hdr; - ref_hdr = ref_hdr->ref_hdr; - } - - return pkt_len; -} - -static inline uint32_t packet_ref_count(odp_packet_hdr_t *pkt_hdr) -{ - /* Breach the atomic type to do a peek at the ref count. This - * is used to bypass atomic operations if ref_count == 1 for - * performance reasons. - */ - return pkt_hdr->ref_count.v; -} - -static inline void packet_ref_count_set(odp_packet_hdr_t *pkt_hdr, uint32_t n) -{ - /* Only used during init when there are no other possible - * references to this pkt, so avoid the "atomic" overhead by - * a controlled breach of the atomic type here. This saves - * over 10% of the pathlength in routines like packet_alloc(). - */ - pkt_hdr->ref_count.v = n; + return pkt_hdr->frame_len; }
static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) { - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - pkt_hdr->frame_len = len; - pkt_hdr->unshared_len = len; }
/* Forward declarations */ diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 1fc715ca..1e5b136a 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -35,10 +35,6 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .frame_len = offsetof(odp_packet_hdr_t, frame_len), .headroom = offsetof(odp_packet_hdr_t, headroom), .tailroom = offsetof(odp_packet_hdr_t, tailroom), - .unshared_len = offsetof(odp_packet_hdr_t, unshared_len), - .ref_hdr = offsetof(odp_packet_hdr_t, ref_hdr), - .ref_offset = offsetof(odp_packet_hdr_t, ref_offset), - .ref_len = offsetof(odp_packet_hdr_t, ref_len), .pool = offsetof(odp_packet_hdr_t, buf_hdr.pool_hdl), .input = offsetof(odp_packet_hdr_t, input), .segcount = offsetof(odp_packet_hdr_t, buf_hdr.segcount), @@ -63,16 +59,6 @@ static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) return (odp_buffer_t)pkt_hdr; }
-static inline void packet_ref_inc(odp_packet_hdr_t *pkt_hdr) -{ - odp_atomic_inc_u32(&pkt_hdr->ref_count); -} - -static inline uint32_t packet_ref_dec(odp_packet_hdr_t *pkt_hdr) -{ - return odp_atomic_fetch_dec_u32(&pkt_hdr->ref_count); -} - static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) { return (odp_packet_hdr_t *)buf_hdl_to_hdr(buf); @@ -94,13 +80,12 @@ static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, return pkt_hdr->buf_hdr.seg[seg_idx].len; }
-static inline uint8_t *packet_seg_data(odp_packet_hdr_t *pkt_hdr, - uint32_t seg_idx) +static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { return pkt_hdr->buf_hdr.seg[seg_idx].data; }
-static inline uint32_t packet_last_seg(odp_packet_hdr_t *pkt_hdr) +static inline uint16_t packet_last_seg(odp_packet_hdr_t *pkt_hdr) { if (CONFIG_PACKET_MAX_SEGS == 1) return 0; @@ -155,7 +140,6 @@ static inline void push_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->headroom -= len; pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; pkt_hdr->buf_hdr.seg[0].data -= len; pkt_hdr->buf_hdr.seg[0].len += len; } @@ -164,7 +148,6 @@ static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->headroom += len; pkt_hdr->frame_len -= len; - pkt_hdr->unshared_len -= len; pkt_hdr->buf_hdr.seg[0].data += len; pkt_hdr->buf_hdr.seg[0].len -= len; } @@ -175,7 +158,6 @@ static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len)
pkt_hdr->tailroom -= len; pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; pkt_hdr->buf_hdr.seg[last].len += len; }
@@ -203,10 +185,6 @@ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; dst->buf_hdr.uarea_size = src->buf_hdr.uarea_size;
- /* reference related metadata */ - dst->ref_len = src->ref_len; - dst->unshared_len = src->unshared_len; - /* segmentation data is not copied: * buf_hdr.seg[] * buf_hdr.segcount @@ -221,15 +199,7 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, int seg = 0; int seg_count = pkt_hdr->buf_hdr.segcount;
- /* Special processing for references */ - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr = pkt_hdr->ref_hdr; - seg_count = pkt_hdr->buf_hdr.segcount; - } - - if (odp_unlikely(offset > pkt_hdr->frame_len)) + if (odp_unlikely(offset >= pkt_hdr->frame_len)) return NULL;
if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || seg_count == 1)) { @@ -271,9 +241,6 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) pkt_hdr->p.l2_offset = 0; pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; - - /* Ensure dummy pkt hdrs used in I/O recv classification are valid */ - pkt_hdr->ref_hdr = NULL; }
static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) @@ -286,7 +253,6 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num)
hdr->buf_hdr.seg[0].data = hdr->buf_hdr.base_data; hdr->buf_hdr.seg[0].len = BASE_LEN; - packet_ref_count_set(hdr, 1);
/* Link segments */ if (CONFIG_PACKET_MAX_SEGS != 1) { @@ -296,7 +262,6 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) for (i = 1; i < num; i++) { odp_buffer_hdr_t *buf_hdr;
- packet_ref_count_set(pkt_hdr[i], 1); buf_hdr = &pkt_hdr[i]->buf_hdr; hdr->buf_hdr.seg[i].hdr = buf_hdr; hdr->buf_hdr.seg[i].data = buf_hdr->base_data; @@ -414,10 +379,9 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, new_hdr->buf_hdr.seg[0].len = seg_len;
packet_seg_copy_md(new_hdr, pkt_hdr); - new_hdr->frame_len = pkt_hdr->frame_len + len; - new_hdr->unshared_len = pkt_hdr->unshared_len + len; - new_hdr->headroom = pool->headroom + offset; - new_hdr->tailroom = pkt_hdr->tailroom; + new_hdr->frame_len = pkt_hdr->frame_len + len; + new_hdr->headroom = pool->headroom + offset; + new_hdr->tailroom = pkt_hdr->tailroom;
pkt_hdr = new_hdr; } else { @@ -430,9 +394,8 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, last = packet_last_seg(pkt_hdr); pkt_hdr->buf_hdr.seg[last].len = seg_len;
- pkt_hdr->frame_len += len; - pkt_hdr->unshared_len += len; - pkt_hdr->tailroom = pool->tailroom + offset; + pkt_hdr->frame_len += len; + pkt_hdr->tailroom = pool->tailroom + offset; }
return pkt_hdr; @@ -440,20 +403,13 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr,
static inline void free_bufs(odp_packet_hdr_t *pkt_hdr, int first, int num) { - int i, nfree; + int i; odp_buffer_hdr_t *buf_hdr[num];
- for (i = 0, nfree = 0; i < num; i++) { - odp_packet_hdr_t *hdr = pkt_hdr->buf_hdr.seg[first + i].hdr; - - if (packet_ref_count(hdr) == 1 || packet_ref_dec(hdr) == 1) { - ODP_ASSERT((packet_ref_count_set(hdr, 0), 1)); - buf_hdr[nfree++] = &hdr->buf_hdr; - } - } + for (i = 0; i < num; i++) + buf_hdr[i] = pkt_hdr->buf_hdr.seg[first + i].hdr;
- if (nfree > 0) - buffer_free_multi(buf_hdr, nfree); + buffer_free_multi(buf_hdr, num); }
static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, @@ -464,19 +420,11 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr,
if (head) { odp_packet_hdr_t *new_hdr; - int i, nfree; + int i; odp_buffer_hdr_t *buf_hdr[num];
- for (i = 0, nfree = 0; i < num; i++) { - new_hdr = pkt_hdr->buf_hdr.seg[i].hdr; - - if (packet_ref_count(new_hdr) == 1 || - packet_ref_dec(new_hdr) == 1) { - ODP_ASSERT((packet_ref_count_set(new_hdr, 0), - 1)); - buf_hdr[nfree++] = &new_hdr->buf_hdr; - } - } + for (i = 0; i < num; i++) + buf_hdr[i] = pkt_hdr->buf_hdr.seg[i].hdr;
/* First remaining segment is the new packet descriptor */ new_hdr = pkt_hdr->buf_hdr.seg[num].hdr; @@ -485,17 +433,15 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, packet_seg_copy_md(new_hdr, pkt_hdr);
/* Tailroom not changed */ - new_hdr->tailroom = pkt_hdr->tailroom; - new_hdr->headroom = seg_headroom(new_hdr, 0); - new_hdr->frame_len = pkt_hdr->frame_len - free_len; - new_hdr->unshared_len = pkt_hdr->unshared_len - free_len; + new_hdr->tailroom = pkt_hdr->tailroom; + new_hdr->headroom = seg_headroom(new_hdr, 0); + new_hdr->frame_len = pkt_hdr->frame_len - free_len;
pull_head(new_hdr, pull_len);
pkt_hdr = new_hdr;
- if (nfree > 0) - buffer_free_multi(buf_hdr, nfree); + buffer_free_multi(buf_hdr, num); } else { /* Free last 'num' bufs */ free_bufs(pkt_hdr, num_remain, num); @@ -504,7 +450,6 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, * of the metadata. */ pkt_hdr->buf_hdr.segcount = num_remain; pkt_hdr->frame_len -= free_len; - pkt_hdr->unshared_len -= free_len; pkt_hdr->tailroom = seg_tailroom(pkt_hdr, num_remain - 1);
pull_tail(pkt_hdr, pull_len); @@ -611,80 +556,48 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, return num; }
-static inline void packet_free(odp_packet_hdr_t *pkt_hdr) +void odp_packet_free(odp_packet_t pkt) { - odp_packet_hdr_t *ref_hdr; - odp_buffer_hdr_t *buf_hdr; - uint32_t ref_count; - int num_seg; - - do { - buf_hdr = &pkt_hdr->buf_hdr; - ref_count = packet_ref_count(pkt_hdr); - num_seg = pkt_hdr->buf_hdr.segcount; - ref_hdr = pkt_hdr->ref_hdr; - ODP_ASSERT(ref_count >= 1); - - if (odp_likely((CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1) && - ref_count == 1)) { - ODP_ASSERT((packet_ref_count_set(pkt_hdr, 0), 1)); - buffer_free_multi(&buf_hdr, 1); - } else { - free_bufs(pkt_hdr, 0, num_seg); - } + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_buffer_t hdl = buffer_handle(pkt_hdr);
- pkt_hdr = ref_hdr; - } while (pkt_hdr); -} + int num_seg = pkt_hdr->buf_hdr.segcount;
-void odp_packet_free(odp_packet_t pkt) -{ - packet_free(packet_hdr(pkt)); + if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) + buffer_free_multi((odp_buffer_hdr_t **)&hdl, 1); + else + free_bufs(pkt_hdr, 0, num_seg); }
void odp_packet_free_multi(const odp_packet_t pkt[], int num) { - odp_packet_hdr_t *pkt_hdr, *ref_hdr, *hdr; - int nbufs = num * CONFIG_PACKET_MAX_SEGS * 2; - odp_buffer_hdr_t *buf_hdr[nbufs]; - int num_seg; - int i, j; - uint32_t ref_count; - int nfree = 0; + if (CONFIG_PACKET_MAX_SEGS == 1) { + buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)pkt, num); + } else { + odp_buffer_hdr_t *buf_hdr[num * CONFIG_PACKET_MAX_SEGS]; + int i; + int j; + int bufs = 0;
- for (i = 0; i < num; i++) { - pkt_hdr = packet_hdr(pkt[i]); + for (i = 0; i < num; i++) { + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); + int num_seg = pkt_hdr->buf_hdr.segcount; + odp_buffer_hdr_t *hdr = &pkt_hdr->buf_hdr;
- do { - num_seg = pkt_hdr->buf_hdr.segcount; - ref_hdr = pkt_hdr->ref_hdr; + buf_hdr[bufs] = hdr; + bufs++;
- /* Make sure we have enough space for this pkt's segs */ - if (nfree + num_seg > nbufs) { - buffer_free_multi(buf_hdr, nfree); - nfree = 0; - } + if (odp_likely(num_seg == 1)) + continue;
- for (j = 0; j < num_seg; j++) { - hdr = pkt_hdr->buf_hdr.seg[j].hdr; - ref_count = packet_ref_count(hdr); - ODP_ASSERT(ref_count >= 1); - - if (ref_count == 1 || - packet_ref_dec(hdr) == 1) { - ODP_ASSERT - ((packet_ref_count_set(hdr, 0), - 1)); - buf_hdr[nfree++] = &hdr->buf_hdr; - } + for (j = 1; j < num_seg; j++) { + buf_hdr[bufs] = hdr->seg[j].hdr; + bufs++; } + }
- pkt_hdr = ref_hdr; - } while (pkt_hdr); + buffer_free_multi(buf_hdr, bufs); } - - if (nfree > 0) - buffer_free_multi(buf_hdr, nfree); }
int odp_packet_reset(odp_packet_t pkt, uint32_t len) @@ -696,9 +609,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) if (odp_unlikely(len > (pool->max_seg_len * num))) return -1;
- if (pkt_hdr->ref_hdr) - packet_free(pkt_hdr->ref_hdr); - reset_seg(pkt_hdr, 0, num);
packet_init(pkt_hdr, len); @@ -732,47 +642,13 @@ odp_event_t odp_packet_to_event(odp_packet_t pkt) uint32_t odp_packet_buf_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t buf_len = 0; - - do { - buf_len += pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr);
- return buf_len; -} - -uint32_t odp_packet_unshared_len(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t pkt_len = 0, offset = 0; - - if (packet_ref_count(pkt_hdr) == 1) - pkt_hdr->unshared_len = pkt_hdr->frame_len; - - do { - if (packet_ref_count(pkt_hdr) > 1) { - if (offset == 0) - pkt_len += pkt_hdr->unshared_len; - break; - } - - pkt_len += pkt_hdr->frame_len - offset; - offset = pkt_hdr->ref_offset; - - if (pkt_hdr->ref_hdr) - offset += (pkt_hdr->ref_hdr->frame_len - - pkt_hdr->ref_len); - - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); - - return pkt_len; + return pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; }
void *odp_packet_tail(odp_packet_t pkt) { - odp_packet_hdr_t *pkt_hdr = packet_last_hdr(pkt, NULL); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
return packet_tail(pkt_hdr); } @@ -968,13 +844,6 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, segs = pkt_hdr->buf_hdr.segcount;
if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { - /* Corner case: fail request if packet has - * references since we cannot shuffle segments - * since another thread may be accessing them - * concurrently */ - if (packet_ref_count(pkt_hdr) > 1) - return -1; - /* Cannot directly add new segments */ odp_packet_hdr_t *new_hdr; int new_segs = 0; @@ -1026,7 +895,6 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len,
pkt_hdr->buf_hdr.segcount = segs; pkt_hdr->frame_len = frame_len; - pkt_hdr->unshared_len = frame_len; pkt_hdr->headroom = offset + pool->headroom; pkt_hdr->tailroom = pool->tailroom;
@@ -1052,16 +920,11 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, push_head(pkt_hdr, len); }
- if (data_ptr || seg_len) { - uint32_t seg_ln = 0; - void *data = packet_map(pkt_hdr, 0, &seg_ln, NULL); - - if (data_ptr) - *data_ptr = data; + if (data_ptr) + *data_ptr = packet_data(pkt_hdr);
- if (seg_len) - *seg_len = seg_ln; - } + if (seg_len) + *seg_len = packet_first_seg_len(pkt_hdr);
return ret; } @@ -1073,8 +936,6 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) if (len > pkt_hdr->frame_len) return NULL;
- ODP_ASSERT(len <= pkt_hdr->unshared_len); - pull_head(pkt_hdr, len); return packet_data(pkt_hdr); } @@ -1082,35 +943,15 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt), *nxt_hdr; + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); uint32_t seg_len = packet_first_seg_len(pkt_hdr); - int ret = 0;
- if (len > packet_len(pkt_hdr)) + if (len > pkt_hdr->frame_len) return -1;
- ODP_ASSERT(len <= odp_packet_unshared_len(*pkt)); - - /* Special processing for references */ - while (len >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - nxt_hdr = pkt_hdr->ref_hdr; - len -= pkt_hdr->frame_len; - len += pkt_hdr->ref_offset + - (nxt_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr->ref_hdr = NULL; - packet_free(pkt_hdr); - pkt_hdr = nxt_hdr; - seg_len = packet_first_seg_len(pkt_hdr); - *pkt = packet_handle(pkt_hdr); - ret = 1; - } - - if (CONFIG_PACKET_MAX_SEGS == 1 || - len < seg_len || - pkt_hdr->buf_hdr.segcount == 1) { + if (len < seg_len) { pull_head(pkt_hdr, len); - } else { + } else if (CONFIG_PACKET_MAX_SEGS != 1) { int num = 0; uint32_t pull_len = 0;
@@ -1125,29 +966,23 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, *pkt = packet_handle(pkt_hdr); }
- if (data_ptr || seg_len_out) { - void *data_head = packet_map(pkt_hdr, 0, &seg_len, NULL); + if (data_ptr) + *data_ptr = packet_data(pkt_hdr);
- if (data_ptr) - *data_ptr = data_head; + if (seg_len_out) + *seg_len_out = packet_first_seg_len(pkt_hdr);
- if (seg_len_out) - *seg_len_out = seg_len; - } - - return ret; + return 0; }
void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) { - odp_packet_hdr_t *pkt_hdr = packet_last_hdr(pkt, NULL); + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); void *old_tail;
if (len > pkt_hdr->tailroom) return NULL;
- ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - old_tail = packet_tail(pkt_hdr); push_tail(pkt_hdr, len);
@@ -1157,14 +992,12 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { - odp_packet_hdr_t *pkt_hdr = packet_last_hdr(*pkt, NULL); + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); uint32_t frame_len = pkt_hdr->frame_len; uint32_t tailroom = pkt_hdr->tailroom; uint32_t tail_off = frame_len; int ret = 0;
- ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); - if (len > tailroom) { pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; int num; @@ -1255,7 +1088,6 @@ void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) if (len > packet_last_seg_len(pkt_hdr)) return NULL;
- ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); pull_tail(pkt_hdr, len);
return packet_tail(pkt_hdr); @@ -1266,34 +1098,17 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, { int last; uint32_t seg_len; - uint32_t offset; - odp_packet_hdr_t *first_hdr = packet_hdr(*pkt); - odp_packet_hdr_t *pkt_hdr, *prev_hdr; + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt);
- if (len > packet_len(first_hdr)) + if (len > pkt_hdr->frame_len) return -1;
- pkt_hdr = packet_last_hdr(*pkt, &offset); - - /* Special processing for references */ - while (len >= pkt_hdr->frame_len - offset && first_hdr->ref_hdr) { - len -= (pkt_hdr->frame_len - offset); - prev_hdr = packet_prev_hdr(first_hdr, pkt_hdr, &offset); - ODP_ASSERT(packet_ref_count(prev_hdr) == 1); - prev_hdr->ref_hdr = NULL; - packet_free(pkt_hdr); - pkt_hdr = prev_hdr; - } - - ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); last = packet_last_seg(pkt_hdr); seg_len = packet_seg_len(pkt_hdr, last);
- if (CONFIG_PACKET_MAX_SEGS == 1 || - len < seg_len || - pkt_hdr->buf_hdr.segcount == 1) { + if (len < seg_len) { pull_tail(pkt_hdr, len); - } else { + } else if (CONFIG_PACKET_MAX_SEGS != 1) { int num = 0; uint32_t pull_len = 0;
@@ -1440,46 +1255,6 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) pkt_hdr->p.input_flags.timestamp = 1; }
-int odp_packet_num_segs(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t segcount = 0, i; - uint32_t seg_offset = 0, offset; - - do { - segcount += pkt_hdr->buf_hdr.segcount - seg_offset; - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - if (pkt_hdr) { - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - } - } while (pkt_hdr); - - return segcount; -} - -odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) -{ - return _odp_packet_seg_from_ndx(odp_packet_num_segs(pkt) - 1); -} - -odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= - packet_last_seg(pkt_hdr))) - return ODP_PACKET_SEG_INVALID; - - return seg + 1; -} - /* * * Segment level @@ -1490,53 +1265,23 @@ odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t seg_offset = 0, offset = 0, i; - uint32_t seg_ndx = _odp_packet_seg_to_ndx(seg); - - while (seg_ndx >= pkt_hdr->buf_hdr.segcount - seg_offset && - pkt_hdr->ref_hdr) { - seg_ndx -= (pkt_hdr->buf_hdr.segcount - seg_offset); - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - }
- if (odp_unlikely(seg_ndx + seg_offset >= pkt_hdr->buf_hdr.segcount)) + if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= + pkt_hdr->buf_hdr.segcount)) return NULL;
- return packet_seg_data(pkt_hdr, seg_ndx + seg_offset) + offset; + return packet_seg_data(pkt_hdr, _odp_packet_seg_to_ndx(seg)); }
uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t seg_offset = 0, offset = 0, i; - uint32_t seg_ndx = _odp_packet_seg_to_ndx(seg); - - while (seg_ndx >= pkt_hdr->buf_hdr.segcount - seg_offset && - pkt_hdr->ref_hdr) { - seg_ndx -= (pkt_hdr->buf_hdr.segcount - seg_offset); - offset = pkt_hdr->ref_offset; - pkt_hdr = pkt_hdr->ref_hdr; - for (i = 0, seg_offset = 0; - i < pkt_hdr->buf_hdr.segcount; - i++, seg_offset++) { - if (offset < pkt_hdr->buf_hdr.seg[i].len) - break; - offset -= pkt_hdr->buf_hdr.seg[i].len; - } - }
- if (odp_unlikely(seg_ndx + seg_offset >= pkt_hdr->buf_hdr.segcount)) + if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >= + pkt_hdr->buf_hdr.segcount)) return 0;
- return packet_seg_len(pkt_hdr, seg_ndx + seg_offset) - offset; + return packet_seg_len(pkt_hdr, _odp_packet_seg_to_ndx(seg)); }
/* @@ -1557,8 +1302,6 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) if (offset > pktlen) return -1;
- ODP_ASSERT(odp_packet_unshared_len(*pkt_ptr) >= offset); - newpkt = odp_packet_alloc(pool->pool_hdl, pktlen + len);
if (newpkt == ODP_PACKET_INVALID) @@ -1582,15 +1325,13 @@ int odp_packet_rem_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) { odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t pktlen = packet_len(pkt_hdr); + uint32_t pktlen = pkt_hdr->frame_len; pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; odp_packet_t newpkt;
- if (offset + len > pktlen) + if (offset > pktlen || offset + len > pktlen) return -1;
- ODP_ASSERT(odp_packet_unshared_len(*pkt_ptr) >= offset + len); - newpkt = odp_packet_alloc(pool->pool_hdl, pktlen - len);
if (newpkt == ODP_PACKET_INVALID) @@ -1624,8 +1365,6 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, if (align > ODP_CACHE_LINE_SIZE) return -1;
- ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); - if (seglen >= len) { misalign = align <= 1 ? 0 : ROUNDUP_ALIGN(uaddr, align) - uaddr; @@ -1665,13 +1404,10 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) uint32_t dst_len = dst_hdr->frame_len; uint32_t src_len = src_hdr->frame_len;
- ODP_ASSERT(packet_ref_count(dst_hdr) == 1); - /* Do a copy if resulting packet would be out of segments or packets - * are from different pools or src is a reference. */ + * are from different pools. */ if (odp_unlikely((dst_segs + src_segs) > CONFIG_PACKET_MAX_SEGS) || - odp_unlikely(dst_pool != src_pool) || - odp_unlikely(packet_ref_count(src_hdr) > 1)) { + odp_unlikely(dst_pool != src_pool)) { if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) { (void)odp_packet_copy_from_pkt(*dst, dst_len, src, 0, src_len); @@ -1686,9 +1422,8 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src)
add_all_segs(dst_hdr, src_hdr);
- dst_hdr->frame_len = dst_len + src_len; - dst_hdr->unshared_len = dst_len + src_len; - dst_hdr->tailroom = src_hdr->tailroom; + dst_hdr->frame_len = dst_len + src_len; + dst_hdr->tailroom = src_hdr->tailroom;
/* Data was not moved in memory */ return 0; @@ -1701,7 +1436,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) if (len >= pktlen || tail == NULL) return -1;
- ODP_ASSERT(odp_packet_unshared_len(*pkt) >= len); *tail = odp_packet_copy_part(*pkt, len, pktlen - len, odp_packet_pool(*pkt));
@@ -1712,109 +1446,6 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) }
/* - * References - */ - -static inline void packet_ref(odp_packet_hdr_t *pkt_hdr) -{ - uint32_t i; - odp_packet_hdr_t *hdr; - - do { - for (i = 0; i < pkt_hdr->buf_hdr.segcount; i++) { - hdr = pkt_hdr->buf_hdr.seg[i].hdr; - packet_ref_inc(hdr); - } - - pkt_hdr = pkt_hdr->ref_hdr; - } while (pkt_hdr); -} - -static inline odp_packet_t packet_splice(odp_packet_hdr_t *pkt_hdr, - uint32_t offset, - odp_packet_hdr_t *ref_hdr) -{ - /* Catch attempted references to stale handles in debug builds */ - ODP_ASSERT(packet_ref_count(pkt_hdr) > 0); - - /* Splicing is from the last section of src pkt */ - while (ref_hdr->ref_hdr) - ref_hdr = ref_hdr->ref_hdr; - - /* Find section where splice begins */ - while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { - offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); - offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); - pkt_hdr = pkt_hdr->ref_hdr; - } - - ref_hdr->ref_hdr = pkt_hdr; - ref_hdr->ref_offset = offset; - ref_hdr->ref_len = pkt_hdr->frame_len; - - if (packet_ref_count(pkt_hdr) == 1 || offset < pkt_hdr->unshared_len) - pkt_hdr->unshared_len = offset; - - packet_ref(pkt_hdr); - return packet_handle(ref_hdr); -} - -odp_packet_t odp_packet_ref_static(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - packet_ref(pkt_hdr); - pkt_hdr->unshared_len = 0; - return pkt; -} - -odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_t hdr; - odp_packet_hdr_t *pkt_hdr; - - if (pkt == ODP_PACKET_INVALID) - return ODP_PACKET_INVALID; - - pkt_hdr = packet_hdr(pkt); - if (offset >= packet_len(pkt_hdr)) - return ODP_PACKET_INVALID; - - hdr = odp_packet_alloc(odp_packet_pool(pkt), 0); - - if (hdr == ODP_PACKET_INVALID) - return ODP_PACKET_INVALID; - - return packet_splice(pkt_hdr, offset, packet_hdr(hdr)); -} - -odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, - odp_packet_t hdr) -{ - odp_packet_hdr_t *pkt_hdr; - - if (pkt == ODP_PACKET_INVALID || - hdr == ODP_PACKET_INVALID || - pkt == hdr) - return ODP_PACKET_INVALID; - - ODP_ASSERT(odp_packet_has_ref(hdr) == 0); - - pkt_hdr = packet_hdr(pkt); - if (offset >= packet_len(pkt_hdr)) - return ODP_PACKET_INVALID; - - return packet_splice(pkt_hdr, offset, packet_hdr(hdr)); -} - -int odp_packet_has_ref(odp_packet_t pkt) -{ - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - - return pkt_hdr->ref_hdr != NULL || packet_ref_count(pkt_hdr) > 1; -} - -/* * * Copy * ******************************************************** @@ -1823,7 +1454,8 @@ int odp_packet_has_ref(odp_packet_t pkt)
odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool) { - uint32_t pktlen = odp_packet_len(pkt); + odp_packet_hdr_t *srchdr = packet_hdr(pkt); + uint32_t pktlen = srchdr->frame_len; odp_packet_t newpkt = odp_packet_alloc(pool, pktlen);
if (newpkt != ODP_PACKET_INVALID) { @@ -1862,7 +1494,7 @@ int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t offset, uint8_t *dstaddr = (uint8_t *)dst; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
- if (offset + len > packet_len(pkt_hdr)) + if (offset + len > pkt_hdr->frame_len) return -1;
while (len > 0) { @@ -1886,11 +1518,9 @@ int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t offset, const uint8_t *srcaddr = (const uint8_t *)src; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
- if (offset + len > packet_len(pkt_hdr)) + if (offset + len > pkt_hdr->frame_len) return -1;
- ODP_ASSERT(odp_packet_unshared_len(pkt) >= offset + len); - while (len > 0) { mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); cpylen = len > seglen ? seglen : len; @@ -1916,12 +1546,10 @@ int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t dst_offset, uint32_t src_seglen = 0; /* GCC */ int overlap;
- if (dst_offset + len > packet_len(dst_hdr) || - src_offset + len > packet_len(src_hdr)) + if (dst_offset + len > dst_hdr->frame_len || + src_offset + len > src_hdr->frame_len) return -1;
- ODP_ASSERT(odp_packet_unshared_len(dst) >= dst_offset + len); - overlap = (dst_hdr == src_hdr && ((dst_offset <= src_offset && dst_offset + len >= src_offset) || @@ -2054,7 +1682,7 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); len += snprintf(&str[len], n - len, - " frame_len %" PRIu32 "\n", packet_len(hdr)); + " frame_len %" PRIu32 "\n", hdr->frame_len); len += snprintf(&str[len], n - len, " input %" PRIu64 "\n", odp_pktio_to_u64(hdr->input)); @@ -2458,6 +2086,80 @@ uint64_t odp_packet_seg_to_u64(odp_packet_seg_t hdl) return _odp_pri(hdl); }
+odp_packet_t odp_packet_ref_static(odp_packet_t pkt) +{ + return odp_packet_copy(pkt, odp_packet_pool(pkt)); +} + +odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) +{ + odp_packet_t new; + int ret; + + new = odp_packet_copy(pkt, odp_packet_pool(pkt)); + + if (new == ODP_PACKET_INVALID) { + ODP_ERR("copy failed\n"); + return ODP_PACKET_INVALID; + } + + ret = odp_packet_trunc_head(&new, offset, NULL, NULL); + + if (ret < 0) { + ODP_ERR("trunk_head failed\n"); + odp_packet_free(new); + return ODP_PACKET_INVALID; + } + + return new; +} + +odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, + odp_packet_t hdr) +{ + odp_packet_t new; + int ret; + + new = odp_packet_copy(pkt, odp_packet_pool(pkt)); + + if (new == ODP_PACKET_INVALID) { + ODP_ERR("copy failed\n"); + return ODP_PACKET_INVALID; + } + + if (offset) { + ret = odp_packet_trunc_head(&new, offset, NULL, NULL); + + if (ret < 0) { + ODP_ERR("trunk_head failed\n"); + odp_packet_free(new); + return ODP_PACKET_INVALID; + } + } + + ret = odp_packet_concat(&hdr, new); + + if (ret < 0) { + ODP_ERR("concat failed\n"); + odp_packet_free(new); + return ODP_PACKET_INVALID; + } + + return hdr; +} + +int odp_packet_has_ref(odp_packet_t pkt) +{ + (void)pkt; + + return 0; +} + +uint32_t odp_packet_unshared_len(odp_packet_t pkt) +{ + return odp_packet_len(pkt); +} + /* Include non-inlined versions of API functions */ #if ODP_ABI_COMPAT == 1 #include <odp/api/plat/packet_inlines_api.h>
-----------------------------------------------------------------------
Summary of changes: include/odp/arch/default/api/abi/packet.h | 2 +- .../include/odp/api/plat/packet_inlines.h | 44 +- .../include/odp/api/plat/packet_inlines_api.h | 16 + .../include/odp/api/plat/packet_types.h | 38 +- .../linux-generic/include/odp_buffer_internal.h | 71 +- .../linux-generic/include/odp_packet_internal.h | 135 +- platform/linux-generic/odp_packet.c | 1518 +++++++++----------- platform/linux-generic/odp_pool.c | 8 + test/common_plat/validation/api/packet/packet.c | 109 +- 9 files changed, 926 insertions(+), 1015 deletions(-)
hooks/post-receive