This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "".

The branch, master has been updated via 251fd5805122a257060a9de62a4c26bc1862a544 (commit) via 8813bda68dac4f6ff448cf783b43e6deafff75ae (commit) via 3d79b95543b6043a9df9f523f81416a30052a9ce (commit) via 3c335400833959b9a96674984c797e8ec4734566 (commit) from dd7bcc7dead9bceba895ad877ad3ac7642d4e717 (commit)

Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below.

- Log ----------------------------------------------------------------- commit 251fd5805122a257060a9de62a4c26bc1862a544 Author: Petri Savolainen petri.savolainen@linaro.org Date: Wed Jun 20 13:31:59 2018 +0300

validation: queue: improve queue pair checks

Output erroneous sequence number. Check that no events were lost.

Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/test/validation/api/queue/queue.c b/test/validation/api/queue/queue.c index f8e0314f..0bd1a06a 100644 --- a/test/validation/api/queue/queue.c +++ b/test/validation/api/queue/queue.c @@ -428,7 +428,11 @@ static int queue_pair_work_loop(void *arg) retry = 0; buf = odp_buffer_from_event(ev); data = odp_buffer_addr(buf); - CU_ASSERT(*data == i); + if (*data != i) { + printf("Seq error: expected %u, recv %u\n", i, *data); + CU_FAIL("Sequence number error"); + } + i++; if (i == burst) i = 0; @@ -458,7 +462,7 @@ static void test_pair(odp_nonblocking_t nonblocking, odp_queue_param_t param; odp_queue_t queue; odp_queue_capability_t capa; - uint32_t max_burst; + uint32_t max_burst, num; odp_pool_t pool; odp_event_t ev; odp_shm_t shm; @@ -526,12 +530,21 @@ static void test_pair(odp_nonblocking_t nonblocking, CU_ASSERT(globals->pair.passed_a); CU_ASSERT(globals->pair.passed_b);

- while ((ev = dequeue_event(globals->pair.queue_a)) != ODP_EVENT_INVALID) + num = 0; + + while ((ev = dequeue_event(globals->pair.queue_a)) + != ODP_EVENT_INVALID) { + num++; odp_event_free(ev); + }

- while ((ev = dequeue_event(globals->pair.queue_b)) != ODP_EVENT_INVALID) + while ((ev = dequeue_event(globals->pair.queue_b)) + != ODP_EVENT_INVALID) { + num++; odp_event_free(ev); + }

+ CU_ASSERT(num == max_burst); CU_ASSERT(odp_queue_destroy(globals->pair.queue_a) == 0); CU_ASSERT(odp_queue_destroy(globals->pair.queue_b) == 0); }

commit 8813bda68dac4f6ff448cf783b43e6deafff75ae Author: Petri Savolainen petri.savolainen@linaro.org Date: Wed Jun 20 10:05:05 2018 +0300

validation: queue: add pair tests for lock-free queues

Add queue pair test cases for lock-free queues.

Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/test/validation/api/queue/queue.c b/test/validation/api/queue/queue.c index f009a24b..f8e0314f 100644 --- a/test/validation/api/queue/queue.c +++ b/test/validation/api/queue/queue.c @@ -556,6 +556,21 @@ static void queue_test_pair_spsc(void) test_pair(ODP_BLOCKING, ODP_QUEUE_OP_MT_UNSAFE, ODP_QUEUE_OP_MT_UNSAFE); }

+static void queue_test_pair_lf(void) +{ + test_pair(ODP_NONBLOCKING_LF, ODP_QUEUE_OP_MT, ODP_QUEUE_OP_MT); +} + +static void queue_test_pair_lf_spmc(void) +{ + test_pair(ODP_NONBLOCKING_LF, ODP_QUEUE_OP_MT_UNSAFE, ODP_QUEUE_OP_MT); +} + +static void queue_test_pair_lf_mpsc(void) +{ + test_pair(ODP_NONBLOCKING_LF, ODP_QUEUE_OP_MT, ODP_QUEUE_OP_MT_UNSAFE); +} + static void queue_test_pair_lf_spsc(void) { test_pair(ODP_NONBLOCKING_LF, ODP_QUEUE_OP_MT_UNSAFE, @@ -944,6 +959,9 @@ odp_testinfo_t queue_suite[] = { ODP_TEST_INFO(queue_test_pair_spmc), ODP_TEST_INFO(queue_test_pair_mpsc), ODP_TEST_INFO(queue_test_pair_spsc), + ODP_TEST_INFO(queue_test_pair_lf), + ODP_TEST_INFO(queue_test_pair_lf_spmc), + ODP_TEST_INFO(queue_test_pair_lf_mpsc), ODP_TEST_INFO(queue_test_pair_lf_spsc), ODP_TEST_INFO(queue_test_param), ODP_TEST_INFO(queue_test_info),

commit 3d79b95543b6043a9df9f523f81416a30052a9ce Author: Petri Savolainen petri.savolainen@linaro.org Date: Wed Jun 20 13:03:18 2018 +0300

linux-gen: queue_lf: fix event ordering issue

New enqueues may happen during a dequeue operation is searching for the lowest counter value. If the first enqueue (with lower counter value) added a node before the current dequeue search index and the second enqueue after the index, dequeue returned these events in wrong order. After finding the lowest counter value, dequeue needs to search nodes before that node again.

Also change CAS operation memory model to acq-rel, so that the first enqueue from a thread is always visible before the second enqueue (from the same thread).

Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index e4902a3f..3ec32524 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -36,25 +36,14 @@ static inline void atomic_zero_u128(u128_t *atomic) */ #include <odp_cpu.h>

-static inline int atomic_cas_rel_u128(u128_t *atomic, u128_t old_val, - u128_t new_val) +static inline int atomic_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, + u128_t new_val) { return __lockfree_compare_exchange_16((__int128 *)atomic, (__int128 *)&old_val, new_val, 0, - __ATOMIC_RELEASE, - __ATOMIC_RELAXED); -} - -static inline int atomic_cas_acq_u128(u128_t *atomic, u128_t old_val, - u128_t new_val) -{ - return __lockfree_compare_exchange_16((__int128 *)atomic, - (__int128 *)&old_val, - new_val, - 0, - __ATOMIC_ACQUIRE, + __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); }

@@ -75,21 +64,12 @@ static inline u128_t atomic_load_u128(u128_t *atomic) return __atomic_load_n(atomic, __ATOMIC_RELAXED); }

-static inline int atomic_cas_rel_u128(u128_t *atomic, u128_t old_val, - u128_t new_val) +static inline int atomic_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, + u128_t new_val) { return __atomic_compare_exchange_n(atomic, &old_val, new_val, 0 /* strong */, - __ATOMIC_RELEASE, - __ATOMIC_RELAXED); -} - -static inline int atomic_cas_acq_u128(u128_t *atomic, u128_t old_val, - u128_t new_val) -{ - return __atomic_compare_exchange_n(atomic, &old_val, new_val, - 0 /* strong */, - __ATOMIC_ACQUIRE, + __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); }

@@ -120,18 +100,17 @@ static inline void atomic_zero_u128(u128_t *atomic) atomic->u64[1] = 0; }

-static inline int atomic_cas_rel_u128(u128_t *atomic, u128_t old_val, - u128_t new_val) +static inline int atomic_cas_acq_rel_u128(u128_t *atomic, u128_t old_val, + u128_t new_val) { - (void)old_val; - *atomic = new_val; - return 1; -} + if (atomic->u64[0] == old_val.u64[0] && + atomic->u64[1] == old_val.u64[1]) { + atomic->u64[0] = new_val.u64[0]; + atomic->u64[1] = new_val.u64[1]; + return 1; + }

-static inline int atomic_cas_acq_u128(u128_t *atomic, u128_t old_val, - u128_t new_val) -{ - return atomic_cas_rel_u128(atomic, old_val, new_val); + return 0; }

static inline int atomic_is_lockfree_u128(void) @@ -146,15 +125,12 @@ typedef union { u128_t u128;

struct { - /* 0: empty, 1: data */ - uint64_t mark: 1; + /* Data with lowest counter value is the head. Empty node has + * counter value 0. */ + uint64_t counter;

- /* A cache aligned pointer fits into 63 bits, since the least - * significant bits are zero. */ - uint64_t ptr: 63; - - /* Data with lowest counter value is the head. */ - uint64_t count; + /* Data pointer */ + uint64_t ptr; } s;

} ring_lf_node_t; @@ -190,30 +166,31 @@ static int queue_lf_enq(void *q_int, odp_buffer_hdr_t *buf_hdr) { queue_entry_t *queue; queue_lf_t *queue_lf; - int i, j, i_node; + int i, j, idx; int found; ring_lf_node_t node_val; ring_lf_node_t new_val; ring_lf_node_t *node; - uint64_t counter;

queue = q_int; queue_lf = queue->s.queue_lf;

- i_node = 0; + new_val.s.ptr = (uintptr_t)buf_hdr; + new_val.s.counter = odp_atomic_fetch_inc_u64(&queue_lf->enq_counter);

- counter = odp_atomic_fetch_inc_u64(&queue_lf->enq_counter); + idx = 0;

for (j = 0; j < ENQ_RETRIES; j++) { found = 0;

/* Find empty node */ for (i = 0; i < RING_LF_SIZE; i++) { - i_node = next_idx(i_node); - node = &queue_lf->node[i_node]; + node = &queue_lf->node[idx]; + idx = next_idx(idx); + node_val.u128 = atomic_load_u128(&node->u128);

- if (node_val.s.mark == 0) { + if (node_val.s.counter == 0) { found = 1; break; } @@ -224,12 +201,8 @@ static int queue_lf_enq(void *q_int, odp_buffer_hdr_t *buf_hdr) return -1;

/* Try to insert data */ - new_val.s.mark = 1; - new_val.s.count = counter; - new_val.s.ptr = ((uintptr_t)buf_hdr) >> 1; - - if (atomic_cas_rel_u128(&node->u128, node_val.u128, - new_val.u128)) + if (atomic_cas_acq_rel_u128(&node->u128, node_val.u128, + new_val.u128)) return 0; }

@@ -251,15 +224,18 @@ static odp_buffer_hdr_t *queue_lf_deq(void *q_int) { queue_entry_t *queue; queue_lf_t *queue_lf; - int i, j, i_node; + int i, j, i_lowest; int found; ring_lf_node_t node_val, old_val, new_val; ring_lf_node_t *node, *old; - uint64_t lowest; + uint64_t lowest, counter; + odp_buffer_hdr_t *buf_hdr;

queue = q_int; queue_lf = queue->s.queue_lf; - i_node = 0; + new_val.s.counter = 0; + new_val.s.ptr = 0; + old = NULL;

for (j = 0; j < DEQ_RETRIES; j++) { found = 0; @@ -268,14 +244,15 @@ static odp_buffer_hdr_t *queue_lf_deq(void *q_int) /* Find the head node. The one with data and * the lowest counter. */ for (i = 0; i < RING_LF_SIZE; i++) { - i_node = next_idx(i_node); - node = &queue_lf->node[i_node]; + node = &queue_lf->node[i]; node_val.u128 = atomic_load_u128(&node->u128); + counter = node_val.s.counter;

- if (node_val.s.mark == 1 && node_val.s.count < lowest) { + if (counter && counter < lowest) { old = node; old_val.u128 = node_val.u128; - lowest = node_val.s.count; + lowest = counter; + i_lowest = i; found = 1; } } @@ -284,13 +261,27 @@ static odp_buffer_hdr_t *queue_lf_deq(void *q_int) if (found == 0) return NULL;

- /* Try to remove data */ - new_val.u128 = old_val.u128; - new_val.s.mark = 0; + /* New data may have been written to the area we searched before + * finding the current lowest. Check that there are no lower + * values. */ + for (i = 0; i < i_lowest; i++) { + node = &queue_lf->node[i]; + node_val.u128 = atomic_load_u128(&node->u128); + counter = node_val.s.counter; + + if (counter && counter < lowest) { + old = node; + old_val.u128 = node_val.u128; + lowest = counter; + } + }

- if (atomic_cas_acq_u128(&old->u128, old_val.u128, - new_val.u128)) - return (void *)(((uintptr_t)old_val.s.ptr) << 1); + buf_hdr = (void *)(uintptr_t)old_val.s.ptr; + + /* Try to remove data */ + if (atomic_cas_acq_rel_u128(&old->u128, old_val.u128, + new_val.u128)) + return buf_hdr; }

return NULL; @@ -363,7 +354,7 @@ static void init_queue(queue_lf_t *queue_lf) { int i;

- odp_atomic_init_u64(&queue_lf->enq_counter, 0); + odp_atomic_init_u64(&queue_lf->enq_counter, 1);

for (i = 0; i < RING_LF_SIZE; i++) atomic_zero_u128(&queue_lf->node[i].u128);

commit 3c335400833959b9a96674984c797e8ec4734566 Author: Petri Savolainen petri.savolainen@linaro.org Date: Tue Jun 19 16:58:08 2018 +0300

linux-gen: queue: fix queue empty check

SPSC and normal queues use different rings.

Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org

diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index f592ef6c..39e2d59c 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -383,6 +383,7 @@ void sched_cb_queue_set_status(uint32_t queue_index, int status)

static int queue_destroy(odp_queue_t handle) { + int empty; queue_entry_t *queue; queue = qentry_from_handle(handle);

@@ -400,7 +401,13 @@ static int queue_destroy(odp_queue_t handle) ODP_ERR("queue "%s" already destroyed\n", queue->s.name); return -1; } - if (ring_st_is_empty(&queue->s.ring_st) == 0) { + + if (queue->s.spsc) + empty = ring_spsc_is_empty(&queue->s.ring_spsc); + else + empty = ring_st_is_empty(&queue->s.ring_st); + + if (!empty) { UNLOCK(queue); ODP_ERR("queue "%s" not empty\n", queue->s.name); return -1;

-----------------------------------------------------------------------

Summary of changes: platform/linux-generic/odp_queue_basic.c | 9 ++- platform/linux-generic/odp_queue_lf.c | 133 ++++++++++++++----------------- test/validation/api/queue/queue.c | 39 ++++++++- 3 files changed, 105 insertions(+), 76 deletions(-)

hooks/post-receive