This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "".
The branch, master has been updated via ee6fdca6b647d1c92ba6d07400c7d1e23c9e0ce3 (commit) via 0070431f72dacef15d104bcf2baa9d70629d3d31 (commit) via 5a43e72ba90dc5b4ea4ae3e8875075440e709655 (commit) via 0ee7d42a7280e7ced1ea2f97e80dc752f28f76b2 (commit) via c7bbbb603fc4cf2e6590af962e2e255241d18276 (commit) via c0d530e01e91fbea195993293eed60f1677e6181 (commit) from 942bbe22b7147e2988aa0b82e094a53421f2c486 (commit)
Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below.
- Log ----------------------------------------------------------------- commit ee6fdca6b647d1c92ba6d07400c7d1e23c9e0ce3 Author: Petri Savolainen petri.savolainen@linaro.org Date: Thu Jul 5 14:27:45 2018 +0300
linux-gen: sched: configurable default burst size
Make default burst sizes configurable. User can set limits for burst size round up in high/low priority. When less than burst_size_xx events are requested, scheduler rounds up the number of events to these limit and stash extra events. Requests with more events are not round down. So, user can use small values in burst size configure for better real-time support, but still request large number of events for better throughput on non real-time threads.
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/config/odp-linux-generic.conf b/config/odp-linux-generic.conf index f5f21b45..85d5414b 100644 --- a/config/odp-linux-generic.conf +++ b/config/odp-linux-generic.conf @@ -55,4 +55,15 @@ sched_basic: { # uneven service level for low thread counts. Typically, optimal # value is the number of threads using the scheduler. prio_spread = 4 + + # Default burst sizes for high and low priority queues. The default + # and higher priority levels are considered as high. Scheduler + # rounds up number of requested events up to these values. In general, + # larger requests are not round down. So, larger bursts than these may + # received when requested. A large burst size improves throughput, + # but decreases application responsiveness to high priority events + # due to head of line blocking cause by a burst of low priority + # events. + burst_size_hi = 32 + burst_size_low = 16 } diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 75434149..df63da72 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -94,7 +94,8 @@ ODP_STATIC_ASSERT((8 * sizeof(pri_mask_t)) >= MAX_SPREAD,
/* Default burst size. Scheduler rounds up number of requested events up to * this value. */ -#define BURST_SIZE CONFIG_BURST_SIZE +#define BURST_SIZE_MAX CONFIG_BURST_SIZE +#define BURST_SIZE_MIN 1
/* Ordered stash size */ #define MAX_ORDERED_STASH 512 @@ -124,7 +125,7 @@ typedef struct { uint16_t spread_round; uint32_t stash_qi; odp_queue_t stash_queue; - odp_event_t stash_ev[BURST_SIZE]; + odp_event_t stash_ev[BURST_SIZE_MAX];
uint32_t grp_epoch; uint16_t num_grp; @@ -179,6 +180,8 @@ typedef struct {
struct { uint8_t num_spread; + uint8_t burst_hi; + uint8_t burst_low; } config;
uint32_t pri_count[NUM_PRIO][MAX_SPREAD]; @@ -252,6 +255,34 @@ static int read_config_file(sched_global_t *sched) }
sched->config.num_spread = val; + ODP_PRINT(" %s: %i\n", str, val); + + str = "sched_basic.burst_size_hi"; + if (!_odp_libconfig_lookup_int(str, &val)) { + ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + + if (val > BURST_SIZE_MAX || val < BURST_SIZE_MIN) { + ODP_ERR("Bad value %s = %u\n", str, val); + return -1; + } + + sched->config.burst_hi = val; + ODP_PRINT(" %s: %i\n", str, val); + + str = "sched_basic.burst_size_low"; + if (!_odp_libconfig_lookup_int(str, &val)) { + ODP_ERR("Config option '%s' not found.\n", str); + return -1; + } + + if (val > BURST_SIZE_MAX || val < BURST_SIZE_MIN) { + ODP_ERR("Bad value %s = %u\n", str, val); + return -1; + } + + sched->config.burst_low = val; ODP_PRINT(" %s: %i\n\n", str, val);
return 0; @@ -767,7 +798,7 @@ static inline int poll_pktin(uint32_t qi, int direct_recv, odp_buffer_hdr_t **hdr_tbl; int ret; void *q_int; - odp_buffer_hdr_t *b_hdr[BURST_SIZE]; + odp_buffer_hdr_t *b_hdr[BURST_SIZE_MAX];
hdr_tbl = (odp_buffer_hdr_t **)ev_tbl;
@@ -775,8 +806,8 @@ static inline int poll_pktin(uint32_t qi, int direct_recv, hdr_tbl = b_hdr;
/* Limit burst to max queue enqueue size */ - if (max_num > BURST_SIZE) - max_num = BURST_SIZE; + if (max_num > BURST_SIZE_MAX) + max_num = BURST_SIZE_MAX; }
pktio_index = sched->queue[qi].pktio_index; @@ -831,6 +862,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], int ret; int id; uint32_t qi; + unsigned int max_burst; int num_spread = sched->config.num_spread; uint32_t ring_mask = sched->ring_mask;
@@ -840,6 +872,10 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], if (sched->pri_mask[prio] == 0) continue;
+ max_burst = sched->config.burst_hi; + if (prio > ODP_SCHED_PRIO_DEFAULT) + max_burst = sched->config.burst_low; + /* Select the first ring based on weights */ id = first;
@@ -849,7 +885,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], odp_queue_t handle; ring_t *ring; int pktin; - unsigned int max_deq = BURST_SIZE; + unsigned int max_deq = max_burst; int stashed = 1; odp_event_t *ev_tbl = sched_local.stash_ev;
@@ -877,24 +913,17 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[],
ordered = queue_is_ordered(qi);
- /* When application's array is larger than our burst + /* When application's array is larger than max burst * size, output all events directly there. Also, ordered * queues are not stashed locally to improve * parallelism. Ordered context can only be released * when the local cache is empty. */ - if (max_num > BURST_SIZE || ordered) { + if (max_num > max_burst || ordered) { stashed = 0; ev_tbl = out_ev; max_deq = max_num; }
- /* Low priorities have smaller burst size to limit - * head of line blocking latency. */ - if (BURST_SIZE > 1 && - odp_unlikely(prio > ODP_SCHED_PRIO_DEFAULT) && - max_deq > BURST_SIZE / 2) - max_deq = BURST_SIZE / 2; - pktin = queue_is_pktin(qi);
num = sched_queue_deq(qi, ev_tbl, max_deq, !pktin);
commit 0070431f72dacef15d104bcf2baa9d70629d3d31 Author: Petri Savolainen petri.savolainen@linaro.org Date: Thu Jul 5 11:14:26 2018 +0300
linux-gen: sched: move basic queue scheduler functions
Moved and renamed basic queue interface functions towards schedulers. Queue and scheduler implementations are tied together as queue enq/deq operations change queue scheduling state. Three schedulers are based on basic queue implementation and include basic queue internal header file already. Move this internal interface into proper internal header file.
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/platform/linux-generic/include/odp_queue_basic_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h index 0a81f3c4..654b9e31 100644 --- a/platform/linux-generic/include/odp_queue_basic_internal.h +++ b/platform/linux-generic/include/odp_queue_basic_internal.h @@ -98,6 +98,13 @@ static inline odp_queue_t queue_from_index(uint32_t queue_id)
void queue_spsc_init(queue_entry_t *queue, uint32_t queue_size);
+/* Functions for schedulers */ +void sched_queue_destroy_finalize(uint32_t queue_index); +void sched_queue_set_status(uint32_t queue_index, int status); +int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int num, + int update_status); +int sched_queue_empty(uint32_t queue_index); + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index 0d095262..8f082aaa 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -82,12 +82,6 @@ int sched_cb_pktin_poll(int pktio_index, int pktin_index, int sched_cb_pktin_poll_old(int pktio_index, int num_queue, int index[]); int sched_cb_pktin_poll_one(int pktio_index, int rx_queue, odp_event_t evts[]); void sched_cb_pktio_stop_finalize(int pktio_index); -odp_queue_t sched_cb_queue_handle(uint32_t queue_index); -void sched_cb_queue_destroy_finalize(uint32_t queue_index); -void sched_cb_queue_set_status(uint32_t queue_index, int status); -int sched_cb_queue_deq_multi(uint32_t queue_index, odp_event_t ev[], int num, - int update_status); -int sched_cb_queue_empty(uint32_t queue_index);
/* API functions */ typedef struct { diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 3962f9f1..89eed3c0 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -357,7 +357,7 @@ static odp_queue_t queue_create(const char *name, return handle; }
-void sched_cb_queue_destroy_finalize(uint32_t queue_index) +void sched_queue_destroy_finalize(uint32_t queue_index) { queue_entry_t *queue = qentry_from_index(queue_index);
@@ -370,7 +370,7 @@ void sched_cb_queue_destroy_finalize(uint32_t queue_index) UNLOCK(queue); }
-void sched_cb_queue_set_status(uint32_t queue_index, int status) +void sched_queue_set_status(uint32_t queue_index, int status) { queue_entry_t *queue = qentry_from_index(queue_index);
@@ -761,8 +761,8 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) return 0; }
-int sched_cb_queue_deq_multi(uint32_t queue_index, odp_event_t ev[], - int max_num, int update_status) +int sched_queue_deq(uint32_t queue_index, odp_event_t ev[], int max_num, + int update_status) { int num_deq; ring_st_t *ring_st; @@ -807,7 +807,7 @@ int sched_cb_queue_deq_multi(uint32_t queue_index, odp_event_t ev[], return num_deq; }
-int sched_cb_queue_empty(uint32_t queue_index) +int sched_queue_empty(uint32_t queue_index) { queue_entry_t *queue = qentry_from_index(queue_index); int ret = 0; diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index fbe5b26a..75434149 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -363,7 +363,7 @@ static int schedule_init_global(void)
static inline void queue_destroy_finalize(uint32_t qi) { - sched_cb_queue_destroy_finalize(qi); + sched_queue_destroy_finalize(qi); }
static int schedule_term_global(void) @@ -384,9 +384,7 @@ static int schedule_term_global(void) odp_event_t events[1]; int num;
- num = sched_cb_queue_deq_multi(qi, - events, - 1, 1); + num = sched_queue_deq(qi, events, 1, 1);
if (num < 0) queue_destroy_finalize(qi); @@ -581,7 +579,7 @@ static void schedule_pktio_start(int pktio_index, int num_pktin, ODP_ASSERT(pktin_idx[i] <= MAX_PKTIN_INDEX);
/* Start polling */ - sched_cb_queue_set_status(qi, QUEUE_STATUS_SCHED); + sched_queue_set_status(qi, QUEUE_STATUS_SCHED); schedule_sched_queue(qi); } } @@ -797,7 +795,7 @@ static inline int poll_pktin(uint32_t qi, int direct_recv, num_pktin = sched->pktio[pktio_index].num_pktin; odp_spinlock_unlock(&sched->pktio_lock);
- sched_cb_queue_set_status(qi, QUEUE_STATUS_NOTSCHED); + sched_queue_set_status(qi, QUEUE_STATUS_NOTSCHED);
if (num_pktin == 0) sched_cb_pktio_stop_finalize(pktio_index); @@ -899,13 +897,12 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[],
pktin = queue_is_pktin(qi);
- num = sched_cb_queue_deq_multi(qi, ev_tbl, max_deq, - !pktin); + num = sched_queue_deq(qi, ev_tbl, max_deq, !pktin);
if (num < 0) { /* Destroyed queue. Continue scheduling the same * priority queue. */ - sched_cb_queue_destroy_finalize(qi); + sched_queue_destroy_finalize(qi); continue; }
diff --git a/platform/linux-generic/odp_schedule_iquery.c b/platform/linux-generic/odp_schedule_iquery.c index e7e23aa5..2501a3f6 100644 --- a/platform/linux-generic/odp_schedule_iquery.c +++ b/platform/linux-generic/odp_schedule_iquery.c @@ -289,10 +289,10 @@ static int schedule_term_global(void) odp_event_t events[1];
if (sched->availables[i]) - count = sched_cb_queue_deq_multi(i, events, 1, 1); + count = sched_queue_deq(i, events, 1, 1);
if (count < 0) - sched_cb_queue_destroy_finalize(i); + sched_queue_destroy_finalize(i); else if (count > 0) ODP_ERR("Queue (%d) not empty\n", i); } @@ -1532,12 +1532,11 @@ static inline int consume_queue(int prio, unsigned int queue_index) if (is_ordered_queue(queue_index)) max = 1;
- count = sched_cb_queue_deq_multi( - queue_index, cache->stash, max, 1); + count = sched_queue_deq(queue_index, cache->stash, max, 1);
if (count < 0) { DO_SCHED_UNLOCK(); - sched_cb_queue_destroy_finalize(queue_index); + sched_queue_destroy_finalize(queue_index); DO_SCHED_LOCK(); return 0; } diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index b6bb3b2b..ae292051 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -228,7 +228,7 @@ static int term_global(void) for (qi = 0; qi < NUM_QUEUE; qi++) { if (sched_global->queue_cmd[qi].s.init) { /* todo: dequeue until empty ? */ - sched_cb_queue_destroy_finalize(qi); + sched_queue_destroy_finalize(qi); } }
@@ -507,7 +507,7 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait,
if (sched_local.cmd) { /* Continue scheduling if queue is not empty */ - if (sched_cb_queue_empty(sched_local.cmd->s.index) == 0) + if (sched_queue_empty(sched_local.cmd->s.index) == 0) add_tail(sched_local.cmd);
sched_local.cmd = NULL; @@ -562,7 +562,7 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait, }
qi = cmd->s.index; - num = sched_cb_queue_deq_multi(qi, events, 1, 1); + num = sched_queue_deq(qi, events, 1, 1);
if (num > 0) { sched_local.cmd = cmd; @@ -575,7 +575,7 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait,
if (num < 0) { /* Destroyed queue */ - sched_cb_queue_destroy_finalize(qi); + sched_queue_destroy_finalize(qi); continue; }
commit 5a43e72ba90dc5b4ea4ae3e8875075440e709655 Author: Petri Savolainen petri.savolainen@linaro.org Date: Thu Jul 5 10:54:16 2018 +0300
linux-gen: queue: rename queue basic internal header file
This is internal interface towards queue basic.
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index a3399011..13101cfd 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -123,7 +123,7 @@ noinst_HEADERS = \ include/odp_pool_internal.h \ include/odp_posix_extensions.h \ include/odp_queue_if.h \ - include/odp_queue_internal.h \ + include/odp_queue_basic_internal.h \ include/odp_queue_lf.h \ include/odp_queue_scalable_internal.h \ include/odp_ring_internal.h \ diff --git a/platform/linux-generic/include/odp_queue_internal.h b/platform/linux-generic/include/odp_queue_basic_internal.h similarity index 97% rename from platform/linux-generic/include/odp_queue_internal.h rename to platform/linux-generic/include/odp_queue_basic_internal.h index bb74f729..0a81f3c4 100644 --- a/platform/linux-generic/include/odp_queue_internal.h +++ b/platform/linux-generic/include/odp_queue_basic_internal.h @@ -4,8 +4,8 @@ * SPDX-License-Identifier: BSD-3-Clause */
-#ifndef ODP_QUEUE_INTERNAL_H_ -#define ODP_QUEUE_INTERNAL_H_ +#ifndef ODP_QUEUE_BASIC_INTERNAL_H_ +#define ODP_QUEUE_BASIC_INTERNAL_H_
#ifdef __cplusplus extern "C" { diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 311fda69..3962f9f1 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -7,7 +7,7 @@ #include "config.h"
#include <odp/api/queue.h> -#include <odp_queue_internal.h> +#include <odp_queue_basic_internal.h> #include <odp_queue_if.h> #include <odp/api/std_types.h> #include <odp/api/align.h> diff --git a/platform/linux-generic/odp_queue_lf.c b/platform/linux-generic/odp_queue_lf.c index 3ec32524..d12a994b 100644 --- a/platform/linux-generic/odp_queue_lf.c +++ b/platform/linux-generic/odp_queue_lf.c @@ -8,7 +8,7 @@ #include <odp/api/atomic.h> #include <odp/api/plat/atomic_inlines.h> #include <odp/api/shared_memory.h> -#include <odp_queue_internal.h> +#include <odp_queue_basic_internal.h> #include <string.h> #include <stdio.h>
diff --git a/platform/linux-generic/odp_queue_spsc.c b/platform/linux-generic/odp_queue_spsc.c index f8b04ca2..3e42b038 100644 --- a/platform/linux-generic/odp_queue_spsc.c +++ b/platform/linux-generic/odp_queue_spsc.c @@ -6,7 +6,7 @@ #include <string.h> #include <stdio.h>
-#include <odp_queue_internal.h> +#include <odp_queue_basic_internal.h> #include <odp_pool_internal.h>
#include "config.h" diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index f45bc8f1..fbe5b26a 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -26,7 +26,7 @@ #include <odp/api/packet_io.h> #include <odp_ring_internal.h> #include <odp_timer_internal.h> -#include <odp_queue_internal.h> +#include <odp_queue_basic_internal.h> #include <odp_libconfig_internal.h>
/* Number of priority levels */ diff --git a/platform/linux-generic/odp_schedule_iquery.c b/platform/linux-generic/odp_schedule_iquery.c index c1e51bb5..e7e23aa5 100644 --- a/platform/linux-generic/odp_schedule_iquery.c +++ b/platform/linux-generic/odp_schedule_iquery.c @@ -25,9 +25,7 @@ #include <odp/api/packet_io.h> #include <odp_config_internal.h> #include <odp_timer_internal.h> - -/* Should remove this dependency */ -#include <odp_queue_internal.h> +#include <odp_queue_basic_internal.h>
/* Number of priority levels */ #define NUM_SCHED_PRIO 8 diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index ac0b35fd..b6bb3b2b 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -20,7 +20,7 @@ #include <odp_config_internal.h> #include <odp_ring_internal.h> #include <odp_timer_internal.h> -#include <odp_queue_internal.h> +#include <odp_queue_basic_internal.h>
#define NUM_THREAD ODP_THREAD_COUNT_MAX #define NUM_QUEUE ODP_CONFIG_QUEUES
commit 0ee7d42a7280e7ced1ea2f97e80dc752f28f76b2 Author: Petri Savolainen petri.savolainen@linaro.org Date: Thu Jul 5 10:33:34 2018 +0300
linux-gen: queue: separate plain and sched dequeues
Separate dequeue functions for plain and scheduled queues. Plain queue dequeue function become simpler and scheduler dequeue function may be optimized later as it is a single consumer dequeue operation.
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/platform/linux-generic/odp_queue_basic.c b/platform/linux-generic/odp_queue_basic.c index 39e2d59c..311fda69 100644 --- a/platform/linux-generic/odp_queue_basic.c +++ b/platform/linux-generic/odp_queue_basic.c @@ -576,10 +576,9 @@ static int queue_enq(odp_queue_t handle, odp_event_t ev) (odp_buffer_hdr_t *)(uintptr_t)ev); }
-static inline int deq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], - int num, int update_status) +static inline int plain_queue_deq(queue_entry_t *queue, + odp_buffer_hdr_t *buf_hdr[], int num) { - int status_sync = sched_fn->status_sync; int num_deq; ring_st_t *ring_st; uint32_t buf_idx[num]; @@ -589,32 +588,17 @@ static inline int deq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], LOCK(queue);
if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { - /* Bad queue, or queue has been destroyed. - * Scheduler finalizes queue destroy after this. */ + /* Bad queue, or queue has been destroyed. */ UNLOCK(queue); return -1; }
num_deq = ring_st_deq_multi(ring_st, buf_idx, num);
- if (num_deq == 0) { - /* Already empty queue */ - if (update_status && queue->s.status == QUEUE_STATUS_SCHED) { - queue->s.status = QUEUE_STATUS_NOTSCHED; - - if (status_sync) - sched_fn->unsched_queue(queue->s.index); - } - - UNLOCK(queue); + UNLOCK(queue);
+ if (num_deq == 0) return 0; - } - - if (status_sync && queue->s.type == ODP_QUEUE_TYPE_SCHED) - sched_fn->save_context(queue->s.index); - - UNLOCK(queue);
buffer_index_to_buf(buf_hdr, buf_idx, num_deq);
@@ -626,7 +610,7 @@ static int queue_int_deq_multi(void *q_int, odp_buffer_hdr_t *buf_hdr[], { queue_entry_t *queue = q_int;
- return deq_multi(queue, buf_hdr, num, 0); + return plain_queue_deq(queue, buf_hdr, num); }
static odp_buffer_hdr_t *queue_int_deq(void *q_int) @@ -635,7 +619,7 @@ static odp_buffer_hdr_t *queue_int_deq(void *q_int) odp_buffer_hdr_t *buf_hdr = NULL; int ret;
- ret = deq_multi(queue, &buf_hdr, 1, 0); + ret = plain_queue_deq(queue, &buf_hdr, 1);
if (ret == 1) return buf_hdr; @@ -777,12 +761,50 @@ static int queue_info(odp_queue_t handle, odp_queue_info_t *info) return 0; }
-int sched_cb_queue_deq_multi(uint32_t queue_index, odp_event_t ev[], int num, - int update_status) +int sched_cb_queue_deq_multi(uint32_t queue_index, odp_event_t ev[], + int max_num, int update_status) { - queue_entry_t *qe = qentry_from_index(queue_index); + int num_deq; + ring_st_t *ring_st; + queue_entry_t *queue = qentry_from_index(queue_index); + int status_sync = sched_fn->status_sync; + uint32_t buf_idx[max_num]; + + ring_st = &queue->s.ring_st; + + LOCK(queue);
- return deq_multi(qe, (odp_buffer_hdr_t **)ev, num, update_status); + if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { + /* Bad queue, or queue has been destroyed. + * Scheduler finalizes queue destroy after this. */ + UNLOCK(queue); + return -1; + } + + num_deq = ring_st_deq_multi(ring_st, buf_idx, max_num); + + if (num_deq == 0) { + /* Already empty queue */ + if (update_status && queue->s.status == QUEUE_STATUS_SCHED) { + queue->s.status = QUEUE_STATUS_NOTSCHED; + + if (status_sync) + sched_fn->unsched_queue(queue->s.index); + } + + UNLOCK(queue); + + return 0; + } + + if (status_sync && queue->s.type == ODP_QUEUE_TYPE_SCHED) + sched_fn->save_context(queue->s.index); + + UNLOCK(queue); + + buffer_index_to_buf((odp_buffer_hdr_t **)ev, buf_idx, num_deq); + + return num_deq; }
int sched_cb_queue_empty(uint32_t queue_index)
commit c7bbbb603fc4cf2e6590af962e2e255241d18276 Author: Petri Savolainen petri.savolainen@linaro.org Date: Wed Jul 4 17:17:46 2018 +0300
linux-gen: sched: support large burst size from pktin
When directly receiving from pktin (atomic or parallel queues), output event handles directly to destination array (scheduler stash or application's array). This avoids extra handle copy and enables supports for large burst sizes.
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index 78ab6135..f45bc8f1 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -762,17 +762,29 @@ static inline int queue_is_pktin(uint32_t queue_index) return sched->queue[queue_index].poll_pktin; }
-static inline int poll_pktin(uint32_t qi, int stash) +static inline int poll_pktin(uint32_t qi, int direct_recv, + odp_event_t ev_tbl[], int max_num) { - odp_buffer_hdr_t *b_hdr[BURST_SIZE]; - int pktio_index, pktin_index, num, num_pktin, i; + int pktio_index, pktin_index, num, num_pktin; + odp_buffer_hdr_t **hdr_tbl; int ret; void *q_int; + odp_buffer_hdr_t *b_hdr[BURST_SIZE]; + + hdr_tbl = (odp_buffer_hdr_t **)ev_tbl; + + if (!direct_recv) { + hdr_tbl = b_hdr; + + /* Limit burst to max queue enqueue size */ + if (max_num > BURST_SIZE) + max_num = BURST_SIZE; + }
pktio_index = sched->queue[qi].pktio_index; pktin_index = sched->queue[qi].pktin_index;
- num = sched_cb_pktin_poll(pktio_index, pktin_index, b_hdr, BURST_SIZE); + num = sched_cb_pktin_poll(pktio_index, pktin_index, hdr_tbl, max_num);
if (num == 0) return 0; @@ -793,12 +805,8 @@ static inline int poll_pktin(uint32_t qi, int stash) return num; }
- if (stash) { - for (i = 0; i < num; i++) - sched_local.stash_ev[i] = event_from_buf_hdr(b_hdr[i]); - + if (direct_recv) return num; - }
q_int = qentry_from_index(qi);
@@ -908,13 +916,16 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], * priorities. Stop scheduling queue when pktio * has been stopped. */ if (pktin) { - int stash = !ordered; - int num_pkt = poll_pktin(qi, stash); + int direct_recv = !ordered; + int num_pkt; + + num_pkt = poll_pktin(qi, direct_recv, + ev_tbl, max_deq);
if (odp_unlikely(num_pkt < 0)) continue;
- if (num_pkt == 0 || !stash) { + if (num_pkt == 0 || !direct_recv) { ring_enq(ring, ring_mask, qi); break; } @@ -922,7 +933,6 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], /* Process packets from an atomic or * parallel queue right away. */ num = num_pkt; - stashed = 1; } else { /* Remove empty queue from scheduling. * Continue scheduling the same priority
commit c0d530e01e91fbea195993293eed60f1677e6181 Author: Petri Savolainen petri.savolainen@linaro.org Date: Wed Jul 4 14:11:59 2018 +0300
linux-gen: sched: support large burst sizes
Small burst sizes are rounded up to CONFIG_BURST_SIZE and excess events are stashed. Larger (>CONFIG_BURST_SIZE) bursts are stored directly into application buffer and nothing is stashed.
Signed-off-by: Petri Savolainen petri.savolainen@linaro.org Reviewed-by: Bill Fischofer bill.fischofer@linaro.org Signed-off-by: Maxim Uvarov maxim.uvarov@linaro.org
diff --git a/platform/linux-generic/odp_schedule_basic.c b/platform/linux-generic/odp_schedule_basic.c index c81e417c..78ab6135 100644 --- a/platform/linux-generic/odp_schedule_basic.c +++ b/platform/linux-generic/odp_schedule_basic.c @@ -92,8 +92,9 @@ ODP_STATIC_ASSERT((8 * sizeof(pri_mask_t)) >= MAX_SPREAD, /* Start of named groups in group mask arrays */ #define SCHED_GROUP_NAMED (ODP_SCHED_GROUP_CONTROL + 1)
-/* Maximum number of dequeues */ -#define MAX_DEQ CONFIG_BURST_SIZE +/* Default burst size. Scheduler rounds up number of requested events up to + * this value. */ +#define BURST_SIZE CONFIG_BURST_SIZE
/* Ordered stash size */ #define MAX_ORDERED_STASH 512 @@ -123,7 +124,7 @@ typedef struct { uint16_t spread_round; uint32_t stash_qi; odp_queue_t stash_queue; - odp_event_t stash_ev[MAX_DEQ]; + odp_event_t stash_ev[BURST_SIZE];
uint32_t grp_epoch; uint16_t num_grp; @@ -763,7 +764,7 @@ static inline int queue_is_pktin(uint32_t queue_index)
static inline int poll_pktin(uint32_t qi, int stash) { - odp_buffer_hdr_t *b_hdr[MAX_DEQ]; + odp_buffer_hdr_t *b_hdr[BURST_SIZE]; int pktio_index, pktin_index, num, num_pktin, i; int ret; void *q_int; @@ -771,7 +772,7 @@ static inline int poll_pktin(uint32_t qi, int stash) pktio_index = sched->queue[qi].pktio_index; pktin_index = sched->queue[qi].pktin_index;
- num = sched_cb_pktin_poll(pktio_index, pktin_index, b_hdr, MAX_DEQ); + num = sched_cb_pktin_poll(pktio_index, pktin_index, b_hdr, BURST_SIZE);
if (num == 0) return 0; @@ -824,7 +825,6 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], int ret; int id; uint32_t qi; - unsigned int max_deq = MAX_DEQ; int num_spread = sched->config.num_spread; uint32_t ring_mask = sched->ring_mask;
@@ -843,6 +843,9 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], odp_queue_t handle; ring_t *ring; int pktin; + unsigned int max_deq = BURST_SIZE; + int stashed = 1; + odp_event_t *ev_tbl = sched_local.stash_ev;
if (id >= num_spread) id = 0; @@ -866,24 +869,30 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], continue; }
- /* Low priorities have smaller batch size to limit - * head of line blocking latency. */ - if (odp_unlikely(MAX_DEQ > 1 && - prio > ODP_SCHED_PRIO_DEFAULT)) - max_deq = MAX_DEQ / 2; - ordered = queue_is_ordered(qi);
- /* Do not cache ordered events locally to improve + /* When application's array is larger than our burst + * size, output all events directly there. Also, ordered + * queues are not stashed locally to improve * parallelism. Ordered context can only be released * when the local cache is empty. */ - if (ordered && max_num < MAX_DEQ) + if (max_num > BURST_SIZE || ordered) { + stashed = 0; + ev_tbl = out_ev; max_deq = max_num; + } + + /* Low priorities have smaller burst size to limit + * head of line blocking latency. */ + if (BURST_SIZE > 1 && + odp_unlikely(prio > ODP_SCHED_PRIO_DEFAULT) && + max_deq > BURST_SIZE / 2) + max_deq = BURST_SIZE / 2;
pktin = queue_is_pktin(qi);
- num = sched_cb_queue_deq_multi(qi, sched_local.stash_ev, - max_deq, !pktin); + num = sched_cb_queue_deq_multi(qi, ev_tbl, max_deq, + !pktin);
if (num < 0) { /* Destroyed queue. Continue scheduling the same @@ -913,6 +922,7 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], /* Process packets from an atomic or * parallel queue right away. */ num = num_pkt; + stashed = 1; } else { /* Remove empty queue from scheduling. * Continue scheduling the same priority @@ -943,10 +953,16 @@ static inline int do_schedule_grp(odp_queue_t *out_queue, odp_event_t out_ev[], }
handle = queue_from_index(qi); - sched_local.stash_num = num; - sched_local.stash_index = 0; - sched_local.stash_queue = handle; - ret = copy_from_stash(out_ev, max_num); + + if (stashed) { + sched_local.stash_num = num; + sched_local.stash_index = 0; + sched_local.stash_queue = handle; + ret = copy_from_stash(out_ev, max_num); + } else { + sched_local.stash_num = 0; + ret = num; + }
/* Output the source queue handle */ if (out_queue)
-----------------------------------------------------------------------
Summary of changes: config/odp-linux-generic.conf | 11 ++ platform/linux-generic/Makefile.am | 2 +- ...queue_internal.h => odp_queue_basic_internal.h} | 11 +- platform/linux-generic/include/odp_schedule_if.h | 6 - platform/linux-generic/odp_queue_basic.c | 84 +++++++++----- platform/linux-generic/odp_queue_lf.c | 2 +- platform/linux-generic/odp_queue_spsc.c | 2 +- platform/linux-generic/odp_schedule_basic.c | 128 +++++++++++++++------ platform/linux-generic/odp_schedule_iquery.c | 13 +-- platform/linux-generic/odp_schedule_sp.c | 10 +- 10 files changed, 176 insertions(+), 93 deletions(-) rename platform/linux-generic/include/{odp_queue_internal.h => odp_queue_basic_internal.h} (86%)
hooks/post-receive