git-lng-odp June 2017

git-lng-odp@lists.linaro.org

1 participants
47 discussions

[COMMIT] branch master updated. v1.15.0.0-15-gceeab69f

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via ceeab69f3af67701adb524c7b9757d19cefb1110 (commit) from 9b78da8e2373ab0530cfbbd72b2229059b57ddf1 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit ceeab69f3af67701adb524c7b9757d19cefb1110 Author: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Date: Mon Jun 26 16:39:35 2017 +0300 travis: add cross-compilation checks Include setup to cross-compile ODP for ARM hard float, ARM64 and PowerPC targets. Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index bae23600..f0567e59 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,12 +38,15 @@ addons: apt: sources: - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.8 packages: - gcc - clang-3.8 - automake autoconf libtool libssl-dev graphviz mscgen doxygen - libpcap-dev + - dpkg-cross equivs + - gcc-aarch64-linux-gnu pkg-config-aarch64-linux-gnu libc6-dev-arm64-cross + - gcc-arm-linux-gnueabihf pkg-config-arm-linux-gnueabihf libc6-dev-armhf-cross + - gcc-powerpc-linux-gnu pkg-config-powerpc-linux-gnu libc6-dev-powerpc-cross # coverity_scan: # project: # name: "$TRAVIS_REPO_SLUG" @@ -62,30 +65,35 @@ env: - CONF="--enable-schedule-sp" - CONF="--enable-schedule-iquery" -install: - - echo 1000 | sudo tee /proc/sys/vm/nr_hugepages - - sudo mkdir -p /mnt/huge - - sudo mount -t hugetlbfs nodev /mnt/huge - - - sudo apt-get -qq update - - sudo apt-get install linux-headers-`uname -r` - - sudo pip install coverage - - gem install asciidoctor - - PATH=${PATH//:\.\/node_modules\/\.bin/} - +before_install: # Install cunit for the validation tests because distro version is too old and fails C99 compile - sudo apt-get remove libcunit1-dev libcunit1 - export CUNIT_VERSION=2.1-3 - curl -sSOL https://github.com/Linaro/libcunit/releases/download/${CUNIT_VERSION}/CUnit… - tar -jxf *.bz2 - pushd CUnit* - - ./bootstrap - - ./configure --enable-debug --enable-automated --enable-basic --enable-console --enable-examples --enable-test + - libtoolize --force --copy + - aclocal + - autoheader + - automake --add-missing --include-deps --copy + - autoconf + - ./configure --enable-debug --enable-automated --enable-basic --enable-console --enable-examples --enable-test $CROSS || cat config.log - make - sudo make install - popd - export LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" +install: + - echo 1000 | sudo tee /proc/sys/vm/nr_hugepages + - sudo mkdir -p /mnt/huge + - sudo mount -t hugetlbfs nodev /mnt/huge + + - sudo apt-get -qq update + - sudo apt-get install linux-headers-`uname -r` + - sudo pip install coverage + - gem install asciidoctor + - PATH=${PATH//:\.\/node_modules\/\.bin/} + # DPDK pktio - TARGET=${TARGET:-"x86_64-native-linuxapp-gcc"} - git -c advice.detachedHead=false clone -q --depth=1 --single-branch --branch=v17.02 http://dpdk.org/git/dpdk dpdk @@ -125,6 +133,69 @@ script: jobs: include: - stage: test + compiler: aarch64-linux-gnu-gcc + env: TEST="aarch64-linux-gnu" CROSS="--host=aarch64-linux-gnu --prefix=/usr/aarch64-linux-gnu" + install: true + script: + - mkdir cross + - pushd cross + # one can not include colon in the command or bad things will happen, so mimic it + - env echo -e 'Provides\x3a multiarch-support, debconf, debconf-2.0' > dummy + - equivs-build -a arm64 dummy + - wget + http://ports.ubuntu.com/pool/main/o/openssl/libssl-dev_1.0.1f-1ubuntu2.22_a… + http://ports.ubuntu.com/pool/main/o/openssl/libssl1.0.0_1.0.1f-1ubuntu2.22_… + http://ports.ubuntu.com/pool/main/z/zlib/zlib1g-dev_1.2.8.dfsg-1ubuntu1_arm… + http://ports.ubuntu.com/pool/main/z/zlib/zlib1g_1.2.8.dfsg-1ubuntu1_arm64.d… + - sudo dpkg-cross -i -M -A -a arm64 *.deb + - popd + - ./bootstrap + - ./configure --prefix=$HOME/odp-install $CROSS + --disable-test-cpp --enable-test-vald --enable-test-helper --enable-test-perf --enable-test-perf-proc --enable-test-example + - make -j $(nproc) + - stage: test + compiler: arm-linux-gnueabihf-gcc + env: TEST="arm-linux-gnueabihf" CROSS="--host=arm-linux-gnueabihf --prefix=/usr/arm-linux-gnueabihf" + install: true + script: + - mkdir cross + - pushd cross + # one can not include colon in the command or bad things will happen, so mimic it + - env echo -e 'Provides\x3a multiarch-support, debconf, debconf-2.0' > dummy + - equivs-build -a armhf dummy + - wget + http://ports.ubuntu.com/pool/main/o/openssl/libssl-dev_1.0.1f-1ubuntu2.22_a… + http://ports.ubuntu.com/pool/main/o/openssl/libssl1.0.0_1.0.1f-1ubuntu2.22_… + http://ports.ubuntu.com/pool/main/z/zlib/zlib1g-dev_1.2.8.dfsg-1ubuntu1_arm… + http://ports.ubuntu.com/pool/main/z/zlib/zlib1g_1.2.8.dfsg-1ubuntu1_armhf.d… + - sudo dpkg-cross -i -M -A -a armhf *.deb + - popd + - ./bootstrap + - ./configure --prefix=$HOME/odp-install $CROSS + --disable-test-cpp --enable-test-vald --enable-test-helper --enable-test-perf --enable-test-perf-proc --enable-test-example + - make -j $(nproc) + - stage: test + compiler: powerpc-linux-gnu-gcc + env: TEST="powerpc-linux-gnu" CROSS="--host=powerpc-linux-gnu --prefix=/usr/powerpc-linux-gnu" + install: true + script: + - mkdir cross + - pushd cross + # one can not include colon in the command or bad things will happen, so mimic it + - env echo -e 'Provides\x3a multiarch-support, debconf, debconf-2.0' > dummy + - equivs-build -a powerpc dummy + - wget + http://ports.ubuntu.com/pool/main/o/openssl/libssl-dev_1.0.1f-1ubuntu2.22_p… + http://ports.ubuntu.com/pool/main/o/openssl/libssl1.0.0_1.0.1f-1ubuntu2.22_… + http://ports.ubuntu.com/pool/main/z/zlib/zlib1g-dev_1.2.8.dfsg-1ubuntu1_pow… + http://ports.ubuntu.com/pool/main/z/zlib/zlib1g_1.2.8.dfsg-1ubuntu1_powerpc… + - sudo dpkg-cross -i -M -A -a powerpc *.deb + - popd + - ./bootstrap + - ./configure --prefix=$HOME/odp-install $CROSS + --disable-test-cpp --enable-test-vald --enable-test-helper --enable-test-perf --enable-test-perf-proc --enable-test-example + - make -j $(nproc) + - stage: test env: TEST=coverage compiler: gcc script: @@ -158,4 +229,5 @@ jobs: - ./scripts/ci-checkpatches.sh ${ODP_PATCHES}; after_failure: + - cat config.log - find . -name 'test-suite.log' -execdir grep -il "FAILED" {} \; -exec echo {} \; -exec cat {} \; ----------------------------------------------------------------------- Summary of changes: .travis.yml | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 86 insertions(+), 14 deletions(-) hooks/post-receive --

7 years, 7 months

[COMMIT] branch api-next updated. v1.15.0.0-387-g7a581304

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via 7a5813042d58598e1c66243d8cfed548302edfc4 (commit) via 759a54bf66317fee539f2b2649c283be937d832a (commit) via 9fd48a9215a7831ca951839b7187bd1eb3f7bb06 (commit) via bf554b2d9472ed8ed04580904d0aa906cdbe1e83 (commit) via ac89f9c568830498b186dff33e908053dbdc88c7 (commit) via 5b08cc290a4a84ed0ffbc4a591427f1891b9ace3 (commit) from 5551c0d76bd721110a0fa4ba1423fe1edb0fd530 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 7a5813042d58598e1c66243d8cfed548302edfc4 Author: Honnappa Nagarahalli <honnappa.nagarahalli(a)arm.com> Date: Fri Jun 23 16:04:41 2017 -0500 travis: add scalable scheduler in CI Added running tests with scalable scheduler to CI Signed-off-by: Honnappa Nagarahalli <honnappa.nagarahalli(a)arm.com> Reviewed-by: Brian Brooks <brian.brooks(a)arm.com> Reviewed-and-tested-by: Yi He <yi.he(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index f74c564f..7304e0ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,6 +62,7 @@ env: - CONF="--disable-abi-compat" - CONF="--enable-schedule-sp" - CONF="--enable-schedule-iquery" + - CONF="--enable-schedule-scalable" install: - echo 1000 | sudo tee /proc/sys/vm/nr_hugepages commit 759a54bf66317fee539f2b2649c283be937d832a Author: Brian Brooks <brian.brooks(a)arm.com> Date: Fri Jun 23 16:04:40 2017 -0500 linux-gen: sched scalable: add scalable scheduler Signed-off-by: Brian Brooks <brian.brooks(a)arm.com> Signed-off-by: Kevin Wang <kevin.wang(a)arm.com> Signed-off-by: Honnappa Nagarahalli <honnappa.nagarahalli(a)arm.com> Signed-off-by: Ola Liljedahl <ola.liljedahl(a)arm.com> Reviewed-and-tested-by: Yi He <yi.he(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 19e2241b..82ab4642 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -185,9 +185,13 @@ noinst_HEADERS = \ ${srcdir}/include/odp_pool_internal.h \ ${srcdir}/include/odp_posix_extensions.h \ ${srcdir}/include/odp_queue_internal.h \ + ${srcdir}/include/odp_queue_scalable_internal.h \ ${srcdir}/include/odp_ring_internal.h \ ${srcdir}/include/odp_queue_if.h \ ${srcdir}/include/odp_schedule_if.h \ + ${srcdir}/include/odp_schedule_scalable.h \ + ${srcdir}/include/odp_schedule_scalable_config.h \ + ${srcdir}/include/odp_schedule_scalable_ordered.h \ ${srcdir}/include/odp_sorted_list_internal.h \ ${srcdir}/include/odp_shm_internal.h \ ${srcdir}/include/odp_time_internal.h \ @@ -259,12 +263,15 @@ __LIB__libodp_linux_la_SOURCES = \ odp_pool.c \ odp_queue.c \ odp_queue_if.c \ + odp_queue_scalable.c \ odp_rwlock.c \ odp_rwlock_recursive.c \ odp_schedule.c \ odp_schedule_if.c \ odp_schedule_sp.c \ odp_schedule_iquery.c \ + odp_schedule_scalable.c \ + odp_schedule_scalable_ordered.c \ odp_shared_memory.c \ odp_sorted_list.c \ odp_spinlock.c \ diff --git a/platform/linux-generic/include/odp/api/plat/schedule_types.h b/platform/linux-generic/include/odp/api/plat/schedule_types.h index 535fd6d0..4e75f9ee 100644 --- a/platform/linux-generic/include/odp/api/plat/schedule_types.h +++ b/platform/linux-generic/include/odp/api/plat/schedule_types.h @@ -18,6 +18,8 @@ extern "C" { #endif +#include <odp/api/std_types.h> + /** @addtogroup odp_scheduler * @{ */ @@ -44,7 +46,7 @@ typedef int odp_schedule_sync_t; typedef int odp_schedule_group_t; /* These must be kept in sync with thread_globals_t in odp_thread.c */ -#define ODP_SCHED_GROUP_INVALID -1 +#define ODP_SCHED_GROUP_INVALID ((odp_schedule_group_t)-1) #define ODP_SCHED_GROUP_ALL 0 #define ODP_SCHED_GROUP_WORKER 1 #define ODP_SCHED_GROUP_CONTROL 2 diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index dadd59e7..3cff0045 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -7,10 +7,6 @@ #ifndef ODP_CONFIG_INTERNAL_H_ #define ODP_CONFIG_INTERNAL_H_ -#ifdef __cplusplus -extern "C" { -#endif - /* * Maximum number of pools */ @@ -22,6 +18,13 @@ extern "C" { #define ODP_CONFIG_QUEUES 1024 /* + * Maximum queue depth. Maximum number of elements that can be stored in a + * queue. This value is used only when the size is not explicitly provided + * during queue creation. + */ +#define CONFIG_QUEUE_SIZE 4096 + +/* * Maximum number of ordered locks per queue */ #define CONFIG_QUEUE_MAX_ORD_LOCKS 4 @@ -139,8 +142,4 @@ extern "C" { */ #define CONFIG_POOL_CACHE_SIZE 256 -#ifdef __cplusplus -} -#endif - #endif diff --git a/platform/linux-generic/include/odp_queue_scalable_internal.h b/platform/linux-generic/include/odp_queue_scalable_internal.h new file mode 100644 index 00000000..f15314b2 --- /dev/null +++ b/platform/linux-generic/include/odp_queue_scalable_internal.h @@ -0,0 +1,104 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_QUEUE_SCALABLE_INTERNAL_H_ +#define ODP_QUEUE_SCALABLE_INTERNAL_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <odp/api/queue.h> +#include <odp_forward_typedefs_internal.h> +#include <odp_queue_if.h> +#include <odp_buffer_internal.h> +#include <odp_align_internal.h> +#include <odp/api/packet_io.h> +#include <odp/api/align.h> +#include <odp/api/hints.h> +#include <odp/api/ticketlock.h> +#include <odp_config_internal.h> +#include <odp_schedule_scalable.h> +#include <odp_schedule_scalable_ordered.h> + +#define QUEUE_STATUS_FREE 0 +#define QUEUE_STATUS_DESTROYED 1 +#define QUEUE_STATUS_READY 2 + +struct queue_entry_s { + sched_elem_t sched_elem; + + odp_ticketlock_t lock ODP_ALIGNED_CACHE; + int status; + + queue_enq_fn_t enqueue ODP_ALIGNED_CACHE; + queue_deq_fn_t dequeue; + queue_enq_multi_fn_t enqueue_multi; + queue_deq_multi_fn_t dequeue_multi; + + uint32_t index; + odp_queue_t handle; + odp_queue_type_t type; + odp_queue_param_t param; + odp_pktin_queue_t pktin; + odp_pktout_queue_t pktout; + char name[ODP_QUEUE_NAME_LEN]; +}; + +union queue_entry_u { + struct queue_entry_s s; + uint8_t pad[ROUNDUP_CACHE_LINE(sizeof(struct queue_entry_s))]; +}; + +int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num); +int _odp_queue_deq_sc(sched_elem_t *q, odp_event_t *evp, int num); +int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num); + +/* Round up memory size to next cache line size to + * align all memory addresses on cache line boundary. + */ +static inline void *shm_pool_alloc_align(_odp_ishm_pool_t *pool, uint32_t size) +{ + void *addr; + + addr = _odp_ishm_pool_alloc(pool, ROUNDUP_CACHE_LINE(size)); + ODP_ASSERT(((uintptr_t)addr & (ODP_CACHE_LINE_SIZE - 1)) == 0); + + return addr; +} + +static inline uint32_t queue_to_id(odp_queue_t handle) +{ + return _odp_typeval(handle) - 1; +} + +static inline queue_entry_t *qentry_from_int(queue_t handle) +{ + return (queue_entry_t *)(void *)(handle); +} + +static inline queue_t qentry_to_int(queue_entry_t *qentry) +{ + return (queue_t)(qentry); +} + +static inline odp_queue_t queue_get_handle(queue_entry_t *queue) +{ + return queue->s.handle; +} + +static inline reorder_window_t *queue_get_rwin(queue_entry_t *queue) +{ + return queue->s.sched_elem.rwin; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index 5d10cd37..5877a1cd 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -12,7 +12,7 @@ extern "C" { #endif #include <odp/api/queue.h> -#include <odp_queue_internal.h> +#include <odp_queue_if.h> #include <odp/api/schedule.h> typedef void (*schedule_pktio_start_fn_t)(int pktio_index, int num_in_queue, diff --git a/platform/linux-generic/include/odp_schedule_scalable.h b/platform/linux-generic/include/odp_schedule_scalable.h new file mode 100644 index 00000000..8a2d70da --- /dev/null +++ b/platform/linux-generic/include/odp_schedule_scalable.h @@ -0,0 +1,139 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_SCHEDULE_SCALABLE_H +#define ODP_SCHEDULE_SCALABLE_H + +#include <odp/api/align.h> +#include <odp/api/schedule.h> +#include <odp/api/ticketlock.h> + +#include <odp_schedule_scalable_config.h> +#include <odp_schedule_scalable_ordered.h> +#include <odp_llqueue.h> + +/* + * ODP_SCHED_PRIO_HIGHEST/NORMAL/LOWEST/DEFAULT are compile time + * constants, but not ODP_SCHED_PRIO_NUM. The current API for this + * is odp_schedule_num_prio(). The other schedulers also define + * this internally as NUM_PRIO. + */ +#define ODP_SCHED_PRIO_NUM 8 + +typedef struct { + union { + struct { + struct llqueue llq; + uint32_t prio; + }; + char line[ODP_CACHE_LINE_SIZE]; + }; +} sched_queue_t ODP_ALIGNED_CACHE; + +#define TICKET_INVALID (uint16_t)(~0U) + +typedef struct { + int32_t numevts; + uint16_t wrr_budget; + uint8_t cur_ticket; + uint8_t nxt_ticket; +} qschedstate_t ODP_ALIGNED(sizeof(uint64_t)); + +typedef uint32_t ringidx_t; + +#ifdef CONFIG_SPLIT_PRODCONS +#define SPLIT_PC ODP_ALIGNED_CACHE +#else +#define SPLIT_PC +#endif + +#define ODP_NO_SCHED_QUEUE (ODP_SCHED_SYNC_ORDERED + 1) + +typedef struct { + struct llnode node; /* must be first */ + sched_queue_t *schedq; +#ifdef CONFIG_QSCHST_LOCK + odp_ticketlock_t qschlock; +#endif + qschedstate_t qschst; + uint16_t pop_deficit; + uint16_t qschst_type; + ringidx_t prod_read SPLIT_PC; + ringidx_t prod_write; + ringidx_t prod_mask; + odp_buffer_hdr_t **prod_ring; + ringidx_t cons_write SPLIT_PC; + ringidx_t cons_read; + reorder_window_t *rwin; + void *user_ctx; +#ifdef CONFIG_SPLIT_PRODCONS + odp_buffer_hdr_t **cons_ring; + ringidx_t cons_mask; + uint16_t cons_type; +#else +#define cons_mask prod_mask +#define cons_ring prod_ring +#define cons_type qschst_type +#endif +} sched_elem_t ODP_ALIGNED_CACHE; + +/* Number of scheduling groups */ +#define MAX_SCHED_GROUP (sizeof(sched_group_mask_t) * CHAR_BIT) + +typedef bitset_t sched_group_mask_t; + +typedef struct { + /* Threads currently associated with the sched group */ + bitset_t thr_actual[ODP_SCHED_PRIO_NUM] ODP_ALIGNED_CACHE; + bitset_t thr_wanted; + /* Used to spread queues over schedq's */ + uint32_t xcount[ODP_SCHED_PRIO_NUM]; + /* Number of schedq's per prio */ + uint32_t xfactor; + char name[ODP_SCHED_GROUP_NAME_LEN]; + /* ODP_SCHED_PRIO_NUM * xfactor. Must be last. */ + sched_queue_t schedq[1] ODP_ALIGNED_CACHE; +} sched_group_t; + +/* Number of reorder contexts per thread */ +#define TS_RVEC_SIZE 16 + +typedef struct { + /* Atomic queue currently being processed or NULL */ + sched_elem_t *atomq; + /* Current reorder context or NULL */ + reorder_context_t *rctx; + uint8_t pause; + uint8_t out_of_order; + uint8_t tidx; + uint8_t pad; + uint32_t dequeued; /* Number of events dequeued from atomic queue */ + uint16_t pktin_next; /* Next pktin tag to poll */ + uint16_t pktin_poll_cnts; + uint16_t ticket; /* Ticket for atomic queue or TICKET_INVALID */ + uint16_t num_schedq; + uint16_t sg_sem; /* Set when sg_wanted is modified by other thread */ +#define SCHEDQ_PER_THREAD (MAX_SCHED_GROUP * ODP_SCHED_PRIO_NUM) + sched_queue_t *schedq_list[SCHEDQ_PER_THREAD]; + /* Current sched_group membership */ + sched_group_mask_t sg_actual[ODP_SCHED_PRIO_NUM]; + /* Future sched_group membership. */ + sched_group_mask_t sg_wanted[ODP_SCHED_PRIO_NUM]; + bitset_t priv_rvec_free; + /* Bitset of free entries in rvec[] */ + bitset_t rvec_free ODP_ALIGNED_CACHE; + /* Reordering contexts to allocate from */ + reorder_context_t rvec[TS_RVEC_SIZE] ODP_ALIGNED_CACHE; +} sched_scalable_thread_state_t ODP_ALIGNED_CACHE; + +void sched_update_enq(sched_elem_t *q, uint32_t actual); +void sched_update_enq_sp(sched_elem_t *q, uint32_t actual); +sched_queue_t *schedq_from_sched_group(odp_schedule_group_t grp, uint32_t prio); +void sched_group_xcount_dec(odp_schedule_group_t grp, uint32_t prio); + +#endif /* ODP_SCHEDULE_SCALABLE_H */ diff --git a/platform/linux-generic/include/odp_schedule_scalable_config.h b/platform/linux-generic/include/odp_schedule_scalable_config.h new file mode 100644 index 00000000..b9a9a55f --- /dev/null +++ b/platform/linux-generic/include/odp_schedule_scalable_config.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_SCHEDULE_SCALABLE_CONFIG_H_ +#define ODP_SCHEDULE_SCALABLE_CONFIG_H_ + +/* + * Default scaling factor for the scheduler group + * + * This scaling factor is used when the application creates a scheduler + * group with no worker threads. + */ +#define CONFIG_DEFAULT_XFACTOR 4 + +/* + * Default weight (in events) for WRR in scalable scheduler + * + * This controls the per-queue weight for WRR between queues of the same + * priority in the scalable scheduler + * A higher value improves throughput while a lower value increases fairness + * and thus likely decreases latency + * + * If WRR is undesired, set the value to ~0 which will use the largest possible + * weight + * + * Note: an API for specifying this on a per-queue basis would be useful but is + * not yet available + */ +#define CONFIG_WRR_WEIGHT 64 + +/* + * Split queue producer/consumer metadata into separate cache lines. + * This is beneficial on e.g. Cortex-A57 but not so much on A53. + */ +#define CONFIG_SPLIT_PRODCONS + +/* + * Use locks to protect queue (ring buffer) and scheduler state updates + * On x86, this decreases overhead noticeably. + */ +#if !defined(__arm__) && !defined(__aarch64__) +#define CONFIG_QSCHST_LOCK +/* Keep all ring buffer/qschst data together when using locks */ +#undef CONFIG_SPLIT_PRODCONS +#endif + +#endif /* ODP_SCHEDULE_SCALABLE_CONFIG_H_ */ diff --git a/platform/linux-generic/include/odp_schedule_scalable_ordered.h b/platform/linux-generic/include/odp_schedule_scalable_ordered.h new file mode 100644 index 00000000..941304b7 --- /dev/null +++ b/platform/linux-generic/include/odp_schedule_scalable_ordered.h @@ -0,0 +1,132 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_SCHEDULE_SCALABLE_ORDERED_H +#define ODP_SCHEDULE_SCALABLE_ORDERED_H + +#include <odp/api/shared_memory.h> + +#include <odp_internal.h> +#include <odp_align_internal.h> +#include <odp_bitset.h> +#include <_ishmpool_internal.h> + +/* High level functioning of reordering + * Datastructures - + * Reorder Window - Every ordered queue is associated with a reorder window. + * Reorder window stores reorder contexts from threads that + * have completed processing out-of-order. + * Reorder Context - Reorder context consists of events that a thread + * wants to enqueue while processing a batch of events + * from an ordered queue. + * + * Algorithm - + * 1) Thread identifies the ordered queue. + * 2) It 'reserves a slot in the reorder window and dequeues the + * events' atomically. Atomicity is achieved by using a ticket-lock + * like design where the reorder window slot is the ticket. + * 3a) Upon order-release/next schedule call, the thread + * checks if it's slot (ticket) equals the head of the reorder window. + * If yes, enqueues the events to the destination queue till + * i) the reorder window is empty or + * ii) there is a gap in the reorder window + * If no, the reorder context is stored in the reorder window at + * the reserved slot. + * 3b) Upon the first enqueue, the thread checks if it's slot (ticket) + * equals the head of the reorder window. + * If yes, enqueues the events immediately to the destination queue + * If no, these (and subsequent) events are stored in the reorder context + * (in the application given order) + */ + +/* Head and change indicator variables are used to synchronise between + * concurrent insert operations in the reorder window. A thread performing + * an in-order insertion must be notified about the newly inserted + * reorder contexts so that it doesn’t halt the retire process too early. + * A thread performing an out-of-order insertion must correspondingly + * notify the thread doing in-order insertion of the new waiting reorder + * context, which may need to be handled by that thread. + * + * Also, an out-of-order insertion may become an in-order insertion if the + * thread doing an in-order insertion completes before this thread completes. + * We need a point of synchronisation where this knowledge and potential state + * change can be transferred between threads. + */ +typedef struct hc { + /* First missing context */ + uint32_t head; + /* Change indicator */ + uint32_t chgi; +} hc_t ODP_ALIGNED(sizeof(uint64_t)); + +/* Number of reorder contects in the reorder window. + * Should be at least one per CPU. + */ +#define RWIN_SIZE 32 +ODP_STATIC_ASSERT(CHECK_IS_POWER2(RWIN_SIZE), "RWIN_SIZE is not a power of 2"); + +typedef struct reorder_context reorder_context_t; + +typedef struct reorder_window { + /* head and change indicator */ + hc_t hc; + uint32_t winmask; + uint32_t tail; + uint32_t turn; + uint32_t olock[CONFIG_QUEUE_MAX_ORD_LOCKS]; + uint16_t lock_count; + /* Reorder contexts in this window */ + reorder_context_t *ring[RWIN_SIZE]; +} reorder_window_t; + +/* Number of events that can be stored in a reorder context. + * This size is chosen so that there is no space left unused at the end + * of the last cache line (for 64b architectures and 64b handles). + */ +#define RC_EVT_SIZE 18 + +struct reorder_context { + /* Reorder window to which this context belongs */ + reorder_window_t *rwin; + /* Pointer to TS->rvec_free */ + bitset_t *rvec_free; + /* Our slot number in the reorder window */ + uint32_t sn; + uint8_t olock_flags; + /* Our index in thread_state rvec array */ + uint8_t idx; + /* Use to link reorder contexts together */ + uint8_t next_idx; + /* Current reorder context to save events in */ + uint8_t cur_idx; + /* Number of events stored in this reorder context */ + uint8_t numevts; + /* Events stored in this context */ + odp_buffer_hdr_t *events[RC_EVT_SIZE]; + queue_entry_t *destq[RC_EVT_SIZE]; +} ODP_ALIGNED_CACHE; + +reorder_window_t *rwin_alloc(_odp_ishm_pool_t *pool, + unsigned lock_count); +int rwin_free(_odp_ishm_pool_t *pool, reorder_window_t *rwin); +bool rwin_reserve(reorder_window_t *rwin, uint32_t *sn); +void rwin_insert(reorder_window_t *rwin, + reorder_context_t *rctx, + uint32_t sn, + void (*callback)(reorder_context_t *)); +void rctx_init(reorder_context_t *rctx, uint16_t idx, + reorder_window_t *rwin, uint32_t sn); +void rctx_free(const reorder_context_t *rctx); +void olock_unlock(const reorder_context_t *rctx, reorder_window_t *rwin, + uint32_t lock_index); +void olock_release(const reorder_context_t *rctx); +void rctx_retire(reorder_context_t *first); +void rctx_release(reorder_context_t *rctx); +int rctx_save(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num); + +#endif /* ODP_SCHEDULE_SCALABLE_ORDERED_H */ diff --git a/platform/linux-generic/m4/odp_schedule.m4 b/platform/linux-generic/m4/odp_schedule.m4 index 91c19f21..d862b8b2 100644 --- a/platform/linux-generic/m4/odp_schedule.m4 +++ b/platform/linux-generic/m4/odp_schedule.m4 @@ -1,13 +1,44 @@ -AC_ARG_ENABLE([schedule-sp], - [ --enable-schedule-sp enable strict priority scheduler], - [if test x$enableval = xyes; then - schedule_sp_enabled=yes - ODP_CFLAGS="$ODP_CFLAGS -DODP_SCHEDULE_SP" - fi]) +# Checks for --enable-schedule-sp and defines ODP_SCHEDULE_SP and adds +# -DODP_SCHEDULE_SP to CFLAGS. +AC_ARG_ENABLE( + [schedule_sp], + [AC_HELP_STRING([--enable-schedule-sp], + [enable strict priority scheduler])], + [if test "x$enableval" = xyes; then + schedule_sp=true + ODP_CFLAGS="$ODP_CFLAGS -DODP_SCHEDULE_SP" + else + schedule_sp=false + fi], + [schedule_sp=false]) +AM_CONDITIONAL([ODP_SCHEDULE_SP], [test x$schedule_sp = xtrue]) -AC_ARG_ENABLE([schedule-iquery], - [ --enable-schedule-iquery enable interests query (sparse bitmap) scheduler], - [if test x$enableval = xyes; then - schedule_iquery_enabled=yes - ODP_CFLAGS="$ODP_CFLAGS -DODP_SCHEDULE_IQUERY" - fi]) +# Checks for --enable-schedule-iquery and defines ODP_SCHEDULE_IQUERY and adds +# -DODP_SCHEDULE_IQUERY to CFLAGS. +AC_ARG_ENABLE( + [schedule_iquery], + [AC_HELP_STRING([--enable-schedule-iquery], + [enable interests query (sparse bitmap) scheduler])], + [if test "x$enableval" = xyes; then + schedule_iquery=true + ODP_CFLAGS="$ODP_CFLAGS -DODP_SCHEDULE_IQUERY" + else + schedule_iquery=false + fi], + [schedule_iquery=false]) +AM_CONDITIONAL([ODP_SCHEDULE_IQUERY], [test x$schedule_iquery = xtrue]) + +# Checks for --enable-schedule-scalable and defines ODP_SCHEDULE_SCALABLE and +# adds -DODP_SCHEDULE_SCALABLE to CFLAGS. +AC_ARG_ENABLE( + [schedule_scalable], + [AC_HELP_STRING([--enable-schedule-scalable], + [enable scalable scheduler])], + [if test "x$enableval" = xyes; then + schedule_scalable=true + ODP_CFLAGS="$ODP_CFLAGS -DODP_SCHEDULE_SCALABLE" + else + schedule_scalable=false + fi], + [schedule_scalable=false]) +AM_CONDITIONAL([ODP_SCHEDULE_SCALABLE], [test x$schedule_scalable = xtrue]) diff --git a/platform/linux-generic/odp_queue_if.c b/platform/linux-generic/odp_queue_if.c index c91f00eb..d7471dfc 100644 --- a/platform/linux-generic/odp_queue_if.c +++ b/platform/linux-generic/odp_queue_if.c @@ -6,11 +6,19 @@ #include <odp_queue_if.h> +extern const queue_api_t queue_scalable_api; +extern const queue_fn_t queue_scalable_fn; + extern const queue_api_t queue_default_api; extern const queue_fn_t queue_default_fn; +#ifdef ODP_SCHEDULE_SCALABLE +const queue_api_t *queue_api = &queue_scalable_api; +const queue_fn_t *queue_fn = &queue_scalable_fn; +#else const queue_api_t *queue_api = &queue_default_api; const queue_fn_t *queue_fn = &queue_default_fn; +#endif odp_queue_t odp_queue_create(const char *name, const odp_queue_param_t *param) { diff --git a/platform/linux-generic/odp_queue_scalable.c b/platform/linux-generic/odp_queue_scalable.c new file mode 100644 index 00000000..f95f5f93 --- /dev/null +++ b/platform/linux-generic/odp_queue_scalable.c @@ -0,0 +1,1022 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/hints.h> +#include <odp/api/plat/ticketlock_inlines.h> +#include <odp/api/queue.h> +#include <odp/api/schedule.h> +#include <odp/api/shared_memory.h> +#include <odp/api/sync.h> +#include <odp/api/traffic_mngr.h> + +#include <odp_internal.h> +#include <odp_config_internal.h> +#include <odp_debug_internal.h> + +#include <odp_buffer_inlines.h> +#include <odp_packet_io_internal.h> +#include <odp_packet_io_queue.h> +#include <odp_pool_internal.h> +#include <odp_queue_scalable_internal.h> +#include <odp_schedule_if.h> +#include <_ishm_internal.h> +#include <_ishmpool_internal.h> + +#include <string.h> +#include <inttypes.h> + +#define NUM_INTERNAL_QUEUES 64 + +#define MIN(a, b) \ + ({ \ + __typeof__(a) tmp_a = (a); \ + __typeof__(b) tmp_b = (b); \ + tmp_a < tmp_b ? tmp_a : tmp_b; \ + }) + +#define LOCK(a) _odp_ticketlock_lock(a) +#define UNLOCK(a) _odp_ticketlock_unlock(a) +#define LOCK_INIT(a) odp_ticketlock_init(a) + +extern __thread sched_scalable_thread_state_t *sched_ts; + +typedef struct queue_table_t { + queue_entry_t queue[ODP_CONFIG_QUEUES]; +} queue_table_t; + +static queue_table_t *queue_tbl; +_odp_ishm_pool_t *queue_shm_pool; + +static inline odp_queue_t queue_from_id(uint32_t queue_id) +{ + return _odp_cast_scalar(odp_queue_t, queue_id + 1); +} + +static queue_t queue_from_ext(odp_queue_t handle); +static int _queue_enq(queue_t handle, odp_buffer_hdr_t *buf_hdr); +static odp_buffer_hdr_t *_queue_deq(queue_t handle); +static int _queue_enq_multi(queue_t handle, odp_buffer_hdr_t *buf_hdr[], + int num); +static int _queue_deq_multi(queue_t handle, odp_buffer_hdr_t *buf_hdr[], + int num); + +static queue_entry_t *get_qentry(uint32_t queue_id) +{ + return &queue_tbl->queue[queue_id]; +} + +static int _odp_queue_disable_enq(sched_elem_t *q) +{ + ringidx_t old_read, old_write, new_write; + uint32_t size; + + old_write = q->prod_write; + size = q->prod_mask + 1; + do { + /* Need __atomic_load to avoid compiler reordering */ + old_read = __atomic_load_n(&q->prod_read, __ATOMIC_ACQUIRE); + if (old_write != old_read) { + /* Queue is not empty, cannot claim all elements + * Cannot disable enqueue. + */ + return -1; + } + /* Claim all elements in ring */ + new_write = old_write + size; + } while (!__atomic_compare_exchange_n(&q->prod_write, + &old_write, /* Updated on failure */ + new_write, + true, + __ATOMIC_RELAXED, + __ATOMIC_RELAXED)); + /* All remaining elements claimed, no one else can enqueue */ + return 0; +} + +static int queue_init(queue_entry_t *queue, const char *name, + const odp_queue_param_t *param) +{ + ringidx_t ring_idx; + sched_elem_t *sched_elem; + uint32_t ring_size; + odp_buffer_hdr_t **ring; + uint32_t size; + + sched_elem = &queue->s.sched_elem; + ring_size = param->size > 0 ? + ROUNDUP_POWER2_U32(param->size) : CONFIG_QUEUE_SIZE; + strncpy(queue->s.name, name ? name : "", ODP_QUEUE_NAME_LEN - 1); + queue->s.name[ODP_QUEUE_NAME_LEN - 1] = 0; + memcpy(&queue->s.param, param, sizeof(odp_queue_param_t)); + + size = ring_size * sizeof(odp_buffer_hdr_t *); + ring = (odp_buffer_hdr_t **)shm_pool_alloc_align(queue_shm_pool, size); + if (NULL == ring) + return -1; + + for (ring_idx = 0; ring_idx < ring_size; ring_idx++) + ring[ring_idx] = NULL; + + queue->s.type = queue->s.param.type; + queue->s.enqueue = _queue_enq; + queue->s.dequeue = _queue_deq; + queue->s.enqueue_multi = _queue_enq_multi; + queue->s.dequeue_multi = _queue_deq_multi; + queue->s.pktin = PKTIN_INVALID; + + sched_elem->node.next = NULL; +#ifdef CONFIG_QSCHST_LOCK + LOCK_INIT(&sched_elem->qschlock); +#endif + sched_elem->qschst.numevts = 0; + sched_elem->qschst.wrr_budget = CONFIG_WRR_WEIGHT; + sched_elem->qschst.cur_ticket = 0; + sched_elem->qschst.nxt_ticket = 0; + sched_elem->pop_deficit = 0; + if (queue->s.type == ODP_QUEUE_TYPE_SCHED) + sched_elem->qschst_type = queue->s.param.sched.sync; + else + sched_elem->qschst_type = ODP_NO_SCHED_QUEUE; + /* 2nd cache line - enqueue */ + sched_elem->prod_read = 0; + sched_elem->prod_write = 0; + sched_elem->prod_ring = ring; + sched_elem->prod_mask = ring_size - 1; + /* 3rd cache line - dequeue */ + sched_elem->cons_read = 0; + sched_elem->cons_write = 0; + sched_elem->rwin = NULL; + sched_elem->schedq = NULL; + sched_elem->user_ctx = queue->s.param.context; +#ifdef CONFIG_SPLIT_PRODCONS + sched_elem->cons_ring = ring; + sched_elem->cons_mask = ring_size - 1; + sched_elem->cons_type = sched_elem->qschst_type; +#endif + + /* Queue initialized successfully, add it to the sched group */ + if (queue->s.type == ODP_QUEUE_TYPE_SCHED) { + if (queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED) { + sched_elem->rwin = + rwin_alloc(queue_shm_pool, + queue->s.param.sched.lock_count); + if (sched_elem->rwin == NULL) { + ODP_ERR("Reorder window not created\n"); + goto rwin_create_failed; + } + } + sched_elem->schedq = + schedq_from_sched_group(param->sched.group, + param->sched.prio); + } + + return 0; + +rwin_create_failed: + _odp_ishm_pool_free(queue_shm_pool, ring); + + return -1; +} + +static int queue_init_global(void) +{ + uint32_t i; + uint64_t pool_size; + uint64_t min_alloc; + uint64_t max_alloc; + + ODP_DBG("Queue init ... "); + + /* Attach to the pool if it exists */ + queue_shm_pool = _odp_ishm_pool_lookup("queue_shm_pool"); + if (queue_shm_pool == NULL) { + /* Create shared memory pool to allocate shared memory for the + * queues. Use the default queue size. + */ + /* Add size of the array holding the queues */ + pool_size = sizeof(queue_table_t); + /* Add storage required for queues */ + pool_size += (CONFIG_QUEUE_SIZE * sizeof(odp_buffer_hdr_t *)) * + ODP_CONFIG_QUEUES; + /* Add the reorder window size */ + pool_size += sizeof(reorder_window_t) * ODP_CONFIG_QUEUES; + /* Choose min_alloc and max_alloc such that buddy allocator is + * is selected. + */ + min_alloc = 0; + max_alloc = CONFIG_QUEUE_SIZE * sizeof(odp_buffer_hdr_t *); + queue_shm_pool = _odp_ishm_pool_create("queue_shm_pool", + pool_size, + min_alloc, max_alloc, + _ODP_ISHM_SINGLE_VA); + if (queue_shm_pool == NULL) { + ODP_ERR("Failed to allocate shared memory pool for" + " queues\n"); + goto queue_shm_pool_create_failed; + } + } + + queue_tbl = (queue_table_t *) + shm_pool_alloc_align(queue_shm_pool, + sizeof(queue_table_t)); + if (queue_tbl == NULL) { + ODP_ERR("Failed to reserve shared memory for queue table\n"); + goto queue_tbl_ishm_alloc_failed; + } + + memset(queue_tbl, 0, sizeof(queue_table_t)); + + for (i = 0; i < ODP_CONFIG_QUEUES; i++) { + /* init locks */ + queue_entry_t *queue; + + queue = get_qentry(i); + LOCK_INIT(&queue->s.lock); + queue->s.index = i; + queue->s.handle = queue_from_id(i); + } + + ODP_DBG("done\n"); + ODP_DBG("Queue init global\n"); + ODP_DBG(" struct queue_entry_s size %zu\n", + sizeof(struct queue_entry_s)); + ODP_DBG(" queue_entry_t size %zu\n", + sizeof(queue_entry_t)); + ODP_DBG("\n"); + + return 0; + +queue_shm_pool_create_failed: + +queue_tbl_ishm_alloc_failed: + _odp_ishm_pool_destroy(queue_shm_pool); + + return -1; +} + +static int queue_term_global(void) +{ + int ret = 0; + int rc = 0; + queue_entry_t *queue; + int i; + + for (i = 0; i < ODP_CONFIG_QUEUES; i++) { + queue = &queue_tbl->queue[i]; + if (__atomic_load_n(&queue->s.status, + __ATOMIC_RELAXED) != QUEUE_STATUS_FREE) { + ODP_ERR("Not destroyed queue: %s\n", queue->s.name); + rc = -1; + } + } + + _odp_ishm_pool_free(queue_shm_pool, queue_tbl); + + ret = _odp_ishm_pool_destroy(queue_shm_pool); + if (ret < 0) { + ODP_ERR("Failed to destroy shared memory pool for queues\n"); + rc = -1; + } + + return rc; +} + +static int queue_init_local(void) +{ + return 0; +} + +static int queue_term_local(void) +{ + return 0; +} + +static int queue_capability(odp_queue_capability_t *capa) +{ + memset(capa, 0, sizeof(odp_queue_capability_t)); + + /* Reserve some queues for internal use */ + capa->max_queues = ODP_CONFIG_QUEUES - NUM_INTERNAL_QUEUES; + capa->max_ordered_locks = sched_fn->max_ordered_locks(); + capa->max_sched_groups = sched_fn->num_grps(); + capa->sched_prios = odp_schedule_num_prio(); + capa->plain.max_num = ODP_CONFIG_QUEUES - NUM_INTERNAL_QUEUES; + capa->plain.max_size = 0; + capa->sched.max_num = ODP_CONFIG_QUEUES - NUM_INTERNAL_QUEUES; + capa->sched.max_size = 0; + + return 0; +} + +static odp_queue_type_t queue_type(odp_queue_t handle) +{ + return qentry_from_int(queue_from_ext(handle))->s.type; +} + +static odp_schedule_sync_t queue_sched_type(odp_queue_t handle) +{ + return qentry_from_int(queue_from_ext(handle))->s.param.sched.sync; +} + +static odp_schedule_prio_t queue_sched_prio(odp_queue_t handle) +{ + return qentry_from_int(queue_from_ext(handle))->s.param.sched.prio; +} + +static odp_schedule_group_t queue_sched_group(odp_queue_t handle) +{ + return qentry_from_int(queue_from_ext(handle))->s.param.sched.group; +} + +static int queue_lock_count(odp_queue_t handle) +{ + queue_entry_t *queue = qentry_from_int(queue_from_ext(handle)); + + return queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED ? + (int)queue->s.param.sched.lock_count : -1; +} + +static odp_queue_t queue_create(const char *name, + const odp_queue_param_t *param) +{ + int queue_idx; + odp_queue_t handle = ODP_QUEUE_INVALID; + queue_entry_t *queue; + odp_queue_param_t default_param; + + if (param == NULL) { + odp_queue_param_init(&default_param); + param = &default_param; + } + + for (queue_idx = 0; queue_idx < ODP_CONFIG_QUEUES; queue_idx++) { + queue = &queue_tbl->queue[queue_idx]; + + if (queue->s.status != QUEUE_STATUS_FREE) + continue; + + LOCK(&queue->s.lock); + if (queue->s.status == QUEUE_STATUS_FREE) { + if (queue_init(queue, name, param)) { + UNLOCK(&queue->s.lock); + return handle; + } + queue->s.status = QUEUE_STATUS_READY; + handle = queue->s.handle; + UNLOCK(&queue->s.lock); + break; + } + UNLOCK(&queue->s.lock); + } + return handle; +} + +static int queue_destroy(odp_queue_t handle) +{ + queue_entry_t *queue; + sched_elem_t *q; + + if (handle == ODP_QUEUE_INVALID) + return -1; + + queue = qentry_from_int(queue_from_ext(handle)); + LOCK(&queue->s.lock); + if (queue->s.status != QUEUE_STATUS_READY) { + UNLOCK(&queue->s.lock); + return -1; + } + q = &queue->s.sched_elem; + +#ifdef CONFIG_QSCHST_LOCK + LOCK(&q->qschlock); +#endif + if (_odp_queue_disable_enq(q)) { + /* Producer side not empty */ +#ifdef CONFIG_QSCHST_LOCK + UNLOCK(&q->qschlock); +#endif + UNLOCK(&queue->s.lock); + return -1; + } + /* Enqueue is now disabled */ + if (q->cons_read != q->cons_write) { + /* Consumer side is not empty + * Roll back previous change, enable enqueue again. + */ + uint32_t size; + + size = q->prod_mask + 1; + __atomic_fetch_sub(&q->prod_write, size, __ATOMIC_RELAXED); +#ifdef CONFIG_QSCHST_LOCK + UNLOCK(&q->qschlock); +#endif + UNLOCK(&queue->s.lock); + return -1; + } +#ifdef CONFIG_QSCHST_LOCK + UNLOCK(&q->qschlock); +#endif + /* Producer and consumer sides empty, enqueue disabled + * Now wait until schedq state is empty and no outstanding tickets + */ + while (__atomic_load_n(&q->qschst.numevts, __ATOMIC_RELAXED) != 0 || + __atomic_load_n(&q->qschst.cur_ticket, __ATOMIC_RELAXED) != + __atomic_load_n(&q->qschst.nxt_ticket, __ATOMIC_RELAXED)) { + sevl(); + while (wfe() && monitor32((uint32_t *)&q->qschst.numevts, + __ATOMIC_RELAXED) != 0) + doze(); + } + + /* Adjust the spread factor for the queues in the schedule group */ + if (queue->s.type == ODP_QUEUE_TYPE_SCHED) + sched_group_xcount_dec(queue->s.param.sched.group, + queue->s.param.sched.prio); + + _odp_ishm_pool_free(queue_shm_pool, q->prod_ring); + + if (queue->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED) { + if (rwin_free(queue_shm_pool, q->rwin) < 0) { + ODP_ERR("Failed to free reorder window\n"); + UNLOCK(&queue->s.lock); + return -1; + } + } + queue->s.status = QUEUE_STATUS_FREE; + UNLOCK(&queue->s.lock); + return 0; +} + +static int queue_context_set(odp_queue_t handle, void *context, + uint32_t len ODP_UNUSED) +{ + odp_mb_full(); + qentry_from_int(queue_from_ext(handle))->s.param.context = context; + odp_mb_full(); + return 0; +} + +static void *queue_context(odp_queue_t handle) +{ + return qentry_from_int(queue_from_ext(handle))->s.param.context; +} + +static odp_queue_t queue_lookup(const char *name) +{ + uint32_t i; + + for (i = 0; i < ODP_CONFIG_QUEUES; i++) { + queue_entry_t *queue = &queue_tbl->queue[i]; + + if (queue->s.status == QUEUE_STATUS_FREE || + queue->s.status == QUEUE_STATUS_DESTROYED) + continue; + + LOCK(&queue->s.lock); + if (strcmp(name, queue->s.name) == 0) { + /* found it */ + UNLOCK(&queue->s.lock); + return queue->s.handle; + } + UNLOCK(&queue->s.lock); + } + + return ODP_QUEUE_INVALID; +} + +#ifndef CONFIG_QSCHST_LOCK +static inline int _odp_queue_enq(sched_elem_t *q, + odp_buffer_hdr_t *buf_hdr[], + int num) +{ + ringidx_t old_read; + ringidx_t old_write; + ringidx_t new_write; + int actual; + uint32_t mask; + odp_buffer_hdr_t **ring; + + mask = q->prod_mask; + ring = q->prod_ring; + + /* Load producer ring state (read & write index) */ + old_write = __atomic_load_n(&q->prod_write, __ATOMIC_RELAXED); + do { + /* Consumer does store-release prod_read, we need + * load-acquire. + */ + old_read = __atomic_load_n(&q->prod_read, __ATOMIC_ACQUIRE); + + actual = MIN(num, (int)((mask + 1) - (old_write - old_read))); + if (odp_unlikely(actual <= 0)) + return 0; + + new_write = old_write + actual; + } while (!__atomic_compare_exchange_n(&q->prod_write, + &old_write, /* Updated on failure */ + new_write, + true, + __ATOMIC_RELAXED, + __ATOMIC_RELAXED)); + +#ifdef CONFIG_SPLIT_PRODCONS + __builtin_prefetch(&q->cons_write, 0, 0); +#endif + /* Store our event(s) in the ring */ + do { + ring[old_write & mask] = *buf_hdr++; + } while (++old_write != new_write); + old_write -= actual; + +#ifdef CONFIG_SPLIT_PRODCONS + __builtin_prefetch(&q->node, 1, 0); +#endif + /* Wait for our turn to signal consumers */ + if (odp_unlikely(__atomic_load_n(&q->cons_write, + __ATOMIC_RELAXED) != old_write)) { + sevl(); + while (wfe() && monitor32(&q->cons_write, + __ATOMIC_RELAXED) != old_write) + doze(); + } + + /* Signal consumers that events are available (release events) + * Enable other producers to continue + */ + /* Wait for writes (to ring slots) to complete */ + atomic_store_release(&q->cons_write, new_write, /*readonly=*/false); + + return actual; +} + +#else + +static inline int _odp_queue_enq_sp(sched_elem_t *q, + odp_buffer_hdr_t *buf_hdr[], + int num) +{ + ringidx_t old_read; + ringidx_t old_write; + ringidx_t new_write; + int actual; + uint32_t mask; + odp_buffer_hdr_t **ring; + + mask = q->prod_mask; + ring = q->prod_ring; + + /* Load producer ring state (read & write index) */ + old_write = q->prod_write; + /* Consumer does store-release prod_read, we need load-acquire */ + old_read = __atomic_load_n(&q->prod_read, __ATOMIC_ACQUIRE); + actual = MIN(num, (int)((mask + 1) - (old_write - old_read))); + if (odp_unlikely(actual <= 0)) + return 0; + + new_write = old_write + actual; + q->prod_write = new_write; + + /* Store our event(s) in the ring */ + do { + ring[old_write & mask] = *buf_hdr++; + } while (++old_write != new_write); + old_write -= actual; + +#ifdef CONFIG_SPLIT_PRODCONS + __builtin_prefetch(&q->node, 1, 0); +#endif + + /* Signal consumers that events are available (release events) + * Enable other producers to continue + */ +#ifdef CONFIG_QSCHST_LOCK + q->cons_write = new_write; +#else + atomic_store_release(&q->cons_write, new_write, /*readonly=*/false); +#endif + + return actual; +} +#endif + +static int _queue_enq_multi(queue_t handle, odp_buffer_hdr_t *buf_hdr[], + int num) +{ + int actual; + queue_entry_t *queue; + sched_scalable_thread_state_t *ts; + + queue = qentry_from_int(handle); + ts = sched_ts; + if (ts && odp_unlikely(ts->out_of_order)) { + actual = rctx_save(queue, buf_hdr, num); + return actual; + } + +#ifdef CONFIG_QSCHST_LOCK + LOCK(&queue->s.sched_elem.qschlock); + actual = _odp_queue_enq_sp(&queue->s.sched_elem, buf_hdr, num); +#else + actual = _odp_queue_enq(&queue->s.sched_elem, buf_hdr, num); +#endif + + if (odp_likely(queue->s.sched_elem.schedq != NULL && actual != 0)) { + /* Perform scheduler related updates. */ +#ifdef CONFIG_QSCHST_LOCK + sched_update_enq_sp(&queue->s.sched_elem, actual); +#else + sched_update_enq(&queue->s.sched_elem, actual); +#endif + } + +#ifdef CONFIG_QSCHST_LOCK + UNLOCK(&queue->s.sched_elem.qschlock); +#endif + return actual; +} + +static int _queue_enq(queue_t handle, odp_buffer_hdr_t *buf_hdr) +{ + return odp_likely( + _queue_enq_multi(handle, &buf_hdr, 1) == 1) ? 0 : -1; +} + +static int queue_enq_multi(odp_queue_t handle, const odp_event_t ev[], int num) +{ + odp_buffer_hdr_t *buf_hdr[QUEUE_MULTI_MAX]; + queue_entry_t *queue; + int i; + + if (num > QUEUE_MULTI_MAX) + num = QUEUE_MULTI_MAX; + + queue = qentry_from_int(queue_from_ext(handle)); + + for (i = 0; i < num; i++) + buf_hdr[i] = buf_hdl_to_hdr(odp_buffer_from_event(ev[i])); + + return queue->s.enqueue_multi(qentry_to_int(queue), buf_hdr, num); +} + +static int queue_enq(odp_queue_t handle, odp_event_t ev) +{ + odp_buffer_hdr_t *buf_hdr; + queue_entry_t *queue; + + queue = qentry_from_int(queue_from_ext(handle)); + buf_hdr = buf_hdl_to_hdr(odp_buffer_from_event(ev)); + + return queue->s.enqueue(qentry_to_int(queue), buf_hdr); +} + +/* Single-consumer dequeue. */ +int _odp_queue_deq_sc(sched_elem_t *q, odp_event_t *evp, int num) +{ + int actual; + ringidx_t old_read; + ringidx_t old_write; + ringidx_t new_read; + uint32_t mask; + odp_buffer_hdr_t **ring; + + /* Load consumer ring state (read & write index). */ + old_read = q->cons_read; + /* Producer does store-release cons_write, we need load-acquire */ + old_write = __atomic_load_n(&q->cons_write, __ATOMIC_ACQUIRE); + actual = MIN(num, (int)(old_write - old_read)); + + if (odp_unlikely(actual <= 0)) + return 0; + +#ifdef CONFIG_SPLIT_PRODCONS + __builtin_prefetch(&q->node, 1, 0); +#endif + new_read = old_read + actual; + q->cons_read = new_read; + + mask = q->cons_mask; + ring = q->cons_ring; + do { + *evp++ = odp_buffer_to_event( + odp_hdr_to_buf(ring[old_read & mask])); + } while (++old_read != new_read); + + /* Signal producers that empty slots are available + * (release ring slots). Enable other consumers to continue. + */ +#ifdef CONFIG_QSCHST_LOCK + q->prod_read = new_read; +#else + /* Wait for loads (from ring slots) to complete. */ + atomic_store_release(&q->prod_read, new_read, /*readonly=*/true); +#endif + return actual; +} + +inline int _odp_queue_deq(sched_elem_t *q, odp_buffer_hdr_t *buf_hdr[], int num) +{ + int actual; + ringidx_t old_read; + ringidx_t old_write; + ringidx_t new_read; + uint32_t mask; + odp_buffer_hdr_t **ring; + odp_buffer_hdr_t **p_buf_hdr; + + mask = q->cons_mask; + ring = q->cons_ring; + + /* Load consumer ring state (read & write index) */ + old_read = __atomic_load_n(&q->cons_read, __ATOMIC_RELAXED); + do { + /* Need __atomic_load to avoid compiler reordering + * Producer does store-release cons_write, we need + * load-acquire. + */ + old_write = __atomic_load_n(&q->cons_write, __ATOMIC_ACQUIRE); + /* Prefetch ring buffer array */ + __builtin_prefetch(&q->cons_ring[old_read & mask], 0, 0); + + actual = MIN(num, (int)(old_write - old_read)); + if (odp_unlikely(actual <= 0)) + return 0; + + /* Attempt to free ring slot(s) */ + new_read = old_read + actual; + } while (!__atomic_compare_exchange_n(&q->cons_read, + &old_read, /* Updated on failure */ + new_read, + true, + __ATOMIC_RELAXED, + __ATOMIC_RELAXED)); +#ifdef CONFIG_SPLIT_PRODCONS + __builtin_prefetch(&q->prod_read, 0, 0); +#endif + p_buf_hdr = buf_hdr; + do { + *p_buf_hdr++ = ring[old_read & mask]; + } while (++old_read != new_read); + old_read -= actual; + +#ifdef CONFIG_SPLIT_PRODCONS + __builtin_prefetch(&q->node, 1, 0); +#endif + /* Wait for our turn to signal producers */ + if (odp_unlikely(__atomic_load_n(&q->prod_read, __ATOMIC_RELAXED) != + old_read)) { + sevl(); + while (wfe() && monitor32(&q->prod_read, + __ATOMIC_RELAXED) != old_read) + doze(); + } + + /* Signal producers that empty slots are available + * (release ring slots) + * Enable other consumers to continue + */ + /* Wait for loads (from ring slots) to complete */ + atomic_store_release(&q->prod_read, new_read, /*readonly=*/true); + + return actual; +} + +inline int _odp_queue_deq_mc(sched_elem_t *q, odp_event_t *evp, int num) +{ + int ret, evt_idx; + odp_buffer_hdr_t *hdr_tbl[QUEUE_MULTI_MAX]; + + if (num > QUEUE_MULTI_MAX) + num = QUEUE_MULTI_MAX; + + ret = _odp_queue_deq(q, hdr_tbl, num); + if (odp_likely(ret != 0)) { + for (evt_idx = 0; evt_idx < num; evt_idx++) + evp[evt_idx] = odp_buffer_to_event( + odp_hdr_to_buf(hdr_tbl[evt_idx])); + } + + return ret; +} + +static int _queue_deq_multi(queue_t handle, odp_buffer_hdr_t *buf_hdr[], + int num) +{ + sched_elem_t *q; + queue_entry_t *queue; + + queue = qentry_from_int(handle); + q = &queue->s.sched_elem; + return _odp_queue_deq(q, buf_hdr, num); +} + +static odp_buffer_hdr_t *_queue_deq(queue_t handle) +{ + sched_elem_t *q; + odp_buffer_hdr_t *buf_hdr; + queue_entry_t *queue; + + queue = qentry_from_int(handle); + q = &queue->s.sched_elem; + if (_odp_queue_deq(q, &buf_hdr, 1) == 1) + return buf_hdr; + else + return NULL; +} + +static int queue_deq_multi(odp_queue_t handle, odp_event_t events[], int num) +{ + queue_entry_t *queue; + odp_buffer_hdr_t *buf_hdr[QUEUE_MULTI_MAX]; + int i, ret; + + if (num > QUEUE_MULTI_MAX) + num = QUEUE_MULTI_MAX; + + queue = qentry_from_int(queue_from_ext(handle)); + + ret = queue->s.dequeue_multi(qentry_to_int(queue), buf_hdr, num); + + for (i = 0; i < ret; i++) + events[i] = odp_buffer_to_event(buf_hdr[i]->handle.handle); + + return ret; +} + +static odp_event_t queue_deq(odp_queue_t handle) +{ + queue_entry_t *queue; + odp_buffer_hdr_t *buf_hdr; + + queue = qentry_from_int(queue_from_ext(handle)); + buf_hdr = queue->s.dequeue(qentry_to_int(queue)); + + if (buf_hdr) + return odp_buffer_to_event(buf_hdr->handle.handle); + + return ODP_EVENT_INVALID; +} + +static void queue_param_init(odp_queue_param_t *params) +{ + memset(params, 0, sizeof(odp_queue_param_t)); + params->type = ODP_QUEUE_TYPE_PLAIN; + params->enq_mode = ODP_QUEUE_OP_MT; + params->deq_mode = ODP_QUEUE_OP_MT; + params->sched.prio = ODP_SCHED_PRIO_DEFAULT; + params->sched.sync = ODP_SCHED_SYNC_PARALLEL; + params->sched.group = ODP_SCHED_GROUP_ALL; +} + +static int queue_info(odp_queue_t handle, odp_queue_info_t *info) +{ + uint32_t queue_id; + queue_entry_t *queue; + int status; + + if (odp_unlikely(info == NULL)) { + ODP_ERR("Unable to store info, NULL ptr given\n"); + return -1; + } + + queue_id = queue_to_id(handle); + + if (odp_unlikely(queue_id >= ODP_CONFIG_QUEUES)) { + ODP_ERR("Invalid queue handle:%" PRIu64 "\n", + odp_queue_to_u64(handle)); + return -1; + } + + queue = get_qentry(queue_id); + + LOCK(&queue->s.lock); + status = queue->s.status; + + if (odp_unlikely(status == QUEUE_STATUS_FREE || + status == QUEUE_STATUS_DESTROYED)) { + UNLOCK(&queue->s.lock); + ODP_ERR("Invalid queue status:%d\n", status); + return -1; + } + + info->name = queue->s.name; + info->param = queue->s.param; + + UNLOCK(&queue->s.lock); + + return 0; +} + +static uint64_t queue_to_u64(odp_queue_t hdl) +{ + return _odp_pri(hdl); +} + +static odp_pktout_queue_t queue_get_pktout(queue_t handle) +{ + return qentry_from_int(handle)->s.pktout; +} + +static void queue_set_pktout(queue_t handle, odp_pktio_t pktio, int index) +{ + qentry_from_int(handle)->s.pktout.pktio = pktio; + qentry_from_int(handle)->s.pktout.index = index; +} + +static odp_pktin_queue_t queue_get_pktin(queue_t handle) +{ + return qentry_from_int(handle)->s.pktin; +} + +static void queue_set_pktin(queue_t handle, odp_pktio_t pktio, int index) +{ + qentry_from_int(handle)->s.pktin.pktio = pktio; + qentry_from_int(handle)->s.pktin.index = index; +} + +static void queue_set_enq_func(queue_t handle, queue_enq_fn_t func) +{ + qentry_from_int(handle)->s.enqueue = func; +} + +static void queue_set_enq_multi_func(queue_t handle, queue_enq_multi_fn_t func) +{ + qentry_from_int(handle)->s.enqueue_multi = func; +} + +static void queue_set_deq_func(queue_t handle, queue_deq_fn_t func) +{ + qentry_from_int(handle)->s.dequeue = func; +} + +static void queue_set_deq_multi_func(queue_t handle, queue_deq_multi_fn_t func) +{ + qentry_from_int(handle)->s.dequeue_multi = func; +} + +static void queue_set_type(queue_t handle, odp_queue_type_t type) +{ + qentry_from_int(handle)->s.type = type; +} + +static queue_t queue_from_ext(odp_queue_t handle) +{ + uint32_t queue_id; + + queue_id = queue_to_id(handle); + return qentry_to_int(get_qentry(queue_id)); +} + +static odp_queue_t queue_to_ext(queue_t handle) +{ + return qentry_from_int(handle)->s.handle; +} + +/* API functions */ +queue_api_t queue_scalable_api = { + .queue_create = queue_create, + .queue_destroy = queue_destroy, + .queue_lookup = queue_lookup, + .queue_capability = queue_capability, + .queue_context_set = queue_context_set, + .queue_context = queue_context, + .queue_enq = queue_enq, + .queue_enq_multi = queue_enq_multi, + .queue_deq = queue_deq, + .queue_deq_multi = queue_deq_multi, + .queue_type = queue_type, + .queue_sched_type = queue_sched_type, + .queue_sched_prio = queue_sched_prio, + .queue_sched_group = queue_sched_group, + .queue_lock_count = queue_lock_count, + .queue_to_u64 = queue_to_u64, + .queue_param_init = queue_param_init, + .queue_info = queue_info +}; + +/* Functions towards internal components */ +queue_fn_t queue_scalable_fn = { + .init_global = queue_init_global, + .term_global = queue_term_global, + .init_local = queue_init_local, + .term_local = queue_term_local, + .from_ext = queue_from_ext, + .to_ext = queue_to_ext, + .enq = _queue_enq, + .enq_multi = _queue_enq_multi, + .deq = _queue_deq, + .deq_multi = _queue_deq_multi, + .get_pktout = queue_get_pktout, + .set_pktout = queue_set_pktout, + .get_pktin = queue_get_pktin, + .set_pktin = queue_set_pktin, + .set_enq_fn = queue_set_enq_func, + .set_enq_multi_fn = queue_set_enq_multi_func, + .set_deq_fn = queue_set_deq_func, + .set_deq_multi_fn = queue_set_deq_multi_func, + .set_type = queue_set_type +}; diff --git a/platform/linux-generic/odp_schedule_if.c b/platform/linux-generic/odp_schedule_if.c index a9ede98d..2f07aafe 100644 --- a/platform/linux-generic/odp_schedule_if.c +++ b/platform/linux-generic/odp_schedule_if.c @@ -15,12 +15,18 @@ extern const schedule_api_t schedule_default_api; extern const schedule_fn_t schedule_iquery_fn; extern const schedule_api_t schedule_iquery_api; +extern const schedule_fn_t schedule_scalable_fn; +extern const schedule_api_t schedule_scalable_api; + #ifdef ODP_SCHEDULE_SP const schedule_fn_t *sched_fn = &schedule_sp_fn; const schedule_api_t *sched_api = &schedule_sp_api; #elif defined(ODP_SCHEDULE_IQUERY) const schedule_fn_t *sched_fn = &schedule_iquery_fn; const schedule_api_t *sched_api = &schedule_iquery_api; +#elif defined(ODP_SCHEDULE_SCALABLE) +const schedule_fn_t *sched_fn = &schedule_scalable_fn; +const schedule_api_t *sched_api = &schedule_scalable_api; #else const schedule_fn_t *sched_fn = &schedule_default_fn; const schedule_api_t *sched_api = &schedule_default_api; diff --git a/platform/linux-generic/odp_schedule_scalable.c b/platform/linux-generic/odp_schedule_scalable.c new file mode 100644 index 00000000..78159b53 --- /dev/null +++ b/platform/linux-generic/odp_schedule_scalable.c @@ -0,0 +1,1980 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/align.h> +#include <odp/api/atomic.h> +#include <odp/api/cpu.h> +#include <odp/api/hints.h> +#include <odp/api/schedule.h> +#include <odp/api/shared_memory.h> +#include <odp/api/sync.h> +#include <odp/api/thread.h> +#include <odp/api/thrmask.h> +#include <odp/api/time.h> + +#include <odp_internal.h> +#include <odp_config_internal.h> +#include <odp_debug_internal.h> +#include <_ishm_internal.h> +#include <_ishmpool_internal.h> + +#include <odp_align_internal.h> +#include <odp_buffer_inlines.h> +#include <odp_llqueue.h> +#include <odp_queue_scalable_internal.h> +#include <odp_schedule_if.h> +#include <odp_bitset.h> +#include <odp_packet_io_internal.h> + +#include <limits.h> +#include <stdbool.h> +#include <string.h> + +#include <odp/api/plat/ticketlock_inlines.h> +#define LOCK(a) _odp_ticketlock_lock((a)) +#define UNLOCK(a) _odp_ticketlock_unlock((a)) + +#define TAG_EMPTY 0U +#define TAG_USED (1U << 15) +#define TAG_BUSY (1U << 31) +#define PKTIO_QUEUE_2_TAG(p, q) ((p) << 16 | (q) | TAG_USED) +#define TAG_2_PKTIO(t) (((t) >> 16) & 0x7FFF) +#define TAG_2_QUEUE(t) ((t) & 0x7FFF) +#define TAG_IS_READY(t) (((t) & (TAG_USED | TAG_BUSY)) == TAG_USED) +#define PKTIN_MAX (ODP_CONFIG_PKTIO_ENTRIES * PKTIO_MAX_QUEUES) +#define MAXTHREADS ATOM_BITSET_SIZE + +static _odp_ishm_pool_t *sched_shm_pool; +static uint32_t pktin_num; +static uint32_t pktin_hi; +static uint16_t pktin_count[ODP_CONFIG_PKTIO_ENTRIES]; +static uint32_t pktin_tags[PKTIN_MAX] ODP_ALIGNED_CACHE; + +#define __atomic_fetch_max(var, v, mo) do { \ + /* Evalulate 'v' once */ \ + __typeof__(v) tmp_v = (v); \ + __typeof__(*var) old_var = \ + __atomic_load_n((var), __ATOMIC_RELAXED); \ + while (tmp_v > old_var) { \ + /* Attempt to store 'v' in '*var' */ \ + if (__atomic_compare_exchange_n((var), &old_var, \ + tmp_v, true, (mo), \ + (mo))) \ + break; \ + } \ + /* v <= old_var, nothing to do */ \ + } while (0) + +ODP_STATIC_ASSERT(ODP_SCHED_PRIO_LOWEST == (ODP_SCHED_PRIO_NUM - 1), + "lowest_prio_does_not_match_with_num_prios"); + +ODP_STATIC_ASSERT((ODP_SCHED_PRIO_NORMAL > 0) && + (ODP_SCHED_PRIO_NORMAL < (ODP_SCHED_PRIO_NUM - 1)), + "normal_prio_is_not_between_highest_and_lowest"); + +ODP_STATIC_ASSERT(CHECK_IS_POWER2(ODP_CONFIG_QUEUES), + "Number_of_queues_is_not_power_of_two"); + +/* + * Scheduler group related variables. + */ +/* Currently used scheduler groups */ +static sched_group_mask_t sg_free; +static sched_group_t *sg_vec[MAX_SCHED_GROUP]; +/* Group lock for MT-safe APIs */ +odp_spinlock_t sched_grp_lock; + +#define SCHED_GROUP_JOIN 0 +#define SCHED_GROUP_LEAVE 1 + +/* + * Per thread state + */ +static sched_scalable_thread_state_t thread_state[MAXTHREADS]; +__thread sched_scalable_thread_state_t *sched_ts; + +/* + * Forward declarations. + */ +static int thread_state_init(int tidx) +{ + sched_scalable_thread_state_t *ts; + uint32_t i; + + ODP_ASSERT(tidx < MAXTHREADS); + ts = &thread_state[tidx]; + ts->atomq = NULL; + ts->rctx = NULL; + ts->pause = false; + ts->out_of_order = false; + ts->tidx = tidx; + ts->dequeued = 0; + ts->pktin_next = 0; + ts->pktin_poll_cnts = 0; + ts->ticket = TICKET_INVALID; + ts->priv_rvec_free = 0; + ts->rvec_free = (1ULL << TS_RVEC_SIZE) - 1; + ts->num_schedq = 0; + ts->sg_sem = 1; /* Start with sched group semaphore changed */ + memset(ts->sg_actual, 0, sizeof(ts->sg_actual)); + for (i = 0; i < TS_RVEC_SIZE; i++) { + ts->rvec[i].rvec_free = &ts->rvec_free; + ts->rvec[i].idx = i; + } + sched_ts = ts; + + return 0; +} + +static void insert_schedq_in_list(sched_scalable_thread_state_t *ts, + sched_queue_t *schedq) +{ + /* Find slot for schedq */ + for (uint32_t i = 0; i < ts->num_schedq; i++) { + /* Lower value is higher priority and closer to start of list */ + if (schedq->prio <= ts->schedq_list[i]->prio) { + /* This is the slot! */ + sched_queue_t *tmp; + + tmp = ts->schedq_list[i]; + ts->schedq_list[i] = schedq; + schedq = tmp; + /* Continue the insertion procedure with the + * new schedq. + */ + } + } + if (ts->num_schedq == SCHEDQ_PER_THREAD) + ODP_ABORT("Too many schedqs\n"); + ts->schedq_list[ts->num_schedq++] = schedq; +} + +static void remove_schedq_from_list(sched_scalable_thread_state_t *ts, + sched_queue_t *schedq) +{ + /* Find schedq */ + for (uint32_t i = 0; i < ts->num_schedq; i++) + if (ts->schedq_list[i] == schedq) { + /* Move remaining schedqs */ + for (uint32_t j = i + 1; j < ts->num_schedq; j++) + ts->schedq_list[j - 1] = ts->schedq_list[j]; + ts->num_schedq--; + return; + } + ODP_ABORT("Cannot find schedq\n"); +} + +/******************************************************************************* + * Scheduler queues + ******************************************************************************/ +#ifndef odp_container_of +#define odp_container_of(pointer, type, member) \ + ((type *)(void *)(((char *)pointer) - offsetof(type, member))) +#endif + +static inline void schedq_init(sched_queue_t *schedq, uint32_t prio) +{ + llqueue_init(&schedq->llq); + schedq->prio = prio; +} + +static inline sched_elem_t *schedq_peek(sched_queue_t *schedq) +{ + struct llnode *ptr; + + ptr = llq_head(&schedq->llq); + return odp_container_of(ptr, sched_elem_t, node); +} + +static inline odp_bool_t schedq_cond_pop(sched_queue_t *schedq, + sched_elem_t *elem) +{ + return llq_dequeue_cond(&schedq->llq, &elem->node); +} + +static inline void schedq_push(sched_queue_t *schedq, sched_elem_t *elem) +{ + llq_enqueue(&schedq->llq, &elem->node); +} + +static inline odp_bool_t schedq_cond_rotate(sched_queue_t *schedq, + sched_elem_t *elem) +{ + return llq_cond_rotate(&schedq->llq, &elem->node); +} + +static inline bool schedq_elem_on_queue(sched_elem_t *elem) +{ + return llq_on_queue(&elem->node); +} + +/******************************************************************************* + * Shared metadata btwn scheduler and queue + ******************************************************************************/ + +void sched_update_enq(sched_elem_t *q, uint32_t actual) +{ + qschedstate_t oss, nss; + uint32_t ticket; + + oss = q->qschst; + /* Update event counter, optionally taking a ticket. */ + do { + ticket = TICKET_INVALID; + nss = oss; + nss.numevts += actual; + if (odp_unlikely(oss.numevts <= 0 && nss.numevts > 0)) + /* E -> NE transition */ + if (q->qschst_type != ODP_SCHED_SYNC_ATOMIC || + oss.cur_ticket == oss.nxt_ticket) + /* Parallel or ordered queues: always take + * ticket. + * Atomic queue: only take ticket if one is + * immediately available. + * Otherwise ticket already taken => queue + * processed by some thread. + */ + ticket = nss.nxt_ticket++; + /* Else queue already was non-empty. */ + /* Attempt to update numevts counter and optionally take ticket. */ + } while (!__atomic_compare_exchange( + &q->qschst, &oss, &nss, + true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + + if (odp_unlikely(ticket != TICKET_INVALID)) { + /* Wait for our turn to update schedq. */ + if (odp_unlikely( + __atomic_load_n(&q->qschst.cur_ticket, + __ATOMIC_ACQUIRE) != ticket)) { + sevl(); + while (wfe() && + monitor8(&q->qschst.cur_ticket, + __ATOMIC_ACQUIRE) != ticket) + doze(); + } + /* Enqueue at end of scheduler queue */ + /* We are here because of empty-to-non-empty transition + * This means queue must be pushed to schedq if possible + * but we can't do that if it already is on the schedq + */ + if (odp_likely(!schedq_elem_on_queue(q) && + q->pop_deficit == 0)) { + /* Queue not already on schedq and no pop deficit means + * we can push queue to schedq */ + schedq_push(q->schedq, q); + } else { + /* Missed push => cancels one missed pop */ + q->pop_deficit--; + } + atomic_store_release(&q->qschst.cur_ticket, ticket + 1, + /*readonly=*/false); + } + /* Else queue was not empty or atomic queue already busy. */ +} + +void sched_update_enq_sp(sched_elem_t *q, uint32_t actual) +{ + qschedstate_t oss, nss; + uint32_t ticket; + + oss = q->qschst; + /* Update event counter, optionally taking a ticket. */ + ticket = TICKET_INVALID; + nss = oss; + nss.numevts += actual; + if (odp_unlikely(oss.numevts <= 0 && nss.numevts > 0)) { + /* E -> NE transition */ + if (q->qschst_type != ODP_SCHED_SYNC_ATOMIC || + oss.cur_ticket == oss.nxt_ticket) { + /* Parallel or ordered queues: always take + * ticket. + * Atomic queue: only take ticket if one is + * immediately available. Otherwise ticket already + * taken => queue owned/processed by some thread + */ + ticket = nss.nxt_ticket++; + } + } + /* Else queue already was non-empty. */ + /* Attempt to update numevts counter and optionally take ticket. */ + q->qschst = nss; + + if (odp_unlikely(ticket != TICKET_INVALID)) { + /* Enqueue at end of scheduler queue */ + /* We are here because of empty-to-non-empty transition + * This means queue must be pushed to schedq if possible + * but we can't do that if it already is on the schedq + */ + if (odp_likely(!schedq_elem_on_queue(q) && + q->pop_deficit == 0)) { + /* Queue not already on schedq and no pop deficit means + * we can push queue to schedq */ + schedq_push(q->schedq, q); + } else { + /* Missed push => cancels one missed pop */ + q->pop_deficit--; + } + q->qschst.cur_ticket = ticket + 1; + } + /* Else queue was not empty or atomic queue already busy. */ +} + +#ifndef CONFIG_QSCHST_LOCK +/* The scheduler is the only entity that performs the dequeue from a queue. */ +static void +sched_update_deq(sched_elem_t *q, + uint32_t actual, + bool atomic) __attribute__((always_inline)); +static inline void +sched_update_deq(sched_elem_t *q, + uint32_t actual, bool atomic) +{ + qschedstate_t oss, nss; + uint32_t ticket; + + if (atomic) { + bool pushed = false; + + /* We own this atomic queue, only we can dequeue from it and + * thus decrease numevts. Other threads may enqueue and thus + * increase numevts. + * This means that numevts can't unexpectedly become 0 and + * invalidate a push operation already performed + */ + oss = q->qschst; + do { + ODP_ASSERT(oss.cur_ticket == sched_ts->ticket); + nss = oss; + nss.numevts -= actual; + if (nss.numevts > 0 && !pushed) { + schedq_push(q->schedq, q); + pushed = true; + } + /* Attempt to release ticket expecting our view of + * numevts to be correct + * Unfortunately nxt_ticket will also be included in + * the CAS operation + */ + nss.cur_ticket = sched_ts->ticket + 1; + } while (odp_unlikely(!__atomic_compare_exchange( + &q->qschst, + &oss, &nss, + true, + __ATOMIC_RELEASE, + __ATOMIC_RELAXED))); + return; + } + + oss = q->qschst; + do { + ticket = TICKET_INVALID; + nss = oss; + nss.numevts -= actual; + nss.wrr_budget -= actual; + if ((oss.numevts > 0 && nss.numevts <= 0) || + oss.wrr_budget <= actual) { + /* If we have emptied parallel/ordered queue or + * exchausted its WRR budget, we need a ticket + * for a later pop. + */ + ticket = nss.nxt_ticket++; + /* Reset wrr_budget as we might also push the + * queue to the schedq. + */ + nss.wrr_budget = CONFIG_WRR_WEIGHT; + } + /* Attempt to update numevts and optionally take ticket. */ + } while (!__atomic_compare_exchange( + &q->qschst, &oss, &nss, + true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + + if (odp_unlikely(ticket != TICKET_INVALID)) { + ODP_ASSERT(q->qschst_type != ODP_SCHED_SYNC_ATOMIC); + /* Wait for our turn to update schedq. */ + if (odp_unlikely( + __atomic_load_n(&q->qschst.cur_ticket, + __ATOMIC_ACQUIRE) != ticket)) { + sevl(); + while (wfe() && + monitor8(&q->qschst.cur_ticket, + __ATOMIC_ACQUIRE) != ticket) + doze(); + } + /* We are here because of non-empty-to-empty transition or + * WRR budget exhausted + * This means the queue must be popped from the schedq, now or + * later + * If there was no NE->E transition but instead the WRR budget + * was exhausted, the queue needs to be moved (popped and + * pushed) to the tail of the schedq + */ + if (oss.numevts > 0 && nss.numevts <= 0) { + /* NE->E transition, need to pop */ + if (!schedq_elem_on_queue(q) || + !schedq_cond_pop(q->schedq, q)) { + /* Queue not at head, failed to dequeue + * Missed a pop. + */ + q->pop_deficit++; + } + } else { + /* WRR budget exhausted + * Need to move queue to tail of schedq if possible + */ + if (odp_likely(schedq_elem_on_queue(q))) { + /* Queue is on schedq, try to move it to + * the tail + */ + (void)schedq_cond_rotate(q->schedq, q); + } + /* Else queue not on schedq or not at head of schedq + * No pop => no push + */ + } + atomic_store_release(&q->qschst.cur_ticket, ticket + 1, + /*readonly=*/false); + } +} +#endif + +#ifdef CONFIG_QSCHST_LOCK +static void +sched_update_deq_sc(sched_elem_t *q, + uint32_t actual, + bool atomic) __attribute__((always_inline)); +static inline void +sched_update_deq_sc(sched_elem_t *q, + uint32_t actual, bool atomic) +{ + qschedstate_t oss, nss; + uint32_t ticket; + + if (atomic) { + ODP_ASSERT(q->qschst.cur_ticket == sched_ts->ticket); + ODP_ASSERT(q->qschst.cur_ticket != q->qschst.nxt_ticket); + q->qschst.numevts -= actual; + q->qschst.cur_ticket = sched_ts->ticket + 1; + if (q->qschst.numevts > 0) + schedq_push(q->schedq, q); + return; + } + + oss = q->qschst; + ticket = TICKET_INVALID; + nss = oss; + nss.numevts -= actual; + nss.wrr_budget -= actual; + if ((oss.numevts > 0 && nss.numevts <= 0) || oss.wrr_budget <= actual) { + /* If we emptied the queue or + * if we have served the maximum number of events + * then we need a ticket for a later pop. + */ + ticket = nss.nxt_ticket++; + /* Also reset wrr_budget as we might also push the + * queue to the schedq. + */ + nss.wrr_budget = CONFIG_WRR_WEIGHT; + } + q->qschst = nss; + + if (ticket != TICKET_INVALID) { + if (oss.numevts > 0 && nss.numevts <= 0) { + /* NE->E transition, need to pop */ + if (!schedq_elem_on_queue(q) || + !schedq_cond_pop(q->schedq, q)) { + /* Queue not at head, failed to dequeue. + * Missed a pop. + */ + q->pop_deficit++; + } + } else { + /* WRR budget exhausted + * Need to move queue to tail of schedq if possible + */ + if (odp_likely(schedq_elem_on_queue(q))) { + /* Queue is on schedq, try to move it to + * the tail + */ + (void)schedq_cond_rotate(q->schedq, q); + } + /* Else queue not on schedq or not at head of schedq + * No pop => no push + */ + } + q->qschst.cur_ticket = ticket + 1; + } +} +#endif + +static inline void sched_update_popd_sc(sched_elem_t *elem) +{ + if (elem->pop_deficit != 0 && + schedq_elem_on_queue(elem) && + schedq_cond_pop(elem->schedq, elem)) + elem->pop_deficit--; +} + +#ifndef CONFIG_QSCHST_LOCK +static inline void sched_update_popd(sched_elem_t *elem) +{ + uint32_t ticket = __atomic_fetch_add(&elem->qschst.nxt_ticket, + 1, + __ATOMIC_RELAXED); + if (odp_unlikely(__atomic_load_n(&elem->qschst.cur_ticket, + __ATOMIC_ACQUIRE) != ticket)) { + sevl(); + while (wfe() && monitor8(&elem->qschst.cur_ticket, + __ATOMIC_ACQUIRE) != ticket) + doze(); + } + sched_update_popd_sc(elem); + atomic_store_release(&elem->qschst.cur_ticket, ticket + 1, + /*readonly=*/false); +} +#endif + +sched_queue_t *schedq_from_sched_group(odp_schedule_group_t grp, uint32_t prio) +{ + uint32_t sgi; + sched_group_t *sg; + uint32_t x; + + ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); + ODP_ASSERT((sg_free & (1ULL << grp)) == 0); + ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); + + sgi = grp; + sg = sg_vec[sgi]; + + /* Use xcount to spread queues over the xfactor schedq's + * per priority. + */ + x = __atomic_fetch_add(&sg->xcount[prio], 1, __ATOMIC_RELAXED); + if (x == 0) { + /* First ODP queue for this priority + * Notify all threads in sg->thr_wanted that they + * should join. + */ + sched_group_mask_t thrds = sg->thr_wanted; + + while (!bitset_is_null(thrds)) { + uint32_t thr; + + thr = bitset_ffs(thrds) - 1; + thrds = bitset_clr(thrds, thr); + /* Notify the thread about membership in this + * group/priority. + */ + atom_bitset_set(&thread_state[thr].sg_wanted[prio], + sgi, __ATOMIC_RELEASE); + __atomic_store_n(&thread_state[thr].sg_sem, 1, + __ATOMIC_RELEASE); + } + } + return &sg->schedq[prio * sg->xfactor + x % sg->xfactor]; +} + +void sched_group_xcount_dec(odp_schedule_group_t grp, uint32_t prio) +{ + uint32_t sgi; + sched_group_t *sg; + uint32_t x; + + ODP_ASSERT(grp >= 0 && grp < (odp_schedule_group_t)MAX_SCHED_GROUP); + ODP_ASSERT((sg_free & (1ULL << grp)) == 0); + ODP_ASSERT(prio < ODP_SCHED_PRIO_NUM); + + sgi = grp; + sg = sg_vec[sgi]; + x = __atomic_sub_fetch(&sg->xcount[prio], 1, __ATOMIC_RELAXED); + + if (x == 0) { + /* Last ODP queue for this priority + * Notify all threads in sg->thr_wanted that they + * should leave. + */ + sched_group_mask_t thrds = sg->thr_wanted; + + while (!bitset_is_null(thrds)) { + uint32_t thr; + + thr = bitset_ffs(thrds) - 1; + thrds = bitset_clr(thrds, thr); + /* Notify the thread about membership in this + * group/priority. + */ + atom_bitset_clr(&thread_state[thr].sg_wanted[prio], + sgi, __ATOMIC_RELEASE); + __atomic_store_n(&thread_state[thr].sg_sem, 1, + __ATOMIC_RELEASE); + } + } +} + +static void update_sg_membership(sched_scalable_thread_state_t *ts) +{ + uint32_t p; + sched_group_mask_t sg_wanted; + sched_group_mask_t added; + sched_group_mask_t removed; + uint32_t sgi; + sched_group_t *sg; + uint32_t x; + + for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { + sg_wanted = atom_bitset_load(&ts->sg_wanted[p], + __ATOMIC_ACQUIRE); + if (!bitset_is_eql(ts->sg_actual[p], sg_wanted)) { + /* Our sched_group membership has changed */ + added = bitset_andn(sg_wanted, ts->sg_actual[p]); + while (!bitset_is_null(added)) { + sgi = bitset_ffs(added) - 1; + sg = sg_vec[sgi]; + for (x = 0; x < sg->xfactor; x++) { + /* Include our thread index to shift + * (rotate) the order of schedq's + */ + insert_schedq_in_list + (ts, + &sg->schedq[p * sg->xfactor + + (x + ts->tidx) % sg->xfactor]); + } + atom_bitset_set(&sg->thr_actual[p], ts->tidx, + __ATOMIC_RELAXED); + added = bitset_clr(added, sgi); + } + removed = bitset_andn(ts->sg_actual[p], sg_wanted); + while (!bitset_is_null(removed)) { + sgi = bitset_ffs(removed) - 1; + sg = sg_vec[sgi]; + for (x = 0; x < sg->xfactor; x++) { + remove_schedq_from_list + (ts, + &sg->schedq[p * + sg->xfactor + x]); + } + atom_bitset_clr(&sg->thr_actual[p], ts->tidx, + __ATOMIC_RELAXED); + removed = bitset_clr(removed, sgi); + } + ts->sg_actual[p] = sg_wanted; + } + } +} + +/******************************************************************************* + * Scheduler + ******************************************************************************/ + +static inline void _schedule_release_atomic(sched_scalable_thread_state_t *ts) +{ +#ifdef CONFIG_QSCHST_LOCK + sched_update_deq_sc(ts->atomq, ts->dequeued, true); + ODP_ASSERT(ts->atomq->qschst.cur_ticket != ts->ticket); + ODP_ASSERT(ts->atomq->qschst.cur_ticket == + ts->atomq->qschst.nxt_ticket); +#else + sched_update_deq(ts->atomq, ts->dequeued, true); +#endif + ts->atomq = NULL; + ts->ticket = TICKET_INVALID; +} + +static inline void _schedule_release_ordered(sched_scalable_thread_state_t *ts) +{ + ts->out_of_order = false; + rctx_release(ts->rctx); + ts->rctx = NULL; +} + +static void pktin_poll(sched_scalable_thread_state_t *ts) +{ + uint32_t i, tag, hi, npolls = 0; + int pktio_index, queue_index; + + hi = __atomic_load_n(&pktin_hi, __ATOMIC_RELAXED); + if (hi == 0) + return; + + for (i = ts->pktin_next; npolls != hi; i = (i + 1) % hi, npolls++) { + tag = __atomic_load_n(&pktin_tags[i], __ATOMIC_RELAXED); + if (!TAG_IS_READY(tag)) + continue; + if (!__atomic_compare_exchange_n(&pktin_tags[i], &tag, + tag | TAG_BUSY, + true, + __ATOMIC_ACQUIRE, + __ATOMIC_RELAXED)) + continue; + /* Tag grabbed */ + pktio_index = TAG_2_PKTIO(tag); + queue_index = TAG_2_QUEUE(tag); + if (odp_unlikely(sched_cb_pktin_poll(pktio_index, + 1, &queue_index))) { + /* Pktio stopped or closed + * Remove tag from pktin_tags + */ + __atomic_store_n(&pktin_tags[i], + TAG_EMPTY, __ATOMIC_RELAXED); + __atomic_fetch_sub(&pktin_num, + 1, __ATOMIC_RELEASE); + /* Call stop_finalize when all queues + * of the pktio have been removed + */ + if (__atomic_sub_fetch(&pktin_count[pktio_index], 1, + __ATOMIC_RELAXED) == 0) + sched_cb_pktio_stop_finalize(pktio_index); + } else { + /* We don't know whether any packets were found and enqueued + * Write back original tag value to release pktin queue + */ + __atomic_store_n(&pktin_tags[i], tag, __ATOMIC_RELAXED); + /* Do not iterate through all pktin queues every time */ + if ((ts->pktin_poll_cnts & 0xf) != 0) + break; + } + } + ODP_ASSERT(i < hi); + ts->pktin_poll_cnts++; + ts->pktin_next = i; +} + +static int _schedule(odp_queue_t *from, odp_event_t ev[], int num_evts) +{ + sched_scalable_thread_state_t *ts; + sched_elem_t *atomq; + int num; + uint32_t i; + + ts = sched_ts; + atomq = ts->atomq; + + /* Once an atomic queue has been scheduled to a thread, it will stay + * on that thread until empty or 'rotated' by WRR + */ + if (atomq != NULL) { + ODP_ASSERT(ts->ticket != TICKET_INVALID); +#ifdef CONFIG_QSCHST_LOCK + LOCK(&atomq->qschlock); +#endif +dequeue_atomic: + ODP_ASSERT(ts->ticket == atomq->qschst.cur_ticket); + ODP_ASSERT(ts->ticket != atomq->qschst.nxt_ticket); + /* Atomic queues can be dequeued without lock since this thread + * has the only reference to the atomic queue being processed. + */ + if (ts->dequeued < atomq->qschst.wrr_budget) { + num = _odp_queue_deq_sc(atomq, ev, num_evts); + if (odp_likely(num != 0)) { +#ifdef CONFIG_QSCHST_LOCK + UNLOCK(&atomq->qschlock); +#endif + ts->dequeued += num; + /* Allow this thread to continue to 'own' this + * atomic queue until all events have been + * processed and the thread re-invokes the + * scheduler. + */ + if (from) + *from = queue_get_handle( + (queue_entry_t *)atomq); + return num; + } + } + /* Atomic queue was empty or interrupted by WRR, release it. */ + _schedule_release_atomic(ts); +#ifdef CONFIG_QSCHST_LOCK + UNLOCK(&atomq->qschlock); +#endif + } + + /* Release any previous reorder context. */ + if (ts->rctx != NULL) + _schedule_release_ordered(ts); + + /* Check for and perform any scheduler group updates. */ + if (odp_unlikely(__atomic_load_n(&ts->sg_sem, __ATOMIC_RELAXED) != 0)) { + (void)__atomic_load_n(&ts->sg_sem, __ATOMIC_ACQUIRE); + ts->sg_sem = 0; + update_sg_membership(ts); + } + + /* Scan our schedq list from beginning to end */ + for (i = 0; i < ts->num_schedq; i++) { + sched_queue_t *schedq = ts->schedq_list[i]; + sched_elem_t *elem; +restart_same: + elem = schedq_peek(schedq); + if (odp_unlikely(elem == NULL)) { + /* Schedq empty, look at next one. */ + continue; + } + + if (elem->cons_type == ODP_SCHED_SYNC_ATOMIC) { + /* Dequeue element only if it is still at head + * of schedq. + */ + if (odp_unlikely(!schedq_cond_pop(schedq, elem))) { + /* Queue not at head of schedq anymore, some + * other thread popped it. + */ + goto restart_same; + } + ts->atomq = elem; + atomq = elem; + ts->dequeued = 0; +#ifdef CONFIG_QSCHST_LOCK + LOCK(&atomq->qschlock); + ts->ticket = atomq->qschst.nxt_ticket++; + ODP_ASSERT(atomq->qschst.cur_ticket == ts->ticket); +#else + /* Dequeued atomic queue from the schedq, only we + * can process it and any qschst updates are our + * responsibility. + */ + /* The ticket taken below will signal producers */ + ts->ticket = __atomic_fetch_add( + &atomq->qschst.nxt_ticket, 1, __ATOMIC_RELAXED); + while (__atomic_load_n( + &atomq->qschst.cur_ticket, + __ATOMIC_ACQUIRE) != ts->ticket) { + /* No need to use WFE, spinning here seems + * very infrequent. + */ + odp_cpu_pause(); + } +#endif + goto dequeue_atomic; + } else if (elem->cons_type == ODP_SCHED_SYNC_PARALLEL) { +#ifdef CONFIG_QSCHST_LOCK + LOCK(&elem->qschlock); + num = _odp_queue_deq_sc(elem, ev, num_evts); + if (odp_likely(num != 0)) { + sched_update_deq_sc(elem, num, false); + UNLOCK(&elem->qschlock); + if (from) + *from = + queue_get_handle((queue_entry_t *)elem); + return num; + } + UNLOCK(&elem->qschlock); +#else + num = _odp_queue_deq_mc(elem, ev, num_evts); + if (odp_likely(num != 0)) { + sched_update_deq(elem, num, false); + if (from) + *from = + queue_get_handle((queue_entry_t *)elem); + return num; + } +#endif + } else if (elem->cons_type == ODP_SCHED_SYNC_ORDERED) { + reorder_window_t *rwin; + reorder_context_t *rctx; + uint32_t sn; + uint32_t idx; + + /* The ordered queue has a reorder window so requires + * order restoration. We must use a reorder context to + * collect all outgoing events. Ensure there is at least + * one available reorder context. + */ + if (odp_unlikely(bitset_is_null(ts->priv_rvec_free))) { + ts->priv_rvec_free = atom_bitset_xchg( + &ts->rvec_free, 0, + __ATOMIC_RELAXED); + if (odp_unlikely(bitset_is_null( + ts->priv_rvec_free))) { + /* No free reorder contexts for + * this thread. Look at next schedq, + * hope we find non-ordered queue. + */ + continue; + } + } + /* rwin_reserve and odp_queue_deq must be atomic or + * there will be a potential race condition. + * Allocate a slot in the reorder window. + */ + rwin = queue_get_rwin((queue_entry_t *)elem); + ODP_ASSERT(rwin != NULL); + if (odp_unlikely(!rwin_reserve(rwin, &sn))) { + /* Reorder window full */ + /* Look at next schedq, find other queue */ + continue; + } + /* Wait for our turn to dequeue */ + if (odp_unlikely(__atomic_load_n(&rwin->turn, + __ATOMIC_ACQUIRE) + != sn)) { + sevl(); + while (wfe() && + monitor32(&rwin->turn, __ATOMIC_ACQUIRE) + != sn) + doze(); + } +#ifdef CONFIG_QSCHST_LOCK + LOCK(&elem->qschlock); +#endif + num = _odp_queue_deq_sc(elem, ev, num_evts); + /* Wait for prod_read write in _odp_queue_dequeue_sc() + * to complete before we signal the next consumer + */ + atomic_store_release(&rwin->turn, sn + 1, + /*readonly=*/false); + /* Find and initialise an unused reorder context. */ + idx = bitset_ffs(ts->priv_rvec_free) - 1; + ts->priv_rvec_free = + bitset_clr(ts->priv_rvec_free, idx); + rctx = &ts->rvec[idx]; + /* Need to initialise reorder context or we can't + * release it later. + */ + rctx_init(rctx, idx, rwin, sn); + + /* Was dequeue successful? */ + if (odp_likely(num != 0)) { + /* Perform scheduler related updates */ +#ifdef CONFIG_QSCHST_LOCK + sched_update_deq_sc(elem, num, + /*atomic=*/false); + UNLOCK(&elem->qschlock); +#else + sched_update_deq(elem, num, /*atomic=*/false); +#endif + + /* Are we in-order or out-of-order? */ + ts->out_of_order = sn != rwin->hc.head; + + ts->rctx = rctx; + if (from) + *from = queue_get_handle( + (queue_entry_t *)elem); + return num; + } +#ifdef CONFIG_QSCHST_LOCK + UNLOCK(&elem->qschlock); +#endif + /* Since a slot was reserved in the reorder window, + * the reorder context needs to be released and + * inserted into the reorder window. + */ + rctx_release(rctx); + ODP_ASSERT(ts->rctx == NULL); + } + /* Dequeue from parallel/ordered queue failed + * Check if we have a queue at the head of the schedq that needs + * to be popped + */ + if (odp_unlikely(__atomic_load_n(&elem->pop_deficit, + __ATOMIC_RELAXED) != 0)) { +#ifdef CONFIG_QSCHST_LOCK + LOCK(&elem->qschlock); + sched_update_popd_sc(elem); + UNLOCK(&elem->qschlock); +#else + sched_update_popd(elem); +#endif + } + } + + pktin_poll(ts); + return 0; +} + +/******************************************************************************/ + +static void schedule_order_lock(unsigned lock_index) +{ + struct reorder_context *rctx = sched_ts->rctx; + + if (odp_unlikely(rctx == NULL || + rctx->rwin == NULL || + lock_index >= rctx->rwin->lock_count)) { + ODP_ERR("Invalid call to odp_schedule_order_lock\n"); + return; + } + if (odp_unlikely(__atomic_load_n(&rctx->rwin->olock[lock_index], + __ATOMIC_ACQUIRE) != rctx->sn)) { + sevl(); + while (wfe() && + monitor32(&rctx->rwin->olock[lock_index], + __ATOMIC_ACQUIRE) != rctx->sn) + doze(); + } +} + +static void schedule_order_unlock(unsigned lock_index) +{ + struct reorder_context *rctx; + + rctx = sched_ts->rctx; + if (odp_unlikely(rctx == NULL || + rctx->rwin == NULL || + lock_index >= rctx->rwin->lock_count || + rctx->rwin->olock[lock_index] != rctx->sn)) { + ODP_ERR("Invalid call to odp_schedule_order_unlock\n"); + return; + } + atomic_store_release(&rctx->rwin->olock[lock_index], + rctx->sn + 1, + /*readonly=*/false); + rctx->olock_flags |= 1U << lock_index; +} + +static void schedule_release_atomic(void) +{ + sched_scalable_thread_state_t *ts; + + ts = sched_ts; + if (odp_likely(ts->atomq != NULL)) { +#ifdef CONFIG_QSCHST_LOCK + sched_elem_t *atomq; + + atomq = ts->atomq; + LOCK(&atomq->qschlock); +#endif + _schedule_release_atomic(ts); +#ifdef CONFIG_QSCHST_LOCK + UNLOCK(&atomq->qschlock); +#endif + } +} + +static void schedule_release_ordered(void) +{ + sched_scalable_thread_state_t *ts; + + ts = sched_ts; + if (ts->rctx != NULL) + _schedule_release_ordered(ts); +} + +static int schedule_multi(odp_queue_t *from, uint64_t wait, odp_event_t ev[], + int num) +{ + sched_scalable_thread_state_t *ts; + int n; + odp_time_t start; + odp_time_t delta; + odp_time_t deadline; + + ts = sched_ts; + if (odp_unlikely(ts->pause)) { + if (ts->atomq != NULL) { +#ifdef CONFIG_QSCHST_LOCK + sched_elem_t *atomq; + + atomq = ts->atomq; + LOCK(&atomq->qschlock); +#endif + _schedule_release_atomic(ts); +#ifdef CONFIG_QSCHST_LOCK + UNLOCK(&atomq->qschlock); +#endif + } else if (ts->rctx != NULL) { + _schedule_release_ordered(ts); + } + return 0; + } + + if (wait == ODP_SCHED_NO_WAIT) + return _schedule(from, ev, num); + + if (wait == ODP_SCHED_WAIT) { + for (;;) { + n = _schedule(from, ev, num); + if (odp_likely(n > 0)) + return n; + } + } + + start = odp_time_local(); + + n = _schedule(from, ev, num); + if (odp_likely(n > 0)) + return n; + + delta = odp_time_local_from_ns(wait); + deadline = odp_time_sum(start, delta); + + while (odp_time_cmp(deadline, odp_time_local()) > 0) { + n = _schedule(from, ev, num); + if (odp_likely(n > 0)) + return n; + } + + return 0; +} + +static odp_event_t schedule(odp_queue_t *from, uint64_t wait) +{ + odp_event_t ev = ODP_EVENT_INVALID; + const int num = 1; + sched_scalable_thread_state_t *ts; + int n; + odp_time_t start; + odp_time_t delta; + odp_time_t deadline; + + ts = sched_ts; + if (odp_unlikely(ts->pause)) { + if (ts->atomq != NULL) { +#ifdef CONFIG_QSCHST_LOCK + sched_elem_t *atomq; + + atomq = ts->atomq; + LOCK(&atomq->qschlock); +#endif + _schedule_release_atomic(ts); +#ifdef CONFIG_QSCHST_LOCK + UNLOCK(&atomq->qschlock); +#endif + } else if (ts->rctx != NULL) { + _schedule_release_ordered(ts); + } + return ev; + } + + if (wait == ODP_SCHED_NO_WAIT) { + (void)_schedule(from, &ev, num); + return ev; + } + + if (wait == ODP_SCHED_WAIT) { + for (;;) { + n = _schedule(from, &ev, num); + if (odp_likely(n > 0)) + return ev; + } + } + + start = odp_time_local(); + + n = _schedule(from, &ev, num); + if (odp_likely(n > 0)) + return ev; + + delta = odp_time_local_from_ns(wait); + deadline = odp_time_sum(start, delta); + + while (odp_time_cmp(deadline, odp_time_local()) > 0) { + n = _schedule(from, &ev, num); + if (odp_likely(n > 0)) + return ev; + } + + return ev; +} + +static void schedule_pause(void) +{ + sched_ts->pause = true; +} + +static void schedule_resume(void) +{ + sched_ts->pause = false; +} + +static uint64_t schedule_wait_time(uint64_t ns) +{ + return ns; +} + +static int schedule_num_prio(void) +{ + return ODP_SCHED_PRIO_NUM; +} + +static int schedule_group_update(sched_group_t *sg, + uint32_t sgi, + const odp_thrmask_t *mask, + int join_leave) +{ + int thr; + uint32_t p; + + /* Internal function, do not validate inputs */ + + /* Notify relevant threads about the change */ + thr = odp_thrmask_first(mask); + while (0 <= thr) { + /* Add thread to scheduler group's wanted thread mask */ + if (join_leave == SCHED_GROUP_JOIN) + atom_bitset_set(&sg->thr_wanted, thr, __ATOMIC_RELAXED); + else + atom_bitset_clr(&sg->thr_wanted, thr, __ATOMIC_RELAXED); + for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { + if (sg->xcount[p] != 0) { + /* This priority level has ODP queues + * Notify the thread about membership in + * this group/priority + */ + if (join_leave == SCHED_GROUP_JOIN) + atom_bitset_set( + &thread_state[thr].sg_wanted[p], + sgi, + __ATOMIC_RELEASE); + else + atom_bitset_clr( + &thread_state[thr].sg_wanted[p], + sgi, + __ATOMIC_RELEASE); + __atomic_store_n(&thread_state[thr].sg_sem, + 1, + __ATOMIC_RELEASE); + } + } + thr = odp_thrmask_next(mask, thr); + } + + return 0; +} + +static int _schedule_group_thrmask(sched_group_t *sg, odp_thrmask_t *mask) +{ + bitset_t bs; + uint32_t bit; + + /* Internal function, do not validate inputs */ + + odp_thrmask_zero(mask); + bs = sg->thr_wanted; + while (!bitset_is_null(bs)) { + bit = bitset_ffs(bs) - 1; + bs = bitset_clr(bs, bit); + odp_thrmask_set(mask, bit); + } + + return 0; +} + +static odp_schedule_group_t schedule_group_create(const char *name, + const odp_thrmask_t *mask) +{ + uint32_t sgi; + sched_group_mask_t free; + uint32_t xfactor; + sched_group_t *sg; + uint32_t p; + uint32_t x; + uint32_t size; + + /* Validate inputs */ + if (mask == NULL) + ODP_ABORT("mask is NULL\n"); + + odp_spinlock_lock(&sched_grp_lock); + + /* Allocate a scheduler group */ + free = atom_bitset_load(&sg_free, __ATOMIC_RELAXED); + do { + /* All sched_groups in use */ + if (bitset_is_null(free)) + goto no_free_sched_group; + + sgi = bitset_ffs(free) - 1; + /* All sched_groups in use */ + if (sgi >= MAX_SCHED_GROUP) + goto no_free_sched_group; + } while (!atom_bitset_cmpxchg(&sg_free, + &free, + bitset_clr(free, sgi), + true, + __ATOMIC_ACQUIRE, + __ATOMIC_ACQUIRE)); + + /* Compute xfactor (spread factor) from the number of threads + * present in the thread mask. Preferable this would be an + * explicit parameter. + */ + xfactor = odp_thrmask_count(mask); + if (xfactor < 1) + xfactor = CONFIG_DEFAULT_XFACTOR; + + size = sizeof(sched_group_t) + + (ODP_SCHED_PRIO_NUM * xfactor - 1) * sizeof(sched_queue_t); + sg = (sched_group_t *)shm_pool_alloc_align(sched_shm_pool, size); + if (sg == NULL) + goto shm_pool_alloc_failed; + + strncpy(sg->name, name ? name : "", ODP_SCHED_GROUP_NAME_LEN - 1); + sg_vec[sgi] = sg; + memset(sg->thr_actual, 0, sizeof(sg->thr_actual)); + sg->thr_wanted = bitset_null(); + sg->xfactor = xfactor; + for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { + sg->xcount[p] = 0; + for (x = 0; x < xfactor; x++) + schedq_init(&sg->schedq[p * xfactor + x], p); + } + if (odp_thrmask_count(mask) != 0) + schedule_group_update(sg, sgi, mask, SCHED_GROUP_JOIN); + + odp_spinlock_unlock(&sched_grp_lock); + + return (odp_schedule_group_t)(sgi); + +shm_pool_alloc_failed: + /* Free the allocated group index */ + atom_bitset_set(&sg_free, sgi, __ATOMIC_RELAXED); + +no_free_sched_group: + odp_spinlock_unlock(&sched_grp_lock); + + return ODP_SCHED_GROUP_INVALID; +} + +static int schedule_group_destroy(odp_schedule_group_t group) +{ + uint32_t sgi; + sched_group_t *sg; + uint32_t p; + int ret = 0; + + /* Validate inputs */ + if (group < 0 || group >= (odp_schedule_group_t)MAX_SCHED_GROUP) { + ret = -1; + goto invalid_group; + } + + if (sched_ts && + odp_unlikely(__atomic_load_n(&sched_ts->sg_sem, + __ATOMIC_RELAXED) != 0)) { + (void)__atomic_load_n(&sched_ts->sg_sem, + __ATOMIC_ACQUIRE); + sched_ts->sg_sem = 0; + update_sg_membership(sched_ts); + } + odp_spinlock_lock(&sched_grp_lock); + + sgi = (uint32_t)group; + if (bitset_is_set(sg_free, sgi)) { + ret = -1; + goto group_not_found; + } + + sg = sg_vec[sgi]; + /* First ensure all threads have processed group_join/group_leave + * requests. + */ + for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { + if (sg->xcount[p] != 0) { + bitset_t wanted = atom_bitset_load( + &sg->thr_wanted, __ATOMIC_RELAXED); + + sevl(); + while (wfe() && + !bitset_is_eql(wanted, + bitset_monitor(&sg->thr_actual[p], + __ATOMIC_RELAXED))) + doze(); + } + /* Else ignore because no ODP queues on this prio */ + } + + /* Check if all threads/queues have left the group */ + for (p = 0; p < ODP_SCHED_PRIO_NUM; p++) { + if (!bitset_is_null(sg->thr_actual[p])) { + ODP_ERR("Group has threads\n"); + ret = -1; + goto thrd_q_present_in_group; + } + if (sg->xcount[p] != 0) { + ODP_ERR("Group has queues\n"); + ret = -1; + goto thrd_q_present_in_group; + } + } + + _odp_ishm_pool_free(sched_shm_pool, sg); + sg_vec[sgi] = NULL; + atom_bitset_set(&sg_free, sgi, __ATOMIC_RELEASE); + + odp_spinlock_unlock(&sched_grp_lock); + + return ret; + +thrd_q_present_in_group: + +group_not_found: + odp_spinlock_unlock(&sched_grp_lock); + +invalid_group: + + return ret; +} + +static odp_schedule_group_t schedule_group_lookup(const char *name) +{ + uint32_t sgi; + odp_schedule_group_t group; + + /* Validate inputs */ + if (name == NULL) + ODP_ABORT("name or mask is NULL\n"); + + group = ODP_SCHED_GROUP_INVALID; + + odp_spinlock_lock(&sched_grp_lock); + + /* Scan through the schedule group array */ + for (sgi = 0; sgi < MAX_SCHED_GROUP; sgi++) { + if ((sg_vec[sgi] != NULL) && + (strncmp(name, sg_vec[sgi]->name, + ODP_SCHED_GROUP_NAME_LEN) == 0)) { + group = (odp_schedule_group_t)sgi; + break; + } + } + + odp_spinlock_unlock(&sched_grp_lock); + + return group; +} + +static int schedule_group_join(odp_schedule_group_t group, + const odp_thrmask_t *mask) +{ + uint32_t sgi; + sched_group_t *sg; + int ret; + + /* Validate inputs */ + if (group < 0 || group >= ((odp_schedule_group_t)MAX_SCHED_GROUP)) + return -1; + + if (mask == NULL) + ODP_ABORT("name or mask is NULL\n"); + + odp_spinlock_lock(&sched_grp_lock); + + sgi = (uint32_t)group; + if (bitset_is_set(sg_free, sgi)) { + odp_spinlock_unlock(&sched_grp_lock); + return -1; + } + + sg = sg_vec[sgi]; + ret = schedule_group_update(sg, sgi, mask, SCHED_GROUP_JOIN); + + odp_spinlock_unlock(&sched_grp_lock); + + return ret; +} + +static int schedule_group_leave(odp_schedule_group_t group, + const odp_thrmask_t *mask) +{ + uint32_t sgi; + sched_group_t *sg; + int ret = 0; + + /* Validate inputs */ + if (group < 0 || group >= (odp_schedule_group_t)MAX_SCHED_GROUP) { + ret = -1; + goto invalid_group; + } + + if (mask == NULL) + ODP_ABORT("name or mask is NULL\n"); + + odp_spinlock_lock(&sched_grp_lock); + + sgi = (uint32_t)group; + if (bitset_is_set(sg_free, sgi)) { + ret = -1; + goto group_not_found; + } + + sg = sg_vec[sgi]; + + ret = schedule_group_update(sg, sgi, mask, SCHED_GROUP_LEAVE); + + odp_spinlock_unlock(&sched_grp_lock); + + return ret; + +group_not_found: + odp_spinlock_unlock(&sched_grp_lock); + +invalid_group: + return ret; +} + +static int schedule_group_thrmask(odp_schedule_group_t group, + odp_thrmask_t *mask) +{ + uint32_t sgi; + sched_group_t *sg; + int ret = 0; + + /* Validate inputs */ + if (group < 0 || group >= ((odp_schedule_group_t)MAX_SCHED_GROUP)) { + ret = -1; + goto invalid_group; + } + + if (mask == NULL) + ODP_ABORT("name or mask is NULL\n"); + + odp_spinlock_lock(&sched_grp_lock); + + sgi = (uint32_t)group; + if (bitset_is_set(sg_free, sgi)) { + ret = -1; + goto group_not_found; + } + + sg = sg_vec[sgi]; + ret = _schedule_group_thrmask(sg, mask); + + odp_spinlock_unlock(&sched_grp_lock); + + return ret; + +group_not_found: + odp_spinlock_unlock(&sched_grp_lock); + +invalid_group: + return ret; +} + +static int schedule_group_info(odp_schedule_group_t group, + odp_schedule_group_info_t *info) +{ + uint32_t sgi; + sched_group_t *sg; + int ret = 0; + + /* Validate inputs */ + if (group < 0 || group >= ((odp_schedule_group_t)MAX_SCHED_GROUP)) { + ret = -1; + goto invalid_group; + } + + if (info == NULL) + ODP_ABORT("name or mask is NULL\n"); + + odp_spinlock_lock(&sched_grp_lock); + + sgi = (uint32_t)group; + if (bitset_is_set(sg_free, sgi)) { + ret = -1; + goto group_not_found; + } + + sg = sg_vec[sgi]; + + ret = _schedule_group_thrmask(sg, &info->thrmask); + + info->name = sg->name; + + odp_spinlock_unlock(&sched_grp_lock); + + return ret; + +group_not_found: + odp_spinlock_unlock(&sched_grp_lock); + +invalid_group: + return ret; +} + +static int schedule_init_global(void) +{ + odp_thrmask_t mask; + odp_schedule_group_t tmp_all; + odp_schedule_group_t tmp_wrkr; + odp_schedule_group_t tmp_ctrl; + uint32_t bits; + uint32_t pool_size; + uint64_t min_alloc; + uint64_t max_alloc; + + /* Attach to the pool if it exists */ + sched_shm_pool = _odp_ishm_pool_lookup("sched_shm_pool"); + if (sched_shm_pool == NULL) { + /* Add storage required for sched groups. Assume worst case + * xfactor of MAXTHREADS. + */ + pool_size = (sizeof(sched_group_t) + + (ODP_SCHED_PRIO_NUM * MAXTHREADS - 1) * + sizeof(sched_queue_t)) * MAX_SCHED_GROUP; + /* Choose min_alloc and max_alloc such that slab allocator + * is selected. + */ + min_alloc = sizeof(sched_group_t) + + (ODP_SCHED_PRIO_NUM * MAXTHREADS - 1) * + sizeof(sched_queue_t); + max_alloc = min_alloc; + sched_shm_pool = _odp_ishm_pool_create("sched_shm_pool", + pool_size, + min_alloc, max_alloc, + _ODP_ISHM_SINGLE_VA); + if (sched_shm_pool == NULL) { + ODP_ERR("Failed to allocate shared memory pool " + "for sched\n"); + goto failed_sched_shm_pool_create; + } + } + + odp_spinlock_init(&sched_grp_lock); + + bits = MAX_SCHED_GROUP; + if (MAX_SCHED_GROUP == sizeof(sg_free) * CHAR_BIT) + sg_free = ~0; + else + sg_free = (1 << bits) - 1; + + for (uint32_t i = 0; i < MAX_SCHED_GROUP; i++) + sg_vec[i] = NULL; + for (uint32_t i = 0; i < MAXTHREADS; i++) { + thread_state[i].sg_sem = 0; + for (uint32_t j = 0; j < ODP_SCHED_PRIO_NUM; j++) + thread_state[i].sg_wanted[j] = bitset_null(); + } + + /* Create sched groups for default GROUP_ALL, GROUP_WORKER and + * GROUP_CONTROL groups. + */ + odp_thrmask_zero(&mask); + tmp_all = odp_schedule_group_create("__group_all", &mask); + if (tmp_all != ODP_SCHED_GROUP_ALL) { + ODP_ERR("Could not create ODP_SCHED_GROUP_ALL()\n"); + goto failed_create_group_all; + } + + tmp_wrkr = odp_schedule_group_create("__group_worker", &mask); + if (tmp_wrkr != ODP_SCHED_GROUP_WORKER) { + ODP_ERR("Could not create ODP_SCHED_GROUP_WORKER()\n"); + goto failed_create_group_worker; + } + + tmp_ctrl = odp_schedule_group_create("__group_control", &mask); + if (tmp_ctrl != ODP_SCHED_GROUP_CONTROL) { + ODP_ERR("Could not create ODP_SCHED_GROUP_CONTROL()\n"); + goto failed_create_group_control; + } + + return 0; + +failed_create_group_control: + if (tmp_ctrl != ODP_SCHED_GROUP_INVALID) + odp_schedule_group_destroy(ODP_SCHED_GROUP_CONTROL); + +failed_create_group_worker: + if (tmp_wrkr != ODP_SCHED_GROUP_INVALID) + odp_schedule_group_destroy(ODP_SCHED_GROUP_WORKER); + +failed_create_group_all: + if (tmp_all != ODP_SCHED_GROUP_INVALID) + odp_schedule_group_destroy(ODP_SCHED_GROUP_ALL); + +failed_sched_shm_pool_create: + + return -1; +} + +static int schedule_term_global(void) +{ + /* Destroy sched groups for default GROUP_ALL, GROUP_WORKER and + * GROUP_CONTROL groups. + */ + if (odp_schedule_group_destroy(ODP_SCHED_GROUP_ALL) != 0) + ODP_ERR("Failed to destroy ODP_SCHED_GROUP_ALL\n"); + if (odp_schedule_group_destroy(ODP_SCHED_GROUP_WORKER) != 0) + ODP_ERR("Failed to destroy ODP_SCHED_GROUP_WORKER\n"); + if (odp_schedule_group_destroy(ODP_SCHED_GROUP_CONTROL) != 0) + ODP_ERR("Failed to destroy ODP_SCHED_GROUP_CONTROL\n"); + + _odp_ishm_pool_destroy(sched_shm_pool); + + return 0; +} + +static int schedule_init_local(void) +{ + int thr_id; + odp_thread_type_t thr_type; + odp_thrmask_t mask; + + thr_id = odp_thread_id(); + if (thread_state_init(thr_id)) + goto failed_to_init_ts; + + /* Add this thread to default schedule groups */ + thr_type = odp_thread_type(); + odp_thrmask_zero(&mask); + odp_thrmask_set(&mask, thr_id); + + if (odp_schedule_group_join(ODP_SCHED_GROUP_ALL, &mask) != 0) { + ODP_ERR("Failed to join ODP_SCHED_GROUP_ALL\n"); + goto failed_to_join_grp_all; + } + if (thr_type == ODP_THREAD_CONTROL) { + if (odp_schedule_group_join(ODP_SCHED_GROUP_CONTROL, + &mask) != 0) { + ODP_ERR("Failed to join ODP_SCHED_GROUP_CONTROL\n"); + goto failed_to_join_grp_ctrl; + } + } else { + if (odp_schedule_group_join(ODP_SCHED_GROUP_WORKER, + &mask) != 0) { + ODP_ERR("Failed to join ODP_SCHED_GROUP_WORKER\n"); + goto failed_to_join_grp_wrkr; + } + } + + return 0; + +failed_to_join_grp_wrkr: + +failed_to_join_grp_ctrl: + odp_schedule_group_leave(ODP_SCHED_GROUP_ALL, &mask); + +failed_to_join_grp_all: +failed_to_init_ts: + + return -1; +} + +static int schedule_term_local(void) +{ + int thr_id; + odp_thread_type_t thr_type; + odp_thrmask_t mask; + int rc = 0; + + /* Remove this thread from default schedule groups */ + thr_id = odp_thread_id(); + thr_type = odp_thread_type(); + odp_thrmask_zero(&mask); + odp_thrmask_set(&mask, thr_id); + + if (odp_schedule_group_leave(ODP_SCHED_GROUP_ALL, &mask) != 0) + ODP_ERR("Failed to leave ODP_SCHED_GROUP_ALL\n"); + if (thr_type == ODP_THREAD_CONTROL) { + if (odp_schedule_group_leave(ODP_SCHED_GROUP_CONTROL, + &mask) != 0) + ODP_ERR("Failed to leave ODP_SCHED_GROUP_CONTROL\n"); + } else { + if (odp_schedule_group_leave(ODP_SCHED_GROUP_WORKER, + &mask) != 0) + ODP_ERR("Failed to leave ODP_SCHED_GROUP_WORKER\n"); + } + + update_sg_membership(sched_ts); + + /* Check if the thread is still part of any groups */ + if (sched_ts->num_schedq != 0) { + ODP_ERR("Thread %d still part of scheduler group(s)\n", + sched_ts->tidx); + rc = -1; + } + + return rc; +} + +static void pktio_start(int pktio_index, int num_in_queue, int in_queue_idx[]) +{ + int i; + uint32_t old, tag, j; + + for (i = 0; i < num_in_queue; i++) { + /* Try to reserve a slot */ + if (__atomic_fetch_add(&pktin_num, + 1, __ATOMIC_RELAXED) >= PKTIN_MAX) { + __atomic_fetch_sub(&pktin_num, 1, __ATOMIC_RELAXED); + ODP_ABORT("Too many pktio in queues for scheduler\n"); + } + /* A slot has been reserved, now we need to find an empty one */ + for (j = 0; ; j = (j + 1) % PKTIN_MAX) { + if (__atomic_load_n(&pktin_tags[j], + __ATOMIC_RELAXED) != TAG_EMPTY) + /* Slot used, continue with next */ + continue; + /* Empty slot found */ + old = TAG_EMPTY; + tag = PKTIO_QUEUE_2_TAG(pktio_index, in_queue_idx[i]); + if (__atomic_compare_exchange_n(&pktin_tags[j], + &old, + tag, + true, + __ATOMIC_RELEASE, + __ATOMIC_RELAXED)) { + /* Success grabbing slot,update high + * watermark + */ + __atomic_fetch_max(&pktin_hi, + j + 1, __ATOMIC_RELAXED); + /* One more tag (queue) for this pktio + * instance + */ + __atomic_fetch_add(&pktin_count[pktio_index], + 1, __ATOMIC_RELAXED); + /* Continue with next RX queue */ + break; + } + /* Failed to grab slot */ + } + } +} + +static int num_grps(void) +{ + return MAX_SCHED_GROUP; +} + +/* + * Stubs for internal scheduler abstraction layer due to absence of NULL + * checking before calling the function pointer. + */ + +static int thr_add(odp_schedule_group_t group, int thr) +{ + /* This function is a schedule_init_local duplicate. */ + (void)group; + (void)thr; + return 0; +} + +static int thr_rem(odp_schedule_group_t group, int thr) +{ + /* This function is a schedule_term_local duplicate. */ + (void)group; + (void)thr; + return 0; +} + +static int init_queue(uint32_t queue_index, + const odp_schedule_param_t *sched_param) +{ + /* Not used in scalable scheduler. */ + (void)queue_index; + (void)sched_param; + return 0; +} + +static void destroy_queue(uint32_t queue_index) +{ + /* Not used in scalable scheduler. */ + (void)queue_index; +} + +static int sched_queue(uint32_t queue_index) +{ + /* Not used in scalable scheduler. */ + (void)queue_index; + return 0; +} + +static int ord_enq_multi(queue_t handle, void *buf_hdr[], int num, + int *ret) + +{ + queue_entry_t *queue; + sched_scalable_thread_state_t *ts; + int actual; + + ts = sched_ts; + if (ts && odp_unlikely(ts->out_of_order)) { + queue = qentry_from_int(handle); + actual = rctx_save(queue, (odp_buffer_hdr_t **)buf_hdr, num); + *ret = actual; + return 1; + } + return 0; +} + +static void schedule_prefetch(int num) +{ + (void)num; +} + +/* Wait until we are in-order (when processing an ordered queue) + * Note: this function may be called also when processing other queue types + */ +static void order_lock(void) +{ + sched_scalable_thread_state_t *ts; + reorder_window_t *rwin; + uint32_t sn; + + ts = sched_ts; + if (odp_unlikely(ts->out_of_order)) { + /* We are processing ordered queue and are currently + * out-of-order. + * We are in-order when our reorder window slot number (sn) + * equals the head of the reorder window. + */ + ODP_ASSERT(ts->rctx != NULL); + rwin = ts->rctx->rwin; + sn = ts->rctx->sn; + sevl(); + /* Use acquire ordering to be on the safe side even if + * this isn't an acquire/release situation (aka lock). + */ + while (wfe() && + monitor32(&rwin->hc.head, __ATOMIC_ACQUIRE) != sn) + doze(); + } +} + +/* This function is unnecessary. + * The next thread becomes in-order when we release our reorder context + * (i.e. when odp_schedule() is called again. + */ +static void order_unlock(void) +{ +} + +static unsigned schedule_max_ordered_locks(void) +{ + return CONFIG_QUEUE_MAX_ORD_LOCKS; +} + +const schedule_fn_t schedule_scalable_fn = { + .pktio_start = pktio_start, + .thr_add = thr_add, + .thr_rem = thr_rem, + .num_grps = num_grps, + .init_queue = init_queue, + .destroy_queue = destroy_queue, + .sched_queue = sched_queue, + .ord_enq_multi = ord_enq_multi, + .init_global = schedule_init_global, + .term_global = schedule_term_global, + .init_local = schedule_init_local, + .term_local = schedule_term_local, + .order_lock = order_lock, + .order_unlock = order_unlock, + .max_ordered_locks = schedule_max_ordered_locks, +}; + +const schedule_api_t schedule_scalable_api = { + .schedule_wait_time = schedule_wait_time, + .schedule = schedule, + .schedule_multi = schedule_multi, + .schedule_pause = schedule_pause, + .schedule_resume = schedule_resume, + .schedule_release_atomic = schedule_release_atomic, + .schedule_release_ordered = schedule_release_ordered, + .schedule_prefetch = schedule_prefetch, + .schedule_num_prio = schedule_num_prio, + .schedule_group_create = schedule_group_create, + .schedule_group_destroy = schedule_group_destroy, + .schedule_group_lookup = schedule_group_lookup, + .schedule_group_join = schedule_group_join, + .schedule_group_leave = schedule_group_leave, + .schedule_group_thrmask = schedule_group_thrmask, + .schedule_group_info = schedule_group_info, + .schedule_order_lock = schedule_order_lock, + .schedule_order_unlock = schedule_order_unlock, +}; diff --git a/platform/linux-generic/odp_schedule_scalable_ordered.c b/platform/linux-generic/odp_schedule_scalable_ordered.c new file mode 100644 index 00000000..90ddb61c --- /dev/null +++ b/platform/linux-generic/odp_schedule_scalable_ordered.c @@ -0,0 +1,345 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/shared_memory.h> +#include <odp_queue_scalable_internal.h> +#include <odp_schedule_if.h> +#include <odp_bitset.h> + +#include <string.h> + +extern __thread sched_scalable_thread_state_t *sched_ts; + +reorder_window_t *rwin_alloc(_odp_ishm_pool_t *pool, unsigned lock_count) +{ + reorder_window_t *rwin; + uint32_t i; + + rwin = (reorder_window_t *) + shm_pool_alloc_align(pool, sizeof(reorder_window_t)); + if (rwin == NULL) + return NULL; + + rwin->hc.head = 0; + rwin->hc.chgi = 0; + rwin->winmask = RWIN_SIZE - 1; + rwin->tail = 0; + rwin->turn = 0; + rwin->lock_count = (uint16_t)lock_count; + memset(rwin->olock, 0, sizeof(rwin->olock)); + for (i = 0; i < RWIN_SIZE; i++) + rwin->ring[i] = NULL; + + return rwin; +} + +int rwin_free(_odp_ishm_pool_t *pool, reorder_window_t *rwin) +{ + return _odp_ishm_pool_free(pool, rwin); +} + +bool rwin_reserve(reorder_window_t *rwin, uint32_t *sn) +{ + uint32_t head; + uint32_t oldt; + uint32_t newt; + uint32_t winmask; + + /* Read head and tail separately */ + oldt = rwin->tail; + winmask = rwin->winmask; + do { + /* Need __atomic_load to avoid compiler reordering */ + head = __atomic_load_n(&rwin->hc.head, __ATOMIC_RELAXED); + if (odp_unlikely(oldt - head >= winmask)) + return false; + + newt = oldt + 1; + } while (!__atomic_compare_exchange(&rwin->tail, + &oldt, + &newt, + true, + __ATOMIC_RELAXED, + __ATOMIC_RELAXED)); + *sn = oldt; + + return true; +} + +void rwin_insert(reorder_window_t *rwin, + reorder_context_t *rctx, + uint32_t sn, + void (*callback)(reorder_context_t *)) +{ + /* Initialise to silence scan-build */ + hc_t old = {0, 0}; + hc_t new; + uint32_t winmask; + + __atomic_load(&rwin->hc, &old, __ATOMIC_ACQUIRE); + winmask = rwin->winmask; + if (old.head != sn) { + /* We are out-of-order. Store context in reorder window, + * releasing its content. + */ + ODP_ASSERT(rwin->ring[sn & winmask] == NULL); + atomic_store_release(&rwin->ring[sn & winmask], + rctx, + /*readonly=*/false); + rctx = NULL; + do { + hc_t new; + + new.head = old.head; + new.chgi = old.chgi + 1; /* Changed value */ + /* Update head & chgi, fail if any has changed */ + if (__atomic_compare_exchange(&rwin->hc, + /* Updated on fail */ + &old, + &new, + true, + /* Rel our ring update */ + __ATOMIC_RELEASE, + __ATOMIC_ACQUIRE)) + /* CAS succeeded => head same (we are not + * in-order), chgi updated. + */ + return; + /* CAS failed => head and/or chgi changed. + * We might not be out-of-order anymore. + */ + } while (old.head != sn); + } + + /* old.head == sn => we are now in-order! */ + ODP_ASSERT(old.head == sn); + /* We are in-order so our responsibility to retire contexts */ + new.head = old.head; + new.chgi = old.chgi + 1; + + /* Retire our in-order context (if we still have it) */ + if (rctx != NULL) { + callback(rctx); + new.head++; + } + + /* Retire in-order contexts in the ring + * The first context might actually be ours (if we were originally + * out-of-order) + */ + do { + for (;;) { + rctx = __atomic_load_n(&rwin->ring[new.head & winmask], + __ATOMIC_ACQUIRE); + if (rctx == NULL) + break; + /* We are the only thread that are in-order + * (until head updated) so don't have to use + * atomic load-and-clear (exchange) + */ + rwin->ring[new.head & winmask] = NULL; + callback(rctx); + new.head++; + } + /* Update head&chgi, fail if chgi has changed (head cannot change) */ + } while (!__atomic_compare_exchange(&rwin->hc, + &old, /* Updated on failure */ + &new, + false, /* weak */ + __ATOMIC_RELEASE, /* Release our ring updates */ + __ATOMIC_ACQUIRE)); +} + +void rctx_init(reorder_context_t *rctx, uint16_t idx, + reorder_window_t *rwin, uint32_t sn) +{ + /* rctx->rvec_free and rctx->idx already initialised in + * thread_state_init function. + */ + ODP_ASSERT(rctx->idx == idx); + rctx->rwin = rwin; + rctx->sn = sn; + rctx->olock_flags = 0; + /* First => no next reorder context */ + rctx->next_idx = idx; + /* Where to store next event */ + rctx->cur_idx = idx; + rctx->numevts = 0; +} + +inline void rctx_free(const reorder_context_t *rctx) +{ + const reorder_context_t *const base = &rctx[-(int)rctx->idx]; + const uint32_t first = rctx->idx; + uint32_t next_idx; + + next_idx = rctx->next_idx; + + ODP_ASSERT(rctx->rwin != NULL); + /* Set free bit */ + if (rctx->rvec_free == &sched_ts->rvec_free) + /* Since it is our own reorder context, we can instead + * perform a non-atomic and relaxed update on our private + * rvec_free. + */ + sched_ts->priv_rvec_free = + bitset_set(sched_ts->priv_rvec_free, rctx->idx); + else + atom_bitset_set(rctx->rvec_free, rctx->idx, __ATOMIC_RELEASE); + + /* Can't dereference rctx after the corresponding free bit is set */ + while (next_idx != first) { + rctx = &base[next_idx]; + next_idx = rctx->next_idx; + /* Set free bit */ + if (rctx->rvec_free == &sched_ts->rvec_free) + sched_ts->priv_rvec_free = + bitset_set(sched_ts->priv_rvec_free, rctx->idx); + else + atom_bitset_set(rctx->rvec_free, rctx->idx, + __ATOMIC_RELEASE); + } +} + +inline void olock_unlock(const reorder_context_t *rctx, reorder_window_t *rwin, + uint32_t lock_index) +{ + if ((rctx->olock_flags & (1U << lock_index)) == 0) { + /* Use relaxed ordering, we are not releasing any updates */ + rwin->olock[lock_index] = rctx->sn + 1; + } +} + +void olock_release(const reorder_context_t *rctx) +{ + reorder_window_t *rwin; + int i; + + rwin = rctx->rwin; + + for (i = 0; i < rwin->lock_count; i++) + olock_unlock(rctx, rwin, i); +} + +static void blocking_enqueue(queue_entry_t *q, odp_buffer_hdr_t **evts, int num) +{ + int actual; + + /* Iterate until all events have been successfully enqueued */ + for (;;) { + /* Attempt to enqueue remaining events */ + actual = q->s.enqueue_multi(qentry_to_int(q), evts, num); + if (odp_unlikely(actual < 0)) + ODP_ERR("Failed to enqueue deferred events\n"); + /* Update for potential partial success */ + evts += actual; + num -= actual; + if (num == 0) + break; + /* Back-off to decrease load on the system */ + odp_cpu_pause(); + } +} + +void rctx_retire(reorder_context_t *first) +{ + reorder_context_t *rctx; + queue_entry_t *q; + uint32_t i; + uint32_t j; + uint32_t num; + + rctx = first; + do { + /* Process all events in this reorder context */ + for (i = 0; i < rctx->numevts;) { + q = rctx->destq[i]; + /* Find index of next different destq */ + j = i + 1; + while (j < rctx->numevts && rctx->destq[j] == q) + j++; + num = j - i; + /* Blocking enqueue of events to this destq */ + blocking_enqueue(q, &rctx->events[i], num); + i += num; + } + /* Update rctx pointer to point to 'next_idx' element */ + rctx += (int)rctx->next_idx - (int)rctx->idx; + } while (rctx != first); + olock_release(first); + rctx_free(first); +} + +void rctx_release(reorder_context_t *rctx) +{ + /* Insert reorder context into reorder window, potentially calling the + * rctx_retire function for all pending reorder_contexts. + */ + rwin_insert(rctx->rwin, rctx, rctx->sn, rctx_retire); +} + +/* Save destination queue and events in the reorder context for deferred + * enqueue. + */ +int rctx_save(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num) +{ + int i; + sched_scalable_thread_state_t *ts; + reorder_context_t *first; + reorder_context_t *cur; + bitset_t next_idx; + + ts = sched_ts; + first = ts->rctx; + ODP_ASSERT(ts->rctx != NULL); + cur = &first[(int)first->cur_idx - (int)first->idx]; + for (i = 0; i < num; i++) { + if (odp_unlikely(cur->numevts == RC_EVT_SIZE)) { + /* No more space in current reorder context + * Try to allocate another. + */ + if (odp_unlikely( + bitset_is_null(ts->priv_rvec_free))) { + ts->priv_rvec_free = + atom_bitset_xchg( + &ts->rvec_free, + 0, + __ATOMIC_RELAXED); + if (odp_unlikely(bitset_is_null( + ts->priv_rvec_free))) + /* Out of reorder contexts. + * Return the number of events + * stored so far. + */ + return i; + } + next_idx = bitset_ffs(ts->priv_rvec_free) - 1; + ts->priv_rvec_free = + bitset_clr(ts->priv_rvec_free, + next_idx); + /* Link current to next (for eventual + * retiring) + */ + cur->next_idx = next_idx; + /* Link first to next (for next call to + * queue_enq_multi()) + */ + first->cur_idx = next_idx; + /* Update current to next */ + cur = &ts->rvec[next_idx]; + rctx_init(cur, next_idx, NULL, 0); + /* The last rctx (so far) */ + cur->next_idx = first->idx; + } + cur->events[cur->numevts] = buf_hdr[i]; + cur->destq[cur->numevts] = queue; + cur->numevts++; + } + /* All events stored. */ + return num; +} commit 9fd48a9215a7831ca951839b7187bd1eb3f7bb06 Author: Brian Brooks <brian.brooks(a)arm.com> Date: Fri Jun 23 16:04:39 2017 -0500 linux-gen: sched scalable: add a concurrent queue Signed-off-by: Ola Liljedahl <ola.liljedahl(a)arm.com> Reviewed-by: Brian Brooks <brian.brooks(a)arm.com> Reviewed-and-tested-by: Yi He <yi.he(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 39322dc1..19e2241b 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -170,6 +170,7 @@ noinst_HEADERS = \ ${srcdir}/include/odp_errno_define.h \ ${srcdir}/include/odp_forward_typedefs_internal.h \ ${srcdir}/include/odp_internal.h \ + ${srcdir}/include/odp_llqueue.h \ ${srcdir}/include/odp_name_table_internal.h \ ${srcdir}/include/odp_packet_internal.h \ ${srcdir}/include/odp_packet_io_internal.h \ diff --git a/platform/linux-generic/include/odp_llqueue.h b/platform/linux-generic/include/odp_llqueue.h new file mode 100644 index 00000000..99b12e66 --- /dev/null +++ b/platform/linux-generic/include/odp_llqueue.h @@ -0,0 +1,311 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_LLQUEUE_H_ +#define ODP_LLQUEUE_H_ + +#include <odp/api/cpu.h> +#include <odp/api/hints.h> +#include <odp/api/spinlock.h> + +#include <odp_config_internal.h> +#include <odp_debug_internal.h> +#include <odp_cpu.h> + +#include <stdint.h> +#include <stdlib.h> + +/****************************************************************************** + * Linked list queues + *****************************************************************************/ + +struct llqueue; +struct llnode; + +static struct llnode *llq_head(struct llqueue *llq); +static void llqueue_init(struct llqueue *llq); +static void llq_enqueue(struct llqueue *llq, struct llnode *node); +static struct llnode *llq_dequeue(struct llqueue *llq); +static odp_bool_t llq_dequeue_cond(struct llqueue *llq, struct llnode *exp); +static odp_bool_t llq_cond_rotate(struct llqueue *llq, struct llnode *node); +static odp_bool_t llq_on_queue(struct llnode *node); + +/****************************************************************************** + * The implementation(s) + *****************************************************************************/ + +#define SENTINEL ((void *)~(uintptr_t)0) + +#ifdef CONFIG_LLDSCD +/* Implement queue operations using double-word LL/SC */ + +/* The scalar equivalent of a double pointer */ +#if __SIZEOF_PTRDIFF_T__ == 4 +typedef uint64_t dintptr_t; +#endif +#if __SIZEOF_PTRDIFF_T__ == 8 +typedef __int128 dintptr_t; +#endif + +struct llnode { + struct llnode *next; +}; + +union llht { + struct { + struct llnode *head, *tail; + } st; + dintptr_t ui; +}; + +struct llqueue { + union llht u; +}; + +static inline struct llnode *llq_head(struct llqueue *llq) +{ + return __atomic_load_n(&llq->u.st.head, __ATOMIC_RELAXED); +} + +static inline void llqueue_init(struct llqueue *llq) +{ + llq->u.st.head = NULL; + llq->u.st.tail = NULL; +} + +static inline void llq_enqueue(struct llqueue *llq, struct llnode *node) +{ + union llht old, neu; + + ODP_ASSERT(node->next == NULL); + node->next = SENTINEL; + do { + old.ui = lld(&llq->u.ui, __ATOMIC_RELAXED); + neu.st.head = old.st.head == NULL ? node : old.st.head; + neu.st.tail = node; + } while (odp_unlikely(scd(&llq->u.ui, neu.ui, __ATOMIC_RELEASE))); + if (old.st.tail != NULL) { + /* List was not empty */ + ODP_ASSERT(old.st.tail->next == SENTINEL); + old.st.tail->next = node; + } +} + +static inline struct llnode *llq_dequeue(struct llqueue *llq) +{ + struct llnode *head; + union llht old, neu; + + /* llq_dequeue() may be used in a busy-waiting fashion + * Read head using plain load to avoid disturbing remote LL/SC + */ + head = __atomic_load_n(&llq->u.st.head, __ATOMIC_ACQUIRE); + if (head == NULL) + return NULL; + /* Read head->next before LL to minimize cache miss latency + * in LL/SC below + */ + (void)__atomic_load_n(&head->next, __ATOMIC_RELAXED); + + do { + old.ui = lld(&llq->u.ui, __ATOMIC_RELAXED); + if (odp_unlikely(old.st.head == NULL)) { + /* Empty list */ + return NULL; + } else if (odp_unlikely(old.st.head == old.st.tail)) { + /* Single-element in list */ + neu.st.head = NULL; + neu.st.tail = NULL; + } else { + /* Multi-element list, dequeue head */ + struct llnode *next; + /* Wait until llq_enqueue() has written true next + * pointer + */ + while ((next = __atomic_load_n(&old.st.head->next, + __ATOMIC_RELAXED)) == + SENTINEL) + odp_cpu_pause(); + neu.st.head = next; + neu.st.tail = old.st.tail; + } + } while (odp_unlikely(scd(&llq->u.ui, neu.ui, __ATOMIC_RELAXED))); + old.st.head->next = NULL; + return old.st.head; +} + +static inline odp_bool_t llq_dequeue_cond(struct llqueue *llq, + struct llnode *exp) +{ + union llht old, neu; + + do { + old.ui = lld(&llq->u.ui, __ATOMIC_ACQUIRE); + if (odp_unlikely(old.st.head == NULL || old.st.head != exp)) { + /* Empty list or wrong head */ + return false; + } else if (odp_unlikely(old.st.head == old.st.tail)) { + /* Single-element in list */ + neu.st.head = NULL; + neu.st.tail = NULL; + } else { + /* Multi-element list, dequeue head */ + struct llnode *next; + + /* Wait until llq_enqueue() has written true next + * pointer */ + while ((next = __atomic_load_n(&old.st.head->next, + __ATOMIC_RELAXED)) == + SENTINEL) + odp_cpu_pause(); + + neu.st.head = next; + neu.st.tail = old.st.tail; + } + } while (odp_unlikely(scd(&llq->u.ui, neu.ui, __ATOMIC_RELAXED))); + old.st.head->next = NULL; + return true; +} + +/* If 'node' is a head of llq then move it to tail */ +static inline odp_bool_t llq_cond_rotate(struct llqueue *llq, + struct llnode *node) +{ + /* Difficult to make this into a single atomic operation + * Instead use existing primitives. + */ + if (odp_likely(llq_dequeue_cond(llq, node))) { + llq_enqueue(llq, node); + return true; + } + return false; +} + +static inline odp_bool_t llq_on_queue(struct llnode *node) +{ + return node->next != NULL; +} + +#else +/* Implement queue operations protected by a spin lock */ + +struct llnode { + struct llnode *next; +}; + +struct llqueue { + struct llnode *head, *tail; + odp_spinlock_t lock; +}; + +static inline struct llnode *llq_head(struct llqueue *llq) +{ + return __atomic_load_n(&llq->head, __ATOMIC_RELAXED); +} + +static inline void llqueue_init(struct llqueue *llq) +{ + llq->head = NULL; + llq->tail = NULL; + odp_spinlock_init(&llq->lock); +} + +static inline void llq_enqueue(struct llqueue *llq, struct llnode *node) +{ + ODP_ASSERT(node->next == NULL); + node->next = SENTINEL; + + odp_spinlock_lock(&llq->lock); + if (llq->head == NULL) { + llq->head = node; + llq->tail = node; + } else { + llq->tail->next = node; + llq->tail = node; + } + odp_spinlock_unlock(&llq->lock); +} + +static inline struct llnode *llq_dequeue(struct llqueue *llq) +{ + struct llnode *head; + struct llnode *node = NULL; + + head = __atomic_load_n(&llq->head, __ATOMIC_RELAXED); + if (head == NULL) + return NULL; + + odp_spinlock_lock(&llq->lock); + if (llq->head != NULL) { + node = llq->head; + if (llq->head == llq->tail) { + ODP_ASSERT(node->next == SENTINEL); + llq->head = NULL; + llq->tail = NULL; + } else { + ODP_ASSERT(node->next != SENTINEL); + llq->head = node->next; + } + node->next = NULL; + } + odp_spinlock_unlock(&llq->lock); + return node; +} + +static inline odp_bool_t llq_dequeue_cond(struct llqueue *llq, + struct llnode *node) +{ + odp_bool_t success = false; + + odp_spinlock_lock(&llq->lock); + if (odp_likely(llq->head != NULL && llq->head == node)) { + success = true; + if (llq->head == llq->tail) { + ODP_ASSERT(node->next == SENTINEL); + llq->head = NULL; + llq->tail = NULL; + } else { + ODP_ASSERT(node->next != SENTINEL); + llq->head = node->next; + } + node->next = NULL; + } + odp_spinlock_unlock(&llq->lock); + return success; +} + +/* If 'node' is a head of llq then move it to tail */ +static inline odp_bool_t llq_cond_rotate(struct llqueue *llq, + struct llnode *node) +{ + odp_bool_t success = false; + + odp_spinlock_lock(&llq->lock); + if (odp_likely(llq->head == node)) { + success = true; + if (llq->tail != node) { + ODP_ASSERT(node->next != SENTINEL); + llq->head = node->next; + llq->tail->next = node; + llq->tail = node; + node->next = SENTINEL; + } + /* Else 'node' is only element on list => nothing to do */ + } + odp_spinlock_unlock(&llq->lock); + return success; +} + +static inline odp_bool_t llq_on_queue(struct llnode *node) +{ + return node->next != NULL; +} + +#endif + +#endif commit bf554b2d9472ed8ed04580904d0aa906cdbe1e83 Author: Brian Brooks <brian.brooks(a)arm.com> Date: Fri Jun 23 16:04:38 2017 -0500 linux-gen: sched scalable: add a bitset Signed-off-by: Ola Liljedahl <ola.liljedahl(a)arm.com> Reviewed-by: Brian Brooks <brian.brooks(a)arm.com> Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli(a)arm.com> Reviewed-and-tested-by: Yi He <yi.he(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 09e25166..39322dc1 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -159,6 +159,7 @@ noinst_HEADERS = \ ${srcdir}/include/odp_atomic_internal.h \ ${srcdir}/include/odp_buffer_inlines.h \ ${srcdir}/include/odp_bitmap_internal.h \ + ${srcdir}/include/odp_bitset.h \ ${srcdir}/include/odp_buffer_internal.h \ ${srcdir}/include/odp_classification_datamodel.h \ ${srcdir}/include/odp_classification_inlines.h \ diff --git a/platform/linux-generic/include/odp_bitset.h b/platform/linux-generic/include/odp_bitset.h new file mode 100644 index 00000000..4b7dd6d6 --- /dev/null +++ b/platform/linux-generic/include/odp_bitset.h @@ -0,0 +1,212 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _ODP_BITSET_H_ +#define _ODP_BITSET_H_ + +#include <odp_cpu.h> + +#include <limits.h> + +/****************************************************************************** + * bitset abstract data type + *****************************************************************************/ +/* This could be a struct of scalars to support larger bit sets */ + +/* + * Size of atomic bit set. This limits the max number of threads, + * scheduler groups and reorder windows. On ARMv8/64-bit and x86-64, the + * (lock-free) max is 128 + */ + +/* Find a suitable data type that supports lock-free atomic operations */ +#if defined(__aarch64__) && defined(__SIZEOF_INT128__) && \ + __SIZEOF_INT128__ == 16 +#define LOCKFREE16 +typedef __int128 bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT128__) + +#elif __GCC_ATOMIC_LLONG_LOCK_FREE == 2 && \ + __SIZEOF_LONG_LONG__ != __SIZEOF_LONG__ +typedef unsigned long long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG_LONG__) + +#elif __GCC_ATOMIC_LONG_LOCK_FREE == 2 && __SIZEOF_LONG__ != __SIZEOF_INT__ +typedef unsigned long bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_LONG__) + +#elif __GCC_ATOMIC_INT_LOCK_FREE == 2 +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) + +#else +/* Target does not support lock-free atomic operations */ +typedef unsigned int bitset_t; +#define ATOM_BITSET_SIZE (CHAR_BIT * __SIZEOF_INT__) +#endif + +#if ATOM_BITSET_SIZE <= 32 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1UL << bit; +} + +/* Return first-bit-set with StdC ffs() semantics */ +static inline uint32_t bitset_ffs(bitset_t b) +{ + return __builtin_ffsl(b); +} + +/* Load-exclusive with memory ordering */ +static inline bitset_t bitset_monitor(bitset_t *bs, int mo) +{ + return monitor32(bs, mo); +} + +#elif ATOM_BITSET_SIZE <= 64 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + return 1ULL << bit; +} + +/* Return first-bit-set with StdC ffs() semantics */ +static inline uint32_t bitset_ffs(bitset_t b) +{ + return __builtin_ffsll(b); +} + +/* Load-exclusive with memory ordering */ +static inline bitset_t bitset_monitor(bitset_t *bs, int mo) +{ + return monitor64(bs, mo); +} + +#elif ATOM_BITSET_SIZE <= 128 + +static inline bitset_t bitset_mask(uint32_t bit) +{ + if (bit < 64) + return 1ULL << bit; + else + return (unsigned __int128)(1ULL << (bit - 64)) << 64; +} + +/* Return first-bit-set with StdC ffs() semantics */ +static inline uint32_t bitset_ffs(bitset_t b) +{ + if ((uint64_t)b != 0) + return __builtin_ffsll((uint64_t)b); + else if ((b >> 64) != 0) + return __builtin_ffsll((uint64_t)(b >> 64)) + 64; + else + return 0; +} + +/* Load-exclusive with memory ordering */ +static inline bitset_t bitset_monitor(bitset_t *bs, int mo) +{ + return monitor128(bs, mo); +} + +#else +#error Unsupported size of bit sets (ATOM_BITSET_SIZE) +#endif + +/* Atomic load with memory ordering */ +static inline bitset_t atom_bitset_load(bitset_t *bs, int mo) +{ +#ifdef LOCKFREE16 + return __lockfree_load_16(bs, mo); +#else + return __atomic_load_n(bs, mo); +#endif +} + +/* Atomic bit set with memory ordering */ +static inline void atom_bitset_set(bitset_t *bs, uint32_t bit, int mo) +{ +#ifdef LOCKFREE16 + (void)__lockfree_fetch_or_16(bs, bitset_mask(bit), mo); +#else + (void)__atomic_fetch_or(bs, bitset_mask(bit), mo); +#endif +} + +/* Atomic bit clear with memory ordering */ +static inline void atom_bitset_clr(bitset_t *bs, uint32_t bit, int mo) +{ +#ifdef LOCKFREE16 + (void)__lockfree_fetch_and_16(bs, ~bitset_mask(bit), mo); +#else + (void)__atomic_fetch_and(bs, ~bitset_mask(bit), mo); +#endif +} + +/* Atomic exchange with memory ordering */ +static inline bitset_t atom_bitset_xchg(bitset_t *bs, bitset_t neu, int mo) +{ +#ifdef LOCKFREE16 + return __lockfree_exchange_16(bs, neu, mo); +#else + return __atomic_exchange_n(bs, neu, mo); +#endif +} + +/* Atomic compare&exchange with memory ordering */ +static inline bitset_t atom_bitset_cmpxchg(bitset_t *bs, bitset_t *old, + bitset_t neu, bool weak, + int mo_success, int mo_failure) +{ +#ifdef LOCKFREE16 + return __lockfree_compare_exchange_16( + bs, old, neu, weak, mo_success, mo_failure); +#else + return __atomic_compare_exchange_n( + bs, old, neu, weak, mo_success, mo_failure); +#endif +} + +/* Return a & ~b */ +static inline bitset_t bitset_andn(bitset_t a, bitset_t b) +{ + return a & ~b; +} + +static inline bool bitset_is_eql(bitset_t a, bitset_t b) +{ + return a == b; +} + +static inline bitset_t bitset_clr(bitset_t bs, uint32_t bit) +{ + return bs & ~bitset_mask(bit); +} + +static inline bitset_t bitset_set(bitset_t bs, uint32_t bit) +{ + return bs | bitset_mask(bit); +} + +static inline bitset_t bitset_null(void) +{ + return 0U; +} + +static inline bool bitset_is_null(bitset_t a) +{ + return a == 0U; +} + +static inline bool bitset_is_set(bitset_t a, uint32_t bit) +{ + return (a & bitset_mask(bit)) != 0; +} + +#endif commit ac89f9c568830498b186dff33e908053dbdc88c7 Author: Brian Brooks <brian.brooks(a)arm.com> Date: Fri Jun 23 16:04:37 2017 -0500 linux-gen: sched scalable: add arch files Signed-off-by: Brian Brooks <brian.brooks(a)arm.com> Signed-off-by: Ola Liljedahl <ola.liljedahl(a)arm.com> Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli(a)arm.com> Reviewed-and-tested-by: Yi He <yi.he(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 2293f7f0..09e25166 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -8,6 +8,7 @@ AM_CFLAGS += -I$(srcdir)/include AM_CFLAGS += -I$(top_srcdir)/include AM_CFLAGS += -I$(top_srcdir)/include/odp/arch/@ARCH_ABI@ AM_CFLAGS += -I$(top_builddir)/include +AM_CFLAGS += -I$(top_srcdir)/arch/@ARCH_DIR@ AM_CFLAGS += -Iinclude AM_CFLAGS += -DSYSCONFDIR=\"@sysconfdir@\" AM_CFLAGS += -D_ODP_PKTIO_IPC @@ -198,6 +199,22 @@ noinst_HEADERS = \ ${srcdir}/include/protocols/udp.h \ ${srcdir}/Makefile.inc +if ARCH_IS_ARM +noinst_HEADERS += ${srcdir}/arch/arm/odp_atomic.h \ + ${srcdir}/arch/arm/odp_cpu.h \ + ${srcdir}/arch/arm/odp_cpu_idling.h \ + ${srcdir}/arch/arm/odp_llsc.h +endif +if ARCH_IS_MIPS64 +noinst_HEADERS += ${srcdir}/arch/mips64/odp_cpu.h +endif +if ARCH_IS_POWERPC +noinst_HEADERS += ${srcdir}/arch/powerpc/odp_cpu.h +endif +if ARCH_IS_X86 +noinst_HEADERS += ${srcdir}/arch/x86/odp_cpu.h +endif + __LIB__libodp_linux_la_SOURCES = \ _fdserver.c \ _ishm.c \ diff --git a/platform/linux-generic/arch/arm/odp_atomic.h b/platform/linux-generic/arch/arm/odp_atomic.h new file mode 100644 index 00000000..3a21a47b --- /dev/null +++ b/platform/linux-generic/arch/arm/odp_atomic.h @@ -0,0 +1,212 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H +#define PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H +#error This file should not be included directly, please include odp_cpu.h +#endif + +#ifdef CONFIG_DMBSTR + +#define atomic_store_release(loc, val, ro) \ +do { \ + _odp_release_barrier(ro); \ + __atomic_store_n(loc, val, __ATOMIC_RELAXED); \ +} while (0) + +#else + +#define atomic_store_release(loc, val, ro) \ + __atomic_store_n(loc, val, __ATOMIC_RELEASE) + +#endif /* CONFIG_DMBSTR */ + +#ifdef __aarch64__ + +#define HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE) +#define HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || (mo) == __ATOMIC_ACQ_REL || \ + (mo) == __ATOMIC_SEQ_CST) + +#define LL_MO(mo) (HAS_ACQ((mo)) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED) +#define SC_MO(mo) (HAS_RLS((mo)) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED) + +#ifndef __ARM_FEATURE_QRDMX /* Feature only available in v8.1a and beyond */ +static inline bool +__lockfree_compare_exchange_16(register __int128 *var, __int128 *exp, + register __int128 neu, bool weak, int mo_success, + int mo_failure) +{ + (void)weak; /* Always do strong CAS or we can't perform atomic read */ + /* Ignore memory ordering for failure, memory order for + * success must be stronger or equal. */ + (void)mo_failure; + register __int128 old; + register __int128 expected; + int ll_mo = LL_MO(mo_success); + int sc_mo = SC_MO(mo_success); + + expected = *exp; + __asm__ volatile("" ::: "memory"); + do { + /* Atomicity of LLD is not guaranteed */ + old = lld(var, ll_mo); + /* Must write back neu or old to verify atomicity of LLD */ + } while (odp_unlikely(scd(var, old == expected ? neu : old, sc_mo))); + *exp = old; /* Always update, atomically read value */ + return old == expected; +} + +static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu, + int mo) +{ + register __int128 old; + int ll_mo = LL_MO(mo); + int sc_mo = SC_MO(mo); + + do { + /* Atomicity of LLD is not guaranteed */ + old = lld(var, ll_mo); + /* Must successfully write back to verify atomicity of LLD */ + } while (odp_unlikely(scd(var, neu, sc_mo))); + return old; +} + +static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask, + int mo) +{ + register __int128 old; + int ll_mo = LL_MO(mo); + int sc_mo = SC_MO(mo); + + do { + /* Atomicity of LLD is not guaranteed */ + old = lld(var, ll_mo); + /* Must successfully write back to verify atomicity of LLD */ + } while (odp_unlikely(scd(var, old & mask, sc_mo))); + return old; +} + +static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask, + int mo) +{ + register __int128 old; + int ll_mo = LL_MO(mo); + int sc_mo = SC_MO(mo); + + do { + /* Atomicity of LLD is not guaranteed */ + old = lld(var, ll_mo); + /* Must successfully write back to verify atomicity of LLD */ + } while (odp_unlikely(scd(var, old | mask, sc_mo))); + return old; +} + +#else + +static inline __int128 casp(__int128 *var, __int128 old, __int128 neu, int mo) +{ + if (mo == __ATOMIC_RELAXED) { + __asm__ volatile("casp %0, %H0, %1, %H1, [%2]" + : "+r" (old) + : "r" (neu), "r" (var) + : "memory"); + } else if (mo == __ATOMIC_ACQUIRE) { + __asm__ volatile("caspa %0, %H0, %1, %H1, [%2]" + : "+r" (old) + : "r" (neu), "r" (var) + : "memory"); + } else if (mo == __ATOMIC_ACQ_REL) { + __asm__ volatile("caspal %0, %H0, %1, %H1, [%2]" + : "+r" (old) + : "r" (neu), "r" (var) + : "memory"); + } else if (mo == __ATOMIC_RELEASE) { + __asm__ volatile("caspl %0, %H0, %1, %H1, [%2]" + : "+r" (old) + : "r" (neu), "r" (var) + : "memory"); + } else { + abort(); + } + return old; +} + +static inline bool +__lockfree_compare_exchange_16(register __int128 *var, __int128 *exp, + register __int128 neu, bool weak, int mo_success, + int mo_failure) +{ + (void)weak; + (void)mo_failure; + __int128 old; + __int128 expected; + + expected = *exp; + old = casp(var, expected, neu, mo_success); + *exp = old; /* Always update, atomically read value */ + return old == expected; +} + +static inline __int128 __lockfree_exchange_16(__int128 *var, __int128 neu, + int mo) +{ + __int128 old; + __int128 expected; + + do { + expected = *var; + old = casp(var, expected, neu, mo); + } while (old != expected); + return old; +} + +static inline __int128 __lockfree_fetch_and_16(__int128 *var, __int128 mask, + int mo) +{ + __int128 old; + __int128 expected; + + do { + expected = *var; + old = casp(var, expected, expected & mask, mo); + } while (old != expected); + return old; +} + +static inline __int128 __lockfree_fetch_or_16(__int128 *var, __int128 mask, + int mo) +{ + __int128 old; + __int128 expected; + + do { + expected = *var; + old = casp(var, expected, expected | mask, mo); + } while (old != expected); + return old; +} + +#endif /* __ARM_FEATURE_QRDMX */ + +static inline __int128 __lockfree_load_16(__int128 *var, int mo) +{ + __int128 old = *var; /* Possibly torn read */ + + /* Do CAS to ensure atomicity + * Either CAS succeeds (writing back the same value) + * Or CAS fails and returns the old value (atomic read) + */ + (void)__lockfree_compare_exchange_16(var, &old, old, false, mo, mo); + return old; +} + +#endif /* __aarch64__ */ + +#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_ATOMIC_H */ diff --git a/platform/linux-generic/arch/arm/odp_cpu.h b/platform/linux-generic/arch/arm/odp_cpu.h new file mode 100644 index 00000000..8ef50da4 --- /dev/null +++ b/platform/linux-generic/arch/arm/odp_cpu.h @@ -0,0 +1,71 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H +#define PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H + +#if !defined(__arm__) && !defined(__aarch64__) +#error Use this file only when compiling for ARM architecture +#endif + +#include <odp_debug_internal.h> + +/* + * Use LLD/SCD atomic primitives instead of lock-based code path in llqueue + * LLD/SCD is on ARM the fastest way to enqueue and dequeue elements from a + * linked list queue. + */ +#define CONFIG_LLDSCD + +/* + * Use DMB;STR instead of STRL on ARM + * On early ARMv8 implementations (e.g. Cortex-A57) this is noticeably more + * performant than using store-release. + * This also allows for load-only barriers (DMB ISHLD) which are much cheaper + * than a full barrier + */ +#define CONFIG_DMBSTR + +/* + * Use ARM event signalling mechanism + * Event signalling minimises spinning (busy waiting) which decreases + * cache coherency traffic when spinning on shared locations (thus faster and + * more scalable) and enables the CPU to enter a sleep state (lower power + * consumption). + */ +#define CONFIG_WFE + +static inline void dmb(void) +{ + __asm__ volatile("dmb" : : : "memory"); +} + +#ifdef __aarch64__ + +/* Only ARMv8 supports DMB ISHLD */ +/* A load only barrier is much cheaper than full barrier */ +#define _odp_release_barrier(ro) \ +do { \ + if (ro) \ + __asm__ volatile("dmb ishld" ::: "memory"); \ + else \ + __asm__ volatile("dmb ish" ::: "memory"); \ +} while (0) + +#else + +#define _odp_release_barrier(ro) \ + __atomic_thread_fence(__ATOMIC_RELEASE) + +#endif /* __aarch64__ */ + +#include "odp_llsc.h" +#include "odp_atomic.h" +#include "odp_cpu_idling.h" + +#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H */ diff --git a/platform/linux-generic/arch/arm/odp_cpu_idling.h b/platform/linux-generic/arch/arm/odp_cpu_idling.h new file mode 100644 index 00000000..3ae1028c --- /dev/null +++ b/platform/linux-generic/arch/arm/odp_cpu_idling.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H +#define PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H +#error This file should not be included directly, please include odp_cpu.h +#endif + +static inline void sevl(void) +{ +#ifdef CONFIG_WFE + __asm__ volatile("sevl" : : : ); +#endif +} + +static inline int wfe(void) +{ +#ifdef CONFIG_WFE + __asm__ volatile("wfe" : : : "memory"); +#endif + return 1; +} + +static inline void doze(void) +{ +#ifndef CONFIG_WFE + /* When using WFE do not stall the pipeline using other means */ + odp_cpu_pause(); +#endif +} + +#ifdef CONFIG_WFE +#ifdef __aarch64__ +#define monitor128(addr, mo) lld((addr), (mo)) +#endif +#define monitor64(addr, mo) ll64((addr), (mo)) +#define monitor32(addr, mo) ll32((addr), (mo)) +#define monitor8(addr, mo) ll8((addr), (mo)) +#else +#define monitor128(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor64(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor32(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor8(addr, mo) __atomic_load_n((addr), (mo)) +#endif + +#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_CPU_IDLING_H */ diff --git a/platform/linux-generic/arch/arm/odp_llsc.h b/platform/linux-generic/arch/arm/odp_llsc.h new file mode 100644 index 00000000..73c4d7b1 --- /dev/null +++ b/platform/linux-generic/arch/arm/odp_llsc.h @@ -0,0 +1,253 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H +#define PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H + +#ifndef PLATFORM_LINUXGENERIC_ARCH_ARM_ODP_CPU_H +#error This file should not be included directly, please include odp_cpu.h +#endif + +#ifdef __arm__ + +static inline uint32_t ll8(uint8_t *var, int mm) +{ + uint8_t old; + + __asm__ volatile("ldrexb %0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + /* Barrier after an acquiring load */ + if (mm == __ATOMIC_ACQUIRE) + dmb(); + return old; +} + +static inline uint32_t ll(uint32_t *var, int mm) +{ + uint32_t old; + + __asm__ volatile("ldrex %0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + /* Barrier after an acquiring load */ + if (mm == __ATOMIC_ACQUIRE) + dmb(); + return old; +} + +#define ll32(a, b) ll((a), (b)) + +/* Return 0 on success, 1 on failure */ +static inline uint32_t sc(uint32_t *var, uint32_t neu, int mm) +{ + uint32_t ret; + + /* Barrier before a releasing store */ + if (mm == __ATOMIC_RELEASE) + dmb(); + __asm__ volatile("strex %0, %1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + return ret; +} + +#define sc32(a, b, c) sc((a), (b), (c)) + +static inline uint64_t lld(uint64_t *var, int mm) +{ + uint64_t old; + + __asm__ volatile("ldrexd %0, %H0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + /* Barrier after an acquiring load */ + if (mm == __ATOMIC_ACQUIRE) + dmb(); + return old; +} + +#define ll64(a, b) lld((a), (b)) + +/* Return 0 on success, 1 on failure */ +static inline uint32_t scd(uint64_t *var, uint64_t neu, int mm) +{ + uint32_t ret; + + /* Barrier before a releasing store */ + if (mm == __ATOMIC_RELEASE) + dmb(); + __asm__ volatile("strexd %0, %1, %H1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + return ret; +} + +#define sc64(a, b, c) scd((a), (b), (c)) + +#endif /* __arm__ */ + +#ifdef __aarch64__ + +static inline uint16_t ll8(uint8_t *var, int mm) +{ + uint16_t old; + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxrb %w0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("ldxrb %w0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + else + ODP_ABORT(); + return old; +} + +static inline uint32_t ll32(uint32_t *var, int mm) +{ + uint32_t old; + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxr %w0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("ldxr %w0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + else + ODP_ABORT(); + return old; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t sc32(uint32_t *var, uint32_t neu, int mm) +{ + uint32_t ret; + + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxr %w0, %w1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("stxr %w0, %w1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + else + ODP_ABORT(); + return ret; +} + +static inline uint64_t ll(uint64_t *var, int mm) +{ + uint64_t old; + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxr %0, [%1]" + : "=&r" (old) + : "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("ldxr %0, [%1]" + : "=&r" (old) + : "r" (var) + : ); + else + ODP_ABORT(); + return old; +} + +#define ll64(a, b) ll((a), (b)) + +/* Return 0 on success, 1 on failure */ +static inline uint32_t sc(uint64_t *var, uint64_t neu, int mm) +{ + uint32_t ret; + + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxr %w0, %1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("stxr %w0, %1, [%2]" + : "=&r" (ret) + : "r" (neu), "r" (var) + : ); + else + ODP_ABORT(); + return ret; +} + +#define sc64(a, b, c) sc((a), (b), (c)) + +union i128 { + __int128 i128; + int64_t i64[2]; +}; + +static inline __int128 lld(__int128 *var, int mm) +{ + union i128 old; + + if (mm == __ATOMIC_ACQUIRE) + __asm__ volatile("ldaxp %0, %1, [%2]" + : "=&r" (old.i64[0]), "=&r" (old.i64[1]) + : "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("ldxp %0, %1, [%2]" + : "=&r" (old.i64[0]), "=&r" (old.i64[1]) + : "r" (var) + : ); + else + ODP_ABORT(); + return old.i128; +} + +/* Return 0 on success, 1 on failure */ +static inline uint32_t scd(__int128 *var, __int128 neu, int mm) +{ + uint32_t ret; + + if (mm == __ATOMIC_RELEASE) + __asm__ volatile("stlxp %w0, %1, %2, [%3]" + : "=&r" (ret) + : "r" (((union i128)neu).i64[0]), + "r" (((union i128)neu).i64[1]), + "r" (var) + : "memory"); + else if (mm == __ATOMIC_RELAXED) + __asm__ volatile("stxp %w0, %1, %2, [%3]" + : "=&r" (ret) + : "r" (((union i128)neu).i64[0]), + "r" (((union i128)neu).i64[1]), + "r" (var) + : ); + else + ODP_ABORT(); + return ret; +} + +#endif /* __aarch64__ */ + +#endif /* PLATFORM_LINUXGENERIC_ARCH_ARM_LLSC_H */ diff --git a/platform/linux-generic/arch/default/odp_cpu.h b/platform/linux-generic/arch/default/odp_cpu.h new file mode 100644 index 00000000..12a44b93 --- /dev/null +++ b/platform/linux-generic/arch/default/odp_cpu.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_DEFAULT_CPU_H_ +#define ODP_DEFAULT_CPU_H_ + +/****************************************************************************** + * Atomics + *****************************************************************************/ + +#define atomic_store_release(loc, val, ro) \ + __atomic_store_n(loc, val, __ATOMIC_RELEASE) + +/****************************************************************************** + * Idle mgmt + *****************************************************************************/ + +static inline void sevl(void) +{ + /* empty */ +} + +static inline int wfe(void) +{ + return 1; +} + +#define monitor128(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor64(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor32(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor8(addr, mo) __atomic_load_n((addr), (mo)) + +static inline void doze(void) +{ + odp_cpu_pause(); +} + +#endif diff --git a/platform/linux-generic/arch/mips64/odp_cpu.h b/platform/linux-generic/arch/mips64/odp_cpu.h new file mode 100644 index 00000000..1097f7bc --- /dev/null +++ b/platform/linux-generic/arch/mips64/odp_cpu.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_MIPS64_CPU_H_ +#define ODP_MIPS64_CPU_H_ + +/****************************************************************************** + * Atomics + *****************************************************************************/ + +#define atomic_store_release(loc, val, ro) \ + __atomic_store_n(loc, val, __ATOMIC_RELEASE) + +/****************************************************************************** + * Idle mgmt + *****************************************************************************/ + +static inline void sevl(void) +{ + /* empty */ +} + +static inline int wfe(void) +{ + return 1; +} + +#define monitor128(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor64(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor32(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor8(addr, mo) __atomic_load_n((addr), (mo)) + +static inline void doze(void) +{ + odp_cpu_pause(); +} + +#endif diff --git a/platform/linux-generic/arch/powerpc/odp_cpu.h b/platform/linux-generic/arch/powerpc/odp_cpu.h new file mode 100644 index 00000000..cd27fe24 --- /dev/null +++ b/platform/linux-generic/arch/powerpc/odp_cpu.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_POWERPC_CPU_H_ +#define ODP_POWERPC_CPU_H_ + +/****************************************************************************** + * Atomics + *****************************************************************************/ + +#define atomic_store_release(loc, val, ro) \ + __atomic_store_n(loc, val, __ATOMIC_RELEASE) + +/****************************************************************************** + * Idle mgmt + *****************************************************************************/ + +static inline void sevl(void) +{ + /* empty */ +} + +static inline int wfe(void) +{ + return 1; +} + +#define monitor128(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor64(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor32(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor8(addr, mo) __atomic_load_n((addr), (mo)) + +static inline void doze(void) +{ + odp_cpu_pause(); +} + +#endif diff --git a/platform/linux-generic/arch/x86/odp_cpu.h b/platform/linux-generic/arch/x86/odp_cpu.h new file mode 100644 index 00000000..b5c89409 --- /dev/null +++ b/platform/linux-generic/arch/x86/odp_cpu.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2017, ARM Limited. All rights reserved. + * + * Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef ODP_X86_CPU_H_ +#define ODP_X86_CPU_H_ + +/****************************************************************************** + * Atomics + *****************************************************************************/ + +#define atomic_store_release(loc, val, ro) \ + __atomic_store_n(loc, val, __ATOMIC_RELEASE) + +/****************************************************************************** + * Idle mgmt + *****************************************************************************/ + +static inline void sevl(void) +{ + /* empty */ +} + +static inline int wfe(void) +{ + return 1; +} + +#define monitor128(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor64(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor32(addr, mo) __atomic_load_n((addr), (mo)) +#define monitor8(addr, mo) __atomic_load_n((addr), (mo)) + +static inline void doze(void) +{ + odp_cpu_pause(); +} + +#endif commit 5b08cc290a4a84ed0ffbc4a591427f1891b9ace3 Author: Brian Brooks <brian.brooks(a)arm.com> Date: Fri Jun 23 16:04:36 2017 -0500 test: odp_pktio_ordered: add queue size Signed-off-by: Brian Brooks <brian.brooks(a)arm.com> Reviewed-and-tested-by: Yi He <yi.he(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/common_plat/performance/odp_pktio_ordered.c b/test/common_plat/performance/odp_pktio_ordered.c index 4bb0bef9..50bfef51 100644 --- a/test/common_plat/performance/odp_pktio_ordered.c +++ b/test/common_plat/performance/odp_pktio_ordered.c @@ -91,6 +91,9 @@ /** Maximum number of pktio queues per interface */ #define MAX_QUEUES 32 +/** Seems to need at least 8192 elements per queue */ +#define QUEUE_SIZE 8192 + /** Maximum number of pktio interfaces */ #define MAX_PKTIOS 8 @@ -1232,6 +1235,7 @@ int main(int argc, char *argv[]) qparam.sched.prio = ODP_SCHED_PRIO_DEFAULT; qparam.sched.sync = ODP_SCHED_SYNC_ATOMIC; qparam.sched.group = ODP_SCHED_GROUP_ALL; + qparam.size = QUEUE_SIZE; gbl_args->flow_qcontext[i][j].idx = i; gbl_args->flow_qcontext[i][j].input_queue = 0; ----------------------------------------------------------------------- Summary of changes: .travis.yml | 1 + platform/linux-generic/Makefile.am | 26 + platform/linux-generic/arch/arm/odp_atomic.h | 212 +++ platform/linux-generic/arch/arm/odp_cpu.h | 71 + platform/linux-generic/arch/arm/odp_cpu_idling.h | 53 + platform/linux-generic/arch/arm/odp_llsc.h | 253 +++ platform/linux-generic/arch/default/odp_cpu.h | 43 + platform/linux-generic/arch/mips64/odp_cpu.h | 43 + platform/linux-generic/arch/powerpc/odp_cpu.h | 43 + platform/linux-generic/arch/x86/odp_cpu.h | 43 + .../include/odp/api/plat/schedule_types.h | 4 +- platform/linux-generic/include/odp_bitset.h | 212 +++ .../linux-generic/include/odp_config_internal.h | 15 +- platform/linux-generic/include/odp_llqueue.h | 311 +++ .../include/odp_queue_scalable_internal.h | 104 + platform/linux-generic/include/odp_schedule_if.h | 2 +- .../linux-generic/include/odp_schedule_scalable.h | 139 ++ .../include/odp_schedule_scalable_config.h | 52 + .../include/odp_schedule_scalable_ordered.h | 132 ++ platform/linux-generic/m4/odp_schedule.m4 | 55 +- platform/linux-generic/odp_queue_if.c | 8 + platform/linux-generic/odp_queue_scalable.c | 1022 ++++++++++ platform/linux-generic/odp_schedule_if.c | 6 + platform/linux-generic/odp_schedule_scalable.c | 1980 ++++++++++++++++++++ .../linux-generic/odp_schedule_scalable_ordered.c | 345 ++++ test/common_plat/performance/odp_pktio_ordered.c | 4 + 26 files changed, 5157 insertions(+), 22 deletions(-) create mode 100644 platform/linux-generic/arch/arm/odp_atomic.h create mode 100644 platform/linux-generic/arch/arm/odp_cpu.h create mode 100644 platform/linux-generic/arch/arm/odp_cpu_idling.h create mode 100644 platform/linux-generic/arch/arm/odp_llsc.h create mode 100644 platform/linux-generic/arch/default/odp_cpu.h create mode 100644 platform/linux-generic/arch/mips64/odp_cpu.h create mode 100644 platform/linux-generic/arch/powerpc/odp_cpu.h create mode 100644 platform/linux-generic/arch/x86/odp_cpu.h create mode 100644 platform/linux-generic/include/odp_bitset.h create mode 100644 platform/linux-generic/include/odp_llqueue.h create mode 100644 platform/linux-generic/include/odp_queue_scalable_internal.h create mode 100644 platform/linux-generic/include/odp_schedule_scalable.h create mode 100644 platform/linux-generic/include/odp_schedule_scalable_config.h create mode 100644 platform/linux-generic/include/odp_schedule_scalable_ordered.h create mode 100644 platform/linux-generic/odp_queue_scalable.c create mode 100644 platform/linux-generic/odp_schedule_scalable.c create mode 100644 platform/linux-generic/odp_schedule_scalable_ordered.c hooks/post-receive --

7 years, 7 months

[COMMIT] branch api-next updated. v1.15.0.0-381-g5551c0d7

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via 5551c0d76bd721110a0fa4ba1423fe1edb0fd530 (commit) via 9b78da8e2373ab0530cfbbd72b2229059b57ddf1 (commit) via 689145bcef5ada4304f1d97d6b72e10bf8308ac0 (commit) via c6a309f00f882fabe70fa535ebe8b765f6e9bd11 (commit) via a4add29291572219143dc667db4e8a7832a5692b (commit) via 497c339f94e35d5fadeff539cebfbf44d698dc7e (commit) via fdd3df3547c78d81d1244eac3fa65512e9023237 (commit) via e5b211403233dda67507f18fd91f9826022fd287 (commit) via d232db102fc77e6cb2fd9d62ba387c22f68c042a (commit) from e2169ff2026af51eec351cd679891c303ab651e0 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 5551c0d76bd721110a0fa4ba1423fe1edb0fd530 Merge: e2169ff2 9b78da8e Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Wed Jun 28 23:01:24 2017 +0300 Merge branch 'master' into api-next diff --cc .travis.yml index 89f028d1,bae23600..f74c564f --- a/.travis.yml +++ b/.travis.yml @@@ -42,9 -42,8 +42,9 @@@ addons packages: - gcc - clang-3.8 - - automake autoconf autoconf-archive libtool libssl-dev graphviz mscgen doxygen + - automake autoconf libtool libssl-dev graphviz mscgen doxygen - libpcap-dev + - libconfig-dev # coverity_scan: # project: # name: "$TRAVIS_REPO_SLUG" ----------------------------------------------------------------------- Summary of changes: .travis.yml | 4 +- DEPENDENCIES | 5 +- Makefile.am | 9 ++- m4/ax_check_compile_flag.m4 | 72 ++++++++++++++++++++++ platform/linux-generic/m4/configure.m4 | 5 +- platform/linux-generic/odp_crypto.c | 43 ++++++++++--- platform/linux-generic/odp_time.c | 4 +- test/common_plat/validation/api/time/Makefile.am | 1 + .../validation/api/traffic_mngr/Makefile.am | 3 +- 9 files changed, 123 insertions(+), 23 deletions(-) create mode 100644 m4/ax_check_compile_flag.m4 hooks/post-receive --

7 years, 7 months

[COMMIT] branch next updated. v1.15.0.0-14-g9b78da8e

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, next has been updated via 9b78da8e2373ab0530cfbbd72b2229059b57ddf1 (commit) via 689145bcef5ada4304f1d97d6b72e10bf8308ac0 (commit) via c6a309f00f882fabe70fa535ebe8b765f6e9bd11 (commit) via a4add29291572219143dc667db4e8a7832a5692b (commit) via 497c339f94e35d5fadeff539cebfbf44d698dc7e (commit) via fdd3df3547c78d81d1244eac3fa65512e9023237 (commit) via e5b211403233dda67507f18fd91f9826022fd287 (commit) via d232db102fc77e6cb2fd9d62ba387c22f68c042a (commit) via e7e92031e08fe525e421ab30223b7398881ed42e (commit) via 09a7800c4c4a093fb962e362952f9cf562d2fc98 (commit) via ddf2ac38b1288283f7b3d7c9e16e36f392685866 (commit) via f4337956ee3be1e783e2389eb5a5ceecb4b0b0ca (commit) via cd1f187d33d65cb54781b5a3c0864229b19c11c3 (commit) via 0d24adacfe17c8e0e1348f19cd8b75b64cb13ccf (commit) via 5dd7d9ed05ee228406cc3c0ae6cd9caa20a71f08 (commit) from 4bda102286cf55b0bc93fdf8f8f276d5528fdf14 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- ----------------------------------------------------------------------- Summary of changes: .travis.yml | 5 +- DEPENDENCIES | 5 +- Makefile.am | 9 +- configure.ac | 3 + doc/application-api-guide/examples.dox | 5 + example/Makefile.am | 1 + example/ipfragreass/.gitignore | 3 + example/ipfragreass/Makefile.am | 23 + example/ipfragreass/odp_ipfragreass.c | 376 ++++++++++ example/ipfragreass/odp_ipfragreass_atomics.h | 55 ++ example/ipfragreass/odp_ipfragreass_atomics_arm.h | 120 ++++ example/ipfragreass/odp_ipfragreass_fragment.c | 99 +++ example/ipfragreass/odp_ipfragreass_fragment.h | 28 + example/ipfragreass/odp_ipfragreass_helpers.c | 124 ++++ example/ipfragreass/odp_ipfragreass_helpers.h | 79 +++ example/ipfragreass/odp_ipfragreass_ip.h | 251 +++++++ example/ipfragreass/odp_ipfragreass_reassemble.c | 771 +++++++++++++++++++++ example/ipfragreass/odp_ipfragreass_reassemble.h | 211 ++++++ example/m4/configure.m4 | 1 + m4/ax_check_compile_flag.m4 | 72 ++ platform/Makefile.inc | 6 +- platform/linux-generic/Makefile.am | 40 +- platform/linux-generic/arch/arm/odp_cpu_arch.c | 38 +- .../linux-generic/include/odp_packet_internal.h | 10 + platform/linux-generic/m4/configure.m4 | 5 +- platform/linux-generic/odp_crypto.c | 43 +- platform/linux-generic/odp_packet_io.c | 8 +- platform/linux-generic/odp_time.c | 4 +- platform/linux-generic/pktio/loop.c | 4 +- test/common_plat/validation/api/time/Makefile.am | 1 + .../validation/api/traffic_mngr/Makefile.am | 3 +- 31 files changed, 2361 insertions(+), 42 deletions(-) create mode 100644 example/ipfragreass/.gitignore create mode 100644 example/ipfragreass/Makefile.am create mode 100644 example/ipfragreass/odp_ipfragreass.c create mode 100644 example/ipfragreass/odp_ipfragreass_atomics.h create mode 100644 example/ipfragreass/odp_ipfragreass_atomics_arm.h create mode 100644 example/ipfragreass/odp_ipfragreass_fragment.c create mode 100644 example/ipfragreass/odp_ipfragreass_fragment.h create mode 100644 example/ipfragreass/odp_ipfragreass_helpers.c create mode 100644 example/ipfragreass/odp_ipfragreass_helpers.h create mode 100644 example/ipfragreass/odp_ipfragreass_ip.h create mode 100644 example/ipfragreass/odp_ipfragreass_reassemble.c create mode 100644 example/ipfragreass/odp_ipfragreass_reassemble.h create mode 100644 m4/ax_check_compile_flag.m4 hooks/post-receive --

7 years, 7 months

[COMMIT] branch master updated. v1.15.0.0-14-g9b78da8e

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via 9b78da8e2373ab0530cfbbd72b2229059b57ddf1 (commit) via 689145bcef5ada4304f1d97d6b72e10bf8308ac0 (commit) from c6a309f00f882fabe70fa535ebe8b765f6e9bd11 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- ----------------------------------------------------------------------- Summary of changes: platform/linux-generic/m4/configure.m4 | 5 +++-- platform/linux-generic/odp_time.c | 4 +--- 2 files changed, 4 insertions(+), 5 deletions(-) hooks/post-receive --

7 years, 7 months

[COMMIT] branch cloud-dev updated. v1.15.0.0-398-g1ba26aa5

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, cloud-dev has been updated via 1ba26aa5650c05718c177842178de6d0f70b7fc1 (commit) via e2169ff2026af51eec351cd679891c303ab651e0 (commit) via e3c2e06c4420f41d7053e615ac1cc6c1dc315407 (commit) via e7e92031e08fe525e421ab30223b7398881ed42e (commit) via f9dcbcc3f45fcd6d825c087fc55c631d6b30fa8c (commit) via 845743bd80da73f2a296f24090dfe058d3be7dec (commit) via bbd08a6c0c31c12275089c3134c9e7e58eb0fafa (commit) via 0468dc01b3ae1517d57cf960b511ab066a9d5f05 (commit) via 274a0b016d8c2eed9f065186ad5458c5edb94dad (commit) via 0ce5d628d817003ca0f96beddece3f618f4593bb (commit) via 65a46f66bfcb6b2034d4b0563facc36cbac1e286 (commit) via 441fb964f22b2baaa97641e6f8dda88d32d153f6 (commit) via d5ff31f14e87c815d3a0e0461206aa2687152dd5 (commit) via 09a7800c4c4a093fb962e362952f9cf562d2fc98 (commit) via ddf2ac38b1288283f7b3d7c9e16e36f392685866 (commit) via 4bcd15e2bfb839ac0ef6419aa9e7b720958638f7 (commit) via f4337956ee3be1e783e2389eb5a5ceecb4b0b0ca (commit) via cd1f187d33d65cb54781b5a3c0864229b19c11c3 (commit) via 58a8ec90b57eae43be6461d1ae407e70a87c65d2 (commit) via ebbd8301f396e3f2f106ed94ce5c3970ec8bacce (commit) via 0d24adacfe17c8e0e1348f19cd8b75b64cb13ccf (commit) via a5ef33a6f2575cd011cb05c3fb1b06d1c017f879 (commit) via 5dd7d9ed05ee228406cc3c0ae6cd9caa20a71f08 (commit) via 4bda102286cf55b0bc93fdf8f8f276d5528fdf14 (commit) via 452c37072103864d7117e31f521b5e620e4937b5 (commit) via 0b5e8c6870b959ad9db426aa067326f4baad5d58 (commit) via b5f5510a9751ad60915f5b9fcf8763a97f77b5ab (commit) via 79ed8c746ba13233c111a61d2d4f17e360dd8023 (commit) via 11a1d56876d14cd715a1906a95a7c3734999bdc0 (commit) via 40c06707ed485e5e957a413d697f4f1370f98837 (commit) via 1b8c663ecd3b8a7c34ae52163961cdb442383d31 (commit) via e24f6b3d0c130af3838ba49661d4bc9cb83e940f (commit) via 9815336c6c7c64df3b40a1fb0c1c668a36d5550b (commit) via e5faf09d8dfa9631caa7ccf1461fd4bc9f69d4c0 (commit) via e248b708ff31467591fb99f7cc8652ab9e3081b2 (commit) via ff6d71903cdfdd68246e3daabedd645bf30640cc (commit) via bddd9fb0a5353c611bb9de78f3355d30147f8efc (commit) via 2156eaa076fe0ca0aaa910c8902d5b60aa3bbed6 (commit) via ca1ebaa8319604d4f5fe303be7150b7278fe9d94 (commit) via 8d4ecf491a0af45c499d38c6e72853e756d69f0e (commit) via b93c17b30d051b114a099ed5219fb2a82f5a864c (commit) via fec16fd81d4084340f0f9a6893c9772000608031 (commit) via 2210dc7d26125ac9b8b51fa4de7b2b72e4e8f13e (commit) via c3d3787ae527291e969a60bc54dd4e2ab220251d (commit) via 14df663ba9534101305ea4f5201fc2a5e55a974f (commit) via dcd53771432aeb838f2584458589b3430104abcf (commit) via 3362bb0eea7b4cdeb97cd2558e9bc5e4ee176852 (commit) via 2239b759ad0cb09e3a2ec3d132580123ed92ab03 (commit) via a28dab7edeeee74db65349231ec400d92d491a22 (commit) via e3a92c5bf48f2279cd4ef5c8d22eafc523445626 (commit) via 93e9e20c17426d3262aa96a589106e635b755090 (commit) via ab3003c87e4cc25e3b26c7bddbfbce61525a94ad (commit) via 93ee286c21a576c265463ca5dcd17b39ef80962b (commit) via 1ebbb8f882ae6506fc9fa2cb02dadae3d35eaf58 (commit) via 1adf94727a92a260f3145c3c777cc8cf3ecb419b (commit) via ad5d48fe658927a7b946533d50fd907e306ec184 (commit) via ce8bebad45c590ad5b06d31d8be9fae8182f944a (commit) via 8e9242391f79ed54430b5fdc37e992addf9a26b8 (commit) via 0344270a085f78d9f6d4fbef3c497750bbcee071 (commit) via 697c3be9b2155379dee458d66335442375a9354d (commit) via 2ffff0bc3e07678214a413b5eb89da13e7624055 (commit) via dfbab7438c8404d5f4af20086e2100d40a5ffc0c (commit) via 60150bba495711e6fdfc948ffb4ad405878fbcbc (commit) via fbe34c754b7abe15100779ce9833a7f200517d9f (commit) via 62f95a564b7b3e08a32e6e72d234eb86292021d0 (commit) via 17f4cddfcbf47f73761cb869ace264cc8444dc08 (commit) via f54860c628c16c4f76d555b9d20c28e04af13751 (commit) via a5116e42abf0452f5ad9fd1920d5377aa0ef36b6 (commit) via 73954b9142d1ddc2dde7c0f9b6596e2d8bac3caf (commit) via 89a00a2ed43a99e0d2d78a5baeff1292e41ecd34 (commit) via 8f07f3de3da7961cd29046d2107f27a0f7391703 (commit) via 4bb3025e76a81130e8e80cd582ae2433d3450e2b (commit) via 629be2dd2d166ed1246b8c1982af3a55a7035657 (commit) via af05215e26c99693c000981fddfaf3cdd2225e3f (commit) via b9676fc5ba51d0f26a9c8c40f0b65a003113f1b5 (commit) via 583dbc9adab460c926339353cc882095594b5b60 (commit) via 867d8fc93d074005b52375995b9beed4a896c1ed (commit) via 87b108d54a697b0c3704342ec7b4d484d04dfdbb (commit) via 20d0d4d1a34725dec053ecd7cd46979345f7d625 (commit) via 8367530231e7cd4c33c63b789a5ed8322b51e58f (commit) via 48ad38ac169c48ac67e0758d3d45eb146255ed09 (commit) via a074bc12f6a960c195daf3ceb28e3f717cfc3aa2 (commit) via 4418eef8a7b7f621047101f284ce5e5df44f0239 (commit) via 464fdc4efb4148dd8d1ec1aeaa2a7ffcb58d7faf (commit) via 7394a5c8dead5edcfa5bda571aceba5945c91d16 (commit) via 93afeeaf4e3dee3099830fccbb53bb32e64e3090 (commit) via fdddf685fcad06f05f3788705968622b23a569ac (commit) via 29b29677ed7d5d396f4aec1d0f3197efbf119113 (commit) via 8c6ca3886ae4ffb111d3a81d09d21b68e0e11349 (commit) via 19c488fdd9f9868cbe60f1ff8558f48c7bba6f04 (commit) via 966c1c956a21f4e70c873a3c3546e65299837457 (commit) via 99478ee923dfd8279a7a5149769aee49df315dde (commit) via be9ca86f431d5eaab0702f7d76db60cd75982e15 (commit) via 2b34cda253fd28f3b373dc3ee66753d47a0c9b33 (commit) via 166e4e6a7f2e289f305203e75c68cea56d015c08 (commit) via e149fb058c6ea718e22c77c6fc930141004dcba2 (commit) via e6e666d533615de703993836b958514208de942a (commit) via 8184de8ef8492a739d30c354628aada2e2bab366 (commit) via 0ee227d51cdb1cc29853dac0376e6e02b6e585de (commit) via 658f6f27ed62c651d8555a4de0ef103191c491b1 (commit) via b8de7dfbadb10da65a0a675db5f588cc3998508a (commit) via 361f0fa4addf83ae9b30d5e705e1f86643a45bab (commit) via cb17e380f1231328cbaaf35635f6db9fb5b99bb8 (commit) via 244fce69b25af28cbe5c19b11bfc3da5aa8da95d (commit) via 974e4477a35742b8021e25692a458d16cd2087f1 (commit) via aadd1d62084191bf2fb909a7dbedee628b197929 (commit) via 248ebf35eaf51d962e9d14a4e2c541d3cbc96268 (commit) via 373f97f88d327ca0e211df33d6fcaad0b9188a1c (commit) via 7cf390d11e69f6b677e6fbfd6414944737406fc3 (commit) via 61a5edca59780fc2a4a0e656e09738145892a3a0 (commit) via 8d19f94fd96eb5a411eafb75c487dc3f972297bd (commit) via 4f97e500a097928e308a415c32a88465adc5f5cc (commit) via 2ea71a57ae4b6122e3e74782aeea2846a757cb14 (commit) via 8b30e228ba2c303b3c69ffa4fbb9f138d2523d9a (commit) via 57783b35f4383bdfdd146f2a3333bbc8bb995896 (commit) via 904631219eab0af46cb99e8a56742401452ac2c3 (commit) via f7bd3a7bbb3613260ef2bad1e3ede5145571dd6c (commit) via d3602cb56413bb31b4535976dfd3dd723a16bfc9 (commit) via 908e27635dfe08e249407f4c4e53aa2c7ce12ea1 (commit) via d80e025c8baebed0305648737e528580b84fae2a (commit) via 33a0dad444fab1c1567083b289eac7727eab3b31 (commit) via ccb2cd5eb7383f46051dff2b1e7675ed54f635b2 (commit) via d8e4647baa7b1159480dfde4d5a2143146e8c6cb (commit) via 20cc38e36a52f70758c3464014f4f00a94d2139d (commit) via 9bb2a439ae2a8751df5a6007a7e242d7bcb963e5 (commit) via a6fa9f3e721dbdb99e2a0bac320f988930e5f874 (commit) via b2261b9485e6993642c5c0c7ea58abb8a2d356a9 (commit) via c969cc4e6f6e2e0cc5c4e407dd696be069cf4525 (commit) via 3a1832865c90865de3fd8bd30103995637e7851c (commit) via ab5a099937bc3d81f610fcfc2a98bfb7e677a06a (commit) via 9f506fecfcd408a8af0df0e9627795d0b1059730 (commit) via c184a9c8d35c10b30014e7538a862a7bc42556e7 (commit) via 9ebcb7c6e2f3226c915c39c81dcd1895dda3639b (commit) via 2f137b2bc9cb9c03a586d2cb324a6a5659098c39 (commit) via 2f1a96d47bb2e6c413ed06749633dcfe33f02b97 (commit) via 4cdaf5527dba63858080c0942ac1c971c615f066 (commit) via 77dff4e05611bc5b06b4bfb5dd86da6d4c3fffa5 (commit) via e6be64e01589f1aa335ea178e8314bf35ad34847 (commit) via 15ea6562d284bd5f2bd20e9b79bd102db106555c (commit) via a48979af5b36c11fb5c7f8d2c6029c12bb191b51 (commit) via a9fc17e1bbef1c3585fd5de47e5023e4d66504f5 (commit) via 7a7e3ff9d50d67b7c79058db98ba27ab5a806437 (commit) via b67bfce457a7546bbae4dad16b5185629eda77e5 (commit) via 87bba414b4444d8247adf3411e8bbefd35899f1b (commit) via 0f47f4c4b48fb08326d6be9c0b1019f738d57911 (commit) via fc815dec86d080ca7d1b8fadddcab1e7c8e4b11b (commit) via 472d34cc0fd2b5f96277b1799583ab6c45375d40 (commit) via 148a700fb9af847a64d732bf7ed9975aa03fefa9 (commit) via ba6cad6319b917c078dd0c20cd6b011637195898 (commit) via 552817483e9d4b6a84d49960920f1de50029f111 (commit) via c5eb1703fe9e7529ae12ecf1799b757e1a992afd (commit) via 2e36f75db71b230021cd907a5aadf39559683d16 (commit) via ea134fe159c0d249e4bed12b7269e8236afa0262 (commit) via 60b9dd435bf1be55794e5be6ee24714d28a5b587 (commit) via f20dee9db3da9425d8764e414b04cdcbd41bece8 (commit) via 3faf025987a1640983080a2743e58fbb3cbac55e (commit) via 599ac6802352c9c56eca33a6004824783becaa6e (commit) via 0a3e5be140e77167173b9af4b72cdad8451aff71 (commit) from c460b6fbaeae6c60d75e8e5a60c34a6260b9e4fa (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 1ba26aa5650c05718c177842178de6d0f70b7fc1 Merge: c460b6fb e2169ff2 Author: Yi He <yi.he(a)linaro.org> Date: Wed Jun 28 09:51:23 2017 +0800 Merge branch 'api-next' into cloud-dev ----------------------------------------------------------------------- Summary of changes: .checkpatch.conf | 2 + .travis.yml | 149 ++-- CHANGELOG | 267 +++++++ DEPENDENCIES | 5 +- Makefile.am | 4 +- configure.ac | 22 +- doc/application-api-guide/examples.dox | 5 + doc/users-guide/users-guide.adoc | 36 +- example/Makefile.am | 1 + example/generator/odp_generator.c | 89 ++- example/ipfragreass/.gitignore | 3 + example/ipfragreass/Makefile.am | 23 + example/ipfragreass/odp_ipfragreass.c | 376 ++++++++++ example/ipfragreass/odp_ipfragreass_atomics.h | 55 ++ example/ipfragreass/odp_ipfragreass_atomics_arm.h | 120 ++++ example/ipfragreass/odp_ipfragreass_fragment.c | 99 +++ example/ipfragreass/odp_ipfragreass_fragment.h | 28 + example/ipfragreass/odp_ipfragreass_helpers.c | 124 ++++ example/ipfragreass/odp_ipfragreass_helpers.h | 79 +++ example/ipfragreass/odp_ipfragreass_ip.h | 251 +++++++ example/ipfragreass/odp_ipfragreass_reassemble.c | 771 +++++++++++++++++++++ example/ipfragreass/odp_ipfragreass_reassemble.h | 211 ++++++ example/ipsec/odp_ipsec_cache.c | 1 + example/m4/configure.m4 | 1 + include/odp/api/spec/crypto.h | 6 +- include/odp/api/spec/event.h | 80 ++- include/odp/api/spec/feature.h | 69 ++ include/odp/api/spec/init.h | 15 + include/odp/api/spec/ipsec.h | 469 +++++++------ include/odp/arch/default/api/abi/event.h | 9 +- include/odp_api.h | 1 + pkgconfig/libodp-linux.pc.in | 2 +- platform/Makefile.inc | 6 +- platform/linux-generic/Makefile.am | 43 +- platform/linux-generic/arch/arm/odp_cpu_arch.c | 38 +- .../include/odp/{drv/hints.h => api/feature.h} | 10 +- .../include/odp/api/plat/event_types.h | 8 +- .../include/odp/api/plat/time_types.h | 1 + .../include/odp_classification_datamodel.h | 4 +- .../linux-generic/include/odp_crypto_internal.h | 17 +- platform/linux-generic/include/odp_internal.h | 2 +- .../linux-generic/include/odp_packet_internal.h | 16 + .../linux-generic/include/odp_packet_io_queue.h | 20 +- platform/linux-generic/include/odp_packet_socket.h | 7 - platform/linux-generic/include/odp_queue_if.h | 104 +++ .../linux-generic/include/odp_queue_internal.h | 41 +- platform/linux-generic/include/odp_schedule_if.h | 2 +- .../linux-generic/include/odp_timer_internal.h | 24 + .../include/odp_traffic_mngr_internal.h | 4 +- platform/linux-generic/m4/configure.m4 | 45 ++ platform/linux-generic/odp_classification.c | 10 +- platform/linux-generic/odp_crypto.c | 628 ++++++++--------- platform/linux-generic/odp_init.c | 24 +- platform/linux-generic/odp_ipsec.c | 67 +- platform/linux-generic/odp_packet.c | 64 +- platform/linux-generic/odp_packet_io.c | 98 +-- platform/linux-generic/odp_queue.c | 342 ++++++--- platform/linux-generic/odp_queue_if.c | 103 +++ platform/linux-generic/odp_schedule.c | 14 +- platform/linux-generic/odp_schedule_iquery.c | 14 +- platform/linux-generic/odp_schedule_sp.c | 7 +- platform/linux-generic/odp_timer.c | 112 ++- platform/linux-generic/odp_traffic_mngr.c | 21 +- platform/linux-generic/pktio/loop.c | 17 +- platform/linux-generic/pktio/socket.c | 183 ++--- scripts/checkpatch.pl | 6 +- scripts/spelling.txt | 151 +++- test/common_plat/performance/odp_crypto.c | 58 +- test/common_plat/performance/odp_l2fwd_run.sh | 3 + test/common_plat/validation/api/crypto/crypto.h | 8 +- .../validation/api/crypto/odp_crypto_test_inp.c | 417 +++++++++-- .../validation/api/crypto/test_vectors.h | 181 +++-- .../validation/api/crypto/test_vectors_len.h | 18 +- test/common_plat/validation/api/init/init.c | 8 +- test/common_plat/validation/api/pktio/pktio.c | 11 +- test/common_plat/validation/api/time/Makefile.am | 10 +- test/common_plat/validation/api/time/time.sh | 42 ++ test/linux-generic/Makefile.am | 2 +- test/linux-generic/pktio_ipc/pktio_ipc_run.sh | 3 - 79 files changed, 5152 insertions(+), 1235 deletions(-) create mode 100644 example/ipfragreass/.gitignore create mode 100644 example/ipfragreass/Makefile.am create mode 100644 example/ipfragreass/odp_ipfragreass.c create mode 100644 example/ipfragreass/odp_ipfragreass_atomics.h create mode 100644 example/ipfragreass/odp_ipfragreass_atomics_arm.h create mode 100644 example/ipfragreass/odp_ipfragreass_fragment.c create mode 100644 example/ipfragreass/odp_ipfragreass_fragment.h create mode 100644 example/ipfragreass/odp_ipfragreass_helpers.c create mode 100644 example/ipfragreass/odp_ipfragreass_helpers.h create mode 100644 example/ipfragreass/odp_ipfragreass_ip.h create mode 100644 example/ipfragreass/odp_ipfragreass_reassemble.c create mode 100644 example/ipfragreass/odp_ipfragreass_reassemble.h create mode 100644 include/odp/api/spec/feature.h copy platform/linux-generic/include/odp/{drv/hints.h => api/feature.h} (61%) create mode 100644 platform/linux-generic/include/odp_queue_if.h create mode 100644 platform/linux-generic/odp_queue_if.c create mode 100755 test/common_plat/validation/api/time/time.sh hooks/post-receive --

7 years, 7 months

[COMMIT] branch master updated. v1.15.0.0-12-gc6a309f0

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via c6a309f00f882fabe70fa535ebe8b765f6e9bd11 (commit) from a4add29291572219143dc667db4e8a7832a5692b (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit c6a309f00f882fabe70fa535ebe8b765f6e9bd11 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Mon Jun 26 18:00:16 2017 +0300 configure: remove dependency of autoconf-archive autoconf-archive is not available on some build environment Number of projects simple have local copy of this file. Do the same thing. Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> Reviewed-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov(a)linaro.org> diff --git a/.travis.yml b/.travis.yml index 2cc8caf7..bae23600 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ addons: packages: - gcc - clang-3.8 - - automake autoconf autoconf-archive libtool libssl-dev graphviz mscgen doxygen + - automake autoconf libtool libssl-dev graphviz mscgen doxygen - libpcap-dev # coverity_scan: # project: diff --git a/DEPENDENCIES b/DEPENDENCIES index 7bcbd5eb..a194cad1 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -8,14 +8,13 @@ Prerequisites for building the OpenDataPlane (ODP) API automake autoconf - autoconf-archive libtool On Debian/Ubuntu systems: - $ sudo apt-get install automake autoconf autoconf-archive libtool + $ sudo apt-get install automake autoconf libtool On CentOS/RedHat/Fedora systems: - $ sudo yum install automake autoconf autoconf-archive libtool + $ sudo yum install automake autoconf libtool 3. Required libraries diff --git a/m4/ax_check_compile_flag.m4 b/m4/ax_check_compile_flag.m4 new file mode 100644 index 00000000..c3a8d695 --- /dev/null +++ b/m4/ax_check_compile_flag.m4 @@ -0,0 +1,72 @@ +# =========================================================================== +# http://www.gnu.org/software/autoconf-archive/ax_check_compile_flag.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CHECK_COMPILE_FLAG(FLAG, [ACTION-SUCCESS], [ACTION-FAILURE], [EXTRA-FLAGS]) +# +# DESCRIPTION +# +# Check whether the given FLAG works with the current language's compiler +# or gives an error. (Warnings, however, are ignored) +# +# ACTION-SUCCESS/ACTION-FAILURE are shell commands to execute on +# success/failure. +# +# If EXTRA-FLAGS is defined, it is added to the current language's default +# flags (e.g. CFLAGS) when the check is done. The check is thus made with +# the flags: "CFLAGS EXTRA-FLAGS FLAG". This can for example be used to +# force the compiler to issue an error when a bad flag is given. +# +# NOTE: Implementation based on AX_CFLAGS_GCC_OPTION. Please keep this +# macro in sync with AX_CHECK_{PREPROC,LINK}_FLAG. +# +# LICENSE +# +# Copyright (c) 2008 Guido U. Draheim <guidod(a)gmx.de> +# Copyright (c) 2011 Maarten Bosmans <mkbosmans(a)gmail.com> +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 2 + +AC_DEFUN([AX_CHECK_COMPILE_FLAG], +[AC_PREREQ(2.59)dnl for _AC_LANG_PREFIX +AS_VAR_PUSHDEF([CACHEVAR],[ax_cv_check_[]_AC_LANG_ABBREV[]flags_$4_$1])dnl +AC_CACHE_CHECK([whether _AC_LANG compiler accepts $1], CACHEVAR, [ + ax_check_save_flags=$[]_AC_LANG_PREFIX[]FLAGS + _AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS $4 $1" + AC_COMPILE_IFELSE([AC_LANG_PROGRAM()], + [AS_VAR_SET(CACHEVAR,[yes])], + [AS_VAR_SET(CACHEVAR,[no])]) + _AC_LANG_PREFIX[]FLAGS=$ax_check_save_flags]) +AS_IF([test x"AS_VAR_GET(CACHEVAR)" = xyes], + [m4_default([$2], :)], + [m4_default([$3], :)]) +AS_VAR_POPDEF([CACHEVAR])dnl +])dnl AX_CHECK_COMPILE_FLAGS ----------------------------------------------------------------------- Summary of changes: .travis.yml | 2 +- DEPENDENCIES | 5 ++-- m4/ax_check_compile_flag.m4 | 72 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 m4/ax_check_compile_flag.m4 hooks/post-receive --

7 years, 7 months

[COMMIT] branch cloud-dev updated. v1.14.0.0-405-gc460b6fb

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, cloud-dev has been updated via c460b6fbaeae6c60d75e8e5a60c34a6260b9e4fa (commit) from 76edf89deaf0fa126f34b6cd58f4373d211f0c33 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit c460b6fbaeae6c60d75e8e5a60c34a6260b9e4fa Author: Bogdan Pricope <bogdan.pricope(a)linaro.org> Date: Thu Jun 22 11:03:47 2017 +0300 ddf: examples: add ddf module skeleton and ddf application Signed-off-by: Bogdan Pricope <bogdan.pricope(a)linaro.org> Reviewed-and-tested-by: Yi He <yi.he(a)linaro.org> Signed-off-by: Yi He <yi.he(a)linaro.org> diff --git a/example/Makefile.am b/example/Makefile.am index dfc07b67..b209585c 100644 --- a/example/Makefile.am +++ b/example/Makefile.am @@ -8,4 +8,6 @@ SUBDIRS = classifier \ switch \ time \ timer \ - traffic_mgmt + traffic_mgmt \ + ddf_ifs \ + ddf_app diff --git a/example/ddf_app/Makefile.am b/example/ddf_app/Makefile.am new file mode 100644 index 00000000..e654202a --- /dev/null +++ b/example/ddf_app/Makefile.am @@ -0,0 +1,10 @@ +include $(top_srcdir)/example/Makefile.inc + +bin_PROGRAMS = odp_ddf_app$(EXEEXT) +odp_ddf_app_LDFLAGS = $(AM_LDFLAGS) -shared +odp_ddf_app_CFLAGS = $(AM_CFLAGS) -I${top_srcdir}/example + +noinst_HEADERS = \ + $(top_srcdir)/example/example_debug.h + +dist_odp_ddf_app_SOURCES = odp_ddf_app.c diff --git a/example/ddf_app/odp_ddf_app.c b/example/ddf_app/odp_ddf_app.c new file mode 100644 index 00000000..af864ee4 --- /dev/null +++ b/example/ddf_app/odp_ddf_app.c @@ -0,0 +1,49 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** enable strtok */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include <stdio.h> +#include <stdlib.h> + +#include <example_debug.h> +#include <odp_api.h> +#include <odp_drv.h> +#include <odp/helper/odph_api.h> + +int main(int argc, char *argv[]) +{ + odp_instance_t instance; + + (void)argc; + (void)argv; + + EXAMPLE_DBG("Start DDF Application...\n"); + + /* Initialize ODP before calling anything else */ + if (odp_init_global(&instance, NULL, NULL)) { + EXAMPLE_ERR("Error: ODP global init failed.\n"); + exit(EXIT_FAILURE); + } + + if (odp_init_local(instance, ODP_THREAD_CONTROL)) { + EXAMPLE_ERR("Error: ODP local init failed.\n"); + exit(EXIT_FAILURE); + } + + /* Print ddf objects*/ + odpdrv_print_all(); + + /* Terminate ODP */ + odp_term_local(); + odp_term_global(instance); + + EXAMPLE_DBG("Exit DDF Application.\n"); + return 0; +} diff --git a/example/ddf_ifs/Makefile.am b/example/ddf_ifs/Makefile.am new file mode 100644 index 00000000..aa892acd --- /dev/null +++ b/example/ddf_ifs/Makefile.am @@ -0,0 +1,27 @@ +LIB = $(top_builddir)/lib + +AM_CPPFLAGS += -I$(srcdir) \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/platform/@with_platform@/include \ + -I$(top_srcdir)/platform/@with_platform@/arch/@ARCH_DIR@ + +lib_LTLIBRARIES = $(LIB)/libddf_ifs.la + +noinst_HEADERS = \ + $(srcdir)/ddf_ifs.h \ + $(srcdir)/ddf_ifs_enumr_class.h \ + $(srcdir)/ddf_ifs_enumr_dpdk.h \ + $(srcdir)/ddf_ifs_enumr_generic.h \ + $(srcdir)/ddf_ifs_dev_dpdk.h \ + $(srcdir)/ddf_ifs_devio_direct.h \ + $(srcdir)/ddf_ifs_driver.h \ + $(srcdir)/ddf_ifs_api.h + +__LIB__libddf_ifs_la_SOURCES = \ + ddf_ifs.c \ + ddf_ifs_enumr_class.c \ + ddf_ifs_enumr_dpdk.c \ + ddf_ifs_enumr_generic.c \ + ddf_ifs_dev_dpdk.c \ + ddf_ifs_devio_direct.c \ + ddf_ifs_driver.c diff --git a/example/ddf_ifs/ddf_ifs.c b/example/ddf_ifs/ddf_ifs.c new file mode 100644 index 00000000..fa2276f7 --- /dev/null +++ b/example/ddf_ifs/ddf_ifs.c @@ -0,0 +1,28 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdio.h> +#include "odp_drv.h" +#include "ddf_ifs.h" +#include "ddf_ifs_enumr_class.h" +#include "ddf_ifs_devio_direct.h" +#include "ddf_ifs_driver.h" + +static ODPDRV_CONSTRUCTOR void ddf_ifs_construct(void) +{ + printf("%s()\n", __func__); + + register_enumerator_class(); + + register_devio_direct(); + + register_driver(); +} + +static __attribute__((destructor)) void ddf_ifs_destruct(void) +{ + printf("%s()\n", __func__); +} diff --git a/example/ddf_ifs/ddf_ifs.h b/example/ddf_ifs/ddf_ifs.h new file mode 100644 index 00000000..ff8bc277 --- /dev/null +++ b/example/ddf_ifs/ddf_ifs.h @@ -0,0 +1,18 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _ODP_DDF_IFS_H_ +#define _ODP_DDF_IFS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*_ODP_DDF_IFS_H_*/ diff --git a/example/ddf_ifs/ddf_ifs_api.h b/example/ddf_ifs/ddf_ifs_api.h new file mode 100644 index 00000000..0d948aea --- /dev/null +++ b/example/ddf_ifs/ddf_ifs_api.h @@ -0,0 +1,25 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _ODP_DDF_IFS_API_H_ +#define _ODP_DDF_IFS_API_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define DDF_IFS_DEV_API_NAME "ifs_api" +#define DDF_IFS_DEV_API_VER 0 +#define DDF_IFS_DEVIO_API_NAME "devio_api" +#define DDF_IFS_DEVIO_API_VER 0 + +#define DDF_IFS_DRV_NAME "ddf_ifs_drv" + +#ifdef __cplusplus +} +#endif + +#endif /*_ODP_DDF_IFS_API_H_*/ diff --git a/example/ddf_ifs/ddf_ifs_dev_dpdk.c b/example/ddf_ifs/ddf_ifs_dev_dpdk.c new file mode 100644 index 00000000..4d08e510 --- /dev/null +++ b/example/ddf_ifs/ddf_ifs_dev_dpdk.c @@ -0,0 +1,25 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdio.h> +#include <string.h> +#include "odp_drv.h" +#include "ddf_ifs_dev_dpdk.h" + +odpdrv_device_t dpdk_device_create(odpdrv_enumr_t enumr, + const char *dev_addr, + void *enum_data) +{ + odpdrv_device_param_t param; + + printf("\t%s(%s)\n", __func__, dev_addr); + + param.enumerator = enumr; + strcpy(param.address, dev_addr); + param.enum_dev = enum_data; + + return odpdrv_device_create(&param); +} diff --git a/example/ddf_ifs/ddf_ifs_dev_dpdk.h b/example/ddf_ifs/ddf_ifs_dev_dpdk.h new file mode 100644 index 00000000..84583a33 --- /dev/null +++ b/example/ddf_ifs/ddf_ifs_dev_dpdk.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _ODP_DDF_IFS_DEV_DPDK_H_ +#define _ODP_DDF_IFS_DEV_DPDK_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "odp_drv.h" + +odpdrv_device_t dpdk_device_create(odpdrv_enumr_t enumr, + const char *dev_addr, + void *enum_data); + +#ifdef __cplusplus +} +#endif + +#endif /*_ODP_DDF_IFS_DEV_DPDK_H_*/ diff --git a/example/ddf_ifs/ddf_ifs_devio_direct.c b/example/ddf_ifs/ddf_ifs_devio_direct.c new file mode 100644 index 00000000..ec451d65 --- /dev/null +++ b/example/ddf_ifs/ddf_ifs_devio_direct.c @@ -0,0 +1,51 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdio.h> +#include "odp_drv.h" +#include "ddf_ifs_api.h" +#include "ddf_ifs_devio_direct.h" + +static odpdrv_devio_t devio_direct; + +static int devio_direct_probe(odpdrv_device_t dev) +{ + (void)dev; + + printf("%s()\n", __func__); + + return 0; +} + +static int devio_direct_remove(void) +{ + printf("%s()\n", __func__); + + return 0; +} + +int register_devio_direct(void) +{ + odpdrv_devio_param_t param = { + .api_name = DDF_IFS_DEVIO_API_NAME, + .api_version = DDF_IFS_DEVIO_API_VER, + .enumr_api_name = DDF_IFS_DEV_API_NAME, + .enumr_api_version = DDF_IFS_DEV_API_VER, + .probe = devio_direct_probe, + .remove = devio_direct_remove, + .ops = NULL + }; + + printf("\t%s()\n", __func__); + + devio_direct = odpdrv_devio_register(&param); + if (ODPDRV_DEVIO_INVALID == devio_direct) { + printf("\tError: failed to %s()\n", __func__); + return -1; + } + + return 0; +} diff --git a/example/ddf_ifs/ddf_ifs_devio_direct.h b/example/ddf_ifs/ddf_ifs_devio_direct.h new file mode 100644 index 00000000..9c004601 --- /dev/null +++ b/example/ddf_ifs/ddf_ifs_devio_direct.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _ODP_DDF_IFS_DEVIO_DIRECT_H_ +#define _ODP_DDF_IFS_DEVIO_DIRECT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int register_devio_direct(void); + +#ifdef __cplusplus +} +#endif + +#endif /*_ODP_DDF_IFS_DEVIO_DIRECT_H_*/ diff --git a/example/ddf_ifs/ddf_ifs_driver.c b/example/ddf_ifs/ddf_ifs_driver.c new file mode 100644 index 00000000..53f94757 --- /dev/null +++ b/example/ddf_ifs/ddf_ifs_driver.c @@ -0,0 +1,69 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdio.h> +#include <string.h> +#include "odp_drv.h" +#include "ddf_ifs_api.h" +#include "ddf_ifs_driver.h" + +static odpdrv_driver_t ddf_ifs_driver; +static int drv_data = 12; + +static int ddf_ifs_driver_probe(odpdrv_device_t dev, odpdrv_devio_t devio, + int devio_idx) +{ + printf("%s(dev, devio, devio_idx = %d)\n", __func__, devio_idx); + + (void)devio; + + odpdrv_device_set_data(dev, (void *)&drv_data); + + return 0; +} + +static int ddf_ifs_driver_unbind(odpdrv_device_t dev, + void (*callback)(odpdrv_device_t dev), + uint32_t flags) +{ + printf("%s()\n", __func__); + + if (!(flags & ODPDRV_DRV_UNBIND_IMMEDIATE)) + return -1; /* unsupported*/ + + callback(dev); + + return 0; +} + +static int ddf_ifs_driver_remove(void) +{ + printf("%s()\n", __func__); + + return 0; +} + +int register_driver(void) +{ + odpdrv_driver_param_t param = { + .name = DDF_IFS_DRV_NAME, + .devios = {{DDF_IFS_DEVIO_API_NAME, DDF_IFS_DEVIO_API_VER}, + {"", 0}, {"", 0} }, + .probe = ddf_ifs_driver_probe, + .unbind = ddf_ifs_driver_unbind, + .remove = ddf_ifs_driver_remove, + }; + + printf("\t%s()\n", __func__); + + ddf_ifs_driver = odpdrv_driver_register(&param); + if (ODPDRV_DRIVER_INVALID == ddf_ifs_driver) { + printf("Error: Failed to %s()\n", __func__); + return -1; + } + + return 0; +} diff --git a/example/ddf_ifs/ddf_ifs_driver.h b/example/ddf_ifs/ddf_ifs_driver.h new file mode 100644 index 00000000..e990c34a --- /dev/null +++ b/example/ddf_ifs/ddf_ifs_driver.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _ODP_DDF_IFS_DRIVER_H_ +#define _ODP_DDF_IFS_DRIVER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int register_driver(void); + +#ifdef __cplusplus +} +#endif + +#endif /*_ODP_DDF_IFS_DRIVER_H_*/ diff --git a/example/ddf_ifs/ddf_ifs_enumr_class.c b/example/ddf_ifs/ddf_ifs_enumr_class.c new file mode 100644 index 00000000..8450dee2 --- /dev/null +++ b/example/ddf_ifs/ddf_ifs_enumr_class.c @@ -0,0 +1,47 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdio.h> +#include "odp_drv.h" +#include "ddf_ifs_enumr_class.h" +#include "ddf_ifs_enumr_dpdk.h" +#include "ddf_ifs_enumr_generic.h" + +static odpdrv_enumr_class_t enumr_class; + +static int enumr_class_ifs_probe(void) +{ + printf("%s()\n", __func__); + + if (register_enumerator_dpdk(enumr_class)) + return -1; + + if (register_enumerator_generic(enumr_class)) + return -1; + + return 0; +} + +static int enumr_class_ifs_remove(void) +{ + printf("%s()\n", __func__); + + return 0; +} + +int register_enumerator_class(void) +{ + odpdrv_enumr_class_param_t param = { + .name = "enumr_class_ifs", + .probe = enumr_class_ifs_probe, + .remove = enumr_class_ifs_remove + }; + + printf("\t%s()\n", __func__); + enumr_class = odpdrv_enumr_class_register(&param); + + return 0; +} diff --git a/example/ddf_ifs/ddf_ifs_enumr_class.h b/example/ddf_ifs/ddf_ifs_enumr_class.h new file mode 100644 index 00000000..a3cb0372 --- /dev/null +++ b/example/ddf_ifs/ddf_ifs_enumr_class.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _ODP_DDF_IFS_ENUMR_CLASS_H_ +#define _ODP_DDF_IFS_ENUMR_CLASS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int register_enumerator_class(void); + +#ifdef __cplusplus +} +#endif + +#endif /*_ODP_DDF_IFS_ENUMR_CLASS_H_*/ diff --git a/example/ddf_ifs/ddf_ifs_enumr_dpdk.c b/example/ddf_ifs/ddf_ifs_enumr_dpdk.c new file mode 100644 index 00000000..782f7b74 --- /dev/null +++ b/example/ddf_ifs/ddf_ifs_enumr_dpdk.c @@ -0,0 +1,86 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdio.h> +#include "odp_drv.h" +#include "ddf_ifs_api.h" +#include "ddf_ifs_enumr_dpdk.h" +#include "ddf_ifs_dev_dpdk.h" + +static odpdrv_enumr_t dpdk_enumr; + +#define TEST_DPDK_DEV_CNT 3 +#define DDF_DPDK_DEV_MAX 10 + +static odpdrv_device_t dpdk_dev[DDF_DPDK_DEV_MAX]; +static int dpdk_dev_cnt; +static int dpdk_enumr_probe(void) +{ + int dpdk_dev_cnt_detected = TEST_DPDK_DEV_CNT; /* detected with + dpdk APIs*/ + char dev_addr[ODPDRV_NAME_ADDR_SZ]; + int i; + + printf("%s() - %d devices found\n", __func__, dpdk_dev_cnt_detected); + + if (dpdk_dev_cnt_detected > DDF_DPDK_DEV_MAX) { + dpdk_dev_cnt_detected = DDF_DPDK_DEV_MAX; + printf("\tWarning: dpdk device count scaled down to %d\n", + dpdk_dev_cnt_detected); + } + for (i = 0; i < dpdk_dev_cnt_detected; i++) { + sprintf(dev_addr, "0000:01:00.%d", i); + dpdk_dev[dpdk_dev_cnt] = dpdk_device_create(dpdk_enumr, + dev_addr, + NULL); + if (dpdk_dev[dpdk_dev_cnt] == ODPDRV_DEVICE_INVALID) + printf("\tError: unable to create device: %s\n", + dev_addr); + else + dpdk_dev_cnt++; + } + + return 0; +} + +static int dpdk_enumr_remove(void) +{ + printf("%s()\n", __func__); + return 0; +} + +static int dpdk_enumr_register_notif(void (*event_handler) (uint64_t event), + int64_t event_mask) +{ + (void)event_handler; + (void)event_mask; + + printf("%s()\n", __func__); + + return 0; +} + +int register_enumerator_dpdk(odpdrv_enumr_class_t enumr_class) +{ + struct odpdrv_enumr_param_t param = { + .enumr_class = enumr_class, + .api_name = DDF_IFS_DEV_API_NAME, + .api_version = DDF_IFS_DEV_API_VER, + .probe = dpdk_enumr_probe, + .remove = dpdk_enumr_remove, + .register_notifier = dpdk_enumr_register_notif + }; + + printf("\t%s()\n", __func__); + + dpdk_enumr = odpdrv_enumr_register(&param); + if (ODPDRV_ENUMR_INVALID == dpdk_enumr) { + printf("\tError: failed to register_enumerator_dpdk()\n"); + return -1; + } + + return 0; +} diff --git a/example/ddf_ifs/ddf_ifs_enumr_dpdk.h b/example/ddf_ifs/ddf_ifs_enumr_dpdk.h new file mode 100644 index 00000000..e670d548 --- /dev/null +++ b/example/ddf_ifs/ddf_ifs_enumr_dpdk.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _ODP_DDF_IFS_ENUMR_DPDK_H_ +#define _ODP_DDF_IFS_ENUMR_DPDK_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int register_enumerator_dpdk(odpdrv_enumr_class_t enumr_class); + +#ifdef __cplusplus +} +#endif + +#endif /*_ODP_DDF_IFS_ENUMR_DPDK_H_*/ diff --git a/example/ddf_ifs/ddf_ifs_enumr_generic.c b/example/ddf_ifs/ddf_ifs_enumr_generic.c new file mode 100644 index 00000000..f5e1a2cb --- /dev/null +++ b/example/ddf_ifs/ddf_ifs_enumr_generic.c @@ -0,0 +1,57 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <stdio.h> +#include "odp_drv.h" +#include "ddf_ifs_api.h" +#include "ddf_ifs_enumr_generic.h" + +static odpdrv_enumr_t gen_enumr; + +static int gen_enumr_probe(void) +{ + printf("%s() - no devices found\n", __func__); + return 0; +} + +static int gen_enumr_remove(void) +{ + printf("%s()\n", __func__); + return 0; +} + +static int gen_enumr_register_notif(void (*event_handler) (uint64_t event), + int64_t event_mask) +{ + (void)event_handler; + (void)event_mask; + + printf("%s()\n", __func__); + + return 0; +} + +int register_enumerator_generic(odpdrv_enumr_class_t enumr_class) +{ + struct odpdrv_enumr_param_t param = { + .enumr_class = enumr_class, + .api_name = DDF_IFS_DEV_API_NAME, + .api_version = DDF_IFS_DEV_API_VER, + .probe = gen_enumr_probe, + .remove = gen_enumr_remove, + .register_notifier = gen_enumr_register_notif + }; + + printf("\t%s()\n", __func__); + + gen_enumr = odpdrv_enumr_register(&param); + if (ODPDRV_ENUMR_INVALID == gen_enumr) { + printf("\tError:failed to %s()\n", __func__); + return -1; + } + + return 0; +} diff --git a/example/ddf_ifs/ddf_ifs_enumr_generic.h b/example/ddf_ifs/ddf_ifs_enumr_generic.h new file mode 100644 index 00000000..0eeb073f --- /dev/null +++ b/example/ddf_ifs/ddf_ifs_enumr_generic.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2017, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef _ODP_DDF_IFS_ENUMR_GENERIC_H_ +#define _ODP_DDF_IFS_ENUMR_GENERIC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int register_enumerator_generic(odpdrv_enumr_class_t enumr_class); + +#ifdef __cplusplus +} +#endif + +#endif /*_ODP_DDF_IFS_ENUMR_GENERIC_H_*/ diff --git a/example/ddf_ifs/odp.conf b/example/ddf_ifs/odp.conf new file mode 100644 index 00000000..14968a7a --- /dev/null +++ b/example/ddf_ifs/odp.conf @@ -0,0 +1,4 @@ +module: +{ + modules = ("libddf_ifs.so.0"); +}; diff --git a/example/m4/configure.m4 b/example/m4/configure.m4 index 620db04f..9e7973b0 100644 --- a/example/m4/configure.m4 +++ b/example/m4/configure.m4 @@ -21,4 +21,6 @@ AC_CONFIG_FILES([example/classifier/Makefile example/time/Makefile example/timer/Makefile example/traffic_mgmt/Makefile + example/ddf_ifs/Makefile + example/ddf_app/Makefile example/Makefile]) ----------------------------------------------------------------------- Summary of changes: example/Makefile.am | 4 +- example/ddf_app/Makefile.am | 10 +++ example/ddf_app/odp_ddf_app.c | 49 ++++++++++++ example/ddf_ifs/Makefile.am | 27 +++++++ example/ddf_ifs/ddf_ifs.c | 28 +++++++ .../x86/cpu_flags.h => example/ddf_ifs/ddf_ifs.h | 8 +- example/ddf_ifs/ddf_ifs_api.h | 25 +++++++ example/ddf_ifs/ddf_ifs_dev_dpdk.c | 25 +++++++ example/ddf_ifs/ddf_ifs_dev_dpdk.h | 24 ++++++ example/ddf_ifs/ddf_ifs_devio_direct.c | 51 +++++++++++++ .../ddf_ifs/ddf_ifs_devio_direct.h | 8 +- example/ddf_ifs/ddf_ifs_driver.c | 69 +++++++++++++++++ .../ddf_ifs/ddf_ifs_driver.h | 8 +- example/ddf_ifs/ddf_ifs_enumr_class.c | 47 ++++++++++++ .../ddf_ifs/ddf_ifs_enumr_class.h | 8 +- example/ddf_ifs/ddf_ifs_enumr_dpdk.c | 86 ++++++++++++++++++++++ .../ddf_ifs/ddf_ifs_enumr_dpdk.h | 8 +- example/ddf_ifs/ddf_ifs_enumr_generic.c | 57 ++++++++++++++ .../ddf_ifs/ddf_ifs_enumr_generic.h | 10 +-- example/ddf_ifs/odp.conf | 4 + example/m4/configure.m4 | 2 + 21 files changed, 530 insertions(+), 28 deletions(-) create mode 100644 example/ddf_app/Makefile.am create mode 100644 example/ddf_app/odp_ddf_app.c create mode 100644 example/ddf_ifs/Makefile.am create mode 100644 example/ddf_ifs/ddf_ifs.c copy platform/linux-generic/arch/x86/cpu_flags.h => example/ddf_ifs/ddf_ifs.h (64%) create mode 100644 example/ddf_ifs/ddf_ifs_api.h create mode 100644 example/ddf_ifs/ddf_ifs_dev_dpdk.c create mode 100644 example/ddf_ifs/ddf_ifs_dev_dpdk.h create mode 100644 example/ddf_ifs/ddf_ifs_devio_direct.c copy platform/linux-generic/arch/x86/cpu_flags.h => example/ddf_ifs/ddf_ifs_devio_direct.h (55%) create mode 100644 example/ddf_ifs/ddf_ifs_driver.c copy platform/linux-generic/arch/x86/cpu_flags.h => example/ddf_ifs/ddf_ifs_driver.h (60%) create mode 100644 example/ddf_ifs/ddf_ifs_enumr_class.c copy platform/linux-generic/arch/x86/cpu_flags.h => example/ddf_ifs/ddf_ifs_enumr_class.h (55%) create mode 100644 example/ddf_ifs/ddf_ifs_enumr_dpdk.c copy platform/linux-generic/arch/x86/cpu_flags.h => example/ddf_ifs/ddf_ifs_enumr_dpdk.h (51%) create mode 100644 example/ddf_ifs/ddf_ifs_enumr_generic.c copy platform/linux-generic/include/drv_driver_internal.h => example/ddf_ifs/ddf_ifs_enumr_generic.h (50%) create mode 100644 example/ddf_ifs/odp.conf hooks/post-receive --

7 years, 7 months

[COMMIT] branch master updated. v1.15.0.0-11-ga4add292

by git＠git-us.linaro.org

7 years, 7 months

[COMMIT] branch api-next updated. v1.15.0.0-372-ge2169ff2

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via e2169ff2026af51eec351cd679891c303ab651e0 (commit) from e3c2e06c4420f41d7053e615ac1cc6c1dc315407 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit e2169ff2026af51eec351cd679891c303ab651e0 Author: Brian Brooks <brian.brooks(a)arm.com> Date: Wed Jun 14 14:34:50 2017 -0500 timer: allow timer processing to run on worker cores Run timer pool processing on worker cores if the application hints that the scheduler will be used. This reduces the latency and jitter of the point at which timer pool processing begins. See [1] for details. [1] https://docs.google.com/document/d/1sY7rOxqCNu-bMqjBiT5_keAIohrX1ZW-eL0oGLA… Signed-off-by: Brian Brooks <brian.brooks(a)arm.com> Reviewed-by: Ola Liljedahl <ola.liljedahl(a)arm.com> Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli(a)arm.com> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index 90e2a629..404792cf 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -108,7 +108,7 @@ int odp_queue_term_global(void); int odp_crypto_init_global(void); int odp_crypto_term_global(void); -int odp_timer_init_global(void); +int odp_timer_init_global(const odp_init_t *params); int odp_timer_term_global(void); int odp_timer_disarm_all(void); diff --git a/platform/linux-generic/include/odp_timer_internal.h b/platform/linux-generic/include/odp_timer_internal.h index 91b12c54..67ee9fef 100644 --- a/platform/linux-generic/include/odp_timer_internal.h +++ b/platform/linux-generic/include/odp_timer_internal.h @@ -20,6 +20,12 @@ #include <odp_pool_internal.h> #include <odp/api/timer.h> +/* Minimum number of nanoseconds between checking timer pools. */ +#define CONFIG_TIMER_RUN_RATELIMIT_NS 100 + +/* Minimum number of scheduling rounds between checking timer pools. */ +#define CONFIG_TIMER_RUN_RATELIMIT_ROUNDS 1 + /** * Internal Timeout header */ @@ -35,4 +41,22 @@ typedef struct { odp_timer_t timer; } odp_timeout_hdr_t; +/* + * Whether to run timer pool processing 'inline' (on worker cores) or in + * background threads (thread-per-timerpool). + * + * If the application will use both scheduler and timer this flag is set + * to true, otherwise false. This application conveys this information via + * the 'not_used' bits in odp_init_t which are passed to odp_global_init(). + */ +extern odp_bool_t inline_timers; + +unsigned _timer_run(void); + +/* Static inline wrapper to minimize modification of schedulers. */ +static inline unsigned timer_run(void) +{ + return inline_timers ? _timer_run() : 0; +} + #endif diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 62a1fbc2..8c17cbb0 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -241,7 +241,7 @@ int odp_init_global(odp_instance_t *instance, } stage = PKTIO_INIT; - if (odp_timer_init_global()) { + if (odp_timer_init_global(params)) { ODP_ERR("ODP timer init failed.\n"); goto init_failed; } diff --git a/platform/linux-generic/odp_schedule.c b/platform/linux-generic/odp_schedule.c index 92634235..814746c7 100644 --- a/platform/linux-generic/odp_schedule.c +++ b/platform/linux-generic/odp_schedule.c @@ -23,6 +23,7 @@ #include <odp/api/packet_io.h> #include <odp_ring_internal.h> #include <odp_queue_internal.h> +#include <odp_timer_internal.h> /* Number of priority levels */ #define NUM_PRIO 8 @@ -998,6 +999,8 @@ static int schedule_loop(odp_queue_t *out_queue, uint64_t wait, int ret; while (1) { + timer_run(); + ret = do_schedule(out_queue, out_ev, max_num); if (ret) diff --git a/platform/linux-generic/odp_schedule_iquery.c b/platform/linux-generic/odp_schedule_iquery.c index 79086843..b33ba7cd 100644 --- a/platform/linux-generic/odp_schedule_iquery.c +++ b/platform/linux-generic/odp_schedule_iquery.c @@ -23,6 +23,7 @@ #include <odp/api/thrmask.h> #include <odp/api/packet_io.h> #include <odp_config_internal.h> +#include <odp_timer_internal.h> /* Number of priority levels */ #define NUM_SCHED_PRIO 8 @@ -719,6 +720,8 @@ static int schedule_loop(odp_queue_t *out_queue, uint64_t wait, odp_time_t next, wtime; while (1) { + timer_run(); + count = do_schedule(out_queue, out_ev, max_num); if (count) diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index 66715c13..14b7aa88 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -15,6 +15,7 @@ #include <odp_align_internal.h> #include <odp_config_internal.h> #include <odp_ring_internal.h> +#include <odp_timer_internal.h> #define NUM_THREAD ODP_THREAD_COUNT_MAX #define NUM_QUEUE ODP_CONFIG_QUEUES @@ -517,6 +518,8 @@ static int schedule_multi(odp_queue_t *from, uint64_t wait, uint32_t qi; int num; + timer_run(); + cmd = sched_cmd(); if (cmd && cmd->s.type == CMD_PKTIO) { diff --git a/platform/linux-generic/odp_timer.c b/platform/linux-generic/odp_timer.c index cf610bfa..bf7f1acd 100644 --- a/platform/linux-generic/odp_timer.c +++ b/platform/linux-generic/odp_timer.c @@ -52,6 +52,7 @@ #include <odp/api/sync.h> #include <odp/api/time.h> #include <odp/api/timer.h> +#include <odp_time_internal.h> #include <odp_timer_internal.h> #define TMO_UNUSED ((uint64_t)0xFFFFFFFFFFFFFFFF) @@ -60,6 +61,8 @@ * for checking the freshness of received timeouts */ #define TMO_INACTIVE ((uint64_t)0x8000000000000000) +odp_bool_t inline_timers = false; + /****************************************************************************** * Mutual exclusion in the absence of CAS16 *****************************************************************************/ @@ -165,7 +168,8 @@ static inline void set_next_free(odp_timer *tim, uint32_t nf) *****************************************************************************/ typedef struct odp_timer_pool_s { -/* Put frequently accessed fields in the first cache line */ + odp_time_t prev_scan; /* Time when previous scan started */ + odp_time_t time_per_tick; /* Time per timer pool tick */ odp_atomic_u64_t cur_tick;/* Current tick value */ uint64_t min_rel_tck; uint64_t max_rel_tck; @@ -242,6 +246,8 @@ static odp_timer_pool_t odp_timer_pool_new(const char *name, ODP_ABORT("%s: timer pool shm-alloc(%zuKB) failed\n", name, (sz0 + sz1 + sz2) / 1024); odp_timer_pool *tp = (odp_timer_pool *)odp_shm_addr(shm); + tp->prev_scan = odp_time_global(); + tp->time_per_tick = odp_time_global_from_ns(param->res_ns); odp_atomic_init_u64(&tp->cur_tick, 0); if (name == NULL) { @@ -276,8 +282,10 @@ static odp_timer_pool_t odp_timer_pool_new(const char *name, tp->tp_idx = tp_idx; odp_spinlock_init(&tp->lock); timer_pool[tp_idx] = tp; - if (tp->param.clk_src == ODP_CLOCK_CPU) - itimer_init(tp); + if (!inline_timers) { + if (tp->param.clk_src == ODP_CLOCK_CPU) + itimer_init(tp); + } return tp; } @@ -306,11 +314,13 @@ static void odp_timer_pool_del(odp_timer_pool *tp) odp_spinlock_lock(&tp->lock); timer_pool[tp->tp_idx] = NULL; - /* Stop timer triggering */ - if (tp->param.clk_src == ODP_CLOCK_CPU) - itimer_fini(tp); + if (!inline_timers) { + /* Stop POSIX itimer signals */ + if (tp->param.clk_src == ODP_CLOCK_CPU) + itimer_fini(tp); - stop_timer_thread(tp); + stop_timer_thread(tp); + } if (tp->num_alloc != 0) { /* It's a programming error to attempt to destroy a */ @@ -671,6 +681,81 @@ static unsigned odp_timer_pool_expire(odp_timer_pool_t tpid, uint64_t tick) } /****************************************************************************** + * Inline timer processing + *****************************************************************************/ + +static unsigned process_timer_pools(void) +{ + odp_timer_pool *tp; + odp_time_t prev_scan, now; + uint64_t nticks; + unsigned nexp = 0; + + for (size_t i = 0; i < MAX_TIMER_POOLS; i++) { + tp = timer_pool[i]; + + if (tp == NULL) + continue; + + /* + * Check the last time this timer pool was expired. If one + * or more periods have passed, attempt to expire it. + */ + prev_scan = tp->prev_scan; + now = odp_time_global(); + + nticks = (now.u64 - prev_scan.u64) / tp->time_per_tick.u64; + + if (nticks < 1) + continue; + + if (__atomic_compare_exchange_n( + &tp->prev_scan.u64, &prev_scan.u64, + prev_scan.u64 + (tp->time_per_tick.u64 * nticks), + false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { + uint64_t tp_tick = _odp_atomic_u64_fetch_add_mm( + &tp->cur_tick, nticks, _ODP_MEMMODEL_RLX); + + if (tp->notify_overrun && nticks > 1) { + ODP_ERR("\n\t%d ticks overrun on timer pool " + "\"%s\", timer resolution too high\n", + nticks - 1, tp->name); + tp->notify_overrun = 0; + } + nexp += odp_timer_pool_expire(tp, tp_tick + nticks); + } + } + return nexp; +} + +static odp_time_t time_per_ratelimit_period; + +unsigned _timer_run(void) +{ + static __thread odp_time_t last_timer_run; + static __thread unsigned timer_run_cnt = + CONFIG_TIMER_RUN_RATELIMIT_ROUNDS; + odp_time_t now; + + /* Rate limit how often this thread checks the timer pools. */ + + if (CONFIG_TIMER_RUN_RATELIMIT_ROUNDS > 1) { + if (--timer_run_cnt) + return 0; + timer_run_cnt = CONFIG_TIMER_RUN_RATELIMIT_ROUNDS; + } + + now = odp_time_global(); + if (odp_time_cmp(odp_time_diff(now, last_timer_run), + time_per_ratelimit_period) == -1) + return 0; + last_timer_run = now; + + /* Check the timer pools. */ + return process_timer_pools(); +} + +/****************************************************************************** * POSIX timer support * Functions that use Linux/POSIX per-process timers and related facilities *****************************************************************************/ @@ -989,7 +1074,7 @@ void odp_timeout_free(odp_timeout_t tmo) odp_buffer_free(odp_buffer_from_event(ev)); } -int odp_timer_init_global(void) +int odp_timer_init_global(const odp_init_t *params) { #ifndef ODP_ATOMIC_U128 uint32_t i; @@ -1000,7 +1085,16 @@ int odp_timer_init_global(void) #endif odp_atomic_init_u32(&num_timer_pools, 0); - block_sigalarm(); + if (params) + inline_timers = + !params->not_used.feat.schedule && + !params->not_used.feat.timer; + + time_per_ratelimit_period = + odp_time_global_from_ns(CONFIG_TIMER_RUN_RATELIMIT_NS); + + if (!inline_timers) + block_sigalarm(); return 0; } ----------------------------------------------------------------------- Summary of changes: platform/linux-generic/include/odp_internal.h | 2 +- .../linux-generic/include/odp_timer_internal.h | 24 +++++ platform/linux-generic/odp_init.c | 2 +- platform/linux-generic/odp_schedule.c | 3 + platform/linux-generic/odp_schedule_iquery.c | 3 + platform/linux-generic/odp_schedule_sp.c | 3 + platform/linux-generic/odp_timer.c | 112 +++++++++++++++++++-- 7 files changed, 138 insertions(+), 11 deletions(-) hooks/post-receive --

7 years, 7 months

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

git-lng-odp June 2017