git-lng-odp September 2016

git-lng-odp@lists.linaro.org

1 participants
20 discussions

[COMMIT] branch master updated. v1.10.1.0-124-g88df261

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via 88df2613cb91022233f9ec973f6ef338eb060f17 (commit) via bc65897481d8ce89a55257b22bf93d05abf74f70 (commit) from e858d688c3ad3ce0d0b3ea22539cac8e6ec844b7 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 88df2613cb91022233f9ec973f6ef338eb060f17 Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Sep 14 11:53:07 2016 +0300 test: sched_latency: add test script Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-and-tested-by: Brian Brooks <brian.brooks(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/common_plat/performance/Makefile.am b/test/common_plat/performance/Makefile.am index f5dd8dd..f184609 100644 --- a/test/common_plat/performance/Makefile.am +++ b/test/common_plat/performance/Makefile.am @@ -9,6 +9,7 @@ COMPILE_ONLY = odp_l2fwd$(EXEEXT) \ odp_scheduling$(EXEEXT) TESTSCRIPTS = odp_l2fwd_run.sh \ + odp_sched_latency_run.sh \ odp_scheduling_run.sh TEST_EXTENSIONS = .sh diff --git a/test/common_plat/performance/odp_sched_latency_run.sh b/test/common_plat/performance/odp_sched_latency_run.sh new file mode 100755 index 0000000..6048f58 --- /dev/null +++ b/test/common_plat/performance/odp_sched_latency_run.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# +# Copyright (c) 2016, Linaro Limited +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +# +# Script that passes command line arguments to odp_sched_latency test when +# launched by 'make check' + +TEST_DIR="${TEST_DIR:-$(dirname $0)}" +ALL=0 + +run() +{ + echo odp_sched_latency_run starts requesting $1 worker threads + echo =============================================== + + $TEST_DIR/odp_sched_latency${EXEEXT} -c $1 || exit $? +} + +run 1 +run 5 +run 8 +run 11 +run $ALL + +exit 0 commit bc65897481d8ce89a55257b22bf93d05abf74f70 Author: Matias Elo <matias.elo(a)nokia.com> Date: Wed Sep 14 11:53:06 2016 +0300 test: perf: add new scheduling latency test Add new scheduling latency benchmark application. The application measures delays (avg, min, max) for high and low priority events. The test has a configurable number of TRAFFIC events and few SAMPLE events (one common or one per priority). The scheduling latency is only measured from the SAMPLE events to minimize measurement overhead. The application's command line arguments enable configuring: - Number of processing threads - Number of high/low priority queues - Number of high/low priority events - Use separate SAMPLE events for each priority - Scheduled queue type (PARALLEL, ATOMIC, ORDERED) Signed-off-by: Matias Elo <matias.elo(a)nokia.com> Reviewed-and-tested-by: Brian Brooks <brian.brooks(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/test/common_plat/performance/.gitignore b/test/common_plat/performance/.gitignore index edcc832..1527d25 100644 --- a/test/common_plat/performance/.gitignore +++ b/test/common_plat/performance/.gitignore @@ -4,4 +4,5 @@ odp_atomic odp_crypto odp_l2fwd odp_pktio_perf +odp_sched_latency odp_scheduling diff --git a/test/common_plat/performance/Makefile.am b/test/common_plat/performance/Makefile.am index d23bb3e..f5dd8dd 100644 --- a/test/common_plat/performance/Makefile.am +++ b/test/common_plat/performance/Makefile.am @@ -5,6 +5,7 @@ TESTS_ENVIRONMENT += TEST_DIR=${builddir} EXECUTABLES = odp_crypto$(EXEEXT) odp_pktio_perf$(EXEEXT) COMPILE_ONLY = odp_l2fwd$(EXEEXT) \ + odp_sched_latency$(EXEEXT) \ odp_scheduling$(EXEEXT) TESTSCRIPTS = odp_l2fwd_run.sh \ @@ -20,6 +21,8 @@ bin_PROGRAMS = $(EXECUTABLES) $(COMPILE_ONLY) odp_crypto_LDFLAGS = $(AM_LDFLAGS) -static odp_crypto_CFLAGS = $(AM_CFLAGS) -I${top_srcdir}/test +odp_sched_latency_LDFLAGS = $(AM_LDFLAGS) -static +odp_sched_latency_CFLAGS = $(AM_CFLAGS) -I${top_srcdir}/test odp_scheduling_LDFLAGS = $(AM_LDFLAGS) -static odp_scheduling_CFLAGS = $(AM_CFLAGS) -I${top_srcdir}/test @@ -27,6 +30,7 @@ noinst_HEADERS = \ $(top_srcdir)/test/test_debug.h dist_odp_crypto_SOURCES = odp_crypto.c +dist_odp_sched_latency_SOURCES = odp_sched_latency.c dist_odp_scheduling_SOURCES = odp_scheduling.c dist_odp_pktio_perf_SOURCES = odp_pktio_perf.c diff --git a/test/common_plat/performance/odp_sched_latency.c b/test/common_plat/performance/odp_sched_latency.c new file mode 100644 index 0000000..063fb21 --- /dev/null +++ b/test/common_plat/performance/odp_sched_latency.c @@ -0,0 +1,767 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * @example odp_sched_latency.c ODP scheduling latency benchmark application + */ + +#include <string.h> +#include <stdlib.h> +#include <inttypes.h> + +#include <test_debug.h> + +/* ODP main header */ +#include <odp_api.h> + +/* ODP helper for Linux apps */ +#include <odp/helper/linux.h> + +/* GNU lib C */ +#include <getopt.h> + +#define MAX_WORKERS 64 /**< Maximum number of worker threads */ +#define MAX_QUEUES 4096 /**< Maximum number of queues */ +#define EVENT_POOL_SIZE (1024 * 1024) /**< Event pool size */ +#define TEST_ROUNDS (4 * 1024 * 1024) /**< Test rounds for each thread */ +#define MAIN_THREAD 1 /**< Thread ID performing maintenance tasks */ + +/* Default values for command line arguments */ +#define SAMPLE_EVENT_PER_PRIO 0 /**< Allocate a separate sample event for + each priority */ +#define HI_PRIO_EVENTS 0 /**< Number of high priority events */ +#define LO_PRIO_EVENTS 32 /**< Number of low priority events */ +#define HI_PRIO_QUEUES 16 /**< Number of high priority queues */ +#define LO_PRIO_QUEUES 64 /**< Number of low priority queues */ + +#define EVENTS_PER_HI_PRIO_QUEUE 0 /**< Alloc HI_PRIO_QUEUES x HI_PRIO_EVENTS + events */ +#define EVENTS_PER_LO_PRIO_QUEUE 1 /**< Alloc LO_PRIO_QUEUES x LO_PRIO_EVENTS + events */ +ODP_STATIC_ASSERT(HI_PRIO_QUEUES <= MAX_QUEUES, "Too many HI priority queues"); +ODP_STATIC_ASSERT(LO_PRIO_QUEUES <= MAX_QUEUES, "Too many LO priority queues"); + +#define CACHE_ALIGN_ROUNDUP(x)\ + ((ODP_CACHE_LINE_SIZE) * \ + (((x) + ODP_CACHE_LINE_SIZE - 1) / (ODP_CACHE_LINE_SIZE))) + +/* Test priorities */ +#define NUM_PRIOS 2 /**< Number of tested priorities */ +#define HI_PRIO 0 +#define LO_PRIO 1 + +/** Test event types */ +typedef enum { + WARM_UP, /**< Warm up event */ + TRAFFIC, /**< Event used only as traffic load */ + SAMPLE /**< Event used to measure latency */ +} event_type_t; + +/** Test event */ +typedef struct { + uint64_t ts; /**< Send timestamp */ + event_type_t type; /**< Message type */ + int src_idx[NUM_PRIOS]; /**< Source ODP queue */ + int prio; /**< Source queue priority */ +} test_event_t; + +/** Test arguments */ +typedef struct { + int cpu_count; /**< CPU count */ + odp_schedule_sync_t sync_type; /**< Scheduler sync type */ + struct { + int queues; /**< Number of scheduling queues */ + int events; /**< Number of events */ + odp_bool_t events_per_queue; /**< Allocate 'queues' x 'events' + test events */ + } prio[NUM_PRIOS]; + odp_bool_t sample_per_prio; /**< Allocate a separate sample event for + each priority */ +} test_args_t; + +/** Latency measurements statistics */ +typedef struct { + uint64_t events; /**< Total number of received events */ + uint64_t sample_events; /**< Number of received sample events */ + uint64_t tot; /**< Total event latency. Sum of all events. */ + uint64_t min; /**< Minimum event latency */ + uint64_t max; /**< Maximum event latency */ +} test_stat_t; + +/** Performance test statistics (per core) */ +typedef union { + test_stat_t prio[NUM_PRIOS]; /**< Test statistics per priority */ + + uint8_t pad[CACHE_ALIGN_ROUNDUP(NUM_PRIOS * sizeof(test_stat_t))]; +} core_stat_t ODP_ALIGNED_CACHE; + +/** Test global variables */ +typedef struct { + core_stat_t core_stat[MAX_WORKERS]; /**< Core specific stats */ + odp_barrier_t barrier; /**< Barrier for thread synchronization */ + odp_pool_t pool; /**< Pool for allocating test events */ + test_args_t args; /**< Parsed command line arguments */ + odp_queue_t queue[NUM_PRIOS][MAX_QUEUES]; /**< Scheduled queues */ +} test_globals_t; + +/** + * Clear all scheduled queues. + * + * Retry to be sure that all buffers have been scheduled. + */ +static void clear_sched_queues(void) +{ + odp_event_t ev; + + while (1) { + ev = odp_schedule(NULL, ODP_SCHED_NO_WAIT); + + if (ev == ODP_EVENT_INVALID) + break; + + odp_event_free(ev); + } +} + +/** + * Enqueue events into queues + * + * @param prio Queue priority (HI_PRIO/LO_PRIO) + * @param num_queues Number of queues + * @param num_events Number of 'TRAFFIC' events + * @param num_samples Number of 'SAMPLE' events + * @param div_events If true, divide 'num_events' between 'num_queues'. if + * false, enqueue 'num_events' to each queue. + * @param globals Test shared data + * + * @retval 0 on success + * @retval -1 on failure + */ +static int enqueue_events(int prio, int num_queues, int num_events, + int num_samples, odp_bool_t div_events, + test_globals_t *globals) +{ + odp_buffer_t buf[num_events + num_samples]; + odp_event_t ev[num_events + num_samples]; + odp_queue_t queue; + test_event_t *event; + int i, j, ret; + int enq_events; + int events_per_queue; + int tot_events; + int rdy_events = 0; + + tot_events = num_events + num_samples; + + if (!num_queues || !tot_events) + return 0; + + events_per_queue = tot_events; + if (div_events) + events_per_queue = (tot_events + num_queues - 1) / num_queues; + + for (i = 0; i < num_queues; i++) { + queue = globals->queue[prio][i]; + + ret = odp_buffer_alloc_multi(globals->pool, buf, + events_per_queue); + if (ret != events_per_queue) { + LOG_ERR("Buffer alloc failed. Try increasing EVENT_POOL_SIZE.\n"); + ret = ret < 0 ? 0 : ret; + odp_buffer_free_multi(buf, ret); + return -1; + } + for (j = 0; j < events_per_queue; j++) { + if (!odp_buffer_is_valid(buf[j])) { + LOG_ERR("Buffer alloc failed\n"); + odp_buffer_free_multi(buf, events_per_queue); + return -1; + } + + event = odp_buffer_addr(buf[j]); + memset(event, 0, sizeof(test_event_t)); + + /* Latency isn't measured from the first processing + * round. */ + if (num_samples > 0) { + event->type = WARM_UP; + num_samples--; + } else { + event->type = TRAFFIC; + } + event->src_idx[prio] = i; + event->prio = prio; + ev[j] = odp_buffer_to_event(buf[j]); + } + + enq_events = 0; + do { + ret = odp_queue_enq_multi(queue, &ev[enq_events], + events_per_queue - + enq_events); + if (ret < 0) { + LOG_ERR("Queue enqueue failed.\n"); + return -1; + } + enq_events += ret; + } while (enq_events < events_per_queue); + + rdy_events += events_per_queue; + if (div_events && rdy_events >= tot_events) + return 0; + } + return 0; +} + +/** + * Print latency measurement results + * + * @param globals Test shared data + */ +static void print_results(test_globals_t *globals) +{ + test_stat_t *lat; + odp_schedule_sync_t stype; + test_stat_t total; + test_args_t *args; + uint64_t avg; + int i, j; + + args = &globals->args; + stype = globals->args.sync_type; + + printf("\n%s queue scheduling latency\n", + (stype == ODP_SCHED_SYNC_ATOMIC) ? "ATOMIC" : + ((stype == ODP_SCHED_SYNC_ORDERED) ? "ORDERED" : "PARALLEL")); + + printf(" LO_PRIO queues: %i\n", args->prio[LO_PRIO].queues); + if (args->prio[LO_PRIO].events_per_queue) + printf(" LO_PRIO event per queue: %i\n", + args->prio[LO_PRIO].events); + else + printf(" LO_PRIO events: %i\n", args->prio[LO_PRIO].events); + + printf(" HI_PRIO queues: %i\n", args->prio[HI_PRIO].queues); + if (args->prio[HI_PRIO].events_per_queue) + printf(" HI_PRIO event per queue: %i\n\n", + args->prio[HI_PRIO].events); + else + printf(" HI_PRIO events: %i\n\n", args->prio[HI_PRIO].events); + + for (i = 0; i < NUM_PRIOS; i++) { + memset(&total, 0, sizeof(test_stat_t)); + total.min = UINT64_MAX; + + printf("%s priority\n" + "Thread Avg[ns] Min[ns] Max[ns] Samples Total\n" + "---------------------------------------------------------------\n", + i == HI_PRIO ? "HIGH" : "LOW"); + for (j = 1; j <= args->cpu_count; j++) { + lat = &globals->core_stat[j].prio[i]; + + if (lat->sample_events == 0) { + printf("%-8d N/A\n", j); + continue; + } + + if (lat->max > total.max) + total.max = lat->max; + if (lat->min < total.min) + total.min = lat->min; + total.tot += lat->tot; + total.sample_events += lat->sample_events; + total.events += lat->events; + + avg = lat->events ? lat->tot / lat->sample_events : 0; + printf("%-8d %-10" PRIu64 " %-10" PRIu64 " " + "%-10" PRIu64 " %-10" PRIu64 " %-10" PRIu64 "\n", + j, avg, lat->min, lat->max, lat->sample_events, + lat->events); + } + printf("---------------------------------------------------------------\n"); + if (total.sample_events == 0) { + printf("Total N/A\n\n"); + continue; + } + avg = total.events ? total.tot / total.sample_events : 0; + printf("Total %-10" PRIu64 " %-10" PRIu64 " %-10" PRIu64 " " + "%-10" PRIu64 " %-10" PRIu64 "\n\n", avg, total.min, + total.max, total.sample_events, total.events); + } +} + +/** + * Measure latency of scheduled ODP events + * + * Schedule and enqueue events until 'TEST_ROUNDS' events have been processed. + * Scheduling latency is measured only from type 'SAMPLE' events. Other events + * are simply enqueued back to the scheduling queues. + * + * For 'TRAFFIC' type events the destination queue is selected from the same + * priority class as source queue. 'SAMPLE' type event may change priority + * depending on the command line arguments. + * + * @param thr Thread ID + * @param globals Test shared data + * + * @retval 0 on success + * @retval -1 on failure + */ +static int test_schedule(int thr, test_globals_t *globals) +{ + odp_event_t ev; + odp_buffer_t buf; + odp_queue_t src_queue; + odp_queue_t dst_queue; + uint64_t latency; + uint32_t i; + test_event_t *event; + test_stat_t *stats; + int dst_idx; + + memset(&globals->core_stat[thr], 0, sizeof(core_stat_t)); + globals->core_stat[thr].prio[HI_PRIO].min = UINT64_MAX; + globals->core_stat[thr].prio[LO_PRIO].min = UINT64_MAX; + + for (i = 0; i < TEST_ROUNDS; i++) { + ev = odp_schedule(&src_queue, ODP_SCHED_WAIT); + + buf = odp_buffer_from_event(ev); + event = odp_buffer_addr(buf); + + stats = &globals->core_stat[thr].prio[event->prio]; + + if (event->type == SAMPLE) { + latency = odp_time_to_ns(odp_time_global()) - event->ts; + + if (latency > stats->max) + stats->max = latency; + if (latency < stats->min) + stats->min = latency; + stats->tot += latency; + stats->sample_events++; + + /* Move sample event to a different priority */ + if (!globals->args.sample_per_prio && + globals->args.prio[!event->prio].queues) + event->prio = !event->prio; + } + + if (odp_unlikely(event->type == WARM_UP)) + event->type = SAMPLE; + else + stats->events++; + + /* Move event to next queue */ + dst_idx = event->src_idx[event->prio] + 1; + if (dst_idx >= globals->args.prio[event->prio].queues) + dst_idx = 0; + event->src_idx[event->prio] = dst_idx; + dst_queue = globals->queue[event->prio][dst_idx]; + + if (event->type == SAMPLE) + event->ts = odp_time_to_ns(odp_time_global()); + + if (odp_queue_enq(dst_queue, ev)) { + LOG_ERR("[%i] Queue enqueue failed.\n", thr); + odp_event_free(ev); + return -1; + } + } + + /* Clear possible locally stored buffers */ + odp_schedule_pause(); + + while (1) { + ev = odp_schedule(&src_queue, ODP_SCHED_NO_WAIT); + + if (ev == ODP_EVENT_INVALID) + break; + + if (odp_queue_enq(src_queue, ev)) { + LOG_ERR("[%i] Queue enqueue failed.\n", thr); + odp_event_free(ev); + return -1; + } + } + + odp_schedule_resume(); + + odp_barrier_wait(&globals->barrier); + + clear_sched_queues(); + + if (thr == MAIN_THREAD) + print_results(globals); + + return 0; +} + +/** + * Worker thread + * + * @param arg Arguments + * + * @retval 0 on success + * @retval -1 on failure + */ +static int run_thread(void *arg ODP_UNUSED) +{ + odp_shm_t shm; + test_globals_t *globals; + test_args_t *args; + int thr; + int sample_events = 0; + + thr = odp_thread_id(); + + shm = odp_shm_lookup("test_globals"); + globals = odp_shm_addr(shm); + + if (globals == NULL) { + LOG_ERR("Shared mem lookup failed\n"); + return -1; + } + + if (thr == MAIN_THREAD) { + args = &globals->args; + + if (enqueue_events(HI_PRIO, args->prio[HI_PRIO].queues, + args->prio[HI_PRIO].events, 1, + !args->prio[HI_PRIO].events_per_queue, + globals)) + return -1; + + if (!args->prio[HI_PRIO].queues || args->sample_per_prio) + sample_events = 1; + + if (enqueue_events(LO_PRIO, args->prio[LO_PRIO].queues, + args->prio[LO_PRIO].events, sample_events, + !args->prio[LO_PRIO].events_per_queue, + globals)) + return -1; + } + + odp_barrier_wait(&globals->barrier); + + if (test_schedule(thr, globals)) + return -1; + + return 0; +} + +/** + * Print usage information + */ +static void usage(void) +{ + printf("\n" + "OpenDataPlane scheduler latency benchmark application.\n" + "\n" + "Usage: ./odp_sched_latency [options]\n" + "Optional OPTIONS:\n" + " -c, --count <number> CPU count\n" + " -l, --lo-prio-queues <number> Number of low priority scheduled queues\n" + " -t, --hi-prio-queues <number> Number of high priority scheduled queues\n" + " -m, --lo-prio-events-per-queue <number> Number of events per low priority queue\n" + " -n, --hi-prio-events-per-queue <number> Number of events per high priority queues\n" + " -o, --lo-prio-events <number> Total number of low priority events (overrides the\n" + " number of events per queue)\n" + " -p, --hi-prio-events <number> Total number of high priority events (overrides the\n" + " number of events per queue)\n" + " -r --sample-per-prio Allocate a separate sample event for each priority. By default\n" + " a single sample event is used and its priority is changed after\n" + " each processing round.\n" + " -s, --sync Scheduled queues' sync type\n" + " 0: ODP_SCHED_SYNC_PARALLEL (default)\n" + " 1: ODP_SCHED_SYNC_ATOMIC\n" + " 2: ODP_SCHED_SYNC_ORDERED\n" + " -h, --help Display help and exit.\n\n" + ); +} + +/** + * Parse arguments + * + * @param argc Argument count + * @param argv Argument vector + * @param args Test arguments + */ +static void parse_args(int argc, char *argv[], test_args_t *args) +{ + int opt; + int long_index; + int i; + + static const struct option longopts[] = { + {"count", required_argument, NULL, 'c'}, + {"lo-prio-queues", required_argument, NULL, 'l'}, + {"hi-prio-queues", required_argument, NULL, 't'}, + {"lo-prio-events-per-queue", required_argument, NULL, 'm'}, + {"hi-prio-events-per-queue", required_argument, NULL, 'n'}, + {"lo-prio-events", required_argument, NULL, 'o'}, + {"hi-prio-events", required_argument, NULL, 'p'}, + {"sample-per-prio", no_argument, NULL, 'r'}, + {"sync", required_argument, NULL, 's'}, + {"help", no_argument, NULL, 'h'}, + {NULL, 0, NULL, 0} + }; + + static const char *shortopts = "+c:s:l:t:m:n:o:p:rh"; + + /* Let helper collect its own arguments (e.g. --odph_proc) */ + odph_parse_options(argc, argv, shortopts, longopts); + + args->sync_type = ODP_SCHED_SYNC_PARALLEL; + args->sample_per_prio = SAMPLE_EVENT_PER_PRIO; + args->prio[LO_PRIO].queues = LO_PRIO_QUEUES; + args->prio[HI_PRIO].queues = HI_PRIO_QUEUES; + args->prio[LO_PRIO].events = LO_PRIO_EVENTS; + args->prio[HI_PRIO].events = HI_PRIO_EVENTS; + args->prio[LO_PRIO].events_per_queue = EVENTS_PER_LO_PRIO_QUEUE; + args->prio[HI_PRIO].events_per_queue = EVENTS_PER_HI_PRIO_QUEUE; + + opterr = 0; /* Do not issue errors on helper options */ + while (1) { + opt = getopt_long(argc, argv, shortopts, longopts, &long_index); + + if (opt == -1) + break; /* No more options */ + + switch (opt) { + case 'c': + args->cpu_count = atoi(optarg); + break; + case 'l': + args->prio[LO_PRIO].queues = atoi(optarg); + break; + case 't': + args->prio[HI_PRIO].queues = atoi(optarg); + break; + case 'm': + args->prio[LO_PRIO].events = atoi(optarg); + args->prio[LO_PRIO].events_per_queue = 1; + break; + case 'n': + args->prio[HI_PRIO].events = atoi(optarg); + args->prio[HI_PRIO].events_per_queue = 1; + break; + case 'o': + args->prio[LO_PRIO].events = atoi(optarg); + args->prio[LO_PRIO].events_per_queue = 0; + break; + case 'p': + args->prio[HI_PRIO].events = atoi(optarg); + args->prio[HI_PRIO].events_per_queue = 0; + break; + case 's': + i = atoi(optarg); + if (i == 1) + args->sync_type = ODP_SCHED_SYNC_ATOMIC; + else if (i == 2) + args->sync_type = ODP_SCHED_SYNC_ORDERED; + else + args->sync_type = ODP_SCHED_SYNC_PARALLEL; + break; + case 'r': + args->sample_per_prio = 1; + break; + case 'h': + usage(); + exit(EXIT_SUCCESS); + break; + + default: + break; + } + } + + /* Make sure arguments are valid */ + if (args->cpu_count > MAX_WORKERS) + args->cpu_count = MAX_WORKERS; + if (args->prio[LO_PRIO].queues > MAX_QUEUES) + args->prio[LO_PRIO].queues = MAX_QUEUES; + if (args->prio[HI_PRIO].queues > MAX_QUEUES) + args->prio[HI_PRIO].queues = MAX_QUEUES; + if (!args->prio[HI_PRIO].queues && !args->prio[LO_PRIO].queues) { + printf("No queues configured\n"); + usage(); + exit(EXIT_FAILURE); + } +} + +/** + * Test main function + */ +int main(int argc, char *argv[]) +{ + odp_instance_t instance; + odph_odpthread_t *thread_tbl; + odph_odpthread_params_t thr_params; + odp_cpumask_t cpumask; + odp_pool_t pool; + odp_pool_param_t params; + odp_shm_t shm; + test_globals_t *globals; + test_args_t args; + char cpumaskstr[ODP_CPUMASK_STR_SIZE]; + int i, j; + int ret = 0; + int num_workers = 0; + + printf("\nODP scheduling latency benchmark starts\n\n"); + + memset(&args, 0, sizeof(args)); + parse_args(argc, argv, &args); + + /* ODP global init */ + if (odp_init_global(&instance, NULL, NULL)) { + LOG_ERR("ODP global init failed.\n"); + return -1; + } + + /* + * Init this thread. It makes also ODP calls when + * setting up resources for worker threads. + */ + if (odp_init_local(instance, ODP_THREAD_CONTROL)) { + LOG_ERR("ODP global init failed.\n"); + return -1; + } + + printf("\n"); + printf("ODP system info\n"); + printf("---------------\n"); + printf("ODP API version: %s\n", odp_version_api_str()); + printf("ODP impl name: %s\n", odp_version_impl_name()); + printf("ODP impl details: %s\n", odp_version_impl_str()); + printf("CPU model: %s\n", odp_cpu_model_str()); + printf("CPU freq (hz): %" PRIu64 "\n", odp_cpu_hz_max()); + printf("Cache line size: %i\n", odp_sys_cache_line_size()); + printf("Max CPU count: %i\n", odp_cpu_count()); + + /* Get default worker cpumask */ + if (args.cpu_count) + num_workers = args.cpu_count; + + num_workers = odp_cpumask_default_worker(&cpumask, num_workers); + args.cpu_count = num_workers; + + (void)odp_cpumask_to_str(&cpumask, cpumaskstr, sizeof(cpumaskstr)); + + printf("Worker threads: %i\n", num_workers); + printf("First CPU: %i\n", odp_cpumask_first(&cpumask)); + printf("CPU mask: %s\n\n", cpumaskstr); + + thread_tbl = calloc(sizeof(odph_odpthread_t), num_workers); + if (!thread_tbl) { + LOG_ERR("no memory for thread_tbl\n"); + return -1; + } + + shm = odp_shm_reserve("test_globals", + sizeof(test_globals_t), ODP_CACHE_LINE_SIZE, 0); + if (shm == ODP_SHM_INVALID) { + LOG_ERR("Shared memory reserve failed.\n"); + return -1; + } + + globals = odp_shm_addr(shm); + memset(globals, 0, sizeof(test_globals_t)); + memcpy(&globals->args, &args, sizeof(test_args_t)); + + /* + * Create event pool + */ + odp_pool_param_init(&params); + params.buf.size = sizeof(test_event_t); + params.buf.align = 0; + params.buf.num = EVENT_POOL_SIZE; + params.type = ODP_POOL_BUFFER; + + pool = odp_pool_create("event_pool", &params); + + if (pool == ODP_POOL_INVALID) { + LOG_ERR("Pool create failed.\n"); + return -1; + } + globals->pool = pool; + + /* + * Create queues for schedule test + */ + for (i = 0; i < NUM_PRIOS; i++) { + char name[] = "sched_XX_YY"; + odp_queue_t queue; + odp_queue_param_t param; + int prio; + + if (i == HI_PRIO) + prio = ODP_SCHED_PRIO_HIGHEST; + else + prio = ODP_SCHED_PRIO_LOWEST; + + name[6] = '0' + (prio / 10); + name[7] = '0' + prio - (10 * (prio / 10)); + + odp_queue_param_init(&param); + param.type = ODP_QUEUE_TYPE_SCHED; + param.sched.prio = prio; + param.sched.sync = args.sync_type; + param.sched.group = ODP_SCHED_GROUP_ALL; + + for (j = 0; j < args.prio[i].queues; j++) { + name[9] = '0' + j / 10; + name[10] = '0' + j - 10 * (j / 10); + + queue = odp_queue_create(name, &param); + + if (queue == ODP_QUEUE_INVALID) { + LOG_ERR("Scheduled queue create failed.\n"); + return -1; + } + + globals->queue[i][j] = queue; + } + } + + odp_barrier_init(&globals->barrier, num_workers); + + /* Create and launch worker threads */ + memset(&thr_params, 0, sizeof(thr_params)); + thr_params.thr_type = ODP_THREAD_WORKER; + thr_params.instance = instance; + thr_params.start = run_thread; + thr_params.arg = NULL; + odph_odpthreads_create(thread_tbl, &cpumask, &thr_params); + + /* Wait for worker threads to terminate */ + odph_odpthreads_join(thread_tbl); + free(thread_tbl); + + printf("ODP scheduling latency test complete\n\n"); + + for (i = 0; i < NUM_PRIOS; i++) { + odp_queue_t queue; + int num_queues; + + num_queues = args.prio[i].queues; + + for (j = 0; j < num_queues; j++) { + queue = globals->queue[i][j]; + ret += odp_queue_destroy(queue); + } + } + + ret += odp_shm_free(shm); + ret += odp_pool_destroy(pool); + ret += odp_term_local(); + ret += odp_term_global(instance); + + return ret; +} ----------------------------------------------------------------------- Summary of changes: test/common_plat/performance/.gitignore | 1 + test/common_plat/performance/Makefile.am | 5 + test/common_plat/performance/odp_sched_latency.c | 767 +++++++++++++++++++++ ..._scheduling_run.sh => odp_sched_latency_run.sh} | 11 +- 4 files changed, 778 insertions(+), 6 deletions(-) create mode 100644 test/common_plat/performance/odp_sched_latency.c copy test/common_plat/performance/{odp_scheduling_run.sh => odp_sched_latency_run.sh} (51%) hooks/post-receive --

8 years, 4 months

[COMMIT] branch master updated. v1.10.1.0-122-ge858d68

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via e858d688c3ad3ce0d0b3ea22539cac8e6ec844b7 (commit) via 93718d4ead55bfbaa8e564c24d1f3b76e60235ce (commit) via 582065e74e2375b5c81ac8fcec9eb02f541f42ff (commit) via 5f4f2e0da6e04637b6b4bd7aa6bb4d4d32680525 (commit) via 92336dd2808af4826371d467588dcb81daafe4cf (commit) via 7bb62b522a5f89d6d19a4c77254222b1c07ab44b (commit) from 566492d067083e870548c78a89f8c65b02ecde89 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit e858d688c3ad3ce0d0b3ea22539cac8e6ec844b7 Author: Petri Savolainen <petri.savolainen(a)nokia.com> Date: Thu Sep 15 16:39:24 2016 +0300 linux-gen: config: increase burst sizes Added main burst size configuration option (CONFIG_BURST_SIZE), which helps to keep various burst sizes in sync. Increased common burst size from 8 to 16. This increases scheduled queue throughput about 30-40%. Signed-off-by: Petri Savolainen <petri.savolainen(a)nokia.com> Reviewed-and-tested-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 69daf94..1c09cd3 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -103,7 +103,7 @@ typedef union odp_buffer_bits_t { }; } odp_buffer_bits_t; -#define BUFFER_BURST_SIZE 8 +#define BUFFER_BURST_SIZE CONFIG_BURST_SIZE /* Common buffer header */ struct odp_buffer_hdr_t { diff --git a/platform/linux-generic/include/odp_config_internal.h b/platform/linux-generic/include/odp_config_internal.h index 989ea08..b7ff610 100644 --- a/platform/linux-generic/include/odp_config_internal.h +++ b/platform/linux-generic/include/odp_config_internal.h @@ -110,6 +110,14 @@ extern "C" { */ #define ODP_CONFIG_SHM_BLOCKS (ODP_CONFIG_POOLS + 48) +/* + * Maximum event burst size + * + * This controls the burst size on various enqueue, dequeue, etc calls. Large + * burst size improves throughput, but may degrade QoS (increase latency). + */ +#define CONFIG_BURST_SIZE 16 + #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp_packet_io_queue.h b/platform/linux-generic/include/odp_packet_io_queue.h index 5843f6a..13b79f3 100644 --- a/platform/linux-generic/include/odp_packet_io_queue.h +++ b/platform/linux-generic/include/odp_packet_io_queue.h @@ -20,9 +20,10 @@ extern "C" { #include <odp_queue_internal.h> #include <odp_buffer_internal.h> +#include <odp_config_internal.h> /** Max nbr of pkts to receive in one burst (keep same as QUEUE_MULTI_MAX) */ -#define ODP_PKTIN_QUEUE_MAX_BURST 16 +#define ODP_PKTIN_QUEUE_MAX_BURST CONFIG_BURST_SIZE /* pktin_deq_multi() depends on the condition: */ ODP_STATIC_ASSERT(ODP_PKTIN_QUEUE_MAX_BURST >= QUEUE_MULTI_MAX, "ODP_PKTIN_DEQ_MULTI_MAX_ERROR"); diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index 1b95627..ca59ade 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -52,7 +52,7 @@ typedef struct _odp_buffer_pool_init_t { } _odp_buffer_pool_init_t; /**< Type of buffer initialization struct */ #define POOL_MAX_LOCAL_CHUNKS 4 -#define POOL_CHUNK_SIZE 32 +#define POOL_CHUNK_SIZE (4 * CONFIG_BURST_SIZE) #define POOL_MAX_LOCAL_BUFS (POOL_MAX_LOCAL_CHUNKS * POOL_CHUNK_SIZE) struct local_cache_s { diff --git a/platform/linux-generic/include/odp_queue_internal.h b/platform/linux-generic/include/odp_queue_internal.h index cb7e3f3..e223d9f 100644 --- a/platform/linux-generic/include/odp_queue_internal.h +++ b/platform/linux-generic/include/odp_queue_internal.h @@ -27,8 +27,9 @@ extern "C" { #include <odp/api/align.h> #include <odp/api/hints.h> #include <odp/api/ticketlock.h> +#include <odp_config_internal.h> -#define QUEUE_MULTI_MAX 8 +#define QUEUE_MULTI_MAX CONFIG_BURST_SIZE #define QUEUE_STATUS_FREE 0 #define QUEUE_STATUS_DESTROYED 1 diff --git a/platform/linux-generic/include/odp_schedule_internal.h b/platform/linux-generic/include/odp_schedule_internal.h index d6cfdb4..02637c2 100644 --- a/platform/linux-generic/include/odp_schedule_internal.h +++ b/platform/linux-generic/include/odp_schedule_internal.h @@ -12,7 +12,7 @@ extern "C" { #endif /* Maximum number of dequeues */ -#define MAX_DEQ 8 +#define MAX_DEQ CONFIG_BURST_SIZE typedef struct { int thr; commit 93718d4ead55bfbaa8e564c24d1f3b76e60235ce Author: Petri Savolainen <petri.savolainen(a)nokia.com> Date: Thu Sep 15 16:39:23 2016 +0300 linux-gen: pool: use inlined ticketlock Use inlined ticketlock calls instead of API calls. Signed-off-by: Petri Savolainen <petri.savolainen(a)nokia.com> Reviewed-and-tested-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index d6717ff..1b95627 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -73,20 +73,10 @@ typedef struct local_cache_t { }; } local_cache_t; -/* Use ticketlock instead of spinlock */ -#define POOL_USE_TICKETLOCK - -#ifdef POOL_USE_TICKETLOCK -#include <odp/api/ticketlock.h> -#define POOL_LOCK(a) odp_ticketlock_lock(a) -#define POOL_UNLOCK(a) odp_ticketlock_unlock(a) +#include <odp/api/plat/ticketlock_inlines.h> +#define POOL_LOCK(a) _odp_ticketlock_lock(a) +#define POOL_UNLOCK(a) _odp_ticketlock_unlock(a) #define POOL_LOCK_INIT(a) odp_ticketlock_init(a) -#else -#include <odp/api/spinlock.h> -#define POOL_LOCK(a) odp_spinlock_lock(a) -#define POOL_UNLOCK(a) odp_spinlock_unlock(a) -#define POOL_LOCK_INIT(a) odp_spinlock_init(a) -#endif /** * ODP Pool stats - Maintain some useful stats regarding pool utilization @@ -105,15 +95,9 @@ typedef struct { } _odp_pool_stats_t; struct pool_entry_s { -#ifdef POOL_USE_TICKETLOCK odp_ticketlock_t lock ODP_ALIGNED_CACHE; odp_ticketlock_t buf_lock; odp_ticketlock_t blk_lock; -#else - odp_spinlock_t lock ODP_ALIGNED_CACHE; - odp_spinlock_t buf_lock; - odp_spinlock_t blk_lock; -#endif char name[ODP_POOL_NAME_LEN]; odp_pool_param_t params; commit 582065e74e2375b5c81ac8fcec9eb02f541f42ff Author: Petri Savolainen <petri.savolainen(a)nokia.com> Date: Thu Sep 15 16:39:22 2016 +0300 linux-gen: queue: use inlined ticketlock Use inlined ticketlock calls instead of API calls. Inlining improves performance and makes it easier to profile functions which use ticketlocks. Cycle consumption of caller functions are more interesting than combined cycle consumption of all ticketlocks. Signed-off-by: Petri Savolainen <petri.savolainen(a)nokia.com> Reviewed-and-tested-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_queue_internal.h b/platform/linux-generic/include/odp_queue_internal.h index 830798b..cb7e3f3 100644 --- a/platform/linux-generic/include/odp_queue_internal.h +++ b/platform/linux-generic/include/odp_queue_internal.h @@ -26,15 +26,7 @@ extern "C" { #include <odp/api/packet_io.h> #include <odp/api/align.h> #include <odp/api/hints.h> - - -#define USE_TICKETLOCK - -#ifdef USE_TICKETLOCK #include <odp/api/ticketlock.h> -#else -#include <odp/api/spinlock.h> -#endif #define QUEUE_MULTI_MAX 8 @@ -57,11 +49,7 @@ typedef int (*deq_multi_func_t)(union queue_entry_u *, odp_buffer_hdr_t **, int); struct queue_entry_s { -#ifdef USE_TICKETLOCK odp_ticketlock_t lock ODP_ALIGNED_CACHE; -#else - odp_spinlock_t lock ODP_ALIGNED_CACHE; -#endif odp_buffer_hdr_t *head; odp_buffer_hdr_t *tail; diff --git a/platform/linux-generic/odp_queue.c b/platform/linux-generic/odp_queue.c index 5b962e9..8667076 100644 --- a/platform/linux-generic/odp_queue.c +++ b/platform/linux-generic/odp_queue.c @@ -27,19 +27,10 @@ #define NUM_INTERNAL_QUEUES 64 -#ifdef USE_TICKETLOCK -#include <odp/api/ticketlock.h> -#define LOCK(a) odp_ticketlock_lock(a) -#define UNLOCK(a) odp_ticketlock_unlock(a) +#include <odp/api/plat/ticketlock_inlines.h> +#define LOCK(a) _odp_ticketlock_lock(a) +#define UNLOCK(a) _odp_ticketlock_unlock(a) #define LOCK_INIT(a) odp_ticketlock_init(a) -#define LOCK_TRY(a) odp_ticketlock_trylock(a) -#else -#include <odp/api/spinlock.h> -#define LOCK(a) odp_spinlock_lock(a) -#define UNLOCK(a) odp_spinlock_unlock(a) -#define LOCK_INIT(a) odp_spinlock_init(a) -#define LOCK_TRY(a) odp_spinlock_trylock(a) -#endif #include <string.h> #include <inttypes.h> commit 5f4f2e0da6e04637b6b4bd7aa6bb4d4d32680525 Author: Petri Savolainen <petri.savolainen(a)nokia.com> Date: Thu Sep 15 16:39:21 2016 +0300 linux-gen: ticketlock: inline ticketlock implementation Moved ticketlock implementation into ticketlock_inlines.h header file, which enables inlined version to be used inside implementation and through API. Signed-off-by: Petri Savolainen <petri.savolainen(a)nokia.com> Reviewed-and-tested-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index e3c0f56..900ac08 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -88,6 +88,7 @@ odpapiplatinclude_HEADERS = \ $(srcdir)/include/odp/api/plat/sync_inlines.h \ $(srcdir)/include/odp/api/plat/thread_types.h \ $(srcdir)/include/odp/api/plat/thrmask_types.h \ + $(srcdir)/include/odp/api/plat/ticketlock_inlines.h \ $(srcdir)/include/odp/api/plat/ticketlock_types.h \ $(srcdir)/include/odp/api/plat/time_types.h \ $(srcdir)/include/odp/api/plat/timer_types.h \ diff --git a/platform/linux-generic/odp_ticketlock.c b/platform/linux-generic/include/odp/api/plat/ticketlock_inlines.h similarity index 83% copy from platform/linux-generic/odp_ticketlock.c copy to platform/linux-generic/include/odp/api/plat/ticketlock_inlines.h index 353af9a..957d22e 100644 --- a/platform/linux-generic/odp_ticketlock.c +++ b/platform/linux-generic/include/odp/api/plat/ticketlock_inlines.h @@ -1,21 +1,24 @@ -/* Copyright (c) 2013, Linaro Limited +/* Copyright (c) 2016, Linaro Limited * All rights reserved. * - * SPDX-License-Identifier: BSD-3-Clause + * SPDX-License-Identifier: BSD-3-Clause */ +/** + * @file + * + * Ticketlock inline functions + */ + +#ifndef _ODP_PLAT_TICKETLOCK_INLINES_H_ +#define _ODP_PLAT_TICKETLOCK_INLINES_H_ + #include <odp/api/ticketlock.h> #include <odp/api/atomic.h> #include <odp/api/sync.h> #include <odp/api/cpu.h> -void odp_ticketlock_init(odp_ticketlock_t *ticketlock) -{ - odp_atomic_init_u32(&ticketlock->next_ticket, 0); - odp_atomic_init_u32(&ticketlock->cur_ticket, 0); -} - -void odp_ticketlock_lock(odp_ticketlock_t *ticketlock) +static inline void _odp_ticketlock_lock(odp_ticketlock_t *ticketlock) { uint32_t ticket; @@ -30,7 +33,7 @@ void odp_ticketlock_lock(odp_ticketlock_t *ticketlock) odp_cpu_pause(); } -int odp_ticketlock_trylock(odp_ticketlock_t *tklock) +static inline int _odp_ticketlock_trylock(odp_ticketlock_t *tklock) { /* We read 'next_ticket' and 'cur_ticket' non-atomically which should * not be a problem as they are not independent of each other. @@ -58,7 +61,7 @@ int odp_ticketlock_trylock(odp_ticketlock_t *tklock) return 0; } -void odp_ticketlock_unlock(odp_ticketlock_t *ticketlock) +static inline void _odp_ticketlock_unlock(odp_ticketlock_t *ticketlock) { /* Release the lock by incrementing 'cur_ticket'. As we are the * lock owner and thus the only thread that is allowed to write @@ -68,10 +71,9 @@ void odp_ticketlock_unlock(odp_ticketlock_t *ticketlock) uint32_t cur = odp_atomic_load_u32(&ticketlock->cur_ticket); odp_atomic_store_rel_u32(&ticketlock->cur_ticket, cur + 1); - } -int odp_ticketlock_is_locked(odp_ticketlock_t *ticketlock) +static inline int _odp_ticketlock_is_locked(odp_ticketlock_t *ticketlock) { /* Compare 'cur_ticket' with 'next_ticket'. Ideally we should read * both variables atomically but the information can become stale @@ -81,3 +83,5 @@ int odp_ticketlock_is_locked(odp_ticketlock_t *ticketlock) return odp_atomic_load_u32(&ticketlock->cur_ticket) != odp_atomic_load_u32(&ticketlock->next_ticket); } + +#endif diff --git a/platform/linux-generic/odp_ticketlock.c b/platform/linux-generic/odp_ticketlock.c index 353af9a..f18d78f 100644 --- a/platform/linux-generic/odp_ticketlock.c +++ b/platform/linux-generic/odp_ticketlock.c @@ -4,10 +4,7 @@ * SPDX-License-Identifier: BSD-3-Clause */ -#include <odp/api/ticketlock.h> -#include <odp/api/atomic.h> -#include <odp/api/sync.h> -#include <odp/api/cpu.h> +#include <odp/api/plat/ticketlock_inlines.h> void odp_ticketlock_init(odp_ticketlock_t *ticketlock) { @@ -15,69 +12,22 @@ void odp_ticketlock_init(odp_ticketlock_t *ticketlock) odp_atomic_init_u32(&ticketlock->cur_ticket, 0); } -void odp_ticketlock_lock(odp_ticketlock_t *ticketlock) +void odp_ticketlock_lock(odp_ticketlock_t *lock) { - uint32_t ticket; - - /* Take a ticket using an atomic increment of 'next_ticket'. - * This can be a relaxed operation but it cannot have the - * acquire semantics since we haven't acquired the lock yet */ - ticket = odp_atomic_fetch_inc_u32(&ticketlock->next_ticket); - - /* Spin waiting for our turn. Use load-acquire so that we acquire - * all stores from the previous lock owner */ - while (ticket != odp_atomic_load_acq_u32(&ticketlock->cur_ticket)) - odp_cpu_pause(); + return _odp_ticketlock_lock(lock); } -int odp_ticketlock_trylock(odp_ticketlock_t *tklock) +int odp_ticketlock_trylock(odp_ticketlock_t *lock) { - /* We read 'next_ticket' and 'cur_ticket' non-atomically which should - * not be a problem as they are not independent of each other. - * 'cur_ticket' is always <= to 'next_ticket' and if we see an - * older value of 'cur_ticket', this only means the lock will - * look busy and trylock will fail. */ - uint32_t next = odp_atomic_load_u32(&tklock->next_ticket); - uint32_t cur = odp_atomic_load_u32(&tklock->cur_ticket); - /* First check that lock is available and possible to take without - * spinning. */ - if (next == cur) { - /* Then try to take the lock by incrementing 'next_ticket' - * but only if it still has the original value which is - * equal to 'cur_ticket'. - * We don't have to include 'cur_ticket' in the comparison - * because it cannot be larger than 'next_ticket' (only - * smaller if the lock is busy). - * If CAS fails, it means some other thread intercepted and - * took a ticket which means the lock is not available - * anymore */ - if (odp_atomic_cas_acq_u32(&tklock->next_ticket, - &next, next + 1)) - return 1; - } - return 0; + return _odp_ticketlock_trylock(lock); } -void odp_ticketlock_unlock(odp_ticketlock_t *ticketlock) +void odp_ticketlock_unlock(odp_ticketlock_t *lock) { - /* Release the lock by incrementing 'cur_ticket'. As we are the - * lock owner and thus the only thread that is allowed to write - * 'cur_ticket', we don't need to do this with an (expensive) - * atomic RMW operation. Instead load-relaxed the current value - * and a store-release of the incremented value */ - uint32_t cur = odp_atomic_load_u32(&ticketlock->cur_ticket); - - odp_atomic_store_rel_u32(&ticketlock->cur_ticket, cur + 1); - + _odp_ticketlock_unlock(lock); } -int odp_ticketlock_is_locked(odp_ticketlock_t *ticketlock) +int odp_ticketlock_is_locked(odp_ticketlock_t *lock) { - /* Compare 'cur_ticket' with 'next_ticket'. Ideally we should read - * both variables atomically but the information can become stale - * immediately anyway so the function can only be used reliably in - * a quiescent system where non-atomic loads should not pose a - * problem */ - return odp_atomic_load_u32(&ticketlock->cur_ticket) != - odp_atomic_load_u32(&ticketlock->next_ticket); + return _odp_ticketlock_is_locked(lock); } commit 92336dd2808af4826371d467588dcb81daafe4cf Author: Petri Savolainen <petri.savolainen(a)nokia.com> Date: Thu Sep 15 16:39:20 2016 +0300 linux-gen: queue: burst enq and deq Added support for a buffer header to carry a burst of buffer pointers. The buffer itself is the last one a burst. Burst are built with a enq_multi call, so single enq operations do not benefit from it. Signed-off-by: Petri Savolainen <petri.savolainen(a)nokia.com> Reviewed-and-tested-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 7b0ef8b..69daf94 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -103,6 +103,8 @@ typedef union odp_buffer_bits_t { }; } odp_buffer_bits_t; +#define BUFFER_BURST_SIZE 8 + /* Common buffer header */ struct odp_buffer_hdr_t { struct odp_buffer_hdr_t *next; /* next buf in a list--keep 1st */ @@ -111,6 +113,11 @@ struct odp_buffer_hdr_t { struct odp_buffer_hdr_t *link; }; odp_buffer_bits_t handle; /* handle */ + + int burst_num; + int burst_first; + struct odp_buffer_hdr_t *burst[BUFFER_BURST_SIZE]; + union { uint32_t all; struct { diff --git a/platform/linux-generic/odp_queue.c b/platform/linux-generic/odp_queue.c index 80d99e8..5b962e9 100644 --- a/platform/linux-generic/odp_queue.c +++ b/platform/linux-generic/odp_queue.c @@ -388,20 +388,48 @@ static inline int enq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], { int sched = 0; int i, ret; - odp_buffer_hdr_t *tail; - - /* Chain input buffers together */ - for (i = 0; i < num - 1; i++) - buf_hdr[i]->next = buf_hdr[i + 1]; - - tail = buf_hdr[num - 1]; - buf_hdr[num - 1]->next = NULL; + odp_buffer_hdr_t *hdr, *tail, *next_hdr; + /* Ordered queues do not use bursts */ if (sched_fn->ord_enq_multi(queue->s.index, (void **)buf_hdr, num, sustain, &ret)) return ret; - /* Handle unordered enqueues */ + /* Optimize the common case of single enqueue */ + if (num == 1) { + tail = buf_hdr[0]; + hdr = tail; + hdr->burst_num = 0; + } else { + int next; + + /* Start from the last buffer header */ + tail = buf_hdr[num - 1]; + hdr = tail; + next = num - 2; + + while (1) { + /* Build a burst. The buffer header carrying + * a burst is the last buffer of the burst. */ + for (i = 0; next >= 0 && i < BUFFER_BURST_SIZE; + i++, next--) + hdr->burst[BUFFER_BURST_SIZE - 1 - i] = + buf_hdr[next]; + + hdr->burst_num = i; + hdr->burst_first = BUFFER_BURST_SIZE - i; + + if (odp_likely(next < 0)) + break; + + /* Get another header and link it */ + next_hdr = hdr; + hdr = buf_hdr[next]; + hdr->next = next_hdr; + next--; + } + } + LOCK(&queue->s.lock); if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { UNLOCK(&queue->s.lock); @@ -411,9 +439,9 @@ static inline int enq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], /* Empty queue */ if (queue->s.head == NULL) - queue->s.head = buf_hdr[0]; + queue->s.head = hdr; else - queue->s.tail->next = buf_hdr[0]; + queue->s.tail->next = hdr; queue->s.tail = tail; @@ -483,9 +511,9 @@ int odp_queue_enq(odp_queue_t handle, odp_event_t ev) static inline int deq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num) { - odp_buffer_hdr_t *hdr; - int i; - uint32_t j; + odp_buffer_hdr_t *hdr, *next; + int i, j; + int updated = 0; LOCK(&queue->s.lock); if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { @@ -506,33 +534,65 @@ static inline int deq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], return 0; } - for (i = 0; i < num && hdr; i++) { - buf_hdr[i] = hdr; - hdr = hdr->next; - buf_hdr[i]->next = NULL; - if (queue_is_ordered(queue)) { - buf_hdr[i]->origin_qe = queue; - buf_hdr[i]->order = queue->s.order_in++; - for (j = 0; j < queue->s.param.sched.lock_count; j++) { - buf_hdr[i]->sync[j] = + for (i = 0; i < num && hdr; ) { + int burst_num = hdr->burst_num; + int first = hdr->burst_first; + + /* First, get bursted buffers */ + for (j = 0; j < burst_num && i < num; j++, i++) { + buf_hdr[i] = hdr->burst[first + j]; + odp_prefetch(buf_hdr[i]); + } + + if (burst_num) { + hdr->burst_num = burst_num - j; + hdr->burst_first = first + j; + } + + if (i == num) + break; + + /* When burst is empty, consume the current buffer header and + * move to the next header */ + buf_hdr[i] = hdr; + next = hdr->next; + hdr->next = NULL; + hdr = next; + updated++; + i++; + } + + /* Ordered queue book keeping inside the lock */ + if (queue_is_ordered(queue)) { + for (j = 0; j < i; j++) { + uint32_t k; + + buf_hdr[j]->origin_qe = queue; + buf_hdr[j]->order = queue->s.order_in++; + for (k = 0; k < queue->s.param.sched.lock_count; k++) { + buf_hdr[j]->sync[k] = odp_atomic_fetch_inc_u64 - (&queue->s.sync_in[j]); + (&queue->s.sync_in[k]); } - buf_hdr[i]->flags.sustain = SUSTAIN_ORDER; - } else { - buf_hdr[i]->origin_qe = NULL; + buf_hdr[j]->flags.sustain = SUSTAIN_ORDER; } } - queue->s.head = hdr; + /* Write head only if updated */ + if (updated) + queue->s.head = hdr; - if (hdr == NULL) { - /* Queue is now empty */ + /* Queue is empty */ + if (hdr == NULL) queue->s.tail = NULL; - } UNLOCK(&queue->s.lock); + /* Init origin_qe for non-ordered queues */ + if (!queue_is_ordered(queue)) + for (j = 0; j < i; j++) + buf_hdr[j]->origin_qe = NULL; + return i; } @@ -543,7 +603,7 @@ int queue_deq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num) odp_buffer_hdr_t *queue_deq(queue_entry_t *queue) { - odp_buffer_hdr_t *buf_hdr; + odp_buffer_hdr_t *buf_hdr = NULL; int ret; ret = deq_multi(queue, &buf_hdr, 1); diff --git a/platform/linux-generic/odp_schedule_ordered.c b/platform/linux-generic/odp_schedule_ordered.c index 92a1cc8..8412183 100644 --- a/platform/linux-generic/odp_schedule_ordered.c +++ b/platform/linux-generic/odp_schedule_ordered.c @@ -457,15 +457,24 @@ int schedule_ordered_queue_enq_multi(uint32_t queue_index, void *p_buf_hdr[], { queue_entry_t *origin_qe; uint64_t order; - int rc; + int i, rc; queue_entry_t *qe = get_qentry(queue_index); - odp_buffer_hdr_t *buf_hdr = p_buf_hdr[0]; + odp_buffer_hdr_t *first_hdr = p_buf_hdr[0]; + odp_buffer_hdr_t **buf_hdr = (odp_buffer_hdr_t **)p_buf_hdr; + + /* Chain input buffers together */ + for (i = 0; i < num - 1; i++) { + buf_hdr[i]->next = buf_hdr[i + 1]; + buf_hdr[i]->burst_num = 0; + } + + buf_hdr[num - 1]->next = NULL; /* Handle ordered enqueues commonly via links */ - get_queue_order(&origin_qe, &order, buf_hdr); + get_queue_order(&origin_qe, &order, first_hdr); if (origin_qe) { - buf_hdr->link = buf_hdr->next; - rc = ordered_queue_enq(qe, buf_hdr, sustain, + first_hdr->link = first_hdr->next; + rc = ordered_queue_enq(qe, first_hdr, sustain, origin_qe, order); *ret = rc == 0 ? num : rc; return 1; commit 7bb62b522a5f89d6d19a4c77254222b1c07ab44b Author: Petri Savolainen <petri.savolainen(a)nokia.com> Date: Thu Sep 15 16:39:19 2016 +0300 linux-gen: queue: reuse enq_ and deq_multi Reuse multi enqueue and dequeue implementations for single enq/deq operations. This enables implementation to concentrate on optimizing the multi operations. Single operations do not suffer a major performance decrease since compiler likely optimizes the inlined code for single operations (num is fixed to 1). Signed-off-by: Petri Savolainen <petri.savolainen(a)nokia.com> Reviewed-and-tested-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/include/odp_schedule_if.h b/platform/linux-generic/include/odp_schedule_if.h index 13cdfb3..df73e70 100644 --- a/platform/linux-generic/include/odp_schedule_if.h +++ b/platform/linux-generic/include/odp_schedule_if.h @@ -30,8 +30,6 @@ typedef int (*schedule_init_queue_fn_t)(uint32_t queue_index, ); typedef void (*schedule_destroy_queue_fn_t)(uint32_t queue_index); typedef int (*schedule_sched_queue_fn_t)(uint32_t queue_index); -typedef int (*schedule_ord_enq_fn_t)(uint32_t queue_index, void *buf_hdr, - int sustain, int *ret); typedef int (*schedule_ord_enq_multi_fn_t)(uint32_t queue_index, void *buf_hdr[], int num, int sustain, int *ret); @@ -48,7 +46,6 @@ typedef struct schedule_fn_t { schedule_init_queue_fn_t init_queue; schedule_destroy_queue_fn_t destroy_queue; schedule_sched_queue_fn_t sched_queue; - schedule_ord_enq_fn_t ord_enq; schedule_ord_enq_multi_fn_t ord_enq_multi; schedule_init_global_fn_t init_global; schedule_term_global_fn_t term_global; diff --git a/platform/linux-generic/odp_queue.c b/platform/linux-generic/odp_queue.c index bec1e51..80d99e8 100644 --- a/platform/linux-generic/odp_queue.c +++ b/platform/linux-generic/odp_queue.c @@ -65,19 +65,6 @@ static inline int queue_is_ordered(queue_entry_t *qe) return qe->s.param.sched.sync == ODP_SCHED_SYNC_ORDERED; } -static inline void queue_add(queue_entry_t *queue, - odp_buffer_hdr_t *buf_hdr) -{ - buf_hdr->next = NULL; - - if (queue->s.head) - queue->s.tail->next = buf_hdr; - else - queue->s.head = buf_hdr; - - queue->s.tail = buf_hdr; -} - queue_entry_t *get_qentry(uint32_t queue_id) { return &queue_tbl->queue[queue_id]; @@ -396,37 +383,8 @@ odp_queue_t odp_queue_lookup(const char *name) return ODP_QUEUE_INVALID; } -int queue_enq(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr, int sustain) -{ - int ret; - - if (sched_fn->ord_enq(queue->s.index, buf_hdr, sustain, &ret)) - return ret; - - LOCK(&queue->s.lock); - - if (odp_unlikely(queue->s.status < QUEUE_STATUS_READY)) { - UNLOCK(&queue->s.lock); - ODP_ERR("Bad queue status\n"); - return -1; - } - - queue_add(queue, buf_hdr); - - if (queue->s.status == QUEUE_STATUS_NOTSCHED) { - queue->s.status = QUEUE_STATUS_SCHED; - UNLOCK(&queue->s.lock); - if (sched_fn->sched_queue(queue->s.index)) - ODP_ABORT("schedule_queue failed\n"); - return 0; - } - - UNLOCK(&queue->s.lock); - return 0; -} - -int queue_enq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], - int num, int sustain) +static inline int enq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], + int num, int sustain) { int sched = 0; int i, ret; @@ -472,6 +430,24 @@ int queue_enq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], return num; /* All events enqueued */ } +int queue_enq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num, + int sustain) +{ + return enq_multi(queue, buf_hdr, num, sustain); +} + +int queue_enq(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr, int sustain) +{ + int ret; + + ret = enq_multi(queue, &buf_hdr, 1, sustain); + + if (ret == 1) + return 0; + else + return -1; +} + int odp_queue_enq_multi(odp_queue_t handle, const odp_event_t ev[], int num) { odp_buffer_hdr_t *buf_hdr[QUEUE_MULTI_MAX]; @@ -504,54 +480,8 @@ int odp_queue_enq(odp_queue_t handle, odp_event_t ev) return queue->s.enqueue(queue, buf_hdr, SUSTAIN_ORDER); } -odp_buffer_hdr_t *queue_deq(queue_entry_t *queue) -{ - odp_buffer_hdr_t *buf_hdr; - uint32_t i; - - LOCK(&queue->s.lock); - - if (queue->s.head == NULL) { - /* Already empty queue */ - if (queue->s.status == QUEUE_STATUS_SCHED) - queue->s.status = QUEUE_STATUS_NOTSCHED; - - UNLOCK(&queue->s.lock); - return NULL; - } - - buf_hdr = queue->s.head; - queue->s.head = buf_hdr->next; - buf_hdr->next = NULL; - - /* Note that order should really be assigned on enq to an - * ordered queue rather than deq, however the logic is simpler - * to do it here and has the same effect. - */ - if (queue_is_ordered(queue)) { - buf_hdr->origin_qe = queue; - buf_hdr->order = queue->s.order_in++; - for (i = 0; i < queue->s.param.sched.lock_count; i++) { - buf_hdr->sync[i] = - odp_atomic_fetch_inc_u64(&queue->s.sync_in[i]); - } - buf_hdr->flags.sustain = SUSTAIN_ORDER; - } else { - buf_hdr->origin_qe = NULL; - } - - if (queue->s.head == NULL) { - /* Queue is now empty */ - queue->s.tail = NULL; - } - - UNLOCK(&queue->s.lock); - - return buf_hdr; -} - - -int queue_deq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num) +static inline int deq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], + int num) { odp_buffer_hdr_t *hdr; int i; @@ -606,6 +536,24 @@ int queue_deq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num) return i; } +int queue_deq_multi(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr[], int num) +{ + return deq_multi(queue, buf_hdr, num); +} + +odp_buffer_hdr_t *queue_deq(queue_entry_t *queue) +{ + odp_buffer_hdr_t *buf_hdr; + int ret; + + ret = deq_multi(queue, &buf_hdr, 1); + + if (ret == 1) + return buf_hdr; + else + return NULL; +} + int odp_queue_deq_multi(odp_queue_t handle, odp_event_t events[], int num) { queue_entry_t *queue; @@ -740,7 +688,7 @@ int sched_cb_queue_deq_multi(uint32_t queue_index, odp_event_t ev[], int num) queue_entry_t *qe = get_qentry(queue_index); odp_buffer_hdr_t *buf_hdr[num]; - ret = queue_deq_multi(qe, buf_hdr, num); + ret = deq_multi(qe, buf_hdr, num); if (ret > 0) for (i = 0; i < ret; i++) diff --git a/platform/linux-generic/odp_schedule.c b/platform/linux-generic/odp_schedule.c index 78982d9..81e79c9 100644 --- a/platform/linux-generic/odp_schedule.c +++ b/platform/linux-generic/odp_schedule.c @@ -1063,7 +1063,6 @@ const schedule_fn_t schedule_default_fn = { .init_queue = schedule_init_queue, .destroy_queue = schedule_destroy_queue, .sched_queue = schedule_sched_queue, - .ord_enq = schedule_ordered_queue_enq, .ord_enq_multi = schedule_ordered_queue_enq_multi, .init_global = schedule_init_global, .term_global = schedule_term_global, diff --git a/platform/linux-generic/odp_schedule_ordered.c b/platform/linux-generic/odp_schedule_ordered.c index 8c1dd7e..92a1cc8 100644 --- a/platform/linux-generic/odp_schedule_ordered.c +++ b/platform/linux-generic/odp_schedule_ordered.c @@ -452,26 +452,6 @@ static int ordered_queue_enq(queue_entry_t *queue, odp_buffer_hdr_t *buf_hdr, return 0; } -int schedule_ordered_queue_enq(uint32_t queue_index, void *p_buf_hdr, - int sustain, int *ret) -{ - queue_entry_t *origin_qe; - uint64_t order; - queue_entry_t *qe = get_qentry(queue_index); - odp_buffer_hdr_t *buf_hdr = p_buf_hdr; - - get_queue_order(&origin_qe, &order, buf_hdr); - - /* Handle enqueues from ordered queues separately */ - if (origin_qe) { - *ret = ordered_queue_enq(qe, buf_hdr, sustain, - origin_qe, order); - return 1; - } - - return 0; -} - int schedule_ordered_queue_enq_multi(uint32_t queue_index, void *p_buf_hdr[], int num, int sustain, int *ret) { diff --git a/platform/linux-generic/odp_schedule_sp.c b/platform/linux-generic/odp_schedule_sp.c index 2e28aa4..879eb5c 100644 --- a/platform/linux-generic/odp_schedule_sp.c +++ b/platform/linux-generic/odp_schedule_sp.c @@ -298,17 +298,6 @@ static int sched_queue(uint32_t qi) return 0; } -static int ord_enq(uint32_t queue_index, void *buf_hdr, int sustain, int *ret) -{ - (void)queue_index; - (void)buf_hdr; - (void)sustain; - (void)ret; - - /* didn't consume the events */ - return 0; -} - static int ord_enq_multi(uint32_t queue_index, void *buf_hdr[], int num, int sustain, int *ret) { @@ -673,7 +662,6 @@ const schedule_fn_t schedule_sp_fn = { .init_queue = init_queue, .destroy_queue = destroy_queue, .sched_queue = sched_queue, - .ord_enq = ord_enq, .ord_enq_multi = ord_enq_multi, .init_global = init_global, .term_global = term_global, ----------------------------------------------------------------------- Summary of changes: platform/linux-generic/Makefile.am | 1 + .../odp/api/plat/ticketlock_inlines.h} | 30 ++- .../linux-generic/include/odp_buffer_internal.h | 7 + .../linux-generic/include/odp_config_internal.h | 8 + .../linux-generic/include/odp_packet_io_queue.h | 3 +- platform/linux-generic/include/odp_pool_internal.h | 24 +- .../linux-generic/include/odp_queue_internal.h | 15 +- platform/linux-generic/include/odp_schedule_if.h | 3 - .../linux-generic/include/odp_schedule_internal.h | 2 +- platform/linux-generic/odp_queue.c | 271 ++++++++++----------- platform/linux-generic/odp_schedule.c | 1 - platform/linux-generic/odp_schedule_ordered.c | 37 +-- platform/linux-generic/odp_schedule_sp.c | 12 - platform/linux-generic/odp_ticketlock.c | 68 +----- 14 files changed, 199 insertions(+), 283 deletions(-) copy platform/linux-generic/{odp_ticketlock.c => include/odp/api/plat/ticketlock_inlines.h} (83%) hooks/post-receive --

8 years, 4 months

[COMMIT] branch master updated. v1.10.1.0-116-g566492d

by git＠git-us.linaro.org

8 years, 4 months

[COMMIT] branch api-next updated. v1.10.1.0-194-gcf6425f

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via cf6425f16f8ee554815f6697a231e398ddb0546c (commit) from 6085237938d93c68710db69f70014eca2238492a (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit cf6425f16f8ee554815f6697a231e398ddb0546c Author: Christophe Milard <christophe.milard(a)linaro.org> Date: Tue Sep 13 14:40:46 2016 +0200 linux-generic: _fdserver: allocating data table dynamicaly The table containing the saved file-descriptors<->{context, key} couples is now dynamically malloc'd in the fd server process, hence avoiding the memory waste which happened in other process when the table was staticaly reserved in all processes. Signed-off-by: Christophe Milard <christophe.milard(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/_fdserver.c b/platform/linux-generic/_fdserver.c index 97661d0..41a630b 100644 --- a/platform/linux-generic/_fdserver.c +++ b/platform/linux-generic/_fdserver.c @@ -73,7 +73,7 @@ typedef struct fdentry_s { uint64_t key; int fd; } fdentry_t; -static fdentry_t fd_table[FDSERVER_MAX_ENTRIES]; +static fdentry_t *fd_table; static int fd_table_nb_entries; /* @@ -622,8 +622,20 @@ int _odp_fdserver_init_global(void) /* TODO: pin the server on appropriate service cpu mask */ /* when (if) we can agree on the usage of service mask */ + /* allocate the space for the file descriptor<->key table: */ + fd_table = malloc(FDSERVER_MAX_ENTRIES * sizeof(fdentry_t)); + if (!fd_table) { + ODP_ERR("maloc failed!\n"); + exit(1); + } + + /* wait for clients requests */ wait_requests(sock); /* Returns when server is stopped */ close(sock); + + /* release the file descriptor table: */ + free(fd_table); + exit(0); } ----------------------------------------------------------------------- Summary of changes: platform/linux-generic/_fdserver.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) hooks/post-receive --

8 years, 5 months

[COMMIT] branch api-next updated. v1.10.1.0-193-g6085237

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via 6085237938d93c68710db69f70014eca2238492a (commit) via 03626c586519818ef4de9a2f733ebfd46a0bf6be (commit) via ea0b2211f1c5d90e1699aa94b3192704fbf6ca91 (commit) from c070eda5f78f9209ef3c7a0fe6dfc0cff210894b (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit 6085237938d93c68710db69f70014eca2238492a Merge: 03626c5 ea0b221 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Wed Sep 14 17:29:27 2016 +0300 Merge branch 'master' into api-next commit 03626c586519818ef4de9a2f733ebfd46a0bf6be Author: Christophe Milard <christophe.milard(a)linaro.org> Date: Tue Sep 13 12:05:18 2016 +0200 linux-generic: _fdserver: fixing comment typo Signed-off-by: Christophe Milard <christophe.milard(a)linaro.org> Reviewed-by: Bill Fischofer <bill.fischofer(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/platform/linux-generic/_fdserver.c b/platform/linux-generic/_fdserver.c index bf36eb2..97661d0 100644 --- a/platform/linux-generic/_fdserver.c +++ b/platform/linux-generic/_fdserver.c @@ -103,7 +103,7 @@ typedef struct fd_server_msg { * Send a fdserver_msg, possibly including a file descriptor, on the socket * This function is used both by: * -the client (sending a FD_REGISTER_REQ with a file descriptor to be shared, - * or FD_LOOKUP_REQ/FD_DEREGISTER_REQ without a file descirptor) + * or FD_LOOKUP_REQ/FD_DEREGISTER_REQ without a file descriptor) * -the server (sending FD_REGISTER_ACK/NACK, FD_LOOKUP_NACK, * FD_DEREGISTER_ACK/NACK... without a fd or a * FD_LOOKUP_ACK with a fd) @@ -165,7 +165,7 @@ static int send_fdserver_msg(int sock, int command, * given socket. * This function is used both by: * -the server (receiving a FD_REGISTER_REQ with a file descriptor to be shared, - * or FD_LOOKUP_REQ, FD_DEREGISTER_REQ without a file descirptor) + * or FD_LOOKUP_REQ, FD_DEREGISTER_REQ without a file descriptor) * -the client (receiving FD_REGISTER_ACK...without a fd or a FD_LOOKUP_ACK with * a fd) * This function make use of the ancillary data (control data) to pass and ----------------------------------------------------------------------- Summary of changes: example/generator/odp_generator.c | 1 + platform/linux-generic/_fdserver.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) hooks/post-receive --

8 years, 5 months

[COMMIT] branch master updated. v1.10.1.0-115-gea0b221

by git＠git-us.linaro.org

8 years, 5 months

[COMMIT] branch api-next updated. v1.10.1.0-190-gc070eda

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, api-next has been updated via c070eda5f78f9209ef3c7a0fe6dfc0cff210894b (commit) via 1d43e53c96246c5625ae619d0667b179ea8bd4b2 (commit) via 032915e5988d0e1da0562193b8bc925600cdb5f6 (commit) via da7a72e86db0cf83f7b391d8516f9b7504f4e27a (commit) via a7f5589f81864c71faf18c3d2c3636320cecb1c6 (commit) via aab7083f4ab4fa2ddd8d1a41a554cead3a0055bb (commit) via 66d26268065eaae370b402dc5b006121c6931c5c (commit) via d6260eeec46915b9d90453cc5add6ac0cac74e52 (commit) via ef8b7b691d4902153a71087ed0f4297cdd8809ff (commit) via de19d9847b76c90473c474a831800d383865a4ba (commit) via 2d63311886330e6acddb55674ff7a7b62d711ef5 (commit) via d96981e81ecbfc54ac8832d3ed2c32f50eaaf9a1 (commit) via 7cd13f48c935a88d56fd7d393fea6f02e5b0365a (commit) via ba2298af4f3912598043938c19bfedc5a8c89567 (commit) via 84c399d72a287115189bc8e8f272e4d7b9eaec47 (commit) from 07bee0be1f22f8a136982935ee06313e5bbc6f3a (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit c070eda5f78f9209ef3c7a0fe6dfc0cff210894b Merge: 07bee0b 1d43e53 Author: Maxim Uvarov <maxim.uvarov(a)linaro.org> Date: Mon Sep 12 18:21:46 2016 +0300 Merge branch 'master' into api-next Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> ----------------------------------------------------------------------- Summary of changes: configure.ac | 7 + example/Makefile.am | 12 +- example/Makefile.inc | 3 +- example/l3fwd/.gitignore | 1 + example/l3fwd/Makefile.am | 20 + example/l3fwd/odp_l3fwd.c | 1086 ++++++++++++++++++++ example/l3fwd/odp_l3fwd_db.c | 494 +++++++++ example/l3fwd/odp_l3fwd_db.h | 141 +++ example/l3fwd/odp_l3fwd_lpm.c | 224 ++++ .../cpu_arch.h => example/l3fwd/odp_l3fwd_lpm.h | 14 +- example/l3fwd/odp_l3fwd_run.sh | 28 + example/{l2fwd_simple => l3fwd}/udp64.pcap | Bin example/m4/configure.m4 | 9 +- helper/Makefile.am | 1 + platform/linux-generic/.gitignore | 1 + platform/linux-generic/Makefile.am | 9 + platform/linux-generic/include/odp/api/atomic.h | 384 +------ platform/linux-generic/include/odp/api/byteorder.h | 111 +- .../odp/api/{atomic.h => plat/atomic_inlines.h} | 153 ++- .../api/{byteorder.h => plat/byteorder_inlines.h} | 36 +- .../include/odp/api/plat/inlines.h.in | 33 + .../api/{std_clib.h => plat/std_clib_inlines.h} | 12 +- .../odp/{drv/sync.h => api/plat/sync_inlines.h} | 16 +- platform/linux-generic/include/odp/api/std_clib.h | 18 +- platform/linux-generic/include/odp/api/sync.h | 18 +- .../linux-generic/include/odp_schedule_internal.h | 5 +- platform/linux-generic/m4/configure.m4 | 3 +- platform/linux-generic/odp_atomic.c | 3 + .../linux-generic/odp_byteorder.c | 10 +- platform/linux-generic/odp_schedule.c | 599 ++++++----- .../linux-generic/odp_std_clib.c | 10 +- .../linux-generic/odp_sync.c | 5 +- platform/linux-generic/pktio/tap.c | 17 +- test/Makefile.inc | 9 +- test/common_plat/performance/odp_l2fwd.c | 6 +- test/linux-generic/Makefile.inc | 10 +- 36 files changed, 2556 insertions(+), 952 deletions(-) create mode 100644 example/l3fwd/.gitignore create mode 100644 example/l3fwd/Makefile.am create mode 100644 example/l3fwd/odp_l3fwd.c create mode 100644 example/l3fwd/odp_l3fwd_db.c create mode 100644 example/l3fwd/odp_l3fwd_db.h create mode 100644 example/l3fwd/odp_l3fwd_lpm.c copy platform/linux-generic/arch/arm/odp/api/cpu_arch.h => example/l3fwd/odp_l3fwd_lpm.h (52%) create mode 100755 example/l3fwd/odp_l3fwd_run.sh copy example/{l2fwd_simple => l3fwd}/udp64.pcap (100%) create mode 100644 platform/linux-generic/.gitignore copy platform/linux-generic/include/odp/api/{atomic.h => plat/atomic_inlines.h} (58%) copy platform/linux-generic/include/odp/api/{byteorder.h => plat/byteorder_inlines.h} (69%) create mode 100644 platform/linux-generic/include/odp/api/plat/inlines.h.in copy platform/linux-generic/include/odp/api/{std_clib.h => plat/std_clib_inlines.h} (52%) copy platform/linux-generic/include/odp/{drv/sync.h => api/plat/sync_inlines.h} (58%) copy test/common_plat/validation/drv/drvshmem/drvshmem_main.c => platform/linux-generic/odp_byteorder.c (52%) copy test/common_plat/validation/drv/drvshmem/drvshmem_main.c => platform/linux-generic/odp_std_clib.c (53%) copy test/linux-generic/validation/api/shmem/shmem_odp.h => platform/linux-generic/odp_sync.c (55%) hooks/post-receive --

8 years, 5 months

[COMMIT] branch master updated. v1.10.1.0-114-g1d43e53

by git＠git-us.linaro.org

8 years, 5 months

[COMMIT] branch master updated. v1.10.1.0-113-g032915e

by git＠git-us.linaro.org

8 years, 5 months

[COMMIT] branch master updated. v1.10.1.0-112-gda7a72e

by git＠git-us.linaro.org

This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "". The branch, master has been updated via da7a72e86db0cf83f7b391d8516f9b7504f4e27a (commit) from a7f5589f81864c71faf18c3d2c3636320cecb1c6 (commit) Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below. - Log ----------------------------------------------------------------- commit da7a72e86db0cf83f7b391d8516f9b7504f4e27a Author: Bill Fischofer <bill.fischofer(a)linaro.org> Date: Tue Aug 30 20:37:25 2016 -0500 linux-generic: configure: add conditional shared library support Add support for the --enable-shared=[yes|no] configure options. --enable-shared=yes is the default and results in suppressing the use of inlines for ODP API functions to enhance ABI compatibility at potentially some performance cost. When --enable-shared=no is specified, inlines are allowed to boost performance at the possible loss of ABI compatibility. This patch addresses Bug https://bugs.linaro.org/show_bug.cgi?id=2490 Note: This patch is a port of the corresponding enhancements made to odp-dpdk with a few changes, mainly that inlines are now part of the odp/api/plat directory, and all ODP APIs are now treated uniformly with regard to whether inlining is permitted. Suggested-by: Zoltan Kiss <zoltan.kiss(a)linaro.org> Signed-off-by: Bill Fischofer <bill.fischofer(a)linaro.org> Reviewed-by: Balakrishna Garapati <balakrishna.garapati(a)linaro.org> Signed-off-by: Maxim Uvarov <maxim.uvarov(a)linaro.org> diff --git a/configure.ac b/configure.ac index 6551287..982aff7 100644 --- a/configure.ac +++ b/configure.ac @@ -176,6 +176,13 @@ AM_CONDITIONAL([test_example], [test x$test_example = xyes ]) AM_CONDITIONAL([HAVE_DOXYGEN], [test "x${DOXYGEN}" = "xdoxygen"]) AM_CONDITIONAL([user_guide], [test "x${user_guides}" = "xyes" ]) AM_CONDITIONAL([HAVE_MSCGEN], [test "x${MSCGEN}" = "xmscgen"]) +if test x$enable_shared != xyes; +then + _ODP_INLINES="_ODP_INLINES" +else + _ODP_INLINES="_ODP_NO_INLINES" +fi +AC_SUBST(_ODP_INLINES) ########################################################################## # Setup doxygen documentation diff --git a/example/Makefile.inc b/example/Makefile.inc index 70ba2c0..9fb2f29 100644 --- a/example/Makefile.inc +++ b/example/Makefile.inc @@ -6,6 +6,7 @@ AM_CFLAGS += \ -I$(top_srcdir)/example \ -I$(top_srcdir)/platform/@with_platform@/include \ -I$(top_srcdir)/include/ \ - -I$(top_srcdir)/helper/include + -I$(top_srcdir)/helper/include \ + -I$(top_builddir)/platform/@with_platform@/include AM_LDFLAGS += -L$(LIB) diff --git a/helper/Makefile.am b/helper/Makefile.am index a82a11a..b6e6a1d 100644 --- a/helper/Makefile.am +++ b/helper/Makefile.am @@ -7,6 +7,7 @@ LIB = $(top_builddir)/lib AM_CFLAGS = -I$(srcdir)/include AM_CFLAGS += -I$(top_srcdir)/platform/@with_platform@/include AM_CFLAGS += -I$(top_srcdir)/include +AM_CFLAGS += -I$(top_builddir)/platform/@with_platform@/include AM_LDFLAGS += -version-number '$(ODPHELPER_LIBSO_VERSION)' diff --git a/platform/linux-generic/.gitignore b/platform/linux-generic/.gitignore new file mode 100644 index 0000000..ec6ca37 --- /dev/null +++ b/platform/linux-generic/.gitignore @@ -0,0 +1 @@ +include/odp/api/plat/inlines.h diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 0cfd0fe..ae3cbba 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -6,6 +6,7 @@ include $(top_srcdir)/platform/@with_platform@/Makefile.inc AM_CFLAGS += -I$(srcdir)/include AM_CFLAGS += -I$(top_srcdir)/include +AM_CFLAGS += -Iinclude include_HEADERS = \ $(top_srcdir)/include/odp.h \ @@ -60,15 +61,18 @@ odpapiinclude_HEADERS = \ odpapiplatincludedir= $(includedir)/odp/api/plat odpapiplatinclude_HEADERS = \ + $(srcdir)/include/odp/api/plat/atomic_inlines.h \ $(srcdir)/include/odp/api/plat/atomic_types.h \ $(srcdir)/include/odp/api/plat/barrier_types.h \ $(srcdir)/include/odp/api/plat/buffer_types.h \ + $(srcdir)/include/odp/api/plat/byteorder_inlines.h \ $(srcdir)/include/odp/api/plat/byteorder_types.h \ $(srcdir)/include/odp/api/plat/classification_types.h \ $(srcdir)/include/odp/api/plat/cpumask_types.h \ $(srcdir)/include/odp/api/plat/crypto_types.h \ $(srcdir)/include/odp/api/plat/event_types.h \ $(srcdir)/include/odp/api/plat/init_types.h \ + $(srcdir)/include/odp/api/plat/inlines.h \ $(srcdir)/include/odp/api/plat/packet_types.h \ $(srcdir)/include/odp/api/plat/packet_io_types.h \ $(srcdir)/include/odp/api/plat/pool_types.h \ @@ -79,7 +83,9 @@ odpapiplatinclude_HEADERS = \ $(srcdir)/include/odp/api/plat/shared_memory_types.h \ $(srcdir)/include/odp/api/plat/spinlock_types.h \ $(srcdir)/include/odp/api/plat/spinlock_recursive_types.h \ + $(srcdir)/include/odp/api/plat/std_clib_inlines.h \ $(srcdir)/include/odp/api/plat/strong_types.h \ + $(srcdir)/include/odp/api/plat/sync_inlines.h \ $(srcdir)/include/odp/api/plat/thread_types.h \ $(srcdir)/include/odp/api/plat/thrmask_types.h \ $(srcdir)/include/odp/api/plat/ticketlock_types.h \ @@ -134,6 +140,7 @@ __LIB__libodp_linux_la_SOURCES = \ odp_atomic.c \ odp_barrier.c \ odp_buffer.c \ + odp_byteorder.c \ odp_classification.c \ odp_cpu.c \ odp_cpumask.c \ @@ -173,6 +180,8 @@ __LIB__libodp_linux_la_SOURCES = \ odp_sorted_list.c \ odp_spinlock.c \ odp_spinlock_recursive.c \ + odp_std_clib.c \ + odp_sync.c \ odp_system_info.c \ odp_thread.c \ odp_thrmask.c \ diff --git a/platform/linux-generic/include/odp/api/atomic.h b/platform/linux-generic/include/odp/api/atomic.h index b487383..c18e68b 100644 --- a/platform/linux-generic/include/odp/api/atomic.h +++ b/platform/linux-generic/include/odp/api/atomic.h @@ -24,388 +24,10 @@ extern "C" { * @{ */ -static inline void odp_atomic_init_u32(odp_atomic_u32_t *atom, uint32_t val) -{ - __atomic_store_n(&atom->v, val, __ATOMIC_RELAXED); -} - -static inline uint32_t odp_atomic_load_u32(odp_atomic_u32_t *atom) -{ - return __atomic_load_n(&atom->v, __ATOMIC_RELAXED); -} - -static inline void odp_atomic_store_u32(odp_atomic_u32_t *atom, - uint32_t val) -{ - __atomic_store_n(&atom->v, val, __ATOMIC_RELAXED); -} - -static inline uint32_t odp_atomic_fetch_add_u32(odp_atomic_u32_t *atom, - uint32_t val) -{ - return __atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); -} - -static inline void odp_atomic_add_u32(odp_atomic_u32_t *atom, - uint32_t val) -{ - (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); -} - -static inline uint32_t odp_atomic_fetch_sub_u32(odp_atomic_u32_t *atom, - uint32_t val) -{ - return __atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); -} - -static inline void odp_atomic_sub_u32(odp_atomic_u32_t *atom, - uint32_t val) -{ - (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); -} - -static inline uint32_t odp_atomic_fetch_inc_u32(odp_atomic_u32_t *atom) -{ - return __atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); -} - -static inline void odp_atomic_inc_u32(odp_atomic_u32_t *atom) -{ - (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); -} - -static inline uint32_t odp_atomic_fetch_dec_u32(odp_atomic_u32_t *atom) -{ - return __atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); -} - -static inline void odp_atomic_dec_u32(odp_atomic_u32_t *atom) -{ - (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); -} - -static inline int odp_atomic_cas_u32(odp_atomic_u32_t *atom, uint32_t *old_val, - uint32_t new_val) -{ - return __atomic_compare_exchange_n(&atom->v, old_val, new_val, - 0 /* strong */, - __ATOMIC_RELAXED, - __ATOMIC_RELAXED); -} - -static inline uint32_t odp_atomic_xchg_u32(odp_atomic_u32_t *atom, - uint32_t new_val) -{ - return __atomic_exchange_n(&atom->v, new_val, __ATOMIC_RELAXED); -} - -static inline void odp_atomic_max_u32(odp_atomic_u32_t *atom, uint32_t new_max) -{ - uint32_t old_val; - - old_val = odp_atomic_load_u32(atom); - - while (new_max > old_val) { - if (odp_atomic_cas_u32(atom, &old_val, new_max)) - break; - } -} - -static inline void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_min) -{ - uint32_t old_val; - - old_val = odp_atomic_load_u32(atom); - - while (new_min < old_val) { - if (odp_atomic_cas_u32(atom, &old_val, new_min)) - break; - } -} - -static inline void odp_atomic_init_u64(odp_atomic_u64_t *atom, uint64_t val) -{ - atom->v = val; -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - __atomic_clear(&atom->lock, __ATOMIC_RELAXED); -#endif -} - -static inline uint64_t odp_atomic_load_u64(odp_atomic_u64_t *atom) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - return ATOMIC_OP(atom, (void)0); -#else - return __atomic_load_n(&atom->v, __ATOMIC_RELAXED); -#endif -} - -static inline void odp_atomic_store_u64(odp_atomic_u64_t *atom, - uint64_t val) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - (void)ATOMIC_OP(atom, atom->v = val); -#else - __atomic_store_n(&atom->v, val, __ATOMIC_RELAXED); -#endif -} - -static inline uint64_t odp_atomic_fetch_add_u64(odp_atomic_u64_t *atom, - uint64_t val) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - return ATOMIC_OP(atom, atom->v += val); -#else - return __atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); -#endif -} - -static inline void odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - (void)ATOMIC_OP(atom, atom->v += val); -#else - (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); -#endif -} - -static inline uint64_t odp_atomic_fetch_sub_u64(odp_atomic_u64_t *atom, - uint64_t val) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - return ATOMIC_OP(atom, atom->v -= val); -#else - return __atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); -#endif -} - -static inline void odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - (void)ATOMIC_OP(atom, atom->v -= val); -#else - (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); -#endif -} - -static inline uint64_t odp_atomic_fetch_inc_u64(odp_atomic_u64_t *atom) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - return ATOMIC_OP(atom, atom->v++); -#else - return __atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); -#endif -} - -static inline void odp_atomic_inc_u64(odp_atomic_u64_t *atom) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - (void)ATOMIC_OP(atom, atom->v++); -#else - (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); -#endif -} - -static inline uint64_t odp_atomic_fetch_dec_u64(odp_atomic_u64_t *atom) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - return ATOMIC_OP(atom, atom->v--); -#else - return __atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); +#include <odp/api/plat/inlines.h> +#ifdef _ODP_INLINES +#include <odp/api/plat/atomic_inlines.h> #endif -} - -static inline void odp_atomic_dec_u64(odp_atomic_u64_t *atom) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - (void)ATOMIC_OP(atom, atom->v--); -#else - (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); -#endif -} - -static inline int odp_atomic_cas_u64(odp_atomic_u64_t *atom, uint64_t *old_val, - uint64_t new_val) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - int ret; - *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); - return ret; -#else - return __atomic_compare_exchange_n(&atom->v, old_val, new_val, - 0 /* strong */, - __ATOMIC_RELAXED, - __ATOMIC_RELAXED); -#endif -} - -static inline uint64_t odp_atomic_xchg_u64(odp_atomic_u64_t *atom, - uint64_t new_val) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - return ATOMIC_OP(atom, atom->v = new_val); -#else - return __atomic_exchange_n(&atom->v, new_val, __ATOMIC_RELAXED); -#endif -} - -static inline void odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t new_max) -{ - uint64_t old_val; - - old_val = odp_atomic_load_u64(atom); - - while (new_max > old_val) { - if (odp_atomic_cas_u64(atom, &old_val, new_max)) - break; - } -} - -static inline void odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t new_min) -{ - uint64_t old_val; - - old_val = odp_atomic_load_u64(atom); - - while (new_min < old_val) { - if (odp_atomic_cas_u64(atom, &old_val, new_min)) - break; - } -} - -static inline uint32_t odp_atomic_load_acq_u32(odp_atomic_u32_t *atom) -{ - return __atomic_load_n(&atom->v, __ATOMIC_ACQUIRE); -} - -static inline void odp_atomic_store_rel_u32(odp_atomic_u32_t *atom, - uint32_t val) -{ - __atomic_store_n(&atom->v, val, __ATOMIC_RELEASE); -} - -static inline void odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, - uint32_t val) -{ - (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); -} - -static inline void odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, - uint32_t val) -{ - (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); -} - -static inline int odp_atomic_cas_acq_u32(odp_atomic_u32_t *atom, - uint32_t *old_val, uint32_t new_val) -{ - return __atomic_compare_exchange_n(&atom->v, old_val, new_val, - 0 /* strong */, - __ATOMIC_ACQUIRE, - __ATOMIC_RELAXED); -} - -static inline int odp_atomic_cas_rel_u32(odp_atomic_u32_t *atom, - uint32_t *old_val, uint32_t new_val) -{ - return __atomic_compare_exchange_n(&atom->v, old_val, new_val, - 0 /* strong */, - __ATOMIC_RELEASE, - __ATOMIC_RELAXED); -} - -static inline int odp_atomic_cas_acq_rel_u32(odp_atomic_u32_t *atom, - uint32_t *old_val, - uint32_t new_val) -{ - return __atomic_compare_exchange_n(&atom->v, old_val, new_val, - 0 /* strong */, - __ATOMIC_ACQ_REL, - __ATOMIC_RELAXED); -} - -static inline uint64_t odp_atomic_load_acq_u64(odp_atomic_u64_t *atom) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - return ATOMIC_OP(atom, (void)0); -#else - return __atomic_load_n(&atom->v, __ATOMIC_ACQUIRE); -#endif -} - -static inline void odp_atomic_store_rel_u64(odp_atomic_u64_t *atom, - uint64_t val) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - (void)ATOMIC_OP(atom, atom->v = val); -#else - __atomic_store_n(&atom->v, val, __ATOMIC_RELEASE); -#endif -} - -static inline void odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - (void)ATOMIC_OP(atom, atom->v += val); -#else - (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); -#endif -} - -static inline void odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - (void)ATOMIC_OP(atom, atom->v -= val); -#else - (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); -#endif -} - -static inline int odp_atomic_cas_acq_u64(odp_atomic_u64_t *atom, - uint64_t *old_val, uint64_t new_val) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - int ret; - *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); - return ret; -#else - return __atomic_compare_exchange_n(&atom->v, old_val, new_val, - 0 /* strong */, - __ATOMIC_ACQUIRE, - __ATOMIC_RELAXED); -#endif -} - -static inline int odp_atomic_cas_rel_u64(odp_atomic_u64_t *atom, - uint64_t *old_val, uint64_t new_val) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - int ret; - *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); - return ret; -#else - return __atomic_compare_exchange_n(&atom->v, old_val, new_val, - 0 /* strong */, - __ATOMIC_RELEASE, - __ATOMIC_RELAXED); -#endif -} - -static inline int odp_atomic_cas_acq_rel_u64(odp_atomic_u64_t *atom, - uint64_t *old_val, - uint64_t new_val) -{ -#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 - int ret; - *old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val)); - return ret; -#else - return __atomic_compare_exchange_n(&atom->v, old_val, new_val, - 0 /* strong */, - __ATOMIC_ACQ_REL, - __ATOMIC_RELAXED); -#endif -} /** * @} diff --git a/platform/linux-generic/include/odp/api/byteorder.h b/platform/linux-generic/include/odp/api/byteorder.h index c347be0..84d1173 100644 --- a/platform/linux-generic/include/odp/api/byteorder.h +++ b/platform/linux-generic/include/odp/api/byteorder.h @@ -17,124 +17,18 @@ extern "C" { #endif -#include <odp/api/plat/byteorder_types.h> #include <odp/api/std_types.h> +#include <odp/api/plat/byteorder_types.h> #include <odp/api/compiler.h> /** @ingroup odp_compiler_optim * @{ */ -static inline uint16_t odp_be_to_cpu_16(odp_u16be_t be16) -{ -#if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return __odp_builtin_bswap16((__odp_force uint16_t)be16); -#else - return (__odp_force uint16_t)be16; -#endif -} - -static inline uint32_t odp_be_to_cpu_32(odp_u32be_t be32) -{ -#if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return __builtin_bswap32((__odp_force uint32_t)be32); -#else - return (__odp_force uint32_t)be32; -#endif -} - -static inline uint64_t odp_be_to_cpu_64(odp_u64be_t be64) -{ -#if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return __builtin_bswap64((__odp_force uint64_t)be64); -#else - return (__odp_force uint64_t)be64; -#endif -} - - -static inline odp_u16be_t odp_cpu_to_be_16(uint16_t cpu16) -{ -#if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force odp_u16be_t)__odp_builtin_bswap16(cpu16); -#else - return (__odp_force odp_u16be_t)cpu16; -#endif -} - -static inline odp_u32be_t odp_cpu_to_be_32(uint32_t cpu32) -{ -#if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force odp_u32be_t)__builtin_bswap32(cpu32); -#else - return (__odp_force odp_u32be_t)cpu32; +#include <odp/api/plat/inlines.h> +#ifdef _ODP_INLINES +#include <odp/api/plat/byteorder_inlines.h> #endif -} - -static inline odp_u64be_t odp_cpu_to_be_64(uint64_t cpu64) -{ -#if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force odp_u64be_t)__builtin_bswap64(cpu64); -#else - return (__odp_force odp_u64be_t)cpu64; -#endif -} - - -static inline uint16_t odp_le_to_cpu_16(odp_u16le_t le16) -{ -#if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint16_t)le16; -#else - return __odp_builtin_bswap16((__odp_force uint16_t)le16); -#endif -} - -static inline uint32_t odp_le_to_cpu_32(odp_u32le_t le32) -{ -#if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint32_t)le32; -#else - return __builtin_bswap32((__odp_force uint32_t)le32); -#endif -} - -static inline uint64_t odp_le_to_cpu_64(odp_u64le_t le64) -{ -#if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force uint64_t)le64; -#else - return __builtin_bswap64((__odp_force uint64_t)le64); -#endif -} - - -static inline odp_u16le_t odp_cpu_to_le_16(uint16_t cpu16) -{ -#if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force odp_u16le_t)cpu16; -#else - return (__odp_force odp_u16le_t)__odp_builtin_bswap16(cpu16); -#endif -} - -static inline odp_u32le_t odp_cpu_to_le_32(uint32_t cpu32) -{ -#if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force odp_u32le_t)cpu32; -#else - return (__odp_force odp_u32le_t)__builtin_bswap32(cpu32); -#endif -} - -static inline odp_u64le_t odp_cpu_to_le_64(uint64_t cpu64) -{ -#if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN - return (__odp_force odp_u64le_t)cpu64; -#else - return (__odp_force odp_u64le_t)__builtin_bswap64(cpu64); -#endif -} /** * @} diff --git a/platform/linux-generic/include/odp/api/atomic.h b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h similarity index 58% copy from platform/linux-generic/include/odp/api/atomic.h copy to platform/linux-generic/include/odp/api/plat/atomic_inlines.h index b487383..4471f2e 100644 --- a/platform/linux-generic/include/odp/api/atomic.h +++ b/platform/linux-generic/include/odp/api/plat/atomic_inlines.h @@ -1,91 +1,75 @@ -/* Copyright (c) 2013, Linaro Limited +/* Copyright (c) 2016, Linaro Limited * All rights reserved. * - * SPDX-License-Identifier: BSD-3-Clause + * SPDX-License-Identifier: BSD-3-Clause */ /** * @file * - * ODP atomic operations + * ODP Atomic inline functions */ -#ifndef ODP_PLAT_ATOMIC_H_ -#define ODP_PLAT_ATOMIC_H_ +#ifndef _ODP_PLAT_ATOMIC_INLINES_H_ +#define _ODP_PLAT_ATOMIC_INLINES_H_ -#ifdef __cplusplus -extern "C" { -#endif - -#include <odp/api/align.h> -#include <odp/api/plat/atomic_types.h> - -/** @ingroup odp_atomic - * @{ - */ - -static inline void odp_atomic_init_u32(odp_atomic_u32_t *atom, uint32_t val) +_STATIC void odp_atomic_init_u32(odp_atomic_u32_t *atom, uint32_t val) { __atomic_store_n(&atom->v, val, __ATOMIC_RELAXED); } -static inline uint32_t odp_atomic_load_u32(odp_atomic_u32_t *atom) +_STATIC uint32_t odp_atomic_load_u32(odp_atomic_u32_t *atom) { return __atomic_load_n(&atom->v, __ATOMIC_RELAXED); } -static inline void odp_atomic_store_u32(odp_atomic_u32_t *atom, - uint32_t val) +_STATIC void odp_atomic_store_u32(odp_atomic_u32_t *atom, uint32_t val) { __atomic_store_n(&atom->v, val, __ATOMIC_RELAXED); } -static inline uint32_t odp_atomic_fetch_add_u32(odp_atomic_u32_t *atom, - uint32_t val) +_STATIC uint32_t odp_atomic_fetch_add_u32(odp_atomic_u32_t *atom, uint32_t val) { return __atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); } -static inline void odp_atomic_add_u32(odp_atomic_u32_t *atom, - uint32_t val) +_STATIC void odp_atomic_add_u32(odp_atomic_u32_t *atom, uint32_t val) { (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED); } -static inline uint32_t odp_atomic_fetch_sub_u32(odp_atomic_u32_t *atom, - uint32_t val) +_STATIC uint32_t odp_atomic_fetch_sub_u32(odp_atomic_u32_t *atom, uint32_t val) { return __atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); } -static inline void odp_atomic_sub_u32(odp_atomic_u32_t *atom, - uint32_t val) +_STATIC void odp_atomic_sub_u32(odp_atomic_u32_t *atom, uint32_t val) { (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED); } -static inline uint32_t odp_atomic_fetch_inc_u32(odp_atomic_u32_t *atom) +_STATIC uint32_t odp_atomic_fetch_inc_u32(odp_atomic_u32_t *atom) { return __atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); } -static inline void odp_atomic_inc_u32(odp_atomic_u32_t *atom) +_STATIC void odp_atomic_inc_u32(odp_atomic_u32_t *atom) { (void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED); } -static inline uint32_t odp_atomic_fetch_dec_u32(odp_atomic_u32_t *atom) +_STATIC uint32_t odp_atomic_fetch_dec_u32(odp_atomic_u32_t *atom) { return __atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); } -static inline void odp_atomic_dec_u32(odp_atomic_u32_t *atom) +_STATIC void odp_atomic_dec_u32(odp_atomic_u32_t *atom) { (void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED); } -static inline int odp_atomic_cas_u32(odp_atomic_u32_t *atom, uint32_t *old_val, - uint32_t new_val) +_STATIC int odp_atomic_cas_u32(odp_atomic_u32_t *atom, uint32_t *old_val, + uint32_t new_val) { return __atomic_compare_exchange_n(&atom->v, old_val, new_val, 0 /* strong */, @@ -93,13 +77,12 @@ static inline int odp_atomic_cas_u32(odp_atomic_u32_t *atom, uint32_t *old_val, __ATOMIC_RELAXED); } -static inline uint32_t odp_atomic_xchg_u32(odp_atomic_u32_t *atom, - uint32_t new_val) +_STATIC uint32_t odp_atomic_xchg_u32(odp_atomic_u32_t *atom, uint32_t new_val) { return __atomic_exchange_n(&atom->v, new_val, __ATOMIC_RELAXED); } -static inline void odp_atomic_max_u32(odp_atomic_u32_t *atom, uint32_t new_max) +_STATIC void odp_atomic_max_u32(odp_atomic_u32_t *atom, uint32_t new_max) { uint32_t old_val; @@ -111,7 +94,7 @@ static inline void odp_atomic_max_u32(odp_atomic_u32_t *atom, uint32_t new_max) } } -static inline void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_min) +_STATIC void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_min) { uint32_t old_val; @@ -123,7 +106,7 @@ static inline void odp_atomic_min_u32(odp_atomic_u32_t *atom, uint32_t new_min) } } -static inline void odp_atomic_init_u64(odp_atomic_u64_t *atom, uint64_t val) +_STATIC void odp_atomic_init_u64(odp_atomic_u64_t *atom, uint64_t val) { atom->v = val; #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 @@ -131,7 +114,7 @@ static inline void odp_atomic_init_u64(odp_atomic_u64_t *atom, uint64_t val) #endif } -static inline uint64_t odp_atomic_load_u64(odp_atomic_u64_t *atom) +_STATIC uint64_t odp_atomic_load_u64(odp_atomic_u64_t *atom) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP(atom, (void)0); @@ -140,8 +123,7 @@ static inline uint64_t odp_atomic_load_u64(odp_atomic_u64_t *atom) #endif } -static inline void odp_atomic_store_u64(odp_atomic_u64_t *atom, - uint64_t val) +_STATIC void odp_atomic_store_u64(odp_atomic_u64_t *atom, uint64_t val) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP(atom, atom->v = val); @@ -150,8 +132,7 @@ static inline void odp_atomic_store_u64(odp_atomic_u64_t *atom, #endif } -static inline uint64_t odp_atomic_fetch_add_u64(odp_atomic_u64_t *atom, - uint64_t val) +_STATIC uint64_t odp_atomic_fetch_add_u64(odp_atomic_u64_t *atom, uint64_t val) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP(atom, atom->v += val); @@ -160,7 +141,7 @@ static inline uint64_t odp_atomic_fetch_add_u64(odp_atomic_u64_t *atom, #endif } -static inline void odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) +_STATIC void odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP(atom, atom->v += val); @@ -169,8 +150,7 @@ static inline void odp_atomic_add_u64(odp_atomic_u64_t *atom, uint64_t val) #endif } -static inline uint64_t odp_atomic_fetch_sub_u64(odp_atomic_u64_t *atom, - uint64_t val) +_STATIC uint64_t odp_atomic_fetch_sub_u64(odp_atomic_u64_t *atom, uint64_t val) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP(atom, atom->v -= val); @@ -179,7 +159,7 @@ static inline uint64_t odp_atomic_fetch_sub_u64(odp_atomic_u64_t *atom, #endif } -static inline void odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) +_STATIC void odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP(atom, atom->v -= val); @@ -188,7 +168,7 @@ static inline void odp_atomic_sub_u64(odp_atomic_u64_t *atom, uint64_t val) #endif } -static inline uint64_t odp_atomic_fetch_inc_u64(odp_atomic_u64_t *atom) +_STATIC uint64_t odp_atomic_fetch_inc_u64(odp_atomic_u64_t *atom) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP(atom, atom->v++); @@ -197,7 +177,7 @@ static inline uint64_t odp_atomic_fetch_inc_u64(odp_atomic_u64_t *atom) #endif } -static inline void odp_atomic_inc_u64(odp_atomic_u64_t *atom) +_STATIC void odp_atomic_inc_u64(odp_atomic_u64_t *atom) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP(atom, atom->v++); @@ -206,7 +186,7 @@ static inline void odp_atomic_inc_u64(odp_atomic_u64_t *atom) #endif } -static inline uint64_t odp_atomic_fetch_dec_u64(odp_atomic_u64_t *atom) +_STATIC uint64_t odp_atomic_fetch_dec_u64(odp_atomic_u64_t *atom) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP(atom, atom->v--); @@ -215,7 +195,7 @@ static inline uint64_t odp_atomic_fetch_dec_u64(odp_atomic_u64_t *atom) #endif } -static inline void odp_atomic_dec_u64(odp_atomic_u64_t *atom) +_STATIC void odp_atomic_dec_u64(odp_atomic_u64_t *atom) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP(atom, atom->v--); @@ -224,8 +204,8 @@ static inline void odp_atomic_dec_u64(odp_atomic_u64_t *atom) #endif } -static inline int odp_atomic_cas_u64(odp_atomic_u64_t *atom, uint64_t *old_val, - uint64_t new_val) +_STATIC int odp_atomic_cas_u64(odp_atomic_u64_t *atom, uint64_t *old_val, + uint64_t new_val) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 int ret; @@ -239,8 +219,7 @@ static inline int odp_atomic_cas_u64(odp_atomic_u64_t *atom, uint64_t *old_val, #endif } -static inline uint64_t odp_atomic_xchg_u64(odp_atomic_u64_t *atom, - uint64_t new_val) +_STATIC uint64_t odp_atomic_xchg_u64(odp_atomic_u64_t *atom, uint64_t new_val) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP(atom, atom->v = new_val); @@ -249,7 +228,7 @@ static inline uint64_t odp_atomic_xchg_u64(odp_atomic_u64_t *atom, #endif } -static inline void odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t new_max) +_STATIC void odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t new_max) { uint64_t old_val; @@ -261,7 +240,7 @@ static inline void odp_atomic_max_u64(odp_atomic_u64_t *atom, uint64_t new_max) } } -static inline void odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t new_min) +_STATIC void odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t new_min) { uint64_t old_val; @@ -273,31 +252,28 @@ static inline void odp_atomic_min_u64(odp_atomic_u64_t *atom, uint64_t new_min) } } -static inline uint32_t odp_atomic_load_acq_u32(odp_atomic_u32_t *atom) +_STATIC uint32_t odp_atomic_load_acq_u32(odp_atomic_u32_t *atom) { return __atomic_load_n(&atom->v, __ATOMIC_ACQUIRE); } -static inline void odp_atomic_store_rel_u32(odp_atomic_u32_t *atom, - uint32_t val) +_STATIC void odp_atomic_store_rel_u32(odp_atomic_u32_t *atom, uint32_t val) { __atomic_store_n(&atom->v, val, __ATOMIC_RELEASE); } -static inline void odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, - uint32_t val) +_STATIC void odp_atomic_add_rel_u32(odp_atomic_u32_t *atom, uint32_t val) { (void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE); } -static inline void odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, - uint32_t val) +_STATIC void odp_atomic_sub_rel_u32(odp_atomic_u32_t *atom, uint32_t val) { (void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE); } -static inline int odp_atomic_cas_acq_u32(odp_atomic_u32_t *atom, - uint32_t *old_val, uint32_t new_val) +_STATIC int odp_atomic_cas_acq_u32(odp_atomic_u32_t *atom, + uint32_t *old_val, uint32_t new_val) { return __atomic_compare_exchange_n(&atom->v, old_val, new_val, 0 /* strong */, @@ -305,8 +281,8 @@ static inline int odp_atomic_cas_acq_u32(odp_atomic_u32_t *atom, __ATOMIC_RELAXED); } -static inline int odp_atomic_cas_rel_u32(odp_atomic_u32_t *atom, - uint32_t *old_val, uint32_t new_val) +_STATIC int odp_atomic_cas_rel_u32(odp_atomic_u32_t *atom, + uint32_t *old_val, uint32_t new_val) { return __atomic_compare_exchange_n(&atom->v, old_val, new_val, 0 /* strong */, @@ -314,9 +290,9 @@ static inline int odp_atomic_cas_rel_u32(odp_atomic_u32_t *atom, __ATOMIC_RELAXED); } -static inline int odp_atomic_cas_acq_rel_u32(odp_atomic_u32_t *atom, - uint32_t *old_val, - uint32_t new_val) +_STATIC int odp_atomic_cas_acq_rel_u32(odp_atomic_u32_t *atom, + uint32_t *old_val, + uint32_t new_val) { return __atomic_compare_exchange_n(&atom->v, old_val, new_val, 0 /* strong */, @@ -324,7 +300,7 @@ static inline int odp_atomic_cas_acq_rel_u32(odp_atomic_u32_t *atom, __ATOMIC_RELAXED); } -static inline uint64_t odp_atomic_load_acq_u64(odp_atomic_u64_t *atom) +_STATIC uint64_t odp_atomic_load_acq_u64(odp_atomic_u64_t *atom) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 return ATOMIC_OP(atom, (void)0); @@ -333,8 +309,7 @@ static inline uint64_t odp_atomic_load_acq_u64(odp_atomic_u64_t *atom) #endif } -static inline void odp_atomic_store_rel_u64(odp_atomic_u64_t *atom, - uint64_t val) +_STATIC void odp_atomic_store_rel_u64(odp_atomic_u64_t *atom, uint64_t val) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP(atom, atom->v = val); @@ -343,7 +318,7 @@ static inline void odp_atomic_store_rel_u64(odp_atomic_u64_t *atom, #endif } -static inline void odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +_STATIC void odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP(atom, atom->v += val); @@ -352,7 +327,7 @@ static inline void odp_atomic_add_rel_u64(odp_atomic_u64_t *atom, uint64_t val) #endif } -static inline void odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) +_STATIC void odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 (void)ATOMIC_OP(atom, atom->v -= val); @@ -361,8 +336,8 @@ static inline void odp_atomic_sub_rel_u64(odp_atomic_u64_t *atom, uint64_t val) #endif } -static inline int odp_atomic_cas_acq_u64(odp_atomic_u64_t *atom, - uint64_t *old_val, uint64_t new_val) +_STATIC int odp_atomic_cas_acq_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, uint64_t new_val) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 int ret; @@ -376,8 +351,8 @@ static inline int odp_atomic_cas_acq_u64(odp_atomic_u64_t *atom, #endif } -static inline int odp_atomic_cas_rel_u64(odp_atomic_u64_t *atom, - uint64_t *old_val, uint64_t new_val) +_STATIC int odp_atomic_cas_rel_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, uint64_t new_val) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 int ret; @@ -391,9 +366,9 @@ static inline int odp_atomic_cas_rel_u64(odp_atomic_u64_t *atom, #endif } -static inline int odp_atomic_cas_acq_rel_u64(odp_atomic_u64_t *atom, - uint64_t *old_val, - uint64_t new_val) +_STATIC int odp_atomic_cas_acq_rel_u64(odp_atomic_u64_t *atom, + uint64_t *old_val, + uint64_t new_val) { #if __GCC_ATOMIC_LLONG_LOCK_FREE < 2 int ret; @@ -407,14 +382,4 @@ static inline int odp_atomic_cas_acq_rel_u64(odp_atomic_u64_t *atom, #endif } -/** - * @} - */ - -#include <odp/api/spec/atomic.h> - -#ifdef __cplusplus -} -#endif - #endif diff --git a/platform/linux-generic/include/odp/api/byteorder.h b/platform/linux-generic/include/odp/api/plat/byteorder_inlines.h similarity index 69% copy from platform/linux-generic/include/odp/api/byteorder.h copy to platform/linux-generic/include/odp/api/plat/byteorder_inlines.h index c347be0..c91a3b1 100644 --- a/platform/linux-generic/include/odp/api/byteorder.h +++ b/platform/linux-generic/include/odp/api/plat/byteorder_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2014, Linaro Limited +/* Copyright (c) 2016, Linaro Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -10,22 +10,18 @@ * ODP byteorder */ -#ifndef ODP_PLAT_BYTEORDER_H_ -#define ODP_PLAT_BYTEORDER_H_ +#ifndef ODP_PLAT_BYTEORDER_INLINES_H_ +#define ODP_PLAT_BYTEORDER_INLINES_H_ #ifdef __cplusplus extern "C" { #endif -#include <odp/api/plat/byteorder_types.h> -#include <odp/api/std_types.h> -#include <odp/api/compiler.h> - /** @ingroup odp_compiler_optim * @{ */ -static inline uint16_t odp_be_to_cpu_16(odp_u16be_t be16) +_STATIC uint16_t odp_be_to_cpu_16(odp_u16be_t be16) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return __odp_builtin_bswap16((__odp_force uint16_t)be16); @@ -34,7 +30,7 @@ static inline uint16_t odp_be_to_cpu_16(odp_u16be_t be16) #endif } -static inline uint32_t odp_be_to_cpu_32(odp_u32be_t be32) +_STATIC uint32_t odp_be_to_cpu_32(odp_u32be_t be32) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return __builtin_bswap32((__odp_force uint32_t)be32); @@ -43,7 +39,7 @@ static inline uint32_t odp_be_to_cpu_32(odp_u32be_t be32) #endif } -static inline uint64_t odp_be_to_cpu_64(odp_u64be_t be64) +_STATIC uint64_t odp_be_to_cpu_64(odp_u64be_t be64) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return __builtin_bswap64((__odp_force uint64_t)be64); @@ -52,8 +48,7 @@ static inline uint64_t odp_be_to_cpu_64(odp_u64be_t be64) #endif } - -static inline odp_u16be_t odp_cpu_to_be_16(uint16_t cpu16) +_STATIC odp_u16be_t odp_cpu_to_be_16(uint16_t cpu16) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force odp_u16be_t)__odp_builtin_bswap16(cpu16); @@ -62,7 +57,7 @@ static inline odp_u16be_t odp_cpu_to_be_16(uint16_t cpu16) #endif } -static inline odp_u32be_t odp_cpu_to_be_32(uint32_t cpu32) +_STATIC odp_u32be_t odp_cpu_to_be_32(uint32_t cpu32) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force odp_u32be_t)__builtin_bswap32(cpu32); @@ -71,7 +66,7 @@ static inline odp_u32be_t odp_cpu_to_be_32(uint32_t cpu32) #endif } -static inline odp_u64be_t odp_cpu_to_be_64(uint64_t cpu64) +_STATIC odp_u64be_t odp_cpu_to_be_64(uint64_t cpu64) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force odp_u64be_t)__builtin_bswap64(cpu64); @@ -80,8 +75,7 @@ static inline odp_u64be_t odp_cpu_to_be_64(uint64_t cpu64) #endif } - -static inline uint16_t odp_le_to_cpu_16(odp_u16le_t le16) +_STATIC uint16_t odp_le_to_cpu_16(odp_u16le_t le16) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force uint16_t)le16; @@ -90,7 +84,7 @@ static inline uint16_t odp_le_to_cpu_16(odp_u16le_t le16) #endif } -static inline uint32_t odp_le_to_cpu_32(odp_u32le_t le32) +_STATIC uint32_t odp_le_to_cpu_32(odp_u32le_t le32) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force uint32_t)le32; @@ -99,7 +93,7 @@ static inline uint32_t odp_le_to_cpu_32(odp_u32le_t le32) #endif } -static inline uint64_t odp_le_to_cpu_64(odp_u64le_t le64) +_STATIC uint64_t odp_le_to_cpu_64(odp_u64le_t le64) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force uint64_t)le64; @@ -108,8 +102,7 @@ static inline uint64_t odp_le_to_cpu_64(odp_u64le_t le64) #endif } - -static inline odp_u16le_t odp_cpu_to_le_16(uint16_t cpu16) +_STATIC odp_u16le_t odp_cpu_to_le_16(uint16_t cpu16) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force odp_u16le_t)cpu16; @@ -118,7 +111,7 @@ static inline odp_u16le_t odp_cpu_to_le_16(uint16_t cpu16) #endif } -static inline odp_u32le_t odp_cpu_to_le_32(uint32_t cpu32) +_STATIC odp_u32le_t odp_cpu_to_le_32(uint32_t cpu32) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force odp_u32le_t)cpu32; @@ -127,7 +120,7 @@ static inline odp_u32le_t odp_cpu_to_le_32(uint32_t cpu32) #endif } -static inline odp_u64le_t odp_cpu_to_le_64(uint64_t cpu64) +_STATIC odp_u64le_t odp_cpu_to_le_64(uint64_t cpu64) { #if ODP_BYTE_ORDER == ODP_LITTLE_ENDIAN return (__odp_force odp_u64le_t)cpu64; @@ -140,8 +133,6 @@ static inline odp_u64le_t odp_cpu_to_le_64(uint64_t cpu64) * @} */ -#include <odp/api/spec/byteorder.h> - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp/api/plat/inlines.h.in b/platform/linux-generic/include/odp/api/plat/inlines.h.in new file mode 100644 index 0000000..5d8c0dc --- /dev/null +++ b/platform/linux-generic/include/odp/api/plat/inlines.h.in @@ -0,0 +1,33 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +/** + * @file + * + * ODP platform inline functions + */ + +#ifndef ODP_PLAT_INLINES_H_ +#define ODP_PLAT_INLINES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define @_ODP_INLINES@ + +#ifdef _ODP_INLINES +#define _STATIC static inline +#else +#define _STATIC +#endif + +#ifdef __cplusplus +} +#endif + + +#endif /* ODP_PLAT_INLINES_H_ */ diff --git a/platform/linux-generic/include/odp/api/std_clib.h b/platform/linux-generic/include/odp/api/plat/std_clib_inlines.h similarity index 52% copy from platform/linux-generic/include/odp/api/std_clib.h copy to platform/linux-generic/include/odp/api/plat/std_clib_inlines.h index 40c0ea8..2e833e4 100644 --- a/platform/linux-generic/include/odp/api/std_clib.h +++ b/platform/linux-generic/include/odp/api/plat/std_clib_inlines.h @@ -1,11 +1,11 @@ -/* Copyright (c) 2015, Linaro Limited +/* Copyright (c) 2016, Linaro Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ -#ifndef ODP_PLAT_STD_CLIB_H_ -#define ODP_PLAT_STD_CLIB_H_ +#ifndef ODP_PLAT_STD_CLIB_INLINE_H_ +#define ODP_PLAT_STD_CLIB_INLINE_H_ #ifdef __cplusplus extern "C" { @@ -14,17 +14,17 @@ extern "C" { #include <odp/api/spec/std_types.h> #include <string.h> -static inline void *odp_memcpy(void *dst, const void *src, size_t num) +_STATIC void *odp_memcpy(void *dst, const void *src, size_t num) { return memcpy(dst, src, num); } -static inline void *odp_memset(void *ptr, int value, size_t num) +_STATIC void *odp_memset(void *ptr, int value, size_t num) { return memset(ptr, value, num); } -static inline int odp_memcmp(const void *ptr1, const void *ptr2, size_t num) +_STATIC int odp_memcmp(const void *ptr1, const void *ptr2, size_t num) { return memcmp(ptr1, ptr2, num); } diff --git a/platform/linux-generic/include/odp/api/sync.h b/platform/linux-generic/include/odp/api/plat/sync_inlines.h similarity index 58% copy from platform/linux-generic/include/odp/api/sync.h copy to platform/linux-generic/include/odp/api/plat/sync_inlines.h index 45fe6bd..245c750 100644 --- a/platform/linux-generic/include/odp/api/sync.h +++ b/platform/linux-generic/include/odp/api/plat/sync_inlines.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013, Linaro Limited +/* Copyright (c) 2016, Linaro Limited * All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause @@ -7,11 +7,11 @@ /** * @file * - * ODP synchronisation + * ODP synchronisation inlines */ -#ifndef ODP_PLAT_SYNC_H_ -#define ODP_PLAT_SYNC_H_ +#ifndef ODP_PLAT_SYNC_INLINE_H_ +#define ODP_PLAT_SYNC_INLINE_H_ #ifdef __cplusplus extern "C" { @@ -21,17 +21,17 @@ extern "C" { * @{ */ -static inline void odp_mb_release(void) +_STATIC void odp_mb_release(void) { __atomic_thread_fence(__ATOMIC_RELEASE); } -static inline void odp_mb_acquire(void) +_STATIC void odp_mb_acquire(void) { __atomic_thread_fence(__ATOMIC_ACQUIRE); } -static inline void odp_mb_full(void) +_STATIC void odp_mb_full(void) { __atomic_thread_fence(__ATOMIC_SEQ_CST); } @@ -40,8 +40,6 @@ static inline void odp_mb_full(void) * @} */ -#include <odp/api/spec/sync.h> - #ifdef __cplusplus } #endif diff --git a/platform/linux-generic/include/odp/api/std_clib.h b/platform/linux-generic/include/odp/api/std_clib.h index 40c0ea8..c498f68 100644 --- a/platform/linux-generic/include/odp/api/std_clib.h +++ b/platform/linux-generic/include/odp/api/std_clib.h @@ -14,20 +14,12 @@ extern "C" { #include <odp/api/spec/std_types.h> #include <string.h> -static inline void *odp_memcpy(void *dst, const void *src, size_t num) -{ - return memcpy(dst, src, num); -} - -static inline void *odp_memset(void *ptr, int value, size_t num) -{ - return memset(ptr, value, num); -} +#include <odp/api/plat/inlines.h> +#ifdef _ODP_INLINES +#include <odp/api/plat/std_clib_inlines.h> +#endif -static inline int odp_memcmp(const void *ptr1, const void *ptr2, size_t num) -{ - return memcmp(ptr1, ptr2, num); -} +#include <odp/api/spec/std_clib.h> #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp/api/sync.h b/platform/linux-generic/include/odp/api/sync.h index 45fe6bd..d2becb9 100644 --- a/platform/linux-generic/include/odp/api/sync.h +++ b/platform/linux-generic/include/odp/api/sync.h @@ -21,20 +21,10 @@ extern "C" { * @{ */ -static inline void odp_mb_release(void) -{ - __atomic_thread_fence(__ATOMIC_RELEASE); -} - -static inline void odp_mb_acquire(void) -{ - __atomic_thread_fence(__ATOMIC_ACQUIRE); -} - -static inline void odp_mb_full(void) -{ - __atomic_thread_fence(__ATOMIC_SEQ_CST); -} +#include <odp/api/plat/inlines.h> +#ifdef _ODP_INLINES +#include <odp/api/plat/sync_inlines.h> +#endif /** * @} diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 1b1b883..6fb05c0 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -36,4 +36,5 @@ m4_include([platform/linux-generic/m4/odp_dpdk.m4]) m4_include([platform/linux-generic/m4/odp_ipc.m4]) m4_include([platform/linux-generic/m4/odp_schedule.m4]) -AC_CONFIG_FILES([platform/linux-generic/Makefile]) +AC_CONFIG_FILES([platform/linux-generic/Makefile + platform/linux-generic/include/odp/api/plat/inlines.h]) diff --git a/platform/linux-generic/odp_atomic.c b/platform/linux-generic/odp_atomic.c index 680e66d..e9a3ed0 100644 --- a/platform/linux-generic/odp_atomic.c +++ b/platform/linux-generic/odp_atomic.c @@ -5,6 +5,9 @@ */ #include <odp/api/atomic.h> +#ifndef _ODP_INLINES +#include <odp/api/plat/atomic_inlines.h> +#endif int odp_atomic_lock_free_u64(odp_atomic_op_t *atomic_op) { diff --git a/platform/linux-generic/odp_byteorder.c b/platform/linux-generic/odp_byteorder.c new file mode 100644 index 0000000..fc87291 --- /dev/null +++ b/platform/linux-generic/odp_byteorder.c @@ -0,0 +1,10 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/byteorder.h> +#ifndef _ODP_INLINES +#include <odp/api/plat/byteorder_inlines.h> +#endif diff --git a/platform/linux-generic/odp_std_clib.c b/platform/linux-generic/odp_std_clib.c new file mode 100644 index 0000000..611ba12 --- /dev/null +++ b/platform/linux-generic/odp_std_clib.c @@ -0,0 +1,10 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/std_clib.h> +#ifndef _ODP_INLINES +#include <odp/api/plat/std_clib_inlines.h> +#endif diff --git a/platform/linux-generic/odp_sync.c b/platform/linux-generic/odp_sync.c new file mode 100644 index 0000000..f31c389 --- /dev/null +++ b/platform/linux-generic/odp_sync.c @@ -0,0 +1,10 @@ +/* Copyright (c) 2016, Linaro Limited + * All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <odp/api/sync.h> +#ifndef _ODP_INLINES +#include <odp/api/plat/sync_inlines.h> +#endif ----------------------------------------------------------------------- Summary of changes: configure.ac | 7 + example/Makefile.inc | 3 +- helper/Makefile.am | 1 + platform/linux-generic/.gitignore | 1 + platform/linux-generic/Makefile.am | 9 + platform/linux-generic/include/odp/api/atomic.h | 384 +-------------------- platform/linux-generic/include/odp/api/byteorder.h | 114 +----- .../odp/api/{atomic.h => plat/atomic_inlines.h} | 153 ++++---- .../api/{byteorder.h => plat/byteorder_inlines.h} | 39 +-- .../include/odp/api/plat/inlines.h.in | 33 ++ .../api/{std_clib.h => plat/std_clib_inlines.h} | 12 +- .../odp/api/{sync.h => plat/sync_inlines.h} | 16 +- platform/linux-generic/include/odp/api/std_clib.h | 18 +- platform/linux-generic/include/odp/api/sync.h | 18 +- platform/linux-generic/m4/configure.m4 | 3 +- platform/linux-generic/odp_atomic.c | 3 + .../linux-generic/odp_byteorder.c | 10 +- .../linux-generic/odp_std_clib.c | 10 +- .../linux-generic/odp_sync.c | 5 +- 19 files changed, 173 insertions(+), 666 deletions(-) create mode 100644 platform/linux-generic/.gitignore copy platform/linux-generic/include/odp/api/{atomic.h => plat/atomic_inlines.h} (58%) copy platform/linux-generic/include/odp/api/{byteorder.h => plat/byteorder_inlines.h} (69%) create mode 100644 platform/linux-generic/include/odp/api/plat/inlines.h.in copy platform/linux-generic/include/odp/api/{std_clib.h => plat/std_clib_inlines.h} (52%) copy platform/linux-generic/include/odp/api/{sync.h => plat/sync_inlines.h} (58%) copy test/linux-generic/ring/ring_main.c => platform/linux-generic/odp_byteorder.c (52%) copy test/linux-generic/ring/ring_main.c => platform/linux-generic/odp_std_clib.c (53%) copy test/linux-generic/validation/api/shmem/shmem_odp.h => platform/linux-generic/odp_sync.c (55%) hooks/post-receive --

8 years, 5 months

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

git-lng-odp September 2016