This is an automated email from the git hooks/post-receive script. It was generated because a ref change was pushed to the repository containing the project "".
The branch, master has been updated via c6d92eeac0e503d53a8e805635ec78a9e9e31338 (commit) via f2ed58011a827959eb81452a99845278acc8a3fb (commit) via 8994bbabb8efb58822b82ad6f03a4f0f332f81cf (commit) via 48e1788231127fd184203d842c95f1560328c7fe (commit) via 7131ada262e45f059b67d5492511a60a3303785a (commit) via ca1003ba1b45a356301587ac31dfca108bf1edcb (commit) via 321f9c88c709c992d395ccff73141d3fa737ab79 (commit) via 77f8e112e5e72d8e6e5bd1c530fccb7557b56124 (commit) via b60c92e93885de2f2b051648632a27413809a01f (commit) via 5355304a9650a2929f080ff66fafeb2792632f06 (commit) from c8d22107bd8b64efc91bc064fe20e7aeeac7efff (commit)
Those revisions listed above that are new to this repository have not appeared on any other notification email; so we list those revisions in full, below.
- Log ----------------------------------------------------------------- commit c6d92eeac0e503d53a8e805635ec78a9e9e31338 Author: Petri Savolainen petri.savolainen@nokia.com Date: Thu Sep 12 10:59:17 2019 +0300
linux-gen: socket_mmap: move locks into ring struct
Locks need to be on separate cachelines to avoid false sharing. Added lock into the ring structure, so that the lock and data it protects are co-located.
FD set function still uses the pktio level lock as it does not touch ring structures.
Signed-off-by: Petri Savolainen petri.savolainen@nokia.com Reviewed-by: Matias Elo matias.elo@nokia.com
diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index ca58eb39a..6535bc72c 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -59,6 +59,7 @@
/** packet mmap ring */ struct ring { + odp_ticketlock_t lock; struct iovec *rd; unsigned int frame_num; int rd_num; @@ -565,6 +566,8 @@ static int sock_mmap_open(odp_pktio_t id ODP_UNUSED, pkt_sock->frame_offset = 0;
pkt_sock->pool = pool; + odp_ticketlock_init(&pkt_sock->rx_ring.lock); + odp_ticketlock_init(&pkt_sock->tx_ring.lock); pkt_sock->rx_ring.shm = ODP_SHM_INVALID; pkt_sock->tx_ring.shm = ODP_SHM_INVALID; pkt_sock->sockfd = mmap_pkt_socket(); @@ -649,10 +652,10 @@ static int sock_mmap_recv(pktio_entry_t *pktio_entry, int index ODP_UNUSED, pkt_sock_mmap_t *const pkt_sock = pkt_priv(pktio_entry); int ret;
- odp_ticketlock_lock(&pktio_entry->s.rxl); + odp_ticketlock_lock(&pkt_sock->rx_ring.lock); ret = pkt_mmap_v2_rx(pktio_entry, pkt_sock, pkt_table, num, pkt_sock->if_mac); - odp_ticketlock_unlock(&pktio_entry->s.rxl); + odp_ticketlock_unlock(&pkt_sock->rx_ring.lock);
return ret; } @@ -748,10 +751,10 @@ static int sock_mmap_send(pktio_entry_t *pktio_entry, int index ODP_UNUSED, int ret; pkt_sock_mmap_t *const pkt_sock = pkt_priv(pktio_entry);
- odp_ticketlock_lock(&pktio_entry->s.txl); + odp_ticketlock_lock(&pkt_sock->tx_ring.lock); ret = pkt_mmap_v2_tx(pkt_sock->tx_ring.sock, &pkt_sock->tx_ring, pkt_table, num); - odp_ticketlock_unlock(&pktio_entry->s.txl); + odp_ticketlock_unlock(&pkt_sock->tx_ring.lock);
return ret; }
commit f2ed58011a827959eb81452a99845278acc8a3fb Author: Petri Savolainen petri.savolainen@nokia.com Date: Thu Sep 12 10:21:59 2019 +0300
linux-gen: socket_mmap: remove redundant header file
Remove odp_packet_socket.h as it is only used by socket_mmap.c. Move relevant content to the c file.
Signed-off-by: Petri Savolainen petri.savolainen@nokia.com Reviewed-by: Matias Elo matias.elo@nokia.com
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am index 517ea258b..9f24bc829 100644 --- a/platform/linux-generic/Makefile.am +++ b/platform/linux-generic/Makefile.am @@ -111,7 +111,6 @@ noinst_HEADERS = \ include/odp_packet_io_internal.h \ include/odp_packet_io_ipc_internal.h \ include/odp_packet_io_ring_internal.h \ - include/odp_packet_socket.h \ include/odp_socket_common.h \ include/odp_packet_io_stats_common.h \ include/odp_packet_io_stats.h \ diff --git a/platform/linux-generic/include/odp_packet_socket.h b/platform/linux-generic/include/odp_packet_socket.h deleted file mode 100644 index 16cee16bc..000000000 --- a/platform/linux-generic/include/odp_packet_socket.h +++ /dev/null @@ -1,62 +0,0 @@ -/* Copyright (c) 2013-2018, Linaro Limited - * Copyright (c) 2013, Nokia Solutions and Networks - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#ifndef ODP_PACKET_SOCKET_H -#define ODP_PACKET_SOCKET_H - -#include <linux/if_packet.h> -#include <linux/if_ether.h> -#include <sys/socket.h> -#include <string.h> -#include <stddef.h> - -#include <odp/api/align.h> -#include <odp/api/buffer.h> -#include <odp/api/debug.h> -#include <odp/api/pool.h> -#include <odp/api/packet.h> -#include <odp/api/packet_io.h> -#include <odp/api/shared_memory.h> - -#include <linux/version.h> - -/* - * Packet socket config: - */ - -/* - * This makes sure that building for kernels older than 3.1 works - * and a fanout requests fails (for invalid packet socket option) - * in runtime if requested - */ -#ifndef PACKET_FANOUT -#define PACKET_FANOUT 18 -#define PACKET_FANOUT_HASH 0 -#endif /* PACKET_FANOUT */ - -/** packet mmap ring */ -struct ring { - struct iovec *rd; - unsigned frame_num; - int rd_num; - - odp_shm_t shm; - int sock; - int type; - int version; - uint8_t *mm_space; - size_t mm_len; - size_t rd_len; - int flen; - - struct tpacket_req req; -}; - -ODP_STATIC_ASSERT(offsetof(struct ring, mm_space) <= ODP_CACHE_LINE_SIZE, - "ERR_STRUCT_RING"); - -#endif diff --git a/platform/linux-generic/pktio/socket.c b/platform/linux-generic/pktio/socket.c index cae838e83..7e195580c 100644 --- a/platform/linux-generic/pktio/socket.c +++ b/platform/linux-generic/pktio/socket.c @@ -35,7 +35,6 @@ #include <linux/sockios.h>
#include <odp_api.h> -#include <odp_packet_socket.h> #include <odp_socket_common.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 54fc7367e..ca58eb39a 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -24,10 +24,10 @@ #include <sys/ioctl.h> #include <errno.h> #include <time.h> +#include <linux/if_packet.h>
#include <odp_api.h> #include <odp/api/plat/packet_inlines.h> -#include <odp_packet_socket.h> #include <odp_socket_common.h> #include <odp_packet_internal.h> #include <odp_packet_io_internal.h> @@ -47,6 +47,37 @@ #define FRAME_MEM_SIZE (4 * 1024 * 1024) #define BLOCK_SIZE (4 * 1024)
+/* + * This makes sure that building for kernels older than 3.1 works + * and a fanout requests fails (for invalid packet socket option) + * in runtime if requested + */ +#ifndef PACKET_FANOUT +#define PACKET_FANOUT 18 +#define PACKET_FANOUT_HASH 0 +#endif + +/** packet mmap ring */ +struct ring { + struct iovec *rd; + unsigned int frame_num; + int rd_num; + + odp_shm_t shm; + int sock; + int type; + int version; + uint8_t *mm_space; + size_t mm_len; + size_t rd_len; + int flen; + + struct tpacket_req req; +}; + +ODP_STATIC_ASSERT(offsetof(struct ring, mm_space) <= ODP_CACHE_LINE_SIZE, + "ERR_STRUCT_RING"); + /** Packet socket using mmap rings for both Rx and Tx */ typedef struct { /** Packet mmap ring for Rx */
commit 8994bbabb8efb58822b82ad6f03a4f0f332f81cf Author: Petri Savolainen petri.savolainen@nokia.com Date: Wed Sep 11 16:58:44 2019 +0300
linux-gen: remove mmap_vlan_ins test
Remove redundant test case for VLAN insertion feature, which was removed.
Signed-off-by: Petri Savolainen petri.savolainen@nokia.com Reviewed-by: Matias Elo matias.elo@nokia.com
diff --git a/platform/linux-generic/m4/configure.m4 b/platform/linux-generic/m4/configure.m4 index 3ab01f2b7..461dfc0fb 100644 --- a/platform/linux-generic/m4/configure.m4 +++ b/platform/linux-generic/m4/configure.m4 @@ -38,7 +38,6 @@ AC_CONFIG_FILES([platform/linux-generic/Makefile platform/linux-generic/test/Makefile platform/linux-generic/test/validation/api/shmem/Makefile platform/linux-generic/test/validation/api/pktio/Makefile - platform/linux-generic/test/mmap_vlan_ins/Makefile platform/linux-generic/test/pktio_ipc/Makefile platform/linux-generic/test/ring/Makefile]) ]) diff --git a/platform/linux-generic/test/Makefile.am b/platform/linux-generic/test/Makefile.am index f2ffe995a..d3193b5e9 100644 --- a/platform/linux-generic/test/Makefile.am +++ b/platform/linux-generic/test/Makefile.am @@ -23,15 +23,11 @@ test_SCRIPTS = $(dist_check_SCRIPTS)
SUBDIRS += validation/api/pktio\ validation/api/shmem\ - mmap_vlan_ins\ pktio_ipc\ ring
if HAVE_PCAP TESTS += validation/api/pktio/pktio_run_pcap.sh - -TESTS += mmap_vlan_ins/mmap_vlan_ins.sh -SUBDIRS += mmap_vlan_ins endif if netmap_support TESTS += validation/api/pktio/pktio_run_netmap.sh diff --git a/platform/linux-generic/test/mmap_vlan_ins/.gitignore b/platform/linux-generic/test/mmap_vlan_ins/.gitignore deleted file mode 100644 index 755fa2ed5..000000000 --- a/platform/linux-generic/test/mmap_vlan_ins/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.pcap -plat_mmap_vlan_ins diff --git a/platform/linux-generic/test/mmap_vlan_ins/Makefile.am b/platform/linux-generic/test/mmap_vlan_ins/Makefile.am deleted file mode 100644 index 8559e55b5..000000000 --- a/platform/linux-generic/test/mmap_vlan_ins/Makefile.am +++ /dev/null @@ -1,16 +0,0 @@ -include $(top_srcdir)/test/Makefile.inc -TESTS_ENVIRONMENT += TEST_DIR=${top_builddir}/test/validation - -dist_check_SCRIPTS = \ - mmap_vlan_ins.sh \ - pktio_env - -dist_check_DATA = vlan.pcap - -test_SCRIPTS = $(dist_check_SCRIPTS) -test_DATA = $(dist_check_DATA) - -test_PROGRAMS = plat_mmap_vlan_ins - -# Clonned from example odp_l2fwd simple -plat_mmap_vlan_ins_SOURCES = mmap_vlan_ins.c diff --git a/platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.c b/platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.c deleted file mode 100644 index 563d3d649..000000000 --- a/platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.c +++ /dev/null @@ -1,250 +0,0 @@ -/* Copyright (c) 2016-2018, Linaro Limited - * All rights reserved. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -#include "config.h" - -#include <stdlib.h> -#include <stdio.h> -#include <getopt.h> -#include <signal.h> - -#include <odp_api.h> -#include <odp/helper/odph_api.h> - -#define POOL_NUM_PKT 100 -#define POOL_SEG_LEN 1500 -#define MAX_PKT_BURST 32 -#define MAX_WORKERS 1 - -typedef struct { - odp_pktio_t if0, if1; - odp_pktin_queue_t if0in, if1in; - odp_pktout_queue_t if0out, if1out; - odph_ethaddr_t src, dst; - int g_ret; -} global_data_t; - -static global_data_t *global; - -static odp_pktio_t create_pktio(const char *name, odp_pool_t pool, - odp_pktin_queue_t *pktin, - odp_pktout_queue_t *pktout) -{ - odp_pktio_param_t pktio_param; - odp_pktin_queue_param_t in_queue_param; - odp_pktout_queue_param_t out_queue_param; - odp_pktio_t pktio; - - odp_pktio_param_init(&pktio_param); - - pktio = odp_pktio_open(name, pool, &pktio_param); - if (pktio == ODP_PKTIO_INVALID) { - printf("Failed to open %s\n", name); - exit(1); - } - - odp_pktin_queue_param_init(&in_queue_param); - odp_pktout_queue_param_init(&out_queue_param); - - in_queue_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE; - - if (odp_pktin_queue_config(pktio, &in_queue_param)) { - printf("Failed to config input queue for %s\n", name); - exit(1); - } - - out_queue_param.op_mode = ODP_PKTIO_OP_MT_UNSAFE; - - if (odp_pktout_queue_config(pktio, &out_queue_param)) { - printf("Failed to config output queue for %s\n", name); - exit(1); - } - - if (odp_pktin_queue(pktio, pktin, 1) != 1) { - printf("pktin queue query failed for %s\n", name); - exit(1); - } - if (odp_pktout_queue(pktio, pktout, 1) != 1) { - printf("pktout queue query failed for %s\n", name); - exit(1); - } - return pktio; -} - -static int run_worker(void *arg ODP_UNUSED) -{ - odp_packet_t pkt_tbl[MAX_PKT_BURST]; - int pkts, sent, tx_drops, i; - int total_pkts = 0; - uint64_t wait_time = odp_pktin_wait_time(2 * ODP_TIME_SEC_IN_NS); - - if (odp_pktio_start(global->if0)) { - printf("unable to start input interface\n"); - exit(1); - } - printf("started input interface\n"); - if (odp_pktio_start(global->if1)) { - printf("unable to start output interface\n"); - exit(1); - } - printf("started output interface\n"); - printf("started all\n"); - - while (1) { - pkts = odp_pktin_recv_tmo(global->if0in, pkt_tbl, MAX_PKT_BURST, - wait_time); - if (odp_unlikely(pkts <= 0)) { - printf("recv tmo!\n"); - break; - } - - for (i = 0; i < pkts; i++) { - odp_packet_t pkt = pkt_tbl[i]; - odph_ethhdr_t *eth; - - if (odp_unlikely(!odp_packet_has_eth(pkt))) { - printf("warning: packet has no eth header\n"); - return 0; - } - eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL); - eth->src = global->src; - eth->dst = global->dst; - } - sent = odp_pktout_send(global->if1out, pkt_tbl, pkts); - if (sent < 0) - sent = 0; - total_pkts += sent; - tx_drops = pkts - sent; - if (odp_unlikely(tx_drops)) - odp_packet_free_multi(&pkt_tbl[sent], tx_drops); - } - - printf("Total send packets: %d\n", total_pkts); - - if (total_pkts < 10) - global->g_ret = -1; - - return 0; -} - -int main(int argc, char **argv) -{ - odp_pool_t pool; - odp_pool_param_t params; - odp_cpumask_t cpumask; - odph_helper_options_t helper_options; - odph_odpthread_t thd[MAX_WORKERS]; - odp_instance_t instance; - odp_init_t init_param; - odph_odpthread_params_t thr_params; - odp_shm_t shm; - int ret; - - /* Let helper collect its own arguments (e.g. --odph_proc) */ - argc = odph_parse_options(argc, argv); - if (odph_options(&helper_options)) { - printf("Error: reading ODP helper options failed.\n"); - exit(EXIT_FAILURE); - } - - if (argc < 3) { - printf("Too few arguments (%i).\n" - "Two interface names needed as arguments.\n", argc); - exit(0); - } - - odp_init_param_init(&init_param); - init_param.mem_model = helper_options.mem_model; - - if (odp_init_global(&instance, &init_param, NULL)) { - printf("Error: ODP global init failed.\n"); - exit(1); - } - - if (odp_init_local(instance, ODP_THREAD_CONTROL)) { - printf("Error: ODP local init failed.\n"); - exit(1); - } - - /* Reserve memory for args from shared mem */ - shm = odp_shm_reserve("_appl_global_data", sizeof(global_data_t), - ODP_CACHE_LINE_SIZE, 0); - if (shm == ODP_SHM_INVALID) { - printf("Error: shared mem alloc failed.\n"); - exit(EXIT_FAILURE); - } - - global = odp_shm_addr(shm); - if (global == NULL) { - printf("Error: shared mem alloc failed.\n"); - exit(EXIT_FAILURE); - } - - memset(global, 0, sizeof(global_data_t)); - - /* Create packet pool */ - odp_pool_param_init(¶ms); - params.pkt.seg_len = POOL_SEG_LEN; - params.pkt.len = POOL_SEG_LEN; - params.pkt.num = POOL_NUM_PKT; - params.type = ODP_POOL_PACKET; - - pool = odp_pool_create("packet pool", ¶ms); - - if (pool == ODP_POOL_INVALID) { - printf("Error: packet pool create failed.\n"); - exit(1); - } - - global->if0 = create_pktio(argv[1], pool, &global->if0in, - &global->if0out); - global->if1 = create_pktio(argv[2], pool, &global->if1in, - &global->if1out); - - odp_cpumask_default_worker(&cpumask, MAX_WORKERS); - - memset(&thr_params, 0, sizeof(thr_params)); - thr_params.start = run_worker; - thr_params.arg = NULL; - thr_params.thr_type = ODP_THREAD_WORKER; - thr_params.instance = instance; - - odph_odpthreads_create(thd, &cpumask, &thr_params); - odph_odpthreads_join(thd); - - ret = global->g_ret; - - if (odp_pktio_stop(global->if0) || odp_pktio_close(global->if0)) { - printf("Error: failed to close interface %s\n", argv[1]); - exit(EXIT_FAILURE); - } - if (odp_pktio_stop(global->if1) || odp_pktio_close(global->if1)) { - printf("Error: failed to close interface %s\n", argv[2]); - exit(EXIT_FAILURE); - } - - if (odp_pool_destroy(pool)) { - printf("Error: pool destroy\n"); - exit(EXIT_FAILURE); - } - - if (odp_shm_free(shm)) { - printf("Error: shm free global data\n"); - exit(EXIT_FAILURE); - } - - if (odp_term_local()) { - printf("Error: term local\n"); - exit(EXIT_FAILURE); - } - - if (odp_term_global(instance)) { - printf("Error: term global\n"); - exit(EXIT_FAILURE); - } - - return ret; -} diff --git a/platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.sh b/platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.sh deleted file mode 100755 index b0eb046b0..000000000 --- a/platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2016-2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause -# - -# -# This test is intend to test pkt_mmap_vlan_insert() feature for -# linux-generic packet mmap pktio. -# -# -export ODP_PKTIO_DISABLE_SOCKET_MMSG=1 - -# directory where platform test sources are, including scripts -TEST_SRC_DIR=$(dirname $0) - -# exit codes expected by automake for skipped tests -TEST_SKIPPED=77 - -# directories where binary can be found: -# -in the validation dir when running make check (intree or out of tree) -# -in the script directory, when running after 'make install', or -# -in the validation when running standalone intree. -# -in the current directory. -# running stand alone out of tree requires setting PATH -PATH=${TEST_DIR}/../mmap_vlan_ins:$PATH -PATH=`pwd`/mmap_vlan_ins:$PATH -PATH=$(dirname $0):$PATH -PATH=.:$PATH - -bin_path=$(which plat_mmap_vlan_ins${EXEEXT}) -if [ -x "$bin_path" ] ; then - echo "running with plat_mmap_vlan_ins: $bin_path" -else - echo "cannot find plat_mmap_vlan_ins: please set you PATH for it." - pwd - exit 1 -fi - - -# Use installed pktio env or for make check take it from platform directory -if [ -f "./pktio_env" ]; then - . ./pktio_env -elif [ -f ${TEST_SRC_DIR}/pktio_env ]; then - . ${TEST_SRC_DIR}/pktio_env -else - echo "BUG: unable to find pktio_env!" - echo "pktio_env has to be in current directory or" - echo " in platform/$ODP_PLATFORM/test." - echo "ODP_PLATFORM="$ODP_PLATFORM"" - exit 1 -fi - -setup_pktio_env -if [ $? -ne 0 ]; then - return 77 # Skip the test -fi - -PCAP_IN=`find . ${TEST_DIR} $(dirname $0) -name vlan.pcap -print -quit` -echo "using PCAP_IN = ${PCAP_IN}" -PCAP_OUT=vlan_out.pcap - -# Listen on veth pipe and write to pcap Send pcap -plat_mmap_vlan_ins${EXEEXT} pktiop0p1 pcap:out=${PCAP_OUT} \ - 00:02:03:04:05:06 00:08:09:0a:0b:0c & -# Send pcap file to veth interface -plat_mmap_vlan_ins${EXEEXT} pcap:in=${PCAP_IN} pktiop1p0 \ - 01:02:03:04:05:06 01:08:09:0a:0b:0c - -# Wait both processes to exit before removing the interfaces -wait $! - -rm -f ${PCAP_OUT} -cleanup_pktio_env - -exit 0 diff --git a/platform/linux-generic/test/mmap_vlan_ins/pktio_env b/platform/linux-generic/test/mmap_vlan_ins/pktio_env deleted file mode 100644 index ba9998e6a..000000000 --- a/platform/linux-generic/test/mmap_vlan_ins/pktio_env +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2015-2018, Linaro Limited -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause -# -# Test script wrapper for running ODP pktio apps on linux-generic. -# -# For linux-generic the default behavior is to create two pairs of -# virtual Ethernet interfaces and provide the names of these via -# environment variables to pktio apps, the interfaces will be removed -# before the script exits. -# -# Note that the creation of virtual Ethernet devices depends on having -# CONFIG_VETH enabled in the kernel, if not enabled the env setup will be skipped. -# -# Network set up -# IF0 <---> IF1 -# IF2 <---> IF3 -IF0=pktiop0p1 -IF1=pktiop1p0 -IF2=pktiop2p3 -IF3=pktiop3p2 - -if [ "$0" = "$BASH_SOURCE" ]; then - echo "Error: Platform specific env file has to be sourced." -fi - -check_for_root() -{ - if [ "$(id -u)" != "0" ]; then - echo "check_for_root(): need to be root to setup VETH" - return 1 - fi - return 0 -} - -# wait for a network interface's operational state to be "up" -wait_for_iface_up() -{ - iface=$1 - cnt=0 - - while [ $cnt -lt 50 ]; do - read operstate < /sys/class/net/$iface/operstate - - if [ $? -ne 0 ]; then - break - elif [ "$operstate" = "up" ]; then - return 0 - fi - - sleep 0.1 - cnt=`expr $cnt + 1` - done - - return 1 -} - -setup_pktio_env() -{ - echo "pktio: setting up test interfaces $IF0, $IF1, $IF2, $IF3." - - check_for_root - if [ $? -ne 0 ]; then - return 1 - fi - - for iface in $IF0 $IF1 $IF2 $IF3; do - ip link show $iface 2> /dev/null - if [ $? -eq 0 ]; then - echo "pktio: interface $iface already exist $?" - return 2 - fi - done - - if [ "$1" = "clean" ]; then - trap cleanup_pktio_env EXIT - fi - - ip link add $IF0 type veth peer name $IF1 - if [ $? -ne 0 ]; then - echo "pktio: error: unable to create veth pair" - return 3 - fi - ip link add $IF2 type veth peer name $IF3 - if [ $? -ne 0 ]; then - echo "pktio: error: unable to create veth pair" - return 4 - fi - - for iface in $IF0 $IF1 $IF2 $IF3; do - ip link set $iface mtu 9216 up - ifconfig $iface -arp - done - - # check that the interface has come up before starting the test - for iface in $IF0 $IF1 $IF2 $IF3; do - wait_for_iface_up $iface - if [ $? -ne 0 ]; then - echo "pktio: interface $iface failed to come up" - return 5 - fi - done -} - -cleanup_pktio_env() -{ - echo "pktio: removing test interfaces $IF0, $IF1, $IF2, $IF3" - check_for_root - if [ $? -ne 0 ]; then - return 1 - fi - - for iface in $IF0 $IF1 $IF2 $IF3; do - ip link del $iface 2> /dev/null - done - return 0 -} diff --git a/platform/linux-generic/test/mmap_vlan_ins/vlan.pcap b/platform/linux-generic/test/mmap_vlan_ins/vlan.pcap deleted file mode 100644 index 106ccb682..000000000 Binary files a/platform/linux-generic/test/mmap_vlan_ins/vlan.pcap and /dev/null differ
commit 48e1788231127fd184203d842c95f1560328c7fe Author: Petri Savolainen petri.savolainen@nokia.com Date: Wed Sep 11 16:31:21 2019 +0300
linux-gen: socket_mmap: break receive loop on empty pool
Break packet receive loop, when pool gets empty. Leave the packet into the ring, so that the next receive call may try to receive it again.
Signed-off-by: Petri Savolainen petri.savolainen@nokia.com Reviewed-by: Matias Elo matias.elo@nokia.com
diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 0a11c1d25..54fc7367e 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -137,6 +137,7 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, unsigned i; unsigned nb_rx; struct ring *ring; + odp_pool_t pool = pkt_sock->pool;
if (pktio_entry->s.config.pktin.bit.ts_all || pktio_entry->s.config.pktin.bit.ts_ptp) @@ -151,7 +152,6 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, odp_packet_t pkt; odp_packet_hdr_t *hdr; odp_packet_hdr_t parsed_hdr; - odp_pool_t pool = pkt_sock->pool; int ret;
tp_hdr = (void *)next_ptr; @@ -177,10 +177,19 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, continue; }
+ ret = packet_alloc_multi(pool, pkt_len, &pkt, 1); + + if (odp_unlikely(ret != 1)) { + /* Stop receiving packets when pool is empty. Leave + * the current frame into the ring. */ + break; + } + /* Don't receive packets sent by ourselves */ eth_hdr = (struct ethhdr *)pkt_buf; if (odp_unlikely(ethaddrs_equal(if_mac, eth_hdr->h_source))) { + odp_packet_free(pkt); tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num; continue; @@ -190,20 +199,13 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, if (cls_classify_packet(pktio_entry, pkt_buf, pkt_len, pkt_len, &pool, &parsed_hdr, true)) { + odp_packet_free(pkt); tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num; continue; } }
- ret = packet_alloc_multi(pool, pkt_len, &pkt, 1); - - if (odp_unlikely(ret != 1)) { - tp_hdr->tp_status = TP_STATUS_KERNEL; - frame_num = next_frame_num; - continue; - } - hdr = packet_hdr(pkt); ret = odp_packet_copy_from_mem(pkt, 0, pkt_len, pkt_buf); if (ret != 0) {
commit 7131ada262e45f059b67d5492511a60a3303785a Author: Petri Savolainen petri.savolainen@nokia.com Date: Wed Sep 11 15:30:11 2019 +0300
linux-gen: socket_mmap: remove vlan insertion
Remove redundant VLAN insertion from packet receive. This function assumed that TP_STATUS_VLAN_VALID status bit is set when Linux has removed VLAN header from packet. This is not always the case. VLAN header is removed also when flag is not set. Also, if user has set up a VLAN interface in Linux, the assumption is that all received packets (from the interface) belong to the VLAN and do not carry VLAN header.
Signed-off-by: Petri Savolainen petri.savolainen@nokia.com Reviewed-by: Matias Elo matias.elo@nokia.com
diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 35f259087..0a11c1d25 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -118,37 +118,6 @@ static int mmap_pkt_socket(void) return sock; }
-static uint8_t *pkt_mmap_vlan_insert(uint8_t *l2_hdr_ptr, - uint16_t mac_offset, - uint16_t vlan_tci, - int *pkt_len_ptr) -{ - _odp_ethhdr_t *eth_hdr; - _odp_vlanhdr_t *vlan_hdr; - uint8_t *new_l2_ptr; - int orig_pkt_len; - - /* First try to see if the mac_offset is large enough to accommodate - * shifting the Ethernet header down to open up space for the IEEE - * 802.1Q vlan header. - */ - if (_ODP_VLANHDR_LEN < mac_offset) { - orig_pkt_len = *pkt_len_ptr; - new_l2_ptr = l2_hdr_ptr - _ODP_VLANHDR_LEN; - memmove(new_l2_ptr, l2_hdr_ptr, _ODP_ETHHDR_LEN); - - eth_hdr = (_odp_ethhdr_t *)new_l2_ptr; - vlan_hdr = (_odp_vlanhdr_t *)(new_l2_ptr + _ODP_ETHHDR_LEN); - vlan_hdr->tci = odp_cpu_to_be_16(vlan_tci); - vlan_hdr->type = eth_hdr->type; - eth_hdr->type = odp_cpu_to_be_16(_ODP_ETHTYPE_VLAN); - *pkt_len_ptr = orig_pkt_len + _ODP_VLANHDR_LEN; - return new_l2_ptr; - } - - return l2_hdr_ptr; -} - static inline unsigned next_frame(unsigned cur_frame, unsigned frame_count) { return odp_unlikely(cur_frame + 1 >= frame_count) ? 0 : cur_frame + 1; @@ -217,12 +186,6 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, continue; }
- if (tp_hdr->tp_status & TP_STATUS_VLAN_VALID) - pkt_buf = pkt_mmap_vlan_insert(pkt_buf, - tp_hdr->tp_mac, - tp_hdr->tp_vlan_tci, - &pkt_len); - if (pktio_cls_enabled(pktio_entry)) { if (cls_classify_packet(pktio_entry, pkt_buf, pkt_len, pkt_len, &pool, &parsed_hdr,
commit ca1003ba1b45a356301587ac31dfca108bf1edcb Author: Petri Savolainen petri.savolainen@nokia.com Date: Tue Sep 10 17:33:36 2019 +0300
linux-gen: socket_mmap: prefetch frame header
Prefetch next frame header and first bytes of data. Also cleaned rx side packet handle usage.
Signed-off-by: Petri Savolainen petri.savolainen@nokia.com Reviewed-by: Matias Elo matias.elo@nokia.com
diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index a9fb44c45..35f259087 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -162,13 +162,12 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, odp_time_t ts_val; odp_time_t *ts = NULL; unsigned frame_num, next_frame_num; - uint8_t *pkt_buf; + uint8_t *pkt_buf, *next_ptr; int pkt_len; struct ethhdr *eth_hdr; unsigned i; unsigned nb_rx; struct ring *ring; - int ret;
if (pktio_entry->s.config.pktin.bit.ts_all || pktio_entry->s.config.pktin.bit.ts_ptp) @@ -176,24 +175,29 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
ring = &pkt_sock->rx_ring; frame_num = ring->frame_num; + next_ptr = ring->rd[frame_num].iov_base;
for (i = 0, nb_rx = 0; i < num; i++) { struct tpacket2_hdr *tp_hdr; + odp_packet_t pkt; odp_packet_hdr_t *hdr; odp_packet_hdr_t parsed_hdr; odp_pool_t pool = pkt_sock->pool; - int pkts; + int ret;
- tp_hdr = ring->rd[frame_num].iov_base; + tp_hdr = (void *)next_ptr;
if (tp_hdr->tp_status == TP_STATUS_KERNEL) break;
+ next_frame_num = next_frame(frame_num, ring->rd_num); + next_ptr = ring->rd[next_frame_num].iov_base; + odp_prefetch(next_ptr); + odp_prefetch(next_ptr + ODP_CACHE_LINE_SIZE); + if (ts != NULL) ts_val = odp_time_global();
- next_frame_num = next_frame(frame_num, ring->rd_num); - pkt_buf = (uint8_t *)(void *)tp_hdr + tp_hdr->tp_mac; pkt_len = tp_hdr->tp_snaplen;
@@ -229,19 +233,18 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, } }
- pkts = packet_alloc_multi(pool, pkt_len, &pkt_table[nb_rx], 1); + ret = packet_alloc_multi(pool, pkt_len, &pkt, 1);
- if (odp_unlikely(pkts != 1)) { - pkt_table[nb_rx] = ODP_PACKET_INVALID; + if (odp_unlikely(ret != 1)) { tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num; continue; } - hdr = packet_hdr(pkt_table[nb_rx]); - ret = odp_packet_copy_from_mem(pkt_table[nb_rx], 0, - pkt_len, pkt_buf); + + hdr = packet_hdr(pkt); + ret = odp_packet_copy_from_mem(pkt, 0, pkt_len, pkt_buf); if (ret != 0) { - odp_packet_free(pkt_table[nb_rx]); + odp_packet_free(pkt); tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num; continue; @@ -260,6 +263,7 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num;
+ pkt_table[nb_rx] = pkt; nb_rx++; }
@@ -272,21 +276,23 @@ static inline int pkt_mmap_v2_tx(int sock, struct ring *ring, uint32_t num) { uint32_t i, pkt_len, num_tx; - uint32_t first_frame_num, frame_num, frame_count; + uint32_t first_frame_num, frame_num, next_frame_num, frame_count; int ret; uint8_t *buf; + void *next_ptr; struct tpacket2_hdr *tp_hdr[num]; int total_len = 0;
- first_frame_num = ring->frame_num; - frame_num = first_frame_num; + frame_num = ring->frame_num; + first_frame_num = frame_num; frame_count = ring->rd_num; + next_ptr = ring->rd[frame_num].iov_base;
if (num > frame_count) num = frame_count;
for (i = 0; i < num; i++) { - tp_hdr[i] = ring->rd[frame_num].iov_base; + tp_hdr[i] = next_ptr;
if (tp_hdr[i]->tp_status != TP_STATUS_AVAILABLE) { if (tp_hdr[i]->tp_status == TP_STATUS_WRONG_FORMAT) { @@ -297,6 +303,10 @@ static inline int pkt_mmap_v2_tx(int sock, struct ring *ring, break; }
+ next_frame_num = next_frame(frame_num, frame_count); + next_ptr = ring->rd[next_frame_num].iov_base; + odp_prefetch(next_ptr); + pkt_len = odp_packet_len(pkt_table[i]); tp_hdr[i]->tp_len = pkt_len; total_len += pkt_len; @@ -307,7 +317,7 @@ static inline int pkt_mmap_v2_tx(int sock, struct ring *ring,
tp_hdr[i]->tp_status = TP_STATUS_SEND_REQUEST;
- frame_num = next_frame(frame_num, frame_count); + frame_num = next_frame_num; }
num = i;
commit 321f9c88c709c992d395ccff73141d3fa737ab79 Author: Petri Savolainen petri.savolainen@nokia.com Date: Mon Sep 9 15:37:38 2019 +0300
linux-gen: socket_mmap: remove redundant union
Clean up receive function code by pointing directly to tpacket2_hdr structure.
Signed-off-by: Petri Savolainen petri.savolainen@nokia.com Reviewed-by: Matias Elo matias.elo@nokia.com
diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 06f80d360..a9fb44c45 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -95,16 +95,6 @@ static int set_pkt_sock_fanout_mmap(pkt_sock_mmap_t *const pkt_sock, return 0; }
-union frame_map { - struct { - struct tpacket2_hdr ODP_ALIGNED(TPACKET_ALIGNMENT) tp_h; - struct sockaddr_ll ODP_ALIGNED(TPACKET_ALIGN(sizeof(struct - tpacket2_hdr))) s_ll; - } *v2; - - void *raw; -}; - static int mmap_pkt_socket(void) { int ver = TPACKET_V2; @@ -128,16 +118,6 @@ static int mmap_pkt_socket(void) return sock; }
-static inline int mmap_rx_kernel_ready(struct tpacket2_hdr *hdr) -{ - return ((hdr->tp_status & TP_STATUS_USER) == TP_STATUS_USER); -} - -static inline void mmap_rx_user_ready(struct tpacket2_hdr *hdr) -{ - hdr->tp_status = TP_STATUS_KERNEL; -} - static uint8_t *pkt_mmap_vlan_insert(uint8_t *l2_hdr_ptr, uint16_t mac_offset, uint16_t vlan_tci, @@ -179,7 +159,6 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, odp_packet_t pkt_table[], unsigned num, unsigned char if_mac[]) { - union frame_map ppd; odp_time_t ts_val; odp_time_t *ts = NULL; unsigned frame_num, next_frame_num; @@ -199,14 +178,15 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, frame_num = ring->frame_num;
for (i = 0, nb_rx = 0; i < num; i++) { + struct tpacket2_hdr *tp_hdr; odp_packet_hdr_t *hdr; odp_packet_hdr_t parsed_hdr; odp_pool_t pool = pkt_sock->pool; int pkts;
- ppd.raw = ring->rd[frame_num].iov_base; + tp_hdr = ring->rd[frame_num].iov_base;
- if (!mmap_rx_kernel_ready(ppd.raw)) + if (tp_hdr->tp_status == TP_STATUS_KERNEL) break;
if (ts != NULL) @@ -214,11 +194,11 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
next_frame_num = next_frame(frame_num, ring->rd_num);
- pkt_buf = (uint8_t *)ppd.raw + ppd.v2->tp_h.tp_mac; - pkt_len = ppd.v2->tp_h.tp_snaplen; + pkt_buf = (uint8_t *)(void *)tp_hdr + tp_hdr->tp_mac; + pkt_len = tp_hdr->tp_snaplen;
if (odp_unlikely(pkt_len > pkt_sock->mtu)) { - mmap_rx_user_ready(ppd.raw); + tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num; ODP_DBG("dropped oversized packet\n"); continue; @@ -228,22 +208,22 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, eth_hdr = (struct ethhdr *)pkt_buf; if (odp_unlikely(ethaddrs_equal(if_mac, eth_hdr->h_source))) { - mmap_rx_user_ready(ppd.raw); /* drop */ + tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num; continue; }
- if (ppd.v2->tp_h.tp_status & TP_STATUS_VLAN_VALID) + if (tp_hdr->tp_status & TP_STATUS_VLAN_VALID) pkt_buf = pkt_mmap_vlan_insert(pkt_buf, - ppd.v2->tp_h.tp_mac, - ppd.v2->tp_h.tp_vlan_tci, + tp_hdr->tp_mac, + tp_hdr->tp_vlan_tci, &pkt_len);
if (pktio_cls_enabled(pktio_entry)) { if (cls_classify_packet(pktio_entry, pkt_buf, pkt_len, pkt_len, &pool, &parsed_hdr, true)) { - mmap_rx_user_ready(ppd.raw); /* drop */ + tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num; continue; } @@ -253,7 +233,7 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
if (odp_unlikely(pkts != 1)) { pkt_table[nb_rx] = ODP_PACKET_INVALID; - mmap_rx_user_ready(ppd.raw); /* drop */ + tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num; continue; } @@ -262,7 +242,7 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, pkt_len, pkt_buf); if (ret != 0) { odp_packet_free(pkt_table[nb_rx]); - mmap_rx_user_ready(ppd.raw); /* drop */ + tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num; continue; } @@ -277,7 +257,7 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry,
packet_set_ts(hdr, ts);
- mmap_rx_user_ready(ppd.raw); + tp_hdr->tp_status = TP_STATUS_KERNEL; frame_num = next_frame_num;
nb_rx++;
commit 77f8e112e5e72d8e6e5bd1c530fccb7557b56124 Author: Petri Savolainen petri.savolainen@nokia.com Date: Mon Sep 9 14:24:20 2019 +0300
linux-gen: socket_mmap: reserve 4MB memory for blocks
Use fixed 4MB memory allocation for RX/TX blocks.
Disable (redundant) fanout configuration. Effective fanout usage would require that each worker (process) creates a socket. Currently, only single socket is created and protected with a lock.
Signed-off-by: Petri Savolainen petri.savolainen@nokia.com Reviewed-by: Matias Elo matias.elo@nokia.com
diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 1f56773c7..06f80d360 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -43,8 +43,9 @@ #include <protocols/eth.h> #include <protocols/ip.h>
-/* Maximum number of packets to store in each RX/TX block */ -#define MAX_PKTS_PER_BLOCK 512 +/* Reserve 4MB memory for frames in a RX/TX ring */ +#define FRAME_MEM_SIZE (4 * 1024 * 1024) +#define BLOCK_SIZE (4 * 1024)
/** Packet socket using mmap rings for both Rx and Tx */ typedef struct { @@ -381,65 +382,58 @@ static inline int pkt_mmap_v2_tx(int sock, struct ring *ring, return num_tx; }
-static void mmap_fill_ring(struct ring *ring, odp_pool_t pool_hdl, int fanout) +static int mmap_setup_ring(pkt_sock_mmap_t *pkt_sock, struct ring *ring, + int type) { - uint32_t num_frames; - int pz = getpagesize(); - pool_t *pool; + odp_shm_t shm; + uint32_t block_size, block_nr, frame_size, frame_nr; + uint32_t ring_size; + int flags; + int sock = pkt_sock->sockfd; + int mtu = pkt_sock->mtu; + int ret = 0;
- if (pool_hdl == ODP_POOL_INVALID) - ODP_ABORT("Invalid pool handle\n"); + ring->sock = sock; + ring->type = type; + ring->version = TPACKET_V2;
- pool = pool_entry_from_hdl(pool_hdl); + frame_size = ROUNDUP_POWER2_U32(mtu + TPACKET_HDRLEN + + TPACKET_ALIGNMENT); + block_size = BLOCK_SIZE; + if (frame_size > block_size) + block_size = frame_size;
- /* Frame has to capture full packet which can fit to the pool block.*/ - ring->req.tp_frame_size = (pool->headroom + pool->seg_len + - pool->tailroom + TPACKET_HDRLEN + - TPACKET_ALIGNMENT + (pz - 1)) & (-pz); + block_nr = FRAME_MEM_SIZE / block_size; + frame_nr = (block_size / frame_size) * block_nr; + ring_size = frame_nr * sizeof(struct iovec); + flags = 0;
- /* Calculate how many pages we need to hold at most MAX_PKTS_PER_BLOCK - * packets and align size to page boundary. - */ - num_frames = pool->num < MAX_PKTS_PER_BLOCK ? pool->num : - MAX_PKTS_PER_BLOCK; - ring->req.tp_block_size = (ring->req.tp_frame_size * num_frames + - (pz - 1)) & (-pz); - - if (!fanout) { - /* Single socket is in use. Use 1 block with buf_num frames. */ - ring->req.tp_block_nr = 1; - } else { - /* Fanout is in use, more likely taffic split accodring to - * number of cpu threads. Use cpu blocks and buf_num frames. */ - ring->req.tp_block_nr = odp_cpu_count(); + if (odp_global_ro.shm_single_va) + flags += ODP_SHM_SINGLE_VA; + + shm = odp_shm_reserve(NULL, ring_size, ODP_CACHE_LINE_SIZE, flags); + + if (shm == ODP_SHM_INVALID) { + ODP_ERR("Reserving shm failed\n"); + return -1; } + ring->shm = shm;
- ring->req.tp_frame_nr = ring->req.tp_block_size / - ring->req.tp_frame_size * ring->req.tp_block_nr; + ring->req.tp_block_size = block_size; + ring->req.tp_block_nr = block_nr; + ring->req.tp_frame_size = frame_size; + ring->req.tp_frame_nr = frame_nr;
ring->mm_len = ring->req.tp_block_size * ring->req.tp_block_nr; ring->rd_num = ring->req.tp_frame_nr; - ring->flen = ring->req.tp_frame_size; -} - -static int mmap_setup_ring(int sock, struct ring *ring, int type, - odp_pool_t pool_hdl, int fanout) -{ - int ret = 0; - int flags = 0; - odp_shm_t shm; - - ring->sock = sock; - ring->type = type; - ring->version = TPACKET_V2; - - mmap_fill_ring(ring, pool_hdl, fanout); + ring->flen = ring->req.tp_frame_size; + ring->rd_len = ring_size;
ODP_DBG(" tp_block_size %u\n", ring->req.tp_block_size); ODP_DBG(" tp_block_nr %u\n", ring->req.tp_block_nr); ODP_DBG(" tp_frame_size %u\n", ring->req.tp_frame_size); ODP_DBG(" tp_frame_nr %u\n", ring->req.tp_frame_nr); - ODP_DBG(" fanout %i\n", fanout); + ODP_DBG(" fanout %i\n", pkt_sock->fanout);
ret = setsockopt(sock, SOL_PACKET, type, &ring->req, sizeof(ring->req)); if (ret == -1) { @@ -448,19 +442,6 @@ static int mmap_setup_ring(int sock, struct ring *ring, int type, return -1; }
- ring->rd_len = ring->rd_num * sizeof(*ring->rd); - - if (odp_global_ro.shm_single_va) - flags += ODP_SHM_SINGLE_VA; - - shm = odp_shm_reserve(NULL, ring->rd_len, ODP_CACHE_LINE_SIZE, flags); - - if (shm == ODP_SHM_INVALID) { - ODP_ERR("Reserving shm failed\n"); - return -1; - } - ring->shm = shm; - ring->rd = odp_shm_addr(shm); if (!ring->rd) { ODP_ERR("Reading shm addr failed\n"); @@ -516,11 +497,17 @@ static int mmap_sock(pkt_sock_mmap_t *pkt_sock)
static int mmap_unmap_sock(pkt_sock_mmap_t *pkt_sock) { + int ret = 0; + if (pkt_sock->rx_ring.shm != ODP_SHM_INVALID) odp_shm_free(pkt_sock->rx_ring.shm); if (pkt_sock->tx_ring.shm != ODP_SHM_INVALID) odp_shm_free(pkt_sock->tx_ring.shm); - return munmap(pkt_sock->mmap_base, pkt_sock->mmap_len); + + if (pkt_sock->mmap_base != MAP_FAILED) + ret = munmap(pkt_sock->mmap_base, pkt_sock->mmap_len); + + return ret; }
static int mmap_bind_sock(pkt_sock_mmap_t *pkt_sock, const char *netdev) @@ -576,12 +563,14 @@ static int sock_mmap_open(odp_pktio_t id ODP_UNUSED, return -1;
pkt_sock_mmap_t *const pkt_sock = pkt_priv(pktio_entry); - int fanout = 1; + int fanout = 0;
/* Init pktio entry */ memset(pkt_sock, 0, sizeof(*pkt_sock)); /* set sockfd to -1, because a valid socked might be initialized to 0 */ pkt_sock->sockfd = -1; + pkt_sock->mmap_base = MAP_FAILED; + pkt_sock->fanout = fanout;
if (pool == ODP_POOL_INVALID) return -1; @@ -600,15 +589,19 @@ static int sock_mmap_open(odp_pktio_t id ODP_UNUSED, if (ret != 0) goto error;
+ pkt_sock->mtu = mtu_get_fd(pkt_sock->sockfd, netdev); + if (!pkt_sock->mtu) + goto error; + + ODP_DBG("MTU size: %i\n", pkt_sock->mtu); + ODP_DBG("TX ring setup:\n"); - ret = mmap_setup_ring(pkt_sock->sockfd, &pkt_sock->tx_ring, - PACKET_TX_RING, pool, fanout); + ret = mmap_setup_ring(pkt_sock, &pkt_sock->tx_ring, PACKET_TX_RING); if (ret != 0) goto error;
ODP_DBG("RX ring setup:\n"); - ret = mmap_setup_ring(pkt_sock->sockfd, &pkt_sock->rx_ring, - PACKET_RX_RING, pool, fanout); + ret = mmap_setup_ring(pkt_sock, &pkt_sock->rx_ring, PACKET_RX_RING); if (ret != 0) goto error;
@@ -620,10 +613,6 @@ static int sock_mmap_open(odp_pktio_t id ODP_UNUSED, if (ret != 0) goto error;
- pkt_sock->mtu = mtu_get_fd(pkt_sock->sockfd, netdev); - if (!pkt_sock->mtu) - goto error; - if_idx = if_nametoindex(netdev); if (if_idx == 0) { __odp_errno = errno; @@ -631,7 +620,6 @@ static int sock_mmap_open(odp_pktio_t id ODP_UNUSED, goto error; }
- pkt_sock->fanout = fanout; if (fanout) { ret = set_pkt_sock_fanout_mmap(pkt_sock, if_idx); if (ret != 0)
commit b60c92e93885de2f2b051648632a27413809a01f Author: Petri Savolainen petri.savolainen@nokia.com Date: Thu Sep 5 10:31:42 2019 +0300
linux-gen: socket_mmap: sync_synchronize not needed
Extra memory barrier is not needed on each receive frame status update. The receive function is guarded by a ticketlock which includes necessary memory barriers.
Signed-off-by: Petri Savolainen petri.savolainen@nokia.com Reviewed-by: Matias Elo matias.elo@nokia.com
diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index f64d5bd68..1f56773c7 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -135,7 +135,6 @@ static inline int mmap_rx_kernel_ready(struct tpacket2_hdr *hdr) static inline void mmap_rx_user_ready(struct tpacket2_hdr *hdr) { hdr->tp_status = TP_STATUS_KERNEL; - __sync_synchronize(); }
static uint8_t *pkt_mmap_vlan_insert(uint8_t *l2_hdr_ptr, @@ -204,13 +203,14 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, odp_pool_t pool = pkt_sock->pool; int pkts;
- if (!mmap_rx_kernel_ready(ring->rd[frame_num].iov_base)) + ppd.raw = ring->rd[frame_num].iov_base; + + if (!mmap_rx_kernel_ready(ppd.raw)) break;
if (ts != NULL) ts_val = odp_time_global();
- ppd.raw = ring->rd[frame_num].iov_base; next_frame_num = next_frame(frame_num, ring->rd_num);
pkt_buf = (uint8_t *)ppd.raw + ppd.v2->tp_h.tp_mac;
commit 5355304a9650a2929f080ff66fafeb2792632f06 Author: Petri Savolainen petri.savolainen@nokia.com Date: Thu Sep 5 10:05:25 2019 +0300
linux-gen: socket_mmap: fix tx side throughput collapse
Fix packet transmit side issue which caused packet rate to collapse on an overloaded system. Apparently tp_status got out of sync between ODP and kernel.
Signed-off-by: Petri Savolainen petri.savolainen@nokia.com Reviewed-by: Matias Elo matias.elo@nokia.com Reported-by: Jari Mustajärvi jari.mustajarvi@nokia-bell-labs.com
diff --git a/platform/linux-generic/pktio/socket_mmap.c b/platform/linux-generic/pktio/socket_mmap.c index 861152288..f64d5bd68 100644 --- a/platform/linux-generic/pktio/socket_mmap.c +++ b/platform/linux-generic/pktio/socket_mmap.c @@ -43,12 +43,6 @@ #include <protocols/eth.h> #include <protocols/ip.h>
-/* Maximum number of retries per sock_mmap_send() call */ -#define TX_RETRIES 10 - -/* Number of nanoseconds to wait between TX retries */ -#define TX_RETRY_NSEC 1000 - /* Maximum number of packets to store in each RX/TX block */ #define MAX_PKTS_PER_BLOCK 512
@@ -144,17 +138,6 @@ static inline void mmap_rx_user_ready(struct tpacket2_hdr *hdr) __sync_synchronize(); }
-static inline int mmap_tx_kernel_ready(struct tpacket2_hdr *hdr) -{ - return !(hdr->tp_status & (TP_STATUS_SEND_REQUEST | TP_STATUS_SENDING)); -} - -static inline void mmap_tx_user_ready(struct tpacket2_hdr *hdr) -{ - hdr->tp_status = TP_STATUS_SEND_REQUEST; - __sync_synchronize(); -} - static uint8_t *pkt_mmap_vlan_insert(uint8_t *l2_hdr_ptr, uint16_t mac_offset, uint16_t vlan_tci, @@ -303,118 +286,99 @@ static inline unsigned pkt_mmap_v2_rx(pktio_entry_t *pktio_entry, return nb_rx; }
-static unsigned handle_pending_frames(int sock, struct ring *ring, int frames) -{ - int i; - int retry = 0; - unsigned nb_tx = 0; - unsigned frame_num; - unsigned frame_count = ring->rd_num; - unsigned first_frame_num = ring->frame_num; - - for (frame_num = first_frame_num, i = 0; i < frames; i++) { - struct tpacket2_hdr *hdr = ring->rd[frame_num].iov_base; - - if (odp_likely(hdr->tp_status == TP_STATUS_AVAILABLE || - hdr->tp_status == TP_STATUS_SENDING)) { - nb_tx++; - } else if (hdr->tp_status == TP_STATUS_SEND_REQUEST) { - if (retry++ < TX_RETRIES) { - struct timespec ts = { .tv_nsec = TX_RETRY_NSEC, - .tv_sec = 0 }; - - sendto(sock, NULL, 0, MSG_DONTWAIT, NULL, 0); - nanosleep(&ts, NULL); - i--; - continue; - } else { - hdr->tp_status = TP_STATUS_AVAILABLE; - } - } else { /* TP_STATUS_WRONG_FORMAT */ - /* Don't try re-sending frames after failure */ - for (; i < frames; i++) { - hdr = ring->rd[frame_num].iov_base; - hdr->tp_status = TP_STATUS_AVAILABLE; - frame_num = next_frame(frame_num, frame_count); - } - break; - } - frame_num = next_frame(frame_num, frame_count); - } - - ring->frame_num = next_frame(first_frame_num + nb_tx - 1, frame_count); - - return nb_tx; -} - -static inline unsigned pkt_mmap_v2_tx(int sock, struct ring *ring, - const odp_packet_t pkt_table[], - unsigned num) +static inline int pkt_mmap_v2_tx(int sock, struct ring *ring, + const odp_packet_t pkt_table[], + uint32_t num) { - union frame_map ppd; - uint32_t pkt_len; - unsigned first_frame_num, frame_num, frame_count; + uint32_t i, pkt_len, num_tx; + uint32_t first_frame_num, frame_num, frame_count; int ret; uint8_t *buf; - unsigned i = 0; - unsigned nb_tx = 0; - int send_errno; + struct tpacket2_hdr *tp_hdr[num]; int total_len = 0;
first_frame_num = ring->frame_num; frame_num = first_frame_num; frame_count = ring->rd_num;
- while (i < num) { - ppd.raw = ring->rd[frame_num].iov_base; - if (!odp_unlikely(mmap_tx_kernel_ready(ppd.raw))) + if (num > frame_count) + num = frame_count; + + for (i = 0; i < num; i++) { + tp_hdr[i] = ring->rd[frame_num].iov_base; + + if (tp_hdr[i]->tp_status != TP_STATUS_AVAILABLE) { + if (tp_hdr[i]->tp_status == TP_STATUS_WRONG_FORMAT) { + ODP_ERR("Socket mmap: wrong format\n"); + return -1; + } + break; + }
pkt_len = odp_packet_len(pkt_table[i]); - ppd.v2->tp_h.tp_snaplen = pkt_len; - ppd.v2->tp_h.tp_len = pkt_len; + tp_hdr[i]->tp_len = pkt_len; total_len += pkt_len;
- buf = (uint8_t *)ppd.raw + TPACKET2_HDRLEN - + buf = (uint8_t *)(void *)tp_hdr[i] + TPACKET2_HDRLEN - sizeof(struct sockaddr_ll); odp_packet_copy_to_mem(pkt_table[i], 0, pkt_len, buf);
- mmap_tx_user_ready(ppd.raw); + tp_hdr[i]->tp_status = TP_STATUS_SEND_REQUEST;
frame_num = next_frame(frame_num, frame_count); - i++; }
- ret = sendto(sock, NULL, 0, MSG_DONTWAIT, NULL, 0); - send_errno = errno; + num = i; + num_tx = num;
- /* On success, the return value indicates the number of bytes sent. On - * failure a value of -1 is returned, even if the failure occurred - * after some of the packets in the ring have already been sent, so we - * need to inspect the packet status to determine which were sent. */ - if (odp_likely(ret == total_len)) { - nb_tx = i; - ring->frame_num = frame_num; - } else { - nb_tx = handle_pending_frames(sock, ring, i); - - if (odp_unlikely(ret == -1 && nb_tx == 0 && - SOCK_ERR_REPORT(send_errno))) { - __odp_errno = send_errno; - /* ENOBUFS indicates that the transmit queue is full, - * which will happen regularly when overloaded so don't - * print it */ - if (errno != ENOBUFS) - ODP_ERR("sendto(pkt mmap): %s\n", - strerror(send_errno)); + /* Ping kernel to send packets */ + ret = send(sock, NULL, 0, MSG_DONTWAIT); + + ring->frame_num = frame_num; + + if (odp_unlikely(ret != total_len)) { + uint32_t tp_status, frame_sum; + + /* Returns -1 when nothing is sent (send() would block) */ + if (ret < 0 && errno != EWOULDBLOCK) { + ODP_ERR("Socket mmap: send failed, ret %i, errno %i\n", + ret, errno); return -1; } + + /* Check how many first packets have been sent + * (TP_STATUS_AVAILABLE or TP_STATUS_SENDING). Assuming that + * the rest will not be sent. */ + for (i = 0; i < num; i++) { + tp_status = tp_hdr[i]->tp_status; + + if (tp_status == TP_STATUS_SEND_REQUEST) + break; + + if (tp_status == TP_STATUS_WRONG_FORMAT) { + ODP_ERR("Socket mmap: wrong format\n"); + break; + } + } + + num_tx = i; + + /* Clear status of not sent packets */ + for (i = num_tx; i < num; i++) + tp_hdr[i]->tp_status = TP_STATUS_AVAILABLE; + + frame_sum = first_frame_num + num_tx; + ring->frame_num = frame_sum; + + if (frame_sum >= frame_count) + ring->frame_num = frame_sum - frame_count; }
- for (i = 0; i < nb_tx; ++i) - odp_packet_free(pkt_table[i]); + /* Free sent packets */ + odp_packet_free_multi(pkt_table, num_tx);
- return nb_tx; + return num_tx; }
static void mmap_fill_ring(struct ring *ring, odp_pool_t pool_hdl, int fanout) @@ -471,6 +435,12 @@ static int mmap_setup_ring(int sock, struct ring *ring, int type,
mmap_fill_ring(ring, pool_hdl, fanout);
+ ODP_DBG(" tp_block_size %u\n", ring->req.tp_block_size); + ODP_DBG(" tp_block_nr %u\n", ring->req.tp_block_nr); + ODP_DBG(" tp_frame_size %u\n", ring->req.tp_frame_size); + ODP_DBG(" tp_frame_nr %u\n", ring->req.tp_frame_nr); + ODP_DBG(" fanout %i\n", fanout); + ret = setsockopt(sock, SOL_PACKET, type, &ring->req, sizeof(ring->req)); if (ret == -1) { __odp_errno = errno; @@ -630,11 +600,13 @@ static int sock_mmap_open(odp_pktio_t id ODP_UNUSED, if (ret != 0) goto error;
+ ODP_DBG("TX ring setup:\n"); ret = mmap_setup_ring(pkt_sock->sockfd, &pkt_sock->tx_ring, PACKET_TX_RING, pool, fanout); if (ret != 0) goto error;
+ ODP_DBG("RX ring setup:\n"); ret = mmap_setup_ring(pkt_sock->sockfd, &pkt_sock->rx_ring, PACKET_RX_RING, pool, fanout); if (ret != 0)
-----------------------------------------------------------------------
Summary of changes: platform/linux-generic/Makefile.am | 1 - platform/linux-generic/include/odp_packet_socket.h | 62 --- platform/linux-generic/m4/configure.m4 | 1 - platform/linux-generic/pktio/socket.c | 1 - platform/linux-generic/pktio/socket_mmap.c | 469 +++++++++------------ platform/linux-generic/test/Makefile.am | 4 - .../linux-generic/test/mmap_vlan_ins/.gitignore | 2 - .../linux-generic/test/mmap_vlan_ins/Makefile.am | 16 - .../test/mmap_vlan_ins/mmap_vlan_ins.c | 250 ----------- .../test/mmap_vlan_ins/mmap_vlan_ins.sh | 78 ---- .../linux-generic/test/mmap_vlan_ins/pktio_env | 120 ------ .../linux-generic/test/mmap_vlan_ins/vlan.pcap | Bin 9728 -> 0 bytes 12 files changed, 209 insertions(+), 795 deletions(-) delete mode 100644 platform/linux-generic/include/odp_packet_socket.h delete mode 100644 platform/linux-generic/test/mmap_vlan_ins/.gitignore delete mode 100644 platform/linux-generic/test/mmap_vlan_ins/Makefile.am delete mode 100644 platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.c delete mode 100755 platform/linux-generic/test/mmap_vlan_ins/mmap_vlan_ins.sh delete mode 100644 platform/linux-generic/test/mmap_vlan_ins/pktio_env delete mode 100644 platform/linux-generic/test/mmap_vlan_ins/vlan.pcap
hooks/post-receive