From: Zijian Zhang zijianzhang@bytedance.com
The MSG_ZEROCOPY flag enables copy avoidance for socket send calls. However, zerocopy is not a free lunch. Apart from the management of user pages, the combination of poll + recvmsg to receive notifications incurs unignorable overhead in the applications. We try to mitigate this overhead with a new notification mechanism based on msg_control. Leveraging the general framework to copy cmsgs to the user space, we copy zerocopy notifications to the user upon returning of sendmsgs.
Signed-off-by: Zijian Zhang zijianzhang@bytedance.com Signed-off-by: Xiaochun Lu xiaochun.lu@bytedance.com --- arch/alpha/include/uapi/asm/socket.h | 2 + arch/mips/include/uapi/asm/socket.h | 2 + arch/parisc/include/uapi/asm/socket.h | 2 + arch/sparc/include/uapi/asm/socket.h | 2 + include/linux/socket.h | 2 +- include/uapi/asm-generic/socket.h | 2 + include/uapi/linux/socket.h | 23 +++++++++ net/core/sock.c | 72 +++++++++++++++++++++++++-- 8 files changed, 102 insertions(+), 5 deletions(-)
diff --git a/arch/alpha/include/uapi/asm/socket.h b/arch/alpha/include/uapi/asm/socket.h index e94f621903fe..7c32d9dbe47f 100644 --- a/arch/alpha/include/uapi/asm/socket.h +++ b/arch/alpha/include/uapi/asm/socket.h @@ -140,6 +140,8 @@ #define SO_PASSPIDFD 76 #define SO_PEERPIDFD 77
+#define SCM_ZC_NOTIFICATION 78 + #if !defined(__KERNEL__)
#if __BITS_PER_LONG == 64 diff --git a/arch/mips/include/uapi/asm/socket.h b/arch/mips/include/uapi/asm/socket.h index 60ebaed28a4c..3f7fade998cb 100644 --- a/arch/mips/include/uapi/asm/socket.h +++ b/arch/mips/include/uapi/asm/socket.h @@ -151,6 +151,8 @@ #define SO_PASSPIDFD 76 #define SO_PEERPIDFD 77
+#define SCM_ZC_NOTIFICATION 78 + #if !defined(__KERNEL__)
#if __BITS_PER_LONG == 64 diff --git a/arch/parisc/include/uapi/asm/socket.h b/arch/parisc/include/uapi/asm/socket.h index be264c2b1a11..77f5bee0fdc9 100644 --- a/arch/parisc/include/uapi/asm/socket.h +++ b/arch/parisc/include/uapi/asm/socket.h @@ -132,6 +132,8 @@ #define SO_PASSPIDFD 0x404A #define SO_PEERPIDFD 0x404B
+#define SCM_ZC_NOTIFICATION 0x404C + #if !defined(__KERNEL__)
#if __BITS_PER_LONG == 64 diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h index 682da3714686..eb44fc515b45 100644 --- a/arch/sparc/include/uapi/asm/socket.h +++ b/arch/sparc/include/uapi/asm/socket.h @@ -133,6 +133,8 @@ #define SO_PASSPIDFD 0x0055 #define SO_PEERPIDFD 0x0056
+#define SCM_ZC_NOTIFICATION 0x0057 + #if !defined(__KERNEL__)
diff --git a/include/linux/socket.h b/include/linux/socket.h index 40173c919d0f..71e3c6ebfed5 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -171,7 +171,7 @@ static inline struct cmsghdr * cmsg_nxthdr (struct msghdr *__msg, struct cmsghdr
static inline bool cmsg_copy_to_user(struct cmsghdr *__cmsg) { - return 0; + return __cmsg->cmsg_type == SCM_ZC_NOTIFICATION; }
static inline size_t msg_data_left(struct msghdr *msg) diff --git a/include/uapi/asm-generic/socket.h b/include/uapi/asm-generic/socket.h index 8ce8a39a1e5f..02e9159c7944 100644 --- a/include/uapi/asm-generic/socket.h +++ b/include/uapi/asm-generic/socket.h @@ -135,6 +135,8 @@ #define SO_PASSPIDFD 76 #define SO_PEERPIDFD 77
+#define SCM_ZC_NOTIFICATION 78 + #if !defined(__KERNEL__)
#if __BITS_PER_LONG == 64 || (defined(__x86_64__) && defined(__ILP32__)) diff --git a/include/uapi/linux/socket.h b/include/uapi/linux/socket.h index d3fcd3b5ec53..b5b5fa9febb1 100644 --- a/include/uapi/linux/socket.h +++ b/include/uapi/linux/socket.h @@ -2,6 +2,8 @@ #ifndef _UAPI_LINUX_SOCKET_H #define _UAPI_LINUX_SOCKET_H
+#include <linux/types.h> + /* * Desired design of maximum size and alignment (see RFC2553) */ @@ -35,4 +37,25 @@ struct __kernel_sockaddr_storage { #define SOCK_TXREHASH_DISABLED 0 #define SOCK_TXREHASH_ENABLED 1
+#define ZC_NOTIFICATION_MAX 16 + +/* + * A zc_info_elem represents a completion notification for sendmsgs in range + * lo to high, zerocopy represents whether the underlying transmission is + * zerocopy or not. + */ +struct zc_info_elem { + __u32 lo; + __u32 hi; + __u8 zerocopy; +}; + +/* + * zc_info is the struct used for the SCM_ZC_NOTIFICATION control message. + */ +struct zc_info { + __u32 size; /* size of the zc_info_elem arr */ + struct zc_info_elem arr[]; +}; + #endif /* _UAPI_LINUX_SOCKET_H */ diff --git a/net/core/sock.c b/net/core/sock.c index b2cbe753af1d..37b1b12623ee 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -1481,10 +1481,12 @@ int sk_setsockopt(struct sock *sk, int level, int optname, ret = -EOPNOTSUPP; } if (!ret) { - if (val < 0 || val > 1) + if (val < 0 || val > 1) { ret = -EINVAL; - else + } else { sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool); + static_branch_enable(&tx_copy_cmsg_to_user_key); + } } break;
@@ -2826,8 +2828,8 @@ struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len, } EXPORT_SYMBOL(sock_alloc_send_pskb);
-int __sock_cmsg_send(struct sock *sk, struct msghdr *msg __always_unused, - struct cmsghdr *cmsg, struct sockcm_cookie *sockc) +int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg, + struct sockcm_cookie *sockc) { u32 tsflags;
@@ -2863,6 +2865,68 @@ int __sock_cmsg_send(struct sock *sk, struct msghdr *msg __always_unused, case SCM_RIGHTS: case SCM_CREDENTIALS: break; + case SCM_ZC_NOTIFICATION: { + struct zc_info *zc = CMSG_DATA(cmsg); + struct sk_buff_head *q, local_q; + int cmsg_data_len, i = 0; + unsigned long flags; + struct sk_buff *skb; + + if (!sock_flag(sk, SOCK_ZEROCOPY) || sk->sk_family == PF_RDS) + return -EINVAL; + + cmsg_data_len = cmsg->cmsg_len - sizeof(struct cmsghdr); + if (cmsg_data_len < sizeof(struct zc_info)) + return -EINVAL; + + if (zc->size > ZC_NOTIFICATION_MAX || + (cmsg_data_len - sizeof(struct zc_info)) != + (zc->size * sizeof(struct zc_info_elem))) + return -EINVAL; + + q = &sk->sk_error_queue; + skb_queue_head_init(&local_q); + + /* Get zerocopy error messages from sk_error_queue, and add them + * to a local queue for later processing. This minimizes the + * code while the spinlock is held and irq is disabled. + */ + spin_lock_irqsave(&q->lock, flags); + skb = skb_peek(q); + while (skb && i < zc->size) { + struct sk_buff *skb_next = skb_peek_next(skb, q); + struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); + + if (serr->ee.ee_errno != 0 || + serr->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY) { + skb = skb_next; + continue; + } + + __skb_unlink(skb, q); + __skb_queue_tail(&local_q, skb); + skb = skb_next; + i++; + } + spin_unlock_irqrestore(&q->lock, flags); + + i = 0; + while ((skb = skb_peek(&local_q)) != NULL) { + struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); + + zc->arr[i].hi = serr->ee.ee_data; + zc->arr[i].lo = serr->ee.ee_info; + zc->arr[i].zerocopy = !(serr->ee.ee_code + & SO_EE_CODE_ZEROCOPY_COPIED); + __skb_unlink(skb, &local_q); + consume_skb(skb); + i++; + } + + zc->size = i; + msg->msg_control_copy_to_user = true; + break; + } default: return -EINVAL; }