Previously, sockmap for AF_UNIX protocol only supports dgram type. This patch add unix stream type support, which is similar to unix_dgram_proto. To support sockmap, dgram and stream cannot share the same unix_proto anymore, because they have different implementations, such as unhash for stream type (which will remove closed or disconnected sockets from the map), so rename unix_proto to unix_dgram_proto and add a new unix_stream_proto.

Also implement stream related sockmap functions. And add dgram key words to those dgram specific functions.

Signed-off-by: Jiang Wang jiang.wang@bytedance.com Reviewed-by: Cong Wang cong.wang@bytedance.com --- include/net/af_unix.h | 8 +++- net/unix/af_unix.c | 83 ++++++++++++++++++++++++++++++++------ net/unix/unix_bpf.c | 93 +++++++++++++++++++++++++++++++++---------- 3 files changed, 147 insertions(+), 37 deletions(-)

diff --git a/include/net/af_unix.h b/include/net/af_unix.h index 435a2c3d5a6f..5d04fbf8aac8 100644 --- a/include/net/af_unix.h +++ b/include/net/af_unix.h @@ -84,6 +84,8 @@ long unix_outq_len(struct sock *sk);

int __unix_dgram_recvmsg(struct sock *sk, struct msghdr *msg, size_t size, int flags); +int __unix_stream_recvmsg(struct sock *sk, struct msghdr *msg, size_t size, + int flags); #ifdef CONFIG_SYSCTL int unix_sysctl_register(struct net *net); void unix_sysctl_unregister(struct net *net); @@ -93,9 +95,11 @@ static inline void unix_sysctl_unregister(struct net *net) {} #endif

#ifdef CONFIG_BPF_SYSCALL -extern struct proto unix_proto; +extern struct proto unix_dgram_proto; +extern struct proto unix_stream_proto;

-int unix_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore); +int unix_dgram_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore); +int unix_stream_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore); void __init unix_bpf_build_proto(void); #else static inline void __init unix_bpf_build_proto(void) diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index c020ad0e8438..5b050a601f46 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -791,17 +791,35 @@ static void unix_close(struct sock *sk, long timeout) */ }

-struct proto unix_proto = { - .name = "UNIX", +static void unix_unhash(struct sock *sk) +{ + /* Nothing to do here, unix socket does not need a ->unhash(). + * This is merely for sockmap. + */ +} + +struct proto unix_dgram_proto = { + .name = "UNIX-DGRAM", + .owner = THIS_MODULE, + .obj_size = sizeof(struct unix_sock), + .close = unix_close, +#ifdef CONFIG_BPF_SYSCALL + .psock_update_sk_prot = unix_dgram_bpf_update_proto, +#endif +}; + +struct proto unix_stream_proto = { + .name = "UNIX-STREAM", .owner = THIS_MODULE, .obj_size = sizeof(struct unix_sock), .close = unix_close, + .unhash = unix_unhash, #ifdef CONFIG_BPF_SYSCALL - .psock_update_sk_prot = unix_bpf_update_proto, + .psock_update_sk_prot = unix_stream_bpf_update_proto, #endif };

-static struct sock *unix_create1(struct net *net, struct socket *sock, int kern) +static struct sock *unix_create1(struct net *net, struct socket *sock, int kern, int type) { struct sock *sk = NULL; struct unix_sock *u; @@ -810,7 +828,11 @@ static struct sock *unix_create1(struct net *net, struct socket *sock, int kern) if (atomic_long_read(&unix_nr_socks) > 2 * get_max_files()) goto out;

- sk = sk_alloc(net, PF_UNIX, GFP_KERNEL, &unix_proto, kern); + if (type == SOCK_STREAM) + sk = sk_alloc(net, PF_UNIX, GFP_KERNEL, &unix_stream_proto, kern); + else /*dgram and seqpacket */ + sk = sk_alloc(net, PF_UNIX, GFP_KERNEL, &unix_dgram_proto, kern); + if (!sk) goto out;

@@ -872,7 +894,7 @@ static int unix_create(struct net *net, struct socket *sock, int protocol, return -ESOCKTNOSUPPORT; }

- return unix_create1(net, sock, kern) ? 0 : -ENOMEM; + return unix_create1(net, sock, kern, sock->type) ? 0 : -ENOMEM; }

static int unix_release(struct socket *sock) @@ -1286,7 +1308,7 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr, err = -ENOMEM;

/* create new sock for complete connection */ - newsk = unix_create1(sock_net(sk), NULL, 0); + newsk = unix_create1(sock_net(sk), NULL, 0, sock->type); if (newsk == NULL) goto out;

@@ -2261,8 +2283,10 @@ static int unix_dgram_recvmsg(struct socket *sock, struct msghdr *msg, size_t si struct sock *sk = sock->sk;

#ifdef CONFIG_BPF_SYSCALL - if (sk->sk_prot != &unix_proto) - return sk->sk_prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT, + const struct proto *prot = READ_ONCE(sk->sk_prot); + + if (prot != &unix_dgram_proto) + return prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT, flags & ~MSG_DONTWAIT, NULL); #endif return __unix_dgram_recvmsg(sk, msg, size, flags); @@ -2580,6 +2604,20 @@ static int unix_stream_read_actor(struct sk_buff *skb, return ret ?: chunk; }

+int __unix_stream_recvmsg(struct sock *sk, struct msghdr *msg, + size_t size, int flags) +{ + struct unix_stream_read_state state = { + .recv_actor = unix_stream_read_actor, + .socket = sk->sk_socket, + .msg = msg, + .size = size, + .flags = flags + }; + + return unix_stream_read_generic(&state, true); +} + static int unix_stream_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, int flags) { @@ -2591,6 +2629,14 @@ static int unix_stream_recvmsg(struct socket *sock, struct msghdr *msg, .flags = flags };

+#ifdef CONFIG_BPF_SYSCALL + struct sock *sk = sock->sk; + const struct proto *prot = READ_ONCE(sk->sk_prot); + + if (prot != &unix_stream_proto) + return prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT, + flags & ~MSG_DONTWAIT, NULL); +#endif return unix_stream_read_generic(&state, true); }

@@ -2651,7 +2697,9 @@ static int unix_shutdown(struct socket *sock, int mode) (sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET)) {

int peer_mode = 0; + const struct proto *prot = READ_ONCE(other->sk_prot);

+ prot->unhash(other); if (mode&RCV_SHUTDOWN) peer_mode |= SEND_SHUTDOWN; if (mode&SEND_SHUTDOWN) @@ -2660,10 +2708,12 @@ static int unix_shutdown(struct socket *sock, int mode) other->sk_shutdown |= peer_mode; unix_state_unlock(other); other->sk_state_change(other); - if (peer_mode == SHUTDOWN_MASK) + if (peer_mode == SHUTDOWN_MASK) { sk_wake_async(other, SOCK_WAKE_WAITD, POLL_HUP); - else if (peer_mode & RCV_SHUTDOWN) + other->sk_state = TCP_CLOSE; + } else if (peer_mode & RCV_SHUTDOWN) { sk_wake_async(other, SOCK_WAKE_WAITD, POLL_IN); + } } if (other) sock_put(other); @@ -3040,7 +3090,13 @@ static int __init af_unix_init(void)

BUILD_BUG_ON(sizeof(struct unix_skb_parms) > sizeof_field(struct sk_buff, cb));

- rc = proto_register(&unix_proto, 1); + rc = proto_register(&unix_dgram_proto, 1); + if (rc != 0) { + pr_crit("%s: Cannot create unix_sock SLAB cache!\n", __func__); + goto out; + } + + rc = proto_register(&unix_stream_proto, 1); if (rc != 0) { pr_crit("%s: Cannot create unix_sock SLAB cache!\n", __func__); goto out; @@ -3056,7 +3112,8 @@ static int __init af_unix_init(void) static void __exit af_unix_exit(void) { sock_unregister(PF_UNIX); - proto_unregister(&unix_proto); + proto_unregister(&unix_dgram_proto); + proto_unregister(&unix_stream_proto); unregister_pernet_subsys(&unix_net_ops); }

diff --git a/net/unix/unix_bpf.c b/net/unix/unix_bpf.c index 20f53575b5c9..b927e2baae50 100644 --- a/net/unix/unix_bpf.c +++ b/net/unix/unix_bpf.c @@ -38,9 +38,18 @@ static int unix_msg_wait_data(struct sock *sk, struct sk_psock *psock, return ret; }

-static int unix_dgram_bpf_recvmsg(struct sock *sk, struct msghdr *msg, - size_t len, int nonblock, int flags, - int *addr_len) +static int __unix_recvmsg(struct sock *sk, struct msghdr *msg, + size_t len, int flags) +{ + if (sk->sk_type == SOCK_DGRAM) + return __unix_dgram_recvmsg(sk, msg, len, flags); + else + return __unix_stream_recvmsg(sk, msg, len, flags); +} + +static int unix_bpf_recvmsg(struct sock *sk, struct msghdr *msg, + size_t len, int nonblock, int flags, + int *addr_len) { struct unix_sock *u = unix_sk(sk); struct sk_psock *psock; @@ -48,14 +57,14 @@ static int unix_dgram_bpf_recvmsg(struct sock *sk, struct msghdr *msg,

psock = sk_psock_get(sk); if (unlikely(!psock)) - return __unix_dgram_recvmsg(sk, msg, len, flags); + return __unix_recvmsg(sk, msg, len, flags);

mutex_lock(&u->iolock); if (!skb_queue_empty(&sk->sk_receive_queue) && sk_psock_queue_empty(psock)) { mutex_unlock(&u->iolock); sk_psock_put(sk, psock); - return __unix_dgram_recvmsg(sk, msg, len, flags); + return __unix_recvmsg(sk, msg, len, flags); }

msg_bytes_ready: @@ -71,7 +80,7 @@ static int unix_dgram_bpf_recvmsg(struct sock *sk, struct msghdr *msg, goto msg_bytes_ready; mutex_unlock(&u->iolock); sk_psock_put(sk, psock); - return __unix_dgram_recvmsg(sk, msg, len, flags); + return __unix_recvmsg(sk, msg, len, flags); } copied = -EAGAIN; } @@ -80,30 +89,55 @@ static int unix_dgram_bpf_recvmsg(struct sock *sk, struct msghdr *msg, return copied; }

-static struct proto *unix_prot_saved __read_mostly; -static DEFINE_SPINLOCK(unix_prot_lock); -static struct proto unix_bpf_prot; +static struct proto *unix_dgram_prot_saved __read_mostly; +static DEFINE_SPINLOCK(unix_dgram_prot_lock); +static struct proto unix_dgram_bpf_prot; + +static struct proto *unix_stream_prot_saved __read_mostly; +static DEFINE_SPINLOCK(unix_stream_prot_lock); +static struct proto unix_stream_bpf_prot;

-static void unix_bpf_rebuild_protos(struct proto *prot, const struct proto *base) +static void unix_dgram_bpf_rebuild_protos(struct proto *prot, const struct proto *base) { *prot = *base; prot->close = sock_map_close; - prot->recvmsg = unix_dgram_bpf_recvmsg; + prot->recvmsg = unix_bpf_recvmsg; +} + +static void unix_stream_bpf_rebuild_protos(struct proto *prot, + const struct proto *base) +{ + *prot = *base; + prot->close = sock_map_close; + prot->recvmsg = unix_bpf_recvmsg; + prot->unhash = sock_map_unhash; +} + +static void unix_dgram_bpf_check_needs_rebuild(struct proto *ops) +{ + if (unlikely(ops != smp_load_acquire(&unix_dgram_prot_saved))) { + spin_lock_bh(&unix_dgram_prot_lock); + if (likely(ops != unix_dgram_prot_saved)) { + unix_dgram_bpf_rebuild_protos(&unix_dgram_bpf_prot, ops); + smp_store_release(&unix_dgram_prot_saved, ops); + } + spin_unlock_bh(&unix_dgram_prot_lock); + } }

-static void unix_bpf_check_needs_rebuild(struct proto *ops) +static void unix_stream_bpf_check_needs_rebuild(struct proto *ops) { - if (unlikely(ops != smp_load_acquire(&unix_prot_saved))) { - spin_lock_bh(&unix_prot_lock); - if (likely(ops != unix_prot_saved)) { - unix_bpf_rebuild_protos(&unix_bpf_prot, ops); - smp_store_release(&unix_prot_saved, ops); + if (unlikely(ops != smp_load_acquire(&unix_stream_prot_saved))) { + spin_lock_bh(&unix_stream_prot_lock); + if (likely(ops != unix_stream_prot_saved)) { + unix_stream_bpf_rebuild_protos(&unix_stream_bpf_prot, ops); + smp_store_release(&unix_stream_prot_saved, ops); } - spin_unlock_bh(&unix_prot_lock); + spin_unlock_bh(&unix_stream_prot_lock); } }

-int unix_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) +int unix_dgram_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) { if (sk->sk_type != SOCK_DGRAM) return -EOPNOTSUPP; @@ -114,12 +148,27 @@ int unix_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) return 0; }

- unix_bpf_check_needs_rebuild(psock->sk_proto); - WRITE_ONCE(sk->sk_prot, &unix_bpf_prot); + unix_dgram_bpf_check_needs_rebuild(psock->sk_proto); + WRITE_ONCE(sk->sk_prot, &unix_dgram_bpf_prot); + return 0; +} + +int unix_stream_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) +{ + if (restore) { + sk->sk_write_space = psock->saved_write_space; + WRITE_ONCE(sk->sk_prot, psock->sk_proto); + return 0; + } + + unix_stream_bpf_check_needs_rebuild(psock->sk_proto); + WRITE_ONCE(sk->sk_prot, &unix_stream_bpf_prot); return 0; }

void __init unix_bpf_build_proto(void) { - unix_bpf_rebuild_protos(&unix_bpf_prot, &unix_proto); + unix_dgram_bpf_rebuild_protos(&unix_dgram_bpf_prot, &unix_dgram_proto); + unix_stream_bpf_rebuild_protos(&unix_stream_bpf_prot, &unix_stream_proto); + }

This is to prepare for adding new unix stream tests. Mostly renames, also pass the socket types as an argument.

Signed-off-by: Jiang Wang jiang.wang@bytedance.com Reviewed-by: Cong Wang cong.wang@bytedance.com Acked-by: John Fastabend john.fastabend@gmail.com --- .../selftests/bpf/prog_tests/sockmap_listen.c | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c index 7a976d43281a..07ed8081f9ae 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c @@ -1692,14 +1692,14 @@ static void test_reuseport(struct test_sockmap_listen *skel, } }

-static int udp_socketpair(int family, int *s, int *c) +static int inet_socketpair(int family, int type, int *s, int *c) { struct sockaddr_storage addr; socklen_t len; int p0, c0; int err;

- p0 = socket_loopback(family, SOCK_DGRAM | SOCK_NONBLOCK); + p0 = socket_loopback(family, type | SOCK_NONBLOCK); if (p0 < 0) return p0;

@@ -1708,7 +1708,7 @@ static int udp_socketpair(int family, int *s, int *c) if (err) goto close_peer0;

- c0 = xsocket(family, SOCK_DGRAM | SOCK_NONBLOCK, 0); + c0 = xsocket(family, type | SOCK_NONBLOCK, 0); if (c0 < 0) { err = c0; goto close_peer0; @@ -1747,10 +1747,10 @@ static void udp_redir_to_connected(int family, int sock_mapfd, int verd_mapfd,

zero_verdict_count(verd_mapfd);

- err = udp_socketpair(family, &p0, &c0); + err = inet_socketpair(family, SOCK_DGRAM, &p0, &c0); if (err) return; - err = udp_socketpair(family, &p1, &c1); + err = inet_socketpair(family, SOCK_DGRAM, &p1, &c1); if (err) goto close_cli0;

@@ -1825,7 +1825,7 @@ static void test_udp_redir(struct test_sockmap_listen *skel, struct bpf_map *map udp_skb_redir_to_connected(skel, map, family); }

-static void udp_unix_redir_to_connected(int family, int sock_mapfd, +static void inet_unix_redir_to_connected(int family, int type, int sock_mapfd, int verd_mapfd, enum redir_mode mode) { const char *log_prefix = redir_mode_str(mode); @@ -1843,7 +1843,7 @@ static void udp_unix_redir_to_connected(int family, int sock_mapfd, return; c0 = sfd[0], p0 = sfd[1];

- err = udp_socketpair(family, &p1, &c1); + err = inet_socketpair(family, SOCK_DGRAM, &p1, &c1); if (err) goto close;

@@ -1897,14 +1897,16 @@ static void udp_unix_skb_redir_to_connected(struct test_sockmap_listen *skel, return;

skel->bss->test_ingress = false; - udp_unix_redir_to_connected(family, sock_map, verdict_map, REDIR_EGRESS); + inet_unix_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map, + REDIR_EGRESS); skel->bss->test_ingress = true; - udp_unix_redir_to_connected(family, sock_map, verdict_map, REDIR_INGRESS); + inet_unix_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map, + REDIR_INGRESS);

xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT); }

-static void unix_udp_redir_to_connected(int family, int sock_mapfd, +static void unix_inet_redir_to_connected(int family, int type, int sock_mapfd, int verd_mapfd, enum redir_mode mode) { const char *log_prefix = redir_mode_str(mode); @@ -1917,7 +1919,7 @@ static void unix_udp_redir_to_connected(int family, int sock_mapfd,

zero_verdict_count(verd_mapfd);

- err = udp_socketpair(family, &p0, &c0); + err = inet_socketpair(family, SOCK_DGRAM, &p0, &c0); if (err) return;

@@ -1972,9 +1974,11 @@ static void unix_udp_skb_redir_to_connected(struct test_sockmap_listen *skel, return;

skel->bss->test_ingress = false; - unix_udp_redir_to_connected(family, sock_map, verdict_map, REDIR_EGRESS); + unix_inet_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map, + REDIR_EGRESS); skel->bss->test_ingress = true; - unix_udp_redir_to_connected(family, sock_map, verdict_map, REDIR_INGRESS); + unix_inet_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map, + REDIR_INGRESS);

xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT); }