diff options
| author | David S. Miller <davem@davemloft.net> | 2023-05-17 08:38:42 +0100 |
|---|---|---|
| committer | David S. Miller <davem@davemloft.net> | 2023-05-17 08:38:42 +0100 |
| commit | 81cf1ade0bb3a42023dd1694ed080c4adb7dafd3 (patch) | |
| tree | ea527346b81a27e1bd3d80a2ea1ff207034e51b4 | |
| parent | 833e24aeb4d9a4803af3b836464df01293ce9041 (diff) | |
| parent | a7533584728d366f11b210d0e8a2fad03109c669 (diff) | |
Merge branch 'tcp-io_uring-zc-opts'
Merge branch 'tcp-io_uring-zc-opts'
Pavel Begunkov says:
====================
minor tcp io_uring zc optimisations
Patch 1 is a simple cleanup, patch 2 gives removes 2 atomics from the
io_uring zc TCP submission path, which yielded extra 0.5% for my
throughput CPU bound tests based on liburing/examples/send-zerocopy.c
====================
Reviewed-by: David Ahern <dsahern@kernel.org>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
| -rw-r--r-- | net/ipv4/tcp.c | 12 |
1 files changed, 7 insertions, 5 deletions
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 4d6392c16b7a..3d18e295bb2f 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1229,13 +1229,11 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) flags = msg->msg_flags; if ((flags & MSG_ZEROCOPY) && size) { - skb = tcp_write_queue_tail(sk); - if (msg->msg_ubuf) { uarg = msg->msg_ubuf; - net_zcopy_get(uarg); zc = sk->sk_route_caps & NETIF_F_SG; } else if (sock_flag(sk, SOCK_ZEROCOPY)) { + skb = tcp_write_queue_tail(sk); uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb)); if (!uarg) { err = -ENOBUFS; @@ -1459,7 +1457,9 @@ out: tcp_push(sk, flags, mss_now, tp->nonagle, size_goal); } out_nopush: - net_zcopy_put(uarg); + /* msg->msg_ubuf is pinned by the caller so we don't take extra refs */ + if (uarg && !msg->msg_ubuf) + net_zcopy_put(uarg); return copied + copied_syn; do_error: @@ -1468,7 +1468,9 @@ do_error: if (copied + copied_syn) goto out; out_err: - net_zcopy_put_abort(uarg, true); + /* msg->msg_ubuf is pinned by the caller so we don't take extra refs */ + if (uarg && !msg->msg_ubuf) + net_zcopy_put_abort(uarg, true); err = sk_stream_error(sk, flags, err); /* make sure we wake any epoll edge trigger waiter */ if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) { |