diff options
| author | Jens Axboe <axboe@kernel.dk> | 2022-07-24 18:40:31 -0600 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2022-07-24 18:40:31 -0600 |
| commit | 32e09298c8b3ff29177c825ab711a4a692d4caad (patch) | |
| tree | 767c8a995d6172596843abdbe03f4b1413f5a105 /net/ipv4/tcp.c | |
| parent | 32346491ddf24599decca06190ebca03ff9de7f8 (diff) | |
| parent | 2829a267fca297983b9302fa2cb833757328c857 (diff) | |
Merge branch 'io_uring-zerocopy-send' of git://git.kernel.org/pub/scm/linux/kernel/git/kuba/linux into for-5.20/io_uring-zerocopy-send
Merge prep net series for io_uring tx zc from the Jakub's tree.
* 'io_uring-zerocopy-send' of git://git.kernel.org/pub/scm/linux/kernel/git/kuba/linux:
net: fix uninitialised msghdr->sg_from_iter
tcp: support externally provided ubufs
ipv6/udp: support externally provided ubufs
ipv4/udp: support externally provided ubufs
net: introduce __skb_fill_page_desc_noacc
net: introduce managed frags infrastructure
net: Allow custom iter handler in msghdr
skbuff: carry external ubuf_info in msghdr
skbuff: add SKBFL_DONT_ORPHAN flag
skbuff: don't mix ubuf_info from different sources
ipv6: avoid partial copy for zc
ipv4: avoid partial copy for zc
Diffstat (limited to 'net/ipv4/tcp.c')
| -rw-r--r-- | net/ipv4/tcp.c | 33 |
1 files changed, 22 insertions, 11 deletions
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 028513d3e2a2..fdb80b9608fe 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1202,17 +1202,23 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) flags = msg->msg_flags; - if (flags & MSG_ZEROCOPY && size && sock_flag(sk, SOCK_ZEROCOPY)) { + if ((flags & MSG_ZEROCOPY) && size) { skb = tcp_write_queue_tail(sk); - uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb)); - if (!uarg) { - err = -ENOBUFS; - goto out_err; - } - zc = sk->sk_route_caps & NETIF_F_SG; - if (!zc) - uarg->zerocopy = 0; + if (msg->msg_ubuf) { + uarg = msg->msg_ubuf; + net_zcopy_get(uarg); + zc = sk->sk_route_caps & NETIF_F_SG; + } else if (sock_flag(sk, SOCK_ZEROCOPY)) { + uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb)); + if (!uarg) { + err = -ENOBUFS; + goto out_err; + } + zc = sk->sk_route_caps & NETIF_F_SG; + if (!zc) + uarg->zerocopy = 0; + } } if (unlikely(flags & MSG_FASTOPEN || inet_sk(sk)->defer_connect) && @@ -1335,8 +1341,13 @@ new_segment: copy = min_t(int, copy, pfrag->size - pfrag->offset); - if (tcp_downgrade_zcopy_pure(sk, skb) || - !sk_wmem_schedule(sk, copy)) + if (unlikely(skb_zcopy_pure(skb) || skb_zcopy_managed(skb))) { + if (tcp_downgrade_zcopy_pure(sk, skb)) + goto wait_for_space; + skb_zcopy_downgrade_managed(skb); + } + + if (!sk_wmem_schedule(sk, copy)) goto wait_for_space; err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb, |