From dfec0ee22c0a4488e4f4c764b2720d3096920383 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 7 May 2018 11:08:22 -0700 Subject: [PATCH 1/6] udp: Record gso_segs when supporting UDP segmentation offload We need to record the number of segments that will be generated when this frame is segmented. The expectation is that if gso_size is set then gso_segs is set as well. Without this some drivers such as ixgbe get confused if they attempt to offload this as they record 0 segments for the entire packet instead of the correct value. Reviewed-by: Eric Dumazet Acked-by: Willem de Bruijn Signed-off-by: Alexander Duyck Signed-off-by: David S. Miller --- net/ipv4/udp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index dd3102a37ef9..e07db83b311e 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -793,6 +793,8 @@ static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4, skb_shinfo(skb)->gso_size = cork->gso_size; skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4; + skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(len - sizeof(uh), + cork->gso_size); goto csum_partial; } From b21c034b3df833b5d9db1cfdc3938dbb0d7995c6 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 7 May 2018 11:08:28 -0700 Subject: [PATCH 2/6] udp: Do not pass MSS as parameter to GSO segmentation There is no point in passing MSS as a parameter for for the GSO segmentation call as it is already available via the shared info for the skb itself. Reviewed-by: Eric Dumazet Acked-by: Willem de Bruijn Signed-off-by: Alexander Duyck Signed-off-by: David S. Miller --- include/net/udp.h | 2 +- net/ipv4/udp_offload.c | 6 ++++-- net/ipv6/udp_offload.c | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/include/net/udp.h b/include/net/udp.h index 05990746810e..8bd83b044ecd 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -176,7 +176,7 @@ int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup); struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, netdev_features_t features, - unsigned int mss, __sum16 check); + __sum16 check); static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb) { diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index 006257092f06..c1afcd2f1a76 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -189,14 +189,16 @@ EXPORT_SYMBOL(skb_udp_tunnel_segment); struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, netdev_features_t features, - unsigned int mss, __sum16 check) + __sum16 check) { struct sock *sk = gso_skb->sk; unsigned int sum_truesize = 0; struct sk_buff *segs, *seg; unsigned int hdrlen; struct udphdr *uh; + unsigned int mss; + mss = skb_shinfo(gso_skb)->gso_size; if (gso_skb->len <= sizeof(*uh) + mss) return ERR_PTR(-EINVAL); @@ -244,7 +246,7 @@ static struct sk_buff *__udp4_gso_segment(struct sk_buff *gso_skb, if (!can_checksum_protocol(features, htons(ETH_P_IP))) return ERR_PTR(-EIO); - return __udp_gso_segment(gso_skb, features, mss, + return __udp_gso_segment(gso_skb, features, udp_v4_check(sizeof(struct udphdr) + mss, iph->saddr, iph->daddr, 0)); } diff --git a/net/ipv6/udp_offload.c b/net/ipv6/udp_offload.c index f7b85b1e6b3e..dea03ec09715 100644 --- a/net/ipv6/udp_offload.c +++ b/net/ipv6/udp_offload.c @@ -26,7 +26,7 @@ static struct sk_buff *__udp6_gso_segment(struct sk_buff *gso_skb, if (!can_checksum_protocol(features, htons(ETH_P_IPV6))) return ERR_PTR(-EIO); - return __udp_gso_segment(gso_skb, features, mss, + return __udp_gso_segment(gso_skb, features, udp_v6_check(sizeof(struct udphdr) + mss, &ip6h->saddr, &ip6h->daddr, 0)); } From 9a0d41b3598ff62ecb26661bbfb1d523586cdea3 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 7 May 2018 11:08:34 -0700 Subject: [PATCH 3/6] udp: Do not pass checksum as a parameter to GSO segmentation This patch is meant to allow us to avoid having to recompute the checksum from scratch and have it passed as a parameter. Instead of taking that approach we can take advantage of the fact that the length that was used to compute the existing checksum is included in the UDP header. Finally to avoid the need to invert the result we can just call csum16_add and csum16_sub directly. By doing this we can avoid a number of instructions in the loop that is handling segmentation. Signed-off-by: Alexander Duyck Acked-by: Willem de Bruijn Signed-off-by: David S. Miller --- include/net/udp.h | 3 +-- net/ipv4/udp_offload.c | 32 ++++++++++++++++++-------------- net/ipv6/udp_offload.c | 7 +------ 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/include/net/udp.h b/include/net/udp.h index 8bd83b044ecd..9289b6425032 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -175,8 +175,7 @@ struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb, int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup); struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, - netdev_features_t features, - __sum16 check); + netdev_features_t features); static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb) { diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index c1afcd2f1a76..92c182e99ddc 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -188,8 +188,7 @@ struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb, EXPORT_SYMBOL(skb_udp_tunnel_segment); struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, - netdev_features_t features, - __sum16 check) + netdev_features_t features) { struct sock *sk = gso_skb->sk; unsigned int sum_truesize = 0; @@ -197,6 +196,8 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, unsigned int hdrlen; struct udphdr *uh; unsigned int mss; + __sum16 check; + __be16 newlen; mss = skb_shinfo(gso_skb)->gso_size; if (gso_skb->len <= sizeof(*uh) + mss) @@ -215,17 +216,25 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, return segs; } + uh = udp_hdr(segs); + + /* compute checksum adjustment based on old length versus new */ + newlen = htons(sizeof(*uh) + mss); + check = csum16_add(csum16_sub(uh->check, uh->len), newlen); + for (seg = segs; seg; seg = seg->next) { uh = udp_hdr(seg); - uh->len = htons(seg->len - hdrlen); - uh->check = check; /* last packet can be partial gso_size */ - if (!seg->next) - csum_replace2(&uh->check, htons(mss), - htons(seg->len - hdrlen - sizeof(*uh))); + if (!seg->next) { + newlen = htons(seg->len - hdrlen); + check = csum16_add(csum16_sub(uh->check, uh->len), + newlen); + } + + uh->len = newlen; + uh->check = check; - uh->check = ~uh->check; seg->destructor = sock_wfree; seg->sk = sk; sum_truesize += seg->truesize; @@ -240,15 +249,10 @@ EXPORT_SYMBOL_GPL(__udp_gso_segment); static struct sk_buff *__udp4_gso_segment(struct sk_buff *gso_skb, netdev_features_t features) { - const struct iphdr *iph = ip_hdr(gso_skb); - unsigned int mss = skb_shinfo(gso_skb)->gso_size; - if (!can_checksum_protocol(features, htons(ETH_P_IP))) return ERR_PTR(-EIO); - return __udp_gso_segment(gso_skb, features, - udp_v4_check(sizeof(struct udphdr) + mss, - iph->saddr, iph->daddr, 0)); + return __udp_gso_segment(gso_skb, features); } static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, diff --git a/net/ipv6/udp_offload.c b/net/ipv6/udp_offload.c index dea03ec09715..61e34f1d2fa2 100644 --- a/net/ipv6/udp_offload.c +++ b/net/ipv6/udp_offload.c @@ -20,15 +20,10 @@ static struct sk_buff *__udp6_gso_segment(struct sk_buff *gso_skb, netdev_features_t features) { - const struct ipv6hdr *ip6h = ipv6_hdr(gso_skb); - unsigned int mss = skb_shinfo(gso_skb)->gso_size; - if (!can_checksum_protocol(features, htons(ETH_P_IPV6))) return ERR_PTR(-EIO); - return __udp_gso_segment(gso_skb, features, - udp_v6_check(sizeof(struct udphdr) + mss, - &ip6h->saddr, &ip6h->daddr, 0)); + return __udp_gso_segment(gso_skb, features); } static struct sk_buff *udp6_ufo_fragment(struct sk_buff *skb, From 0ad6509571e06b302d519f2f05e616ac8c1a10d7 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 7 May 2018 11:08:40 -0700 Subject: [PATCH 4/6] udp: Partially unroll handling of first segment and last segment This patch allows us to take care of unrolling the first segment and the last segment of the loop for processing the segmented skb. Part of the motivation for this is that it makes it easier to process the fact that the first fame and all of the frames in between should be mostly identical in terms of header data, and the last frame has differences in the length and partial checksum. In addition I am dropping the header length calculation since we don't really need it for anything but the last frame and it can be easily obtained by just pulling the data_len and offset of tail from the transport header. Signed-off-by: Alexander Duyck Reviewed-by: Eric Dumazet Acked-by: Willem de Bruijn Signed-off-by: David S. Miller --- net/ipv4/udp_offload.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index 92c182e99ddc..b15c78ac3f23 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -193,7 +193,6 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, struct sock *sk = gso_skb->sk; unsigned int sum_truesize = 0; struct sk_buff *segs, *seg; - unsigned int hdrlen; struct udphdr *uh; unsigned int mss; __sum16 check; @@ -203,7 +202,6 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, if (gso_skb->len <= sizeof(*uh) + mss) return ERR_PTR(-EINVAL); - hdrlen = gso_skb->data - skb_mac_header(gso_skb); skb_pull(gso_skb, sizeof(*uh)); /* clear destructor to avoid skb_segment assigning it to tail */ @@ -216,30 +214,37 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, return segs; } - uh = udp_hdr(segs); + seg = segs; + uh = udp_hdr(seg); /* compute checksum adjustment based on old length versus new */ newlen = htons(sizeof(*uh) + mss); check = csum16_add(csum16_sub(uh->check, uh->len), newlen); - for (seg = segs; seg; seg = seg->next) { - uh = udp_hdr(seg); + for (;;) { + seg->destructor = sock_wfree; + seg->sk = sk; + sum_truesize += seg->truesize; - /* last packet can be partial gso_size */ - if (!seg->next) { - newlen = htons(seg->len - hdrlen); - check = csum16_add(csum16_sub(uh->check, uh->len), - newlen); - } + if (!seg->next) + break; uh->len = newlen; uh->check = check; - seg->destructor = sock_wfree; - seg->sk = sk; - sum_truesize += seg->truesize; + seg = seg->next; + uh = udp_hdr(seg); } + /* last packet can be partial gso_size, account for that in checksum */ + newlen = htons(skb_tail_pointer(seg) - skb_transport_header(seg) + + seg->data_len); + check = csum16_add(csum16_sub(uh->check, uh->len), newlen); + + uh->len = newlen; + uh->check = check; + + /* update refcount for the packet */ refcount_add(sum_truesize - gso_skb->truesize, &sk->sk_wmem_alloc); return segs; From 6053d0f189064302420930f9ef9022e24a04946a Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 7 May 2018 11:08:46 -0700 Subject: [PATCH 5/6] udp: Add support for software checksum and GSO_PARTIAL with GSO offload This patch adds support for a software provided checksum and GSO_PARTIAL segmentation support. With this we can offload UDP segmentation on devices that only have partial support for tunnels. Since we are no longer needing the hardware checksum we can drop the checks in the segmentation code that were verifying if it was present. Signed-off-by: Alexander Duyck Acked-by: Willem de Bruijn Signed-off-by: David S. Miller --- net/ipv4/udp_offload.c | 29 +++++++++++++++++++---------- net/ipv6/udp_offload.c | 11 +---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index b15c78ac3f23..d4f2daca0c33 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -214,6 +214,13 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, return segs; } + /* GSO partial and frag_list segmentation only requires splitting + * the frame into an MSS multiple and possibly a remainder, both + * cases return a GSO skb. So update the mss now. + */ + if (skb_is_gso(segs)) + mss *= skb_shinfo(segs)->gso_segs; + seg = segs; uh = udp_hdr(seg); @@ -232,6 +239,12 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, uh->len = newlen; uh->check = check; + if (seg->ip_summed == CHECKSUM_PARTIAL) + gso_reset_checksum(seg, ~check); + else + uh->check = gso_make_checksum(seg, ~check) ? : + CSUM_MANGLED_0; + seg = seg->next; uh = udp_hdr(seg); } @@ -244,6 +257,11 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, uh->len = newlen; uh->check = check; + if (seg->ip_summed == CHECKSUM_PARTIAL) + gso_reset_checksum(seg, ~check); + else + uh->check = gso_make_checksum(seg, ~check) ? : CSUM_MANGLED_0; + /* update refcount for the packet */ refcount_add(sum_truesize - gso_skb->truesize, &sk->sk_wmem_alloc); @@ -251,15 +269,6 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, } EXPORT_SYMBOL_GPL(__udp_gso_segment); -static struct sk_buff *__udp4_gso_segment(struct sk_buff *gso_skb, - netdev_features_t features) -{ - if (!can_checksum_protocol(features, htons(ETH_P_IP))) - return ERR_PTR(-EIO); - - return __udp_gso_segment(gso_skb, features); -} - static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, netdev_features_t features) { @@ -283,7 +292,7 @@ static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb, goto out; if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) - return __udp4_gso_segment(skb, features); + return __udp_gso_segment(skb, features); mss = skb_shinfo(skb)->gso_size; if (unlikely(skb->len <= mss)) diff --git a/net/ipv6/udp_offload.c b/net/ipv6/udp_offload.c index 61e34f1d2fa2..03a2ff3fe1e6 100644 --- a/net/ipv6/udp_offload.c +++ b/net/ipv6/udp_offload.c @@ -17,15 +17,6 @@ #include #include "ip6_offload.h" -static struct sk_buff *__udp6_gso_segment(struct sk_buff *gso_skb, - netdev_features_t features) -{ - if (!can_checksum_protocol(features, htons(ETH_P_IPV6))) - return ERR_PTR(-EIO); - - return __udp_gso_segment(gso_skb, features); -} - static struct sk_buff *udp6_ufo_fragment(struct sk_buff *skb, netdev_features_t features) { @@ -58,7 +49,7 @@ static struct sk_buff *udp6_ufo_fragment(struct sk_buff *skb, goto out; if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) - return __udp6_gso_segment(skb, features); + return __udp_gso_segment(skb, features); /* Do software UFO. Complete and fill in the UDP checksum as HW cannot * do checksum of UDP packets sent as multiple IP fragments. From 04d55b257cbb9eb6c722410251d3b7df00835c6c Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 7 May 2018 11:08:52 -0700 Subject: [PATCH 6/6] udp: Do not copy destructor if one is not present This patch makes it so that if a destructor is not present we avoid trying to update the skb socket or any reference counting that would be associated with the NULL socket and/or descriptor. By doing this we can support traffic coming from another namespace without any issues. Acked-by: Willem de Bruijn Signed-off-by: Alexander Duyck Reviewed-by: Eric Dumazet Signed-off-by: David S. Miller --- net/ipv4/udp_offload.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index d4f2daca0c33..ede2a7305b90 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -195,6 +195,7 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, struct sk_buff *segs, *seg; struct udphdr *uh; unsigned int mss; + bool copy_dtor; __sum16 check; __be16 newlen; @@ -205,12 +206,14 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, skb_pull(gso_skb, sizeof(*uh)); /* clear destructor to avoid skb_segment assigning it to tail */ - WARN_ON_ONCE(gso_skb->destructor != sock_wfree); - gso_skb->destructor = NULL; + copy_dtor = gso_skb->destructor == sock_wfree; + if (copy_dtor) + gso_skb->destructor = NULL; segs = skb_segment(gso_skb, features); if (unlikely(IS_ERR_OR_NULL(segs))) { - gso_skb->destructor = sock_wfree; + if (copy_dtor) + gso_skb->destructor = sock_wfree; return segs; } @@ -229,9 +232,11 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, check = csum16_add(csum16_sub(uh->check, uh->len), newlen); for (;;) { - seg->destructor = sock_wfree; - seg->sk = sk; - sum_truesize += seg->truesize; + if (copy_dtor) { + seg->destructor = sock_wfree; + seg->sk = sk; + sum_truesize += seg->truesize; + } if (!seg->next) break; @@ -263,8 +268,9 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, uh->check = gso_make_checksum(seg, ~check) ? : CSUM_MANGLED_0; /* update refcount for the packet */ - refcount_add(sum_truesize - gso_skb->truesize, &sk->sk_wmem_alloc); - + if (copy_dtor) + refcount_add(sum_truesize - gso_skb->truesize, + &sk->sk_wmem_alloc); return segs; } EXPORT_SYMBOL_GPL(__udp_gso_segment);