From 0c0a5ef809f9150e9229e7b13e43183b681b7a39 Mon Sep 17 00:00:00 2001

From: Eric Dumazet <edumazet@google.com>

Date: Mon, 25 Oct 2021 09:48:16 -0700

Subject: [PATCH] tcp: move inet->rx_dst_ifindex to sk->sk_rx_dst_ifindex

Git-commit: 0c0a5ef809f9150e9229e7b13e43183b681b7a39

Patch-mainline: v5.16-rc1

References: bsc#1207361

Increase cache locality by moving rx_dst_ifindex next to sk->sk_rx_dst

This is part of an effort to reduce cache line misses in TCP fast path.

This removes one cache line miss in early demux.

Signed-off-by: Eric Dumazet <edumazet@google.com>

Acked-by: Soheil Hassas Yeganeh <soheil@google.com>

Signed-off-by: Jakub Kicinski <kuba@kernel.org>

Signed-off-by: Jiri Wiesner <jwiesner@suse.de>

---

 include/net/inet_sock.h | 3 +--

 include/net/sock.h      | 3 +++

 net/ipv4/tcp_ipv4.c     | 6 +++---

 net/ipv6/tcp_ipv6.c     | 6 +++---

 4 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h

index 89163ef8cf4b..9e1111f5915b 100644

--- a/include/net/inet_sock.h

+++ b/include/net/inet_sock.h

@@ -207,11 +207,10 @@ struct inet_sock {

 	__be32			inet_saddr;

 	__s16			uc_ttl;

 	__u16			cmsg_flags;

+	struct ip_options_rcu __rcu	*inet_opt;

 	__be16			inet_sport;

 	__u16			inet_id;

-	struct ip_options_rcu __rcu	*inet_opt;

-	int			rx_dst_ifindex;

 	__u8			tos;

 	__u8			min_ttl;

 	__u8			mc_ttl;

diff --git a/include/net/sock.h b/include/net/sock.h

index 596ba85611bc..0bfb3f138bda 100644

--- a/include/net/sock.h

+++ b/include/net/sock.h

@@ -259,6 +259,7 @@ struct bpf_local_storage;

   *	@sk_rcvbuf: size of receive buffer in bytes

   *	@sk_wq: sock wait queue and async head

   *	@sk_rx_dst: receive input route used by early demux

+  *	@sk_rx_dst_ifindex: ifindex for @sk_rx_dst

   *	@sk_dst_cache: destination cache

   *	@sk_dst_pending_confirm: need to confirm neighbour

   *	@sk_policy: flow policy

@@ -430,6 +431,8 @@ struct sock {

 	struct xfrm_policy __rcu *sk_policy[2];

 #endif

 	struct dst_entry	*sk_rx_dst;

+	int			sk_rx_dst_ifindex;

+

 	struct dst_entry __rcu	*sk_dst_cache;

 	atomic_t		sk_omem_alloc;

 	int			sk_sndbuf;

diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c

index 5e6d8cb82ce5..2bdc32c1afb6 100644

--- a/net/ipv4/tcp_ipv4.c

+++ b/net/ipv4/tcp_ipv4.c

@@ -1703,7 +1703,7 @@ int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)

 		sock_rps_save_rxhash(sk, skb);

 		sk_mark_napi_id(sk, skb);

 		if (dst) {

-			if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||

+			if (sk->sk_rx_dst_ifindex != skb->skb_iif ||

 			    !INDIRECT_CALL_1(dst->ops->check, ipv4_dst_check,

 					     dst, 0)) {

 				dst_release(dst);

@@ -1788,7 +1788,7 @@ int tcp_v4_early_demux(struct sk_buff *skb)

 			if (dst)

 				dst = dst_check(dst, 0);

 			if (dst &&

-			    inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)

+			    sk->sk_rx_dst_ifindex == skb->skb_iif)

 				skb_dst_set_noref(skb, dst);

 		}

 	}

@@ -2195,7 +2195,7 @@ void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)

 	if (dst && dst_hold_safe(dst)) {

 		sk->sk_rx_dst = dst;

-		inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;

+		sk->sk_rx_dst_ifindex = skb->skb_iif;

 	}

 }

 EXPORT_SYMBOL(inet_sk_rx_dst_set);

diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c

index 1ef27cd7dbff..3e8669b6d636 100644

--- a/net/ipv6/tcp_ipv6.c

+++ b/net/ipv6/tcp_ipv6.c

@@ -108,7 +108,7 @@ static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)

 		const struct rt6_info *rt = (const struct rt6_info *)dst;

 		sk->sk_rx_dst = dst;

-		inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;

+		sk->sk_rx_dst_ifindex = skb->skb_iif;

 		tcp_inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt);

 	}

 }

@@ -1509,7 +1509,7 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)

 		sock_rps_save_rxhash(sk, skb);

 		sk_mark_napi_id(sk, skb);

 		if (dst) {

-			if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif ||

+			if (sk->sk_rx_dst_ifindex != skb->skb_iif ||

 			    INDIRECT_CALL_1(dst->ops->check, ip6_dst_check,

 					    dst, np->rx_dst_cookie) == NULL) {

 				dst_release(dst);

@@ -1874,7 +1874,7 @@ INDIRECT_CALLABLE_SCOPE void tcp_v6_early_demux(struct sk_buff *skb)

 			if (dst)

 				dst = dst_check(dst, tcp_inet6_sk(sk)->rx_dst_cookie);

 			if (dst &&

-			    inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)

+			    sk->sk_rx_dst_ifindex == skb->skb_iif)

 				skb_dst_set_noref(skb, dst);

 		}

 	}

-- 

2.35.3