/net/ipv6/inet6_hashtables.c

  1// SPDX-License-Identifier: GPL-2.0-or-later
  2/*
  3 * INET		An implementation of the TCP/IP protocol suite for the LINUX
  4 *		operating system.  INET is implemented using the BSD Socket
  5 *		interface as the means of communication with the user level.
  6 *
  7 *		Generic INET6 transport hashtables
  8 *
  9 * Authors:	Lotsa people, from code originally in tcp, generalised here
 10 *		by Arnaldo Carvalho de Melo <acme@mandriva.com>
 11 */
 12
 13#include <linux/module.h>
 14#include <linux/random.h>
 15
 16#include <net/addrconf.h>
 17#include <net/inet_connection_sock.h>
 18#include <net/inet_hashtables.h>
 19#include <net/inet6_hashtables.h>
 20#include <net/secure_seq.h>
 21#include <net/ip.h>
 22#include <net/sock_reuseport.h>
 23
 24u32 inet6_ehashfn(const struct net *net,
 25		  const struct in6_addr *laddr, const u16 lport,
 26		  const struct in6_addr *faddr, const __be16 fport)
 27{
 28	static u32 inet6_ehash_secret __read_mostly;
 29	static u32 ipv6_hash_secret __read_mostly;
 30
 31	u32 lhash, fhash;
 32
 33	net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret));
 34	net_get_random_once(&ipv6_hash_secret, sizeof(ipv6_hash_secret));
 35
 36	lhash = (__force u32)laddr->s6_addr32[3];
 37	fhash = __ipv6_addr_jhash(faddr, ipv6_hash_secret);
 38
 39	return __inet6_ehashfn(lhash, lport, fhash, fport,
 40			       inet6_ehash_secret + net_hash_mix(net));
 41}
 42
 43/*
 44 * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so
 45 * we need not check it for TCP lookups anymore, thanks Alexey. -DaveM
 46 *
 47 * The sockhash lock must be held as a reader here.
 48 */
 49struct sock *__inet6_lookup_established(struct net *net,
 50					struct inet_hashinfo *hashinfo,
 51					   const struct in6_addr *saddr,
 52					   const __be16 sport,
 53					   const struct in6_addr *daddr,
 54					   const u16 hnum,
 55					   const int dif, const int sdif)
 56{
 57	struct sock *sk;
 58	const struct hlist_nulls_node *node;
 59	const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
 60	/* Optimize here for direct hit, only listening connections can
 61	 * have wildcards anyways.
 62	 */
 63	unsigned int hash = inet6_ehashfn(net, daddr, hnum, saddr, sport);
 64	unsigned int slot = hash & hashinfo->ehash_mask;
 65	struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
 66
 67
 68begin:
 69	sk_nulls_for_each_rcu(sk, node, &head->chain) {
 70		if (sk->sk_hash != hash)
 71			continue;
 72		if (!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif))
 73			continue;
 74		if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))
 75			goto out;
 76
 77		if (unlikely(!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif))) {
 78			sock_gen_put(sk);
 79			goto begin;
 80		}
 81		goto found;
 82	}
 83	if (get_nulls_value(node) != slot)
 84		goto begin;
 85out:
 86	sk = NULL;
 87found:
 88	return sk;
 89}
 90EXPORT_SYMBOL(__inet6_lookup_established);
 91
 92static inline int compute_score(struct sock *sk, struct net *net,
 93				const unsigned short hnum,
 94				const struct in6_addr *daddr,
 95				const int dif, const int sdif, bool exact_dif)
 96{
 97	int score = -1;
 98
 99	if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum &&
100	    sk->sk_family == PF_INET6) {
101		if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))
102			return -1;
103
104		if (!inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))
105			return -1;
106
107		score = 1;
108		if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
109			score++;
110	}
111	return score;
112}
113
114/* called with rcu_read_lock() */
115static struct sock *inet6_lhash2_lookup(struct net *net,
116		struct inet_listen_hashbucket *ilb2,
117		struct sk_buff *skb, int doff,
118		const struct in6_addr *saddr,
119		const __be16 sport, const struct in6_addr *daddr,
120		const unsigned short hnum, const int dif, const int sdif)
121{
122	bool exact_dif = inet6_exact_dif_match(net, skb);
123	struct inet_connection_sock *icsk;
124	struct sock *sk, *result = NULL;
125	int score, hiscore = 0;
126	u32 phash = 0;
127
128	inet_lhash2_for_each_icsk_rcu(icsk, &ilb2->head) {
129		sk = (struct sock *)icsk;
130		score = compute_score(sk, net, hnum, daddr, dif, sdif,
131				      exact_dif);
132		if (score > hiscore) {
133			if (sk->sk_reuseport) {
134				phash = inet6_ehashfn(net, daddr, hnum,
135						      saddr, sport);
136				result = reuseport_select_sock(sk, phash,
137							       skb, doff);
138				if (result)
139					return result;
140			}
141			result = sk;
142			hiscore = score;
143		}
144	}
145
146	return result;
147}
148
149struct sock *inet6_lookup_listener(struct net *net,
150		struct inet_hashinfo *hashinfo,
151		struct sk_buff *skb, int doff,
152		const struct in6_addr *saddr,
153		const __be16 sport, const struct in6_addr *daddr,
154		const unsigned short hnum, const int dif, const int sdif)
155{
156	struct inet_listen_hashbucket *ilb2;
157	struct sock *result = NULL;
158	unsigned int hash2;
159
160	hash2 = ipv6_portaddr_hash(net, daddr, hnum);
161	ilb2 = inet_lhash2_bucket(hashinfo, hash2);
162
163	result = inet6_lhash2_lookup(net, ilb2, skb, doff,
164				     saddr, sport, daddr, hnum,
165				     dif, sdif);
166	if (result)
167		goto done;
168
169	/* Lookup lhash2 with in6addr_any */
170	hash2 = ipv6_portaddr_hash(net, &in6addr_any, hnum);
171	ilb2 = inet_lhash2_bucket(hashinfo, hash2);
172
173	result = inet6_lhash2_lookup(net, ilb2, skb, doff,
174				     saddr, sport, &in6addr_any, hnum,
175				     dif, sdif);
176done:
177	if (IS_ERR(result))
178		return NULL;
179	return result;
180}
181EXPORT_SYMBOL_GPL(inet6_lookup_listener);
182
183struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo,
184			  struct sk_buff *skb, int doff,
185			  const struct in6_addr *saddr, const __be16 sport,
186			  const struct in6_addr *daddr, const __be16 dport,
187			  const int dif)
188{
189	struct sock *sk;
190	bool refcounted;
191
192	sk = __inet6_lookup(net, hashinfo, skb, doff, saddr, sport, daddr,
193			    ntohs(dport), dif, 0, &refcounted);
194	if (sk && !refcounted && !refcount_inc_not_zero(&sk->sk_refcnt))
195		sk = NULL;
196	return sk;
197}
198EXPORT_SYMBOL_GPL(inet6_lookup);
199
200static int __inet6_check_established(struct inet_timewait_death_row *death_row,
201				     struct sock *sk, const __u16 lport,
202				     struct inet_timewait_sock **twp)
203{
204	struct inet_hashinfo *hinfo = death_row->hashinfo;
205	struct inet_sock *inet = inet_sk(sk);
206	const struct in6_addr *daddr = &sk->sk_v6_rcv_saddr;
207	const struct in6_addr *saddr = &sk->sk_v6_daddr;
208	const int dif = sk->sk_bound_dev_if;
209	struct net *net = sock_net(sk);
210	const int sdif = l3mdev_master_ifindex_by_index(net, dif);
211	const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport);
212	const unsigned int hash = inet6_ehashfn(net, daddr, lport, saddr,
213						inet->inet_dport);
214	struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
215	spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
216	struct sock *sk2;
217	const struct hlist_nulls_node *node;
218	struct inet_timewait_sock *tw = NULL;
219
220	spin_lock(lock);
221
222	sk_nulls_for_each(sk2, node, &head->chain) {
223		if (sk2->sk_hash != hash)
224			continue;
225
226		if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports,
227				       dif, sdif))) {
228			if (sk2->sk_state == TCP_TIME_WAIT) {
229				tw = inet_twsk(sk2);
230				if (twsk_unique(sk, sk2, twp))
231					break;
232			}
233			goto not_unique;
234		}
235	}
236
237	/* Must record num and sport now. Otherwise we will see
238	 * in hash table socket with a funny identity.
239	 */
240	inet->inet_num = lport;
241	inet->inet_sport = htons(lport);
242	sk->sk_hash = hash;
243	WARN_ON(!sk_unhashed(sk));
244	__sk_nulls_add_node_rcu(sk, &head->chain);
245	if (tw) {
246		sk_nulls_del_node_init_rcu((struct sock *)tw);
247		__NET_INC_STATS(net, LINUX_MIB_TIMEWAITRECYCLED);
248	}
249	spin_unlock(lock);
250	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
251
252	if (twp) {
253		*twp = tw;
254	} else if (tw) {
255		/* Silly. Should hash-dance instead... */
256		inet_twsk_deschedule_put(tw);
257	}
258	return 0;
259
260not_unique:
261	spin_unlock(lock);
262	return -EADDRNOTAVAIL;
263}
264
265static u32 inet6_sk_port_offset(const struct sock *sk)
266{
267	const struct inet_sock *inet = inet_sk(sk);
268
269	return secure_ipv6_port_ephemeral(sk->sk_v6_rcv_saddr.s6_addr32,
270					  sk->sk_v6_daddr.s6_addr32,
271					  inet->inet_dport);
272}
273
274int inet6_hash_connect(struct inet_timewait_death_row *death_row,
275		       struct sock *sk)
276{
277	u32 port_offset = 0;
278
279	if (!inet_sk(sk)->inet_num)
280		port_offset = inet6_sk_port_offset(sk);
281	return __inet_hash_connect(death_row, sk, port_offset,
282				   __inet6_check_established);
283}
284EXPORT_SYMBOL_GPL(inet6_hash_connect);
285
286int inet6_hash(struct sock *sk)
287{
288	int err = 0;
289
290	if (sk->sk_state != TCP_CLOSE) {
291		local_bh_disable();
292		err = __inet_hash(sk, NULL);
293		local_bh_enable();
294	}
295
296	return err;
297}
298EXPORT_SYMBOL_GPL(inet6_hash);