/net/ipv6/inet6_hashtables.c

http://github.com/mirrors/linux · C · 298 lines · 233 code · 39 blank · 26 comment · 36 complexity · 478021a44dffddc459bce93faba3b47f MD5 · raw file

  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * INET An implementation of the TCP/IP protocol suite for the LINUX
  4. * operating system. INET is implemented using the BSD Socket
  5. * interface as the means of communication with the user level.
  6. *
  7. * Generic INET6 transport hashtables
  8. *
  9. * Authors: Lotsa people, from code originally in tcp, generalised here
  10. * by Arnaldo Carvalho de Melo <acme@mandriva.com>
  11. */
  12. #include <linux/module.h>
  13. #include <linux/random.h>
  14. #include <net/addrconf.h>
  15. #include <net/inet_connection_sock.h>
  16. #include <net/inet_hashtables.h>
  17. #include <net/inet6_hashtables.h>
  18. #include <net/secure_seq.h>
  19. #include <net/ip.h>
  20. #include <net/sock_reuseport.h>
  21. u32 inet6_ehashfn(const struct net *net,
  22. const struct in6_addr *laddr, const u16 lport,
  23. const struct in6_addr *faddr, const __be16 fport)
  24. {
  25. static u32 inet6_ehash_secret __read_mostly;
  26. static u32 ipv6_hash_secret __read_mostly;
  27. u32 lhash, fhash;
  28. net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret));
  29. net_get_random_once(&ipv6_hash_secret, sizeof(ipv6_hash_secret));
  30. lhash = (__force u32)laddr->s6_addr32[3];
  31. fhash = __ipv6_addr_jhash(faddr, ipv6_hash_secret);
  32. return __inet6_ehashfn(lhash, lport, fhash, fport,
  33. inet6_ehash_secret + net_hash_mix(net));
  34. }
  35. /*
  36. * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so
  37. * we need not check it for TCP lookups anymore, thanks Alexey. -DaveM
  38. *
  39. * The sockhash lock must be held as a reader here.
  40. */
  41. struct sock *__inet6_lookup_established(struct net *net,
  42. struct inet_hashinfo *hashinfo,
  43. const struct in6_addr *saddr,
  44. const __be16 sport,
  45. const struct in6_addr *daddr,
  46. const u16 hnum,
  47. const int dif, const int sdif)
  48. {
  49. struct sock *sk;
  50. const struct hlist_nulls_node *node;
  51. const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
  52. /* Optimize here for direct hit, only listening connections can
  53. * have wildcards anyways.
  54. */
  55. unsigned int hash = inet6_ehashfn(net, daddr, hnum, saddr, sport);
  56. unsigned int slot = hash & hashinfo->ehash_mask;
  57. struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
  58. begin:
  59. sk_nulls_for_each_rcu(sk, node, &head->chain) {
  60. if (sk->sk_hash != hash)
  61. continue;
  62. if (!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif))
  63. continue;
  64. if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))
  65. goto out;
  66. if (unlikely(!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif))) {
  67. sock_gen_put(sk);
  68. goto begin;
  69. }
  70. goto found;
  71. }
  72. if (get_nulls_value(node) != slot)
  73. goto begin;
  74. out:
  75. sk = NULL;
  76. found:
  77. return sk;
  78. }
  79. EXPORT_SYMBOL(__inet6_lookup_established);
  80. static inline int compute_score(struct sock *sk, struct net *net,
  81. const unsigned short hnum,
  82. const struct in6_addr *daddr,
  83. const int dif, const int sdif, bool exact_dif)
  84. {
  85. int score = -1;
  86. if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum &&
  87. sk->sk_family == PF_INET6) {
  88. if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))
  89. return -1;
  90. if (!inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))
  91. return -1;
  92. score = 1;
  93. if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
  94. score++;
  95. }
  96. return score;
  97. }
  98. /* called with rcu_read_lock() */
  99. static struct sock *inet6_lhash2_lookup(struct net *net,
  100. struct inet_listen_hashbucket *ilb2,
  101. struct sk_buff *skb, int doff,
  102. const struct in6_addr *saddr,
  103. const __be16 sport, const struct in6_addr *daddr,
  104. const unsigned short hnum, const int dif, const int sdif)
  105. {
  106. bool exact_dif = inet6_exact_dif_match(net, skb);
  107. struct inet_connection_sock *icsk;
  108. struct sock *sk, *result = NULL;
  109. int score, hiscore = 0;
  110. u32 phash = 0;
  111. inet_lhash2_for_each_icsk_rcu(icsk, &ilb2->head) {
  112. sk = (struct sock *)icsk;
  113. score = compute_score(sk, net, hnum, daddr, dif, sdif,
  114. exact_dif);
  115. if (score > hiscore) {
  116. if (sk->sk_reuseport) {
  117. phash = inet6_ehashfn(net, daddr, hnum,
  118. saddr, sport);
  119. result = reuseport_select_sock(sk, phash,
  120. skb, doff);
  121. if (result)
  122. return result;
  123. }
  124. result = sk;
  125. hiscore = score;
  126. }
  127. }
  128. return result;
  129. }
  130. struct sock *inet6_lookup_listener(struct net *net,
  131. struct inet_hashinfo *hashinfo,
  132. struct sk_buff *skb, int doff,
  133. const struct in6_addr *saddr,
  134. const __be16 sport, const struct in6_addr *daddr,
  135. const unsigned short hnum, const int dif, const int sdif)
  136. {
  137. struct inet_listen_hashbucket *ilb2;
  138. struct sock *result = NULL;
  139. unsigned int hash2;
  140. hash2 = ipv6_portaddr_hash(net, daddr, hnum);
  141. ilb2 = inet_lhash2_bucket(hashinfo, hash2);
  142. result = inet6_lhash2_lookup(net, ilb2, skb, doff,
  143. saddr, sport, daddr, hnum,
  144. dif, sdif);
  145. if (result)
  146. goto done;
  147. /* Lookup lhash2 with in6addr_any */
  148. hash2 = ipv6_portaddr_hash(net, &in6addr_any, hnum);
  149. ilb2 = inet_lhash2_bucket(hashinfo, hash2);
  150. result = inet6_lhash2_lookup(net, ilb2, skb, doff,
  151. saddr, sport, &in6addr_any, hnum,
  152. dif, sdif);
  153. done:
  154. if (IS_ERR(result))
  155. return NULL;
  156. return result;
  157. }
  158. EXPORT_SYMBOL_GPL(inet6_lookup_listener);
  159. struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo,
  160. struct sk_buff *skb, int doff,
  161. const struct in6_addr *saddr, const __be16 sport,
  162. const struct in6_addr *daddr, const __be16 dport,
  163. const int dif)
  164. {
  165. struct sock *sk;
  166. bool refcounted;
  167. sk = __inet6_lookup(net, hashinfo, skb, doff, saddr, sport, daddr,
  168. ntohs(dport), dif, 0, &refcounted);
  169. if (sk && !refcounted && !refcount_inc_not_zero(&sk->sk_refcnt))
  170. sk = NULL;
  171. return sk;
  172. }
  173. EXPORT_SYMBOL_GPL(inet6_lookup);
  174. static int __inet6_check_established(struct inet_timewait_death_row *death_row,
  175. struct sock *sk, const __u16 lport,
  176. struct inet_timewait_sock **twp)
  177. {
  178. struct inet_hashinfo *hinfo = death_row->hashinfo;
  179. struct inet_sock *inet = inet_sk(sk);
  180. const struct in6_addr *daddr = &sk->sk_v6_rcv_saddr;
  181. const struct in6_addr *saddr = &sk->sk_v6_daddr;
  182. const int dif = sk->sk_bound_dev_if;
  183. struct net *net = sock_net(sk);
  184. const int sdif = l3mdev_master_ifindex_by_index(net, dif);
  185. const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport);
  186. const unsigned int hash = inet6_ehashfn(net, daddr, lport, saddr,
  187. inet->inet_dport);
  188. struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
  189. spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
  190. struct sock *sk2;
  191. const struct hlist_nulls_node *node;
  192. struct inet_timewait_sock *tw = NULL;
  193. spin_lock(lock);
  194. sk_nulls_for_each(sk2, node, &head->chain) {
  195. if (sk2->sk_hash != hash)
  196. continue;
  197. if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports,
  198. dif, sdif))) {
  199. if (sk2->sk_state == TCP_TIME_WAIT) {
  200. tw = inet_twsk(sk2);
  201. if (twsk_unique(sk, sk2, twp))
  202. break;
  203. }
  204. goto not_unique;
  205. }
  206. }
  207. /* Must record num and sport now. Otherwise we will see
  208. * in hash table socket with a funny identity.
  209. */
  210. inet->inet_num = lport;
  211. inet->inet_sport = htons(lport);
  212. sk->sk_hash = hash;
  213. WARN_ON(!sk_unhashed(sk));
  214. __sk_nulls_add_node_rcu(sk, &head->chain);
  215. if (tw) {
  216. sk_nulls_del_node_init_rcu((struct sock *)tw);
  217. __NET_INC_STATS(net, LINUX_MIB_TIMEWAITRECYCLED);
  218. }
  219. spin_unlock(lock);
  220. sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  221. if (twp) {
  222. *twp = tw;
  223. } else if (tw) {
  224. /* Silly. Should hash-dance instead... */
  225. inet_twsk_deschedule_put(tw);
  226. }
  227. return 0;
  228. not_unique:
  229. spin_unlock(lock);
  230. return -EADDRNOTAVAIL;
  231. }
  232. static u32 inet6_sk_port_offset(const struct sock *sk)
  233. {
  234. const struct inet_sock *inet = inet_sk(sk);
  235. return secure_ipv6_port_ephemeral(sk->sk_v6_rcv_saddr.s6_addr32,
  236. sk->sk_v6_daddr.s6_addr32,
  237. inet->inet_dport);
  238. }
  239. int inet6_hash_connect(struct inet_timewait_death_row *death_row,
  240. struct sock *sk)
  241. {
  242. u32 port_offset = 0;
  243. if (!inet_sk(sk)->inet_num)
  244. port_offset = inet6_sk_port_offset(sk);
  245. return __inet_hash_connect(death_row, sk, port_offset,
  246. __inet6_check_established);
  247. }
  248. EXPORT_SYMBOL_GPL(inet6_hash_connect);
  249. int inet6_hash(struct sock *sk)
  250. {
  251. int err = 0;
  252. if (sk->sk_state != TCP_CLOSE) {
  253. local_bh_disable();
  254. err = __inet_hash(sk, NULL);
  255. local_bh_enable();
  256. }
  257. return err;
  258. }
  259. EXPORT_SYMBOL_GPL(inet6_hash);