/net/netfilter/ipvs/ip_vs_xmit.c

   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * ip_vs_xmit.c: various packet transmitters for IPVS
   4 *
   5 * Authors:     Wensong Zhang <wensong@linuxvirtualserver.org>
   6 *              Julian Anastasov <ja@ssi.bg>
   7 *
   8 * Changes:
   9 *
  10 * Description of forwarding methods:
  11 * - all transmitters are called from LOCAL_IN (remote clients) and
  12 * LOCAL_OUT (local clients) but for ICMP can be called from FORWARD
  13 * - not all connections have destination server, for example,
  14 * connections in backup server when fwmark is used
  15 * - bypass connections use daddr from packet
  16 * - we can use dst without ref while sending in RCU section, we use
  17 * ref when returning NF_ACCEPT for NAT-ed packet via loopback
  18 * LOCAL_OUT rules:
  19 * - skb->dev is NULL, skb->protocol is not set (both are set in POST_ROUTING)
  20 * - skb->pkt_type is not set yet
  21 * - the only place where we can see skb->sk != NULL
  22 */
  23
  24#define KMSG_COMPONENT "IPVS"
  25#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  26
  27#include <linux/kernel.h>
  28#include <linux/slab.h>
  29#include <linux/tcp.h>                  /* for tcphdr */
  30#include <net/ip.h>
  31#include <net/gue.h>
  32#include <net/gre.h>
  33#include <net/tcp.h>                    /* for csum_tcpudp_magic */
  34#include <net/udp.h>
  35#include <net/icmp.h>                   /* for icmp_send */
  36#include <net/route.h>                  /* for ip_route_output */
  37#include <net/ipv6.h>
  38#include <net/ip6_route.h>
  39#include <net/ip_tunnels.h>
  40#include <net/ip6_checksum.h>
  41#include <net/addrconf.h>
  42#include <linux/icmpv6.h>
  43#include <linux/netfilter.h>
  44#include <linux/netfilter_ipv4.h>
  45
  46#include <net/ip_vs.h>
  47
  48enum {
  49	IP_VS_RT_MODE_LOCAL	= 1, /* Allow local dest */
  50	IP_VS_RT_MODE_NON_LOCAL	= 2, /* Allow non-local dest */
  51	IP_VS_RT_MODE_RDR	= 4, /* Allow redirect from remote daddr to
  52				      * local
  53				      */
  54	IP_VS_RT_MODE_CONNECT	= 8, /* Always bind route to saddr */
  55	IP_VS_RT_MODE_KNOWN_NH	= 16,/* Route via remote addr */
  56	IP_VS_RT_MODE_TUNNEL	= 32,/* Tunnel mode */
  57};
  58
  59static inline struct ip_vs_dest_dst *ip_vs_dest_dst_alloc(void)
  60{
  61	return kmalloc(sizeof(struct ip_vs_dest_dst), GFP_ATOMIC);
  62}
  63
  64static inline void ip_vs_dest_dst_free(struct ip_vs_dest_dst *dest_dst)
  65{
  66	kfree(dest_dst);
  67}
  68
  69/*
  70 *      Destination cache to speed up outgoing route lookup
  71 */
  72static inline void
  73__ip_vs_dst_set(struct ip_vs_dest *dest, struct ip_vs_dest_dst *dest_dst,
  74		struct dst_entry *dst, u32 dst_cookie)
  75{
  76	struct ip_vs_dest_dst *old;
  77
  78	old = rcu_dereference_protected(dest->dest_dst,
  79					lockdep_is_held(&dest->dst_lock));
  80
  81	if (dest_dst) {
  82		dest_dst->dst_cache = dst;
  83		dest_dst->dst_cookie = dst_cookie;
  84	}
  85	rcu_assign_pointer(dest->dest_dst, dest_dst);
  86
  87	if (old)
  88		call_rcu(&old->rcu_head, ip_vs_dest_dst_rcu_free);
  89}
  90
  91static inline struct ip_vs_dest_dst *
  92__ip_vs_dst_check(struct ip_vs_dest *dest)
  93{
  94	struct ip_vs_dest_dst *dest_dst = rcu_dereference(dest->dest_dst);
  95	struct dst_entry *dst;
  96
  97	if (!dest_dst)
  98		return NULL;
  99	dst = dest_dst->dst_cache;
 100	if (dst->obsolete &&
 101	    dst->ops->check(dst, dest_dst->dst_cookie) == NULL)
 102		return NULL;
 103	return dest_dst;
 104}
 105
 106static inline bool
 107__mtu_check_toobig_v6(const struct sk_buff *skb, u32 mtu)
 108{
 109	if (IP6CB(skb)->frag_max_size) {
 110		/* frag_max_size tell us that, this packet have been
 111		 * defragmented by netfilter IPv6 conntrack module.
 112		 */
 113		if (IP6CB(skb)->frag_max_size > mtu)
 114			return true; /* largest fragment violate MTU */
 115	}
 116	else if (skb->len > mtu && !skb_is_gso(skb)) {
 117		return true; /* Packet size violate MTU size */
 118	}
 119	return false;
 120}
 121
 122/* Get route to daddr, update *saddr, optionally bind route to saddr */
 123static struct rtable *do_output_route4(struct net *net, __be32 daddr,
 124				       int rt_mode, __be32 *saddr)
 125{
 126	struct flowi4 fl4;
 127	struct rtable *rt;
 128	bool loop = false;
 129
 130	memset(&fl4, 0, sizeof(fl4));
 131	fl4.daddr = daddr;
 132	fl4.flowi4_flags = (rt_mode & IP_VS_RT_MODE_KNOWN_NH) ?
 133			   FLOWI_FLAG_KNOWN_NH : 0;
 134
 135retry:
 136	rt = ip_route_output_key(net, &fl4);
 137	if (IS_ERR(rt)) {
 138		/* Invalid saddr ? */
 139		if (PTR_ERR(rt) == -EINVAL && *saddr &&
 140		    rt_mode & IP_VS_RT_MODE_CONNECT && !loop) {
 141			*saddr = 0;
 142			flowi4_update_output(&fl4, 0, 0, daddr, 0);
 143			goto retry;
 144		}
 145		IP_VS_DBG_RL("ip_route_output error, dest: %pI4\n", &daddr);
 146		return NULL;
 147	} else if (!*saddr && rt_mode & IP_VS_RT_MODE_CONNECT && fl4.saddr) {
 148		ip_rt_put(rt);
 149		*saddr = fl4.saddr;
 150		flowi4_update_output(&fl4, 0, 0, daddr, fl4.saddr);
 151		loop = true;
 152		goto retry;
 153	}
 154	*saddr = fl4.saddr;
 155	return rt;
 156}
 157
 158#ifdef CONFIG_IP_VS_IPV6
 159static inline int __ip_vs_is_local_route6(struct rt6_info *rt)
 160{
 161	return rt->dst.dev && rt->dst.dev->flags & IFF_LOOPBACK;
 162}
 163#endif
 164
 165static inline bool crosses_local_route_boundary(int skb_af, struct sk_buff *skb,
 166						int rt_mode,
 167						bool new_rt_is_local)
 168{
 169	bool rt_mode_allow_local = !!(rt_mode & IP_VS_RT_MODE_LOCAL);
 170	bool rt_mode_allow_non_local = !!(rt_mode & IP_VS_RT_MODE_NON_LOCAL);
 171	bool rt_mode_allow_redirect = !!(rt_mode & IP_VS_RT_MODE_RDR);
 172	bool source_is_loopback;
 173	bool old_rt_is_local;
 174
 175#ifdef CONFIG_IP_VS_IPV6
 176	if (skb_af == AF_INET6) {
 177		int addr_type = ipv6_addr_type(&ipv6_hdr(skb)->saddr);
 178
 179		source_is_loopback =
 180			(!skb->dev || skb->dev->flags & IFF_LOOPBACK) &&
 181			(addr_type & IPV6_ADDR_LOOPBACK);
 182		old_rt_is_local = __ip_vs_is_local_route6(
 183			(struct rt6_info *)skb_dst(skb));
 184	} else
 185#endif
 186	{
 187		source_is_loopback = ipv4_is_loopback(ip_hdr(skb)->saddr);
 188		old_rt_is_local = skb_rtable(skb)->rt_flags & RTCF_LOCAL;
 189	}
 190
 191	if (unlikely(new_rt_is_local)) {
 192		if (!rt_mode_allow_local)
 193			return true;
 194		if (!rt_mode_allow_redirect && !old_rt_is_local)
 195			return true;
 196	} else {
 197		if (!rt_mode_allow_non_local)
 198			return true;
 199		if (source_is_loopback)
 200			return true;
 201	}
 202	return false;
 203}
 204
 205static inline void maybe_update_pmtu(int skb_af, struct sk_buff *skb, int mtu)
 206{
 207	struct sock *sk = skb->sk;
 208	struct rtable *ort = skb_rtable(skb);
 209
 210	if (!skb->dev && sk && sk_fullsock(sk))
 211		ort->dst.ops->update_pmtu(&ort->dst, sk, NULL, mtu, true);
 212}
 213
 214static inline bool ensure_mtu_is_adequate(struct netns_ipvs *ipvs, int skb_af,
 215					  int rt_mode,
 216					  struct ip_vs_iphdr *ipvsh,
 217					  struct sk_buff *skb, int mtu)
 218{
 219#ifdef CONFIG_IP_VS_IPV6
 220	if (skb_af == AF_INET6) {
 221		struct net *net = ipvs->net;
 222
 223		if (unlikely(__mtu_check_toobig_v6(skb, mtu))) {
 224			if (!skb->dev)
 225				skb->dev = net->loopback_dev;
 226			/* only send ICMP too big on first fragment */
 227			if (!ipvsh->fragoffs && !ip_vs_iph_icmp(ipvsh))
 228				icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
 229			IP_VS_DBG(1, "frag needed for %pI6c\n",
 230				  &ipv6_hdr(skb)->saddr);
 231			return false;
 232		}
 233	} else
 234#endif
 235	{
 236		/* If we're going to tunnel the packet and pmtu discovery
 237		 * is disabled, we'll just fragment it anyway
 238		 */
 239		if ((rt_mode & IP_VS_RT_MODE_TUNNEL) && !sysctl_pmtu_disc(ipvs))
 240			return true;
 241
 242		if (unlikely(ip_hdr(skb)->frag_off & htons(IP_DF) &&
 243			     skb->len > mtu && !skb_is_gso(skb) &&
 244			     !ip_vs_iph_icmp(ipvsh))) {
 245			icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED,
 246				  htonl(mtu));
 247			IP_VS_DBG(1, "frag needed for %pI4\n",
 248				  &ip_hdr(skb)->saddr);
 249			return false;
 250		}
 251	}
 252
 253	return true;
 254}
 255
 256static inline bool decrement_ttl(struct netns_ipvs *ipvs,
 257				 int skb_af,
 258				 struct sk_buff *skb)
 259{
 260	struct net *net = ipvs->net;
 261
 262#ifdef CONFIG_IP_VS_IPV6
 263	if (skb_af == AF_INET6) {
 264		struct dst_entry *dst = skb_dst(skb);
 265
 266		/* check and decrement ttl */
 267		if (ipv6_hdr(skb)->hop_limit <= 1) {
 268			struct inet6_dev *idev = __in6_dev_get_safely(skb->dev);
 269
 270			/* Force OUTPUT device used as source address */
 271			skb->dev = dst->dev;
 272			icmpv6_send(skb, ICMPV6_TIME_EXCEED,
 273				    ICMPV6_EXC_HOPLIMIT, 0);
 274			__IP6_INC_STATS(net, idev, IPSTATS_MIB_INHDRERRORS);
 275
 276			return false;
 277		}
 278
 279		/* don't propagate ttl change to cloned packets */
 280		if (skb_ensure_writable(skb, sizeof(struct ipv6hdr)))
 281			return false;
 282
 283		ipv6_hdr(skb)->hop_limit--;
 284	} else
 285#endif
 286	{
 287		if (ip_hdr(skb)->ttl <= 1) {
 288			/* Tell the sender its packet died... */
 289			__IP_INC_STATS(net, IPSTATS_MIB_INHDRERRORS);
 290			icmp_send(skb, ICMP_TIME_EXCEEDED, ICMP_EXC_TTL, 0);
 291			return false;
 292		}
 293
 294		/* don't propagate ttl change to cloned packets */
 295		if (skb_ensure_writable(skb, sizeof(struct iphdr)))
 296			return false;
 297
 298		/* Decrease ttl */
 299		ip_decrease_ttl(ip_hdr(skb));
 300	}
 301
 302	return true;
 303}
 304
 305/* Get route to destination or remote server */
 306static int
 307__ip_vs_get_out_rt(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb,
 308		   struct ip_vs_dest *dest,
 309		   __be32 daddr, int rt_mode, __be32 *ret_saddr,
 310		   struct ip_vs_iphdr *ipvsh)
 311{
 312	struct net *net = ipvs->net;
 313	struct ip_vs_dest_dst *dest_dst;
 314	struct rtable *rt;			/* Route to the other host */
 315	int mtu;
 316	int local, noref = 1;
 317
 318	if (dest) {
 319		dest_dst = __ip_vs_dst_check(dest);
 320		if (likely(dest_dst))
 321			rt = (struct rtable *) dest_dst->dst_cache;
 322		else {
 323			dest_dst = ip_vs_dest_dst_alloc();
 324			spin_lock_bh(&dest->dst_lock);
 325			if (!dest_dst) {
 326				__ip_vs_dst_set(dest, NULL, NULL, 0);
 327				spin_unlock_bh(&dest->dst_lock);
 328				goto err_unreach;
 329			}
 330			rt = do_output_route4(net, dest->addr.ip, rt_mode,
 331					      &dest_dst->dst_saddr.ip);
 332			if (!rt) {
 333				__ip_vs_dst_set(dest, NULL, NULL, 0);
 334				spin_unlock_bh(&dest->dst_lock);
 335				ip_vs_dest_dst_free(dest_dst);
 336				goto err_unreach;
 337			}
 338			__ip_vs_dst_set(dest, dest_dst, &rt->dst, 0);
 339			spin_unlock_bh(&dest->dst_lock);
 340			IP_VS_DBG(10, "new dst %pI4, src %pI4, refcnt=%d\n",
 341				  &dest->addr.ip, &dest_dst->dst_saddr.ip,
 342				  atomic_read(&rt->dst.__refcnt));
 343		}
 344		if (ret_saddr)
 345			*ret_saddr = dest_dst->dst_saddr.ip;
 346	} else {
 347		__be32 saddr = htonl(INADDR_ANY);
 348
 349		noref = 0;
 350
 351		/* For such unconfigured boxes avoid many route lookups
 352		 * for performance reasons because we do not remember saddr
 353		 */
 354		rt_mode &= ~IP_VS_RT_MODE_CONNECT;
 355		rt = do_output_route4(net, daddr, rt_mode, &saddr);
 356		if (!rt)
 357			goto err_unreach;
 358		if (ret_saddr)
 359			*ret_saddr = saddr;
 360	}
 361
 362	local = (rt->rt_flags & RTCF_LOCAL) ? 1 : 0;
 363	if (unlikely(crosses_local_route_boundary(skb_af, skb, rt_mode,
 364						  local))) {
 365		IP_VS_DBG_RL("We are crossing local and non-local addresses"
 366			     " daddr=%pI4\n", &daddr);
 367		goto err_put;
 368	}
 369
 370	if (unlikely(local)) {
 371		/* skb to local stack, preserve old route */
 372		if (!noref)
 373			ip_rt_put(rt);
 374		return local;
 375	}
 376
 377	if (!decrement_ttl(ipvs, skb_af, skb))
 378		goto err_put;
 379
 380	if (likely(!(rt_mode & IP_VS_RT_MODE_TUNNEL))) {
 381		mtu = dst_mtu(&rt->dst);
 382	} else {
 383		mtu = dst_mtu(&rt->dst) - sizeof(struct iphdr);
 384		if (!dest)
 385			goto err_put;
 386		if (dest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
 387			mtu -= sizeof(struct udphdr) + sizeof(struct guehdr);
 388			if ((dest->tun_flags &
 389			     IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM) &&
 390			    skb->ip_summed == CHECKSUM_PARTIAL)
 391				mtu -= GUE_PLEN_REMCSUM + GUE_LEN_PRIV;
 392		} else if (dest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE) {
 393			__be16 tflags = 0;
 394
 395			if (dest->tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM)
 396				tflags |= TUNNEL_CSUM;
 397			mtu -= gre_calc_hlen(tflags);
 398		}
 399		if (mtu < 68) {
 400			IP_VS_DBG_RL("%s(): mtu less than 68\n", __func__);
 401			goto err_put;
 402		}
 403		maybe_update_pmtu(skb_af, skb, mtu);
 404	}
 405
 406	if (!ensure_mtu_is_adequate(ipvs, skb_af, rt_mode, ipvsh, skb, mtu))
 407		goto err_put;
 408
 409	skb_dst_drop(skb);
 410	if (noref)
 411		skb_dst_set_noref(skb, &rt->dst);
 412	else
 413		skb_dst_set(skb, &rt->dst);
 414
 415	return local;
 416
 417err_put:
 418	if (!noref)
 419		ip_rt_put(rt);
 420	return -1;
 421
 422err_unreach:
 423	dst_link_failure(skb);
 424	return -1;
 425}
 426
 427#ifdef CONFIG_IP_VS_IPV6
 428static struct dst_entry *
 429__ip_vs_route_output_v6(struct net *net, struct in6_addr *daddr,
 430			struct in6_addr *ret_saddr, int do_xfrm, int rt_mode)
 431{
 432	struct dst_entry *dst;
 433	struct flowi6 fl6 = {
 434		.daddr = *daddr,
 435	};
 436
 437	if (rt_mode & IP_VS_RT_MODE_KNOWN_NH)
 438		fl6.flowi6_flags = FLOWI_FLAG_KNOWN_NH;
 439
 440	dst = ip6_route_output(net, NULL, &fl6);
 441	if (dst->error)
 442		goto out_err;
 443	if (!ret_saddr)
 444		return dst;
 445	if (ipv6_addr_any(&fl6.saddr) &&
 446	    ipv6_dev_get_saddr(net, ip6_dst_idev(dst)->dev,
 447			       &fl6.daddr, 0, &fl6.saddr) < 0)
 448		goto out_err;
 449	if (do_xfrm) {
 450		dst = xfrm_lookup(net, dst, flowi6_to_flowi(&fl6), NULL, 0);
 451		if (IS_ERR(dst)) {
 452			dst = NULL;
 453			goto out_err;
 454		}
 455	}
 456	*ret_saddr = fl6.saddr;
 457	return dst;
 458
 459out_err:
 460	dst_release(dst);
 461	IP_VS_DBG_RL("ip6_route_output error, dest: %pI6\n", daddr);
 462	return NULL;
 463}
 464
 465/*
 466 * Get route to destination or remote server
 467 */
 468static int
 469__ip_vs_get_out_rt_v6(struct netns_ipvs *ipvs, int skb_af, struct sk_buff *skb,
 470		      struct ip_vs_dest *dest,
 471		      struct in6_addr *daddr, struct in6_addr *ret_saddr,
 472		      struct ip_vs_iphdr *ipvsh, int do_xfrm, int rt_mode)
 473{
 474	struct net *net = ipvs->net;
 475	struct ip_vs_dest_dst *dest_dst;
 476	struct rt6_info *rt;			/* Route to the other host */
 477	struct dst_entry *dst;
 478	int mtu;
 479	int local, noref = 1;
 480
 481	if (dest) {
 482		dest_dst = __ip_vs_dst_check(dest);
 483		if (likely(dest_dst))
 484			rt = (struct rt6_info *) dest_dst->dst_cache;
 485		else {
 486			u32 cookie;
 487
 488			dest_dst = ip_vs_dest_dst_alloc();
 489			spin_lock_bh(&dest->dst_lock);
 490			if (!dest_dst) {
 491				__ip_vs_dst_set(dest, NULL, NULL, 0);
 492				spin_unlock_bh(&dest->dst_lock);
 493				goto err_unreach;
 494			}
 495			dst = __ip_vs_route_output_v6(net, &dest->addr.in6,
 496						      &dest_dst->dst_saddr.in6,
 497						      do_xfrm, rt_mode);
 498			if (!dst) {
 499				__ip_vs_dst_set(dest, NULL, NULL, 0);
 500				spin_unlock_bh(&dest->dst_lock);
 501				ip_vs_dest_dst_free(dest_dst);
 502				goto err_unreach;
 503			}
 504			rt = (struct rt6_info *) dst;
 505			cookie = rt6_get_cookie(rt);
 506			__ip_vs_dst_set(dest, dest_dst, &rt->dst, cookie);
 507			spin_unlock_bh(&dest->dst_lock);
 508			IP_VS_DBG(10, "new dst %pI6, src %pI6, refcnt=%d\n",
 509				  &dest->addr.in6, &dest_dst->dst_saddr.in6,
 510				  atomic_read(&rt->dst.__refcnt));
 511		}
 512		if (ret_saddr)
 513			*ret_saddr = dest_dst->dst_saddr.in6;
 514	} else {
 515		noref = 0;
 516		dst = __ip_vs_route_output_v6(net, daddr, ret_saddr, do_xfrm,
 517					      rt_mode);
 518		if (!dst)
 519			goto err_unreach;
 520		rt = (struct rt6_info *) dst;
 521	}
 522
 523	local = __ip_vs_is_local_route6(rt);
 524
 525	if (unlikely(crosses_local_route_boundary(skb_af, skb, rt_mode,
 526						  local))) {
 527		IP_VS_DBG_RL("We are crossing local and non-local addresses"
 528			     " daddr=%pI6\n", daddr);
 529		goto err_put;
 530	}
 531
 532	if (unlikely(local)) {
 533		/* skb to local stack, preserve old route */
 534		if (!noref)
 535			dst_release(&rt->dst);
 536		return local;
 537	}
 538
 539	if (!decrement_ttl(ipvs, skb_af, skb))
 540		goto err_put;
 541
 542	/* MTU checking */
 543	if (likely(!(rt_mode & IP_VS_RT_MODE_TUNNEL)))
 544		mtu = dst_mtu(&rt->dst);
 545	else {
 546		mtu = dst_mtu(&rt->dst) - sizeof(struct ipv6hdr);
 547		if (!dest)
 548			goto err_put;
 549		if (dest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
 550			mtu -= sizeof(struct udphdr) + sizeof(struct guehdr);
 551			if ((dest->tun_flags &
 552			     IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM) &&
 553			    skb->ip_summed == CHECKSUM_PARTIAL)
 554				mtu -= GUE_PLEN_REMCSUM + GUE_LEN_PRIV;
 555		} else if (dest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE) {
 556			__be16 tflags = 0;
 557
 558			if (dest->tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM)
 559				tflags |= TUNNEL_CSUM;
 560			mtu -= gre_calc_hlen(tflags);
 561		}
 562		if (mtu < IPV6_MIN_MTU) {
 563			IP_VS_DBG_RL("%s(): mtu less than %d\n", __func__,
 564				     IPV6_MIN_MTU);
 565			goto err_put;
 566		}
 567		maybe_update_pmtu(skb_af, skb, mtu);
 568	}
 569
 570	if (!ensure_mtu_is_adequate(ipvs, skb_af, rt_mode, ipvsh, skb, mtu))
 571		goto err_put;
 572
 573	skb_dst_drop(skb);
 574	if (noref)
 575		skb_dst_set_noref(skb, &rt->dst);
 576	else
 577		skb_dst_set(skb, &rt->dst);
 578
 579	return local;
 580
 581err_put:
 582	if (!noref)
 583		dst_release(&rt->dst);
 584	return -1;
 585
 586err_unreach:
 587	/* The ip6_link_failure function requires the dev field to be set
 588	 * in order to get the net (further for the sake of fwmark
 589	 * reflection).
 590	 */
 591	if (!skb->dev)
 592		skb->dev = skb_dst(skb)->dev;
 593
 594	dst_link_failure(skb);
 595	return -1;
 596}
 597#endif
 598
 599
 600/* return NF_ACCEPT to allow forwarding or other NF_xxx on error */
 601static inline int ip_vs_tunnel_xmit_prepare(struct sk_buff *skb,
 602					    struct ip_vs_conn *cp)
 603{
 604	int ret = NF_ACCEPT;
 605
 606	skb->ipvs_property = 1;
 607	if (unlikely(cp->flags & IP_VS_CONN_F_NFCT))
 608		ret = ip_vs_confirm_conntrack(skb);
 609	if (ret == NF_ACCEPT) {
 610		nf_reset_ct(skb);
 611		skb_forward_csum(skb);
 612	}
 613	return ret;
 614}
 615
 616/* In the event of a remote destination, it's possible that we would have
 617 * matches against an old socket (particularly a TIME-WAIT socket). This
 618 * causes havoc down the line (ip_local_out et. al. expect regular sockets
 619 * and invalid memory accesses will happen) so simply drop the association
 620 * in this case.
 621*/
 622static inline void ip_vs_drop_early_demux_sk(struct sk_buff *skb)
 623{
 624	/* If dev is set, the packet came from the LOCAL_IN callback and
 625	 * not from a local TCP socket.
 626	 */
 627	if (skb->dev)
 628		skb_orphan(skb);
 629}
 630
 631/* return NF_STOLEN (sent) or NF_ACCEPT if local=1 (not sent) */
 632static inline int ip_vs_nat_send_or_cont(int pf, struct sk_buff *skb,
 633					 struct ip_vs_conn *cp, int local)
 634{
 635	int ret = NF_STOLEN;
 636
 637	skb->ipvs_property = 1;
 638	if (likely(!(cp->flags & IP_VS_CONN_F_NFCT)))
 639		ip_vs_notrack(skb);
 640	else
 641		ip_vs_update_conntrack(skb, cp, 1);
 642
 643	/* Remove the early_demux association unless it's bound for the
 644	 * exact same port and address on this host after translation.
 645	 */
 646	if (!local || cp->vport != cp->dport ||
 647	    !ip_vs_addr_equal(cp->af, &cp->vaddr, &cp->daddr))
 648		ip_vs_drop_early_demux_sk(skb);
 649
 650	if (!local) {
 651		skb_forward_csum(skb);
 652		NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb,
 653			NULL, skb_dst(skb)->dev, dst_output);
 654	} else
 655		ret = NF_ACCEPT;
 656
 657	return ret;
 658}
 659
 660/* return NF_STOLEN (sent) or NF_ACCEPT if local=1 (not sent) */
 661static inline int ip_vs_send_or_cont(int pf, struct sk_buff *skb,
 662				     struct ip_vs_conn *cp, int local)
 663{
 664	int ret = NF_STOLEN;
 665
 666	skb->ipvs_property = 1;
 667	if (likely(!(cp->flags & IP_VS_CONN_F_NFCT)))
 668		ip_vs_notrack(skb);
 669	if (!local) {
 670		ip_vs_drop_early_demux_sk(skb);
 671		skb_forward_csum(skb);
 672		NF_HOOK(pf, NF_INET_LOCAL_OUT, cp->ipvs->net, NULL, skb,
 673			NULL, skb_dst(skb)->dev, dst_output);
 674	} else
 675		ret = NF_ACCEPT;
 676	return ret;
 677}
 678
 679
 680/*
 681 *      NULL transmitter (do nothing except return NF_ACCEPT)
 682 */
 683int
 684ip_vs_null_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
 685		struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
 686{
 687	/* we do not touch skb and do not need pskb ptr */
 688	return ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 1);
 689}
 690
 691
 692/*
 693 *      Bypass transmitter
 694 *      Let packets bypass the destination when the destination is not
 695 *      available, it may be only used in transparent cache cluster.
 696 */
 697int
 698ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
 699		  struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
 700{
 701	struct iphdr  *iph = ip_hdr(skb);
 702
 703	EnterFunction(10);
 704
 705	if (__ip_vs_get_out_rt(cp->ipvs, cp->af, skb, NULL, iph->daddr,
 706			       IP_VS_RT_MODE_NON_LOCAL, NULL, ipvsh) < 0)
 707		goto tx_error;
 708
 709	ip_send_check(iph);
 710
 711	/* Another hack: avoid icmp_send in ip_fragment */
 712	skb->ignore_df = 1;
 713
 714	ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 0);
 715
 716	LeaveFunction(10);
 717	return NF_STOLEN;
 718
 719 tx_error:
 720	kfree_skb(skb);
 721	LeaveFunction(10);
 722	return NF_STOLEN;
 723}
 724
 725#ifdef CONFIG_IP_VS_IPV6
 726int
 727ip_vs_bypass_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
 728		     struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
 729{
 730	struct ipv6hdr *iph = ipv6_hdr(skb);
 731
 732	EnterFunction(10);
 733
 734	if (__ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, NULL,
 735				  &iph->daddr, NULL,
 736				  ipvsh, 0, IP_VS_RT_MODE_NON_LOCAL) < 0)
 737		goto tx_error;
 738
 739	/* Another hack: avoid icmp_send in ip_fragment */
 740	skb->ignore_df = 1;
 741
 742	ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 0);
 743
 744	LeaveFunction(10);
 745	return NF_STOLEN;
 746
 747 tx_error:
 748	kfree_skb(skb);
 749	LeaveFunction(10);
 750	return NF_STOLEN;
 751}
 752#endif
 753
 754/*
 755 *      NAT transmitter (only for outside-to-inside nat forwarding)
 756 *      Not used for related ICMP
 757 */
 758int
 759ip_vs_nat_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
 760	       struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
 761{
 762	struct rtable *rt;		/* Route to the other host */
 763	int local, rc, was_input;
 764
 765	EnterFunction(10);
 766
 767	/* check if it is a connection of no-client-port */
 768	if (unlikely(cp->flags & IP_VS_CONN_F_NO_CPORT)) {
 769		__be16 _pt, *p;
 770
 771		p = skb_header_pointer(skb, ipvsh->len, sizeof(_pt), &_pt);
 772		if (p == NULL)
 773			goto tx_error;
 774		ip_vs_conn_fill_cport(cp, *p);
 775		IP_VS_DBG(10, "filled cport=%d\n", ntohs(*p));
 776	}
 777
 778	was_input = rt_is_input_route(skb_rtable(skb));
 779	local = __ip_vs_get_out_rt(cp->ipvs, cp->af, skb, cp->dest, cp->daddr.ip,
 780				   IP_VS_RT_MODE_LOCAL |
 781				   IP_VS_RT_MODE_NON_LOCAL |
 782				   IP_VS_RT_MODE_RDR, NULL, ipvsh);
 783	if (local < 0)
 784		goto tx_error;
 785	rt = skb_rtable(skb);
 786	/*
 787	 * Avoid duplicate tuple in reply direction for NAT traffic
 788	 * to local address when connection is sync-ed
 789	 */
 790#if IS_ENABLED(CONFIG_NF_CONNTRACK)
 791	if (cp->flags & IP_VS_CONN_F_SYNC && local) {
 792		enum ip_conntrack_info ctinfo;
 793		struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
 794
 795		if (ct) {
 796			IP_VS_DBG_RL_PKT(10, AF_INET, pp, skb, ipvsh->off,
 797					 "ip_vs_nat_xmit(): "
 798					 "stopping DNAT to local address");
 799			goto tx_error;
 800		}
 801	}
 802#endif
 803
 804	/* From world but DNAT to loopback address? */
 805	if (local && ipv4_is_loopback(cp->daddr.ip) && was_input) {
 806		IP_VS_DBG_RL_PKT(1, AF_INET, pp, skb, ipvsh->off,
 807				 "ip_vs_nat_xmit(): stopping DNAT to loopback "
 808				 "address");
 809		goto tx_error;
 810	}
 811
 812	/* copy-on-write the packet before mangling it */
 813	if (skb_ensure_writable(skb, sizeof(struct iphdr)))
 814		goto tx_error;
 815
 816	if (skb_cow(skb, rt->dst.dev->hard_header_len))
 817		goto tx_error;
 818
 819	/* mangle the packet */
 820	if (pp->dnat_handler && !pp->dnat_handler(skb, pp, cp, ipvsh))
 821		goto tx_error;
 822	ip_hdr(skb)->daddr = cp->daddr.ip;
 823	ip_send_check(ip_hdr(skb));
 824
 825	IP_VS_DBG_PKT(10, AF_INET, pp, skb, ipvsh->off, "After DNAT");
 826
 827	/* FIXME: when application helper enlarges the packet and the length
 828	   is larger than the MTU of outgoing device, there will be still
 829	   MTU problem. */
 830
 831	/* Another hack: avoid icmp_send in ip_fragment */
 832	skb->ignore_df = 1;
 833
 834	rc = ip_vs_nat_send_or_cont(NFPROTO_IPV4, skb, cp, local);
 835
 836	LeaveFunction(10);
 837	return rc;
 838
 839  tx_error:
 840	kfree_skb(skb);
 841	LeaveFunction(10);
 842	return NF_STOLEN;
 843}
 844
 845#ifdef CONFIG_IP_VS_IPV6
 846int
 847ip_vs_nat_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
 848		  struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
 849{
 850	struct rt6_info *rt;		/* Route to the other host */
 851	int local, rc;
 852
 853	EnterFunction(10);
 854
 855	/* check if it is a connection of no-client-port */
 856	if (unlikely(cp->flags & IP_VS_CONN_F_NO_CPORT && !ipvsh->fragoffs)) {
 857		__be16 _pt, *p;
 858		p = skb_header_pointer(skb, ipvsh->len, sizeof(_pt), &_pt);
 859		if (p == NULL)
 860			goto tx_error;
 861		ip_vs_conn_fill_cport(cp, *p);
 862		IP_VS_DBG(10, "filled cport=%d\n", ntohs(*p));
 863	}
 864
 865	local = __ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, cp->dest,
 866				      &cp->daddr.in6,
 867				      NULL, ipvsh, 0,
 868				      IP_VS_RT_MODE_LOCAL |
 869				      IP_VS_RT_MODE_NON_LOCAL |
 870				      IP_VS_RT_MODE_RDR);
 871	if (local < 0)
 872		goto tx_error;
 873	rt = (struct rt6_info *) skb_dst(skb);
 874	/*
 875	 * Avoid duplicate tuple in reply direction for NAT traffic
 876	 * to local address when connection is sync-ed
 877	 */
 878#if IS_ENABLED(CONFIG_NF_CONNTRACK)
 879	if (cp->flags & IP_VS_CONN_F_SYNC && local) {
 880		enum ip_conntrack_info ctinfo;
 881		struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
 882
 883		if (ct) {
 884			IP_VS_DBG_RL_PKT(10, AF_INET6, pp, skb, ipvsh->off,
 885					 "ip_vs_nat_xmit_v6(): "
 886					 "stopping DNAT to local address");
 887			goto tx_error;
 888		}
 889	}
 890#endif
 891
 892	/* From world but DNAT to loopback address? */
 893	if (local && skb->dev && !(skb->dev->flags & IFF_LOOPBACK) &&
 894	    ipv6_addr_type(&cp->daddr.in6) & IPV6_ADDR_LOOPBACK) {
 895		IP_VS_DBG_RL_PKT(1, AF_INET6, pp, skb, ipvsh->off,
 896				 "ip_vs_nat_xmit_v6(): "
 897				 "stopping DNAT to loopback address");
 898		goto tx_error;
 899	}
 900
 901	/* copy-on-write the packet before mangling it */
 902	if (skb_ensure_writable(skb, sizeof(struct ipv6hdr)))
 903		goto tx_error;
 904
 905	if (skb_cow(skb, rt->dst.dev->hard_header_len))
 906		goto tx_error;
 907
 908	/* mangle the packet */
 909	if (pp->dnat_handler && !pp->dnat_handler(skb, pp, cp, ipvsh))
 910		goto tx_error;
 911	ipv6_hdr(skb)->daddr = cp->daddr.in6;
 912
 913	IP_VS_DBG_PKT(10, AF_INET6, pp, skb, ipvsh->off, "After DNAT");
 914
 915	/* FIXME: when application helper enlarges the packet and the length
 916	   is larger than the MTU of outgoing device, there will be still
 917	   MTU problem. */
 918
 919	/* Another hack: avoid icmp_send in ip_fragment */
 920	skb->ignore_df = 1;
 921
 922	rc = ip_vs_nat_send_or_cont(NFPROTO_IPV6, skb, cp, local);
 923
 924	LeaveFunction(10);
 925	return rc;
 926
 927tx_error:
 928	LeaveFunction(10);
 929	kfree_skb(skb);
 930	return NF_STOLEN;
 931}
 932#endif
 933
 934/* When forwarding a packet, we must ensure that we've got enough headroom
 935 * for the encapsulation packet in the skb.  This also gives us an
 936 * opportunity to figure out what the payload_len, dsfield, ttl, and df
 937 * values should be, so that we won't need to look at the old ip header
 938 * again
 939 */
 940static struct sk_buff *
 941ip_vs_prepare_tunneled_skb(struct sk_buff *skb, int skb_af,
 942			   unsigned int max_headroom, __u8 *next_protocol,
 943			   __u32 *payload_len, __u8 *dsfield, __u8 *ttl,
 944			   __be16 *df)
 945{
 946	struct sk_buff *new_skb = NULL;
 947	struct iphdr *old_iph = NULL;
 948	__u8 old_dsfield;
 949#ifdef CONFIG_IP_VS_IPV6
 950	struct ipv6hdr *old_ipv6h = NULL;
 951#endif
 952
 953	ip_vs_drop_early_demux_sk(skb);
 954
 955	if (skb_headroom(skb) < max_headroom || skb_cloned(skb)) {
 956		new_skb = skb_realloc_headroom(skb, max_headroom);
 957		if (!new_skb)
 958			goto error;
 959		if (skb->sk)
 960			skb_set_owner_w(new_skb, skb->sk);
 961		consume_skb(skb);
 962		skb = new_skb;
 963	}
 964
 965#ifdef CONFIG_IP_VS_IPV6
 966	if (skb_af == AF_INET6) {
 967		old_ipv6h = ipv6_hdr(skb);
 968		*next_protocol = IPPROTO_IPV6;
 969		if (payload_len)
 970			*payload_len =
 971				ntohs(old_ipv6h->payload_len) +
 972				sizeof(*old_ipv6h);
 973		old_dsfield = ipv6_get_dsfield(old_ipv6h);
 974		*ttl = old_ipv6h->hop_limit;
 975		if (df)
 976			*df = 0;
 977	} else
 978#endif
 979	{
 980		old_iph = ip_hdr(skb);
 981		/* Copy DF, reset fragment offset and MF */
 982		if (df)
 983			*df = (old_iph->frag_off & htons(IP_DF));
 984		*next_protocol = IPPROTO_IPIP;
 985
 986		/* fix old IP header checksum */
 987		ip_send_check(old_iph);
 988		old_dsfield = ipv4_get_dsfield(old_iph);
 989		*ttl = old_iph->ttl;
 990		if (payload_len)
 991			*payload_len = ntohs(old_iph->tot_len);
 992	}
 993
 994	/* Implement full-functionality option for ECN encapsulation */
 995	*dsfield = INET_ECN_encapsulate(old_dsfield, old_dsfield);
 996
 997	return skb;
 998error:
 999	kfree_skb(skb);
1000	return ERR_PTR(-ENOMEM);
1001}
1002
1003static inline int __tun_gso_type_mask(int encaps_af, int orig_af)
1004{
1005	switch (encaps_af) {
1006	case AF_INET:
1007		return SKB_GSO_IPXIP4;
1008	case AF_INET6:
1009		return SKB_GSO_IPXIP6;
1010	default:
1011		return 0;
1012	}
1013}
1014
1015static int
1016ipvs_gue_encap(struct net *net, struct sk_buff *skb,
1017	       struct ip_vs_conn *cp, __u8 *next_protocol)
1018{
1019	__be16 dport;
1020	__be16 sport = udp_flow_src_port(net, skb, 0, 0, false);
1021	struct udphdr  *udph;	/* Our new UDP header */
1022	struct guehdr  *gueh;	/* Our new GUE header */
1023	size_t hdrlen, optlen = 0;
1024	void *data;
1025	bool need_priv = false;
1026
1027	if ((cp->dest->tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM) &&
1028	    skb->ip_summed == CHECKSUM_PARTIAL) {
1029		optlen += GUE_PLEN_REMCSUM + GUE_LEN_PRIV;
1030		need_priv = true;
1031	}
1032
1033	hdrlen = sizeof(struct guehdr) + optlen;
1034
1035	skb_push(skb, hdrlen);
1036
1037	gueh = (struct guehdr *)skb->data;
1038
1039	gueh->control = 0;
1040	gueh->version = 0;
1041	gueh->hlen = optlen >> 2;
1042	gueh->flags = 0;
1043	gueh->proto_ctype = *next_protocol;
1044
1045	data = &gueh[1];
1046
1047	if (need_priv) {
1048		__be32 *flags = data;
1049		u16 csum_start = skb_checksum_start_offset(skb);
1050		__be16 *pd;
1051
1052		gueh->flags |= GUE_FLAG_PRIV;
1053		*flags = 0;
1054		data += GUE_LEN_PRIV;
1055
1056		if (csum_start < hdrlen)
1057			return -EINVAL;
1058
1059		csum_start -= hdrlen;
1060		pd = data;
1061		pd[0] = htons(csum_start);
1062		pd[1] = htons(csum_start + skb->csum_offset);
1063
1064		if (!skb_is_gso(skb)) {
1065			skb->ip_summed = CHECKSUM_NONE;
1066			skb->encapsulation = 0;
1067		}
1068
1069		*flags |= GUE_PFLAG_REMCSUM;
1070		data += GUE_PLEN_REMCSUM;
1071	}
1072
1073	skb_push(skb, sizeof(struct udphdr));
1074	skb_reset_transport_header(skb);
1075
1076	udph = udp_hdr(skb);
1077
1078	dport = cp->dest->tun_port;
1079	udph->dest = dport;
1080	udph->source = sport;
1081	udph->len = htons(skb->len);
1082	udph->check = 0;
1083
1084	*next_protocol = IPPROTO_UDP;
1085
1086	return 0;
1087}
1088
1089static void
1090ipvs_gre_encap(struct net *net, struct sk_buff *skb,
1091	       struct ip_vs_conn *cp, __u8 *next_protocol)
1092{
1093	__be16 proto = *next_protocol == IPPROTO_IPIP ?
1094				htons(ETH_P_IP) : htons(ETH_P_IPV6);
1095	__be16 tflags = 0;
1096	size_t hdrlen;
1097
1098	if (cp->dest->tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM)
1099		tflags |= TUNNEL_CSUM;
1100
1101	hdrlen = gre_calc_hlen(tflags);
1102	gre_build_header(skb, hdrlen, tflags, proto, 0, 0);
1103
1104	*next_protocol = IPPROTO_GRE;
1105}
1106
1107/*
1108 *   IP Tunneling transmitter
1109 *
1110 *   This function encapsulates the packet in a new IP packet, its
1111 *   destination will be set to cp->daddr. Most code of this function
1112 *   is taken from ipip.c.
1113 *
1114 *   It is used in VS/TUN cluster. The load balancer selects a real
1115 *   server from a cluster based on a scheduling algorithm,
1116 *   encapsulates the request packet and forwards it to the selected
1117 *   server. For example, all real servers are configured with
1118 *   "ifconfig tunl0 <Virtual IP Address> up". When the server receives
1119 *   the encapsulated packet, it will decapsulate the packet, processe
1120 *   the request and return the response packets directly to the client
1121 *   without passing the load balancer. This can greatly increase the
1122 *   scalability of virtual server.
1123 *
1124 *   Used for ANY protocol
1125 */
1126int
1127ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
1128		  struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
1129{
1130	struct netns_ipvs *ipvs = cp->ipvs;
1131	struct net *net = ipvs->net;
1132	struct rtable *rt;			/* Route to the other host */
1133	__be32 saddr;				/* Source for tunnel */
1134	struct net_device *tdev;		/* Device to other host */
1135	__u8 next_protocol = 0;
1136	__u8 dsfield = 0;
1137	__u8 ttl = 0;
1138	__be16 df = 0;
1139	__be16 *dfp = NULL;
1140	struct iphdr  *iph;			/* Our new IP header */
1141	unsigned int max_headroom;		/* The extra header space needed */
1142	int ret, local;
1143	int tun_type, gso_type;
1144	int tun_flags;
1145
1146	EnterFunction(10);
1147
1148	local = __ip_vs_get_out_rt(ipvs, cp->af, skb, cp->dest, cp->daddr.ip,
1149				   IP_VS_RT_MODE_LOCAL |
1150				   IP_VS_RT_MODE_NON_LOCAL |
1151				   IP_VS_RT_MODE_CONNECT |
1152				   IP_VS_RT_MODE_TUNNEL, &saddr, ipvsh);
1153	if (local < 0)
1154		goto tx_error;
1155	if (local)
1156		return ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 1);
1157
1158	rt = skb_rtable(skb);
1159	tdev = rt->dst.dev;
1160
1161	/*
1162	 * Okay, now see if we can stuff it in the buffer as-is.
1163	 */
1164	max_headroom = LL_RESERVED_SPACE(tdev) + sizeof(struct iphdr);
1165
1166	tun_type = cp->dest->tun_type;
1167	tun_flags = cp->dest->tun_flags;
1168
1169	if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
1170		size_t gue_hdrlen, gue_optlen = 0;
1171
1172		if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM) &&
1173		    skb->ip_summed == CHECKSUM_PARTIAL) {
1174			gue_optlen += GUE_PLEN_REMCSUM + GUE_LEN_PRIV;
1175		}
1176		gue_hdrlen = sizeof(struct guehdr) + gue_optlen;
1177
1178		max_headroom += sizeof(struct udphdr) + gue_hdrlen;
1179	} else if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE) {
1180		size_t gre_hdrlen;
1181		__be16 tflags = 0;
1182
1183		if (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM)
1184			tflags |= TUNNEL_CSUM;
1185		gre_hdrlen = gre_calc_hlen(tflags);
1186
1187		max_headroom += gre_hdrlen;
1188	}
1189
1190	/* We only care about the df field if sysctl_pmtu_disc(ipvs) is set */
1191	dfp = sysctl_pmtu_disc(ipvs) ? &df : NULL;
1192	skb = ip_vs_prepare_tunneled_skb(skb, cp->af, max_headroom,
1193					 &next_protocol, NULL, &dsfield,
1194					 &ttl, dfp);
1195	if (IS_ERR(skb))
1196		goto tx_error;
1197
1198	gso_type = __tun_gso_type_mask(AF_INET, cp->af);
1199	if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
1200		if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM) ||
1201		    (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM))
1202			gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1203		else
1204			gso_type |= SKB_GSO_UDP_TUNNEL;
1205		if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM) &&
1206		    skb->ip_summed == CHECKSUM_PARTIAL) {
1207			gso_type |= SKB_GSO_TUNNEL_REMCSUM;
1208		}
1209	} else if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE) {
1210		if (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM)
1211			gso_type |= SKB_GSO_GRE_CSUM;
1212		else
1213			gso_type |= SKB_GSO_GRE;
1214	}
1215
1216	if (iptunnel_handle_offloads(skb, gso_type))
1217		goto tx_error;
1218
1219	skb->transport_header = skb->network_header;
1220
1221	skb_set_inner_ipproto(skb, next_protocol);
1222
1223	if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
1224		bool check = false;
1225
1226		if (ipvs_gue_encap(net, skb, cp, &next_protocol))
1227			goto tx_error;
1228
1229		if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM) ||
1230		    (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM))
1231			check = true;
1232
1233		udp_set_csum(!check, skb, saddr, cp->daddr.ip, skb->len);
1234	} else if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE)
1235		ipvs_gre_encap(net, skb, cp, &next_protocol);
1236
1237	skb_push(skb, sizeof(struct iphdr));
1238	skb_reset_network_header(skb);
1239	memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
1240
1241	/*
1242	 *	Push down and install the IPIP header.
1243	 */
1244	iph			=	ip_hdr(skb);
1245	iph->version		=	4;
1246	iph->ihl		=	sizeof(struct iphdr)>>2;
1247	iph->frag_off		=	df;
1248	iph->protocol		=	next_protocol;
1249	iph->tos		=	dsfield;
1250	iph->daddr		=	cp->daddr.ip;
1251	iph->saddr		=	saddr;
1252	iph->ttl		=	ttl;
1253	ip_select_ident(net, skb, NULL);
1254
1255	/* Another hack: avoid icmp_send in ip_fragment */
1256	skb->ignore_df = 1;
1257
1258	ret = ip_vs_tunnel_xmit_prepare(skb, cp);
1259	if (ret == NF_ACCEPT)
1260		ip_local_out(net, skb->sk, skb);
1261	else if (ret == NF_DROP)
1262		kfree_skb(skb);
1263
1264	LeaveFunction(10);
1265
1266	return NF_STOLEN;
1267
1268  tx_error:
1269	if (!IS_ERR(skb))
1270		kfree_skb(skb);
1271	LeaveFunction(10);
1272	return NF_STOLEN;
1273}
1274
1275#ifdef CONFIG_IP_VS_IPV6
1276int
1277ip_vs_tunnel_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
1278		     struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
1279{
1280	struct netns_ipvs *ipvs = cp->ipvs;
1281	struct net *net = ipvs->net;
1282	struct rt6_info *rt;		/* Route to the other host */
1283	struct in6_addr saddr;		/* Source for tunnel */
1284	struct net_device *tdev;	/* Device to other host */
1285	__u8 next_protocol = 0;
1286	__u32 payload_len = 0;
1287	__u8 dsfield = 0;
1288	__u8 ttl = 0;
1289	struct ipv6hdr  *iph;		/* Our new IP header */
1290	unsigned int max_headroom;	/* The extra header space needed */
1291	int ret, local;
1292	int tun_type, gso_type;
1293	int tun_flags;
1294
1295	EnterFunction(10);
1296
1297	local = __ip_vs_get_out_rt_v6(ipvs, cp->af, skb, cp->dest,
1298				      &cp->daddr.in6,
1299				      &saddr, ipvsh, 1,
1300				      IP_VS_RT_MODE_LOCAL |
1301				      IP_VS_RT_MODE_NON_LOCAL |
1302				      IP_VS_RT_MODE_TUNNEL);
1303	if (local < 0)
1304		goto tx_error;
1305	if (local)
1306		return ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 1);
1307
1308	rt = (struct rt6_info *) skb_dst(skb);
1309	tdev = rt->dst.dev;
1310
1311	/*
1312	 * Okay, now see if we can stuff it in the buffer as-is.
1313	 */
1314	max_headroom = LL_RESERVED_SPACE(tdev) + sizeof(struct ipv6hdr);
1315
1316	tun_type = cp->dest->tun_type;
1317	tun_flags = cp->dest->tun_flags;
1318
1319	if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
1320		size_t gue_hdrlen, gue_optlen = 0;
1321
1322		if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM) &&
1323		    skb->ip_summed == CHECKSUM_PARTIAL) {
1324			gue_optlen += GUE_PLEN_REMCSUM + GUE_LEN_PRIV;
1325		}
1326		gue_hdrlen = sizeof(struct guehdr) + gue_optlen;
1327
1328		max_headroom += sizeof(struct udphdr) + gue_hdrlen;
1329	} else if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE) {
1330		size_t gre_hdrlen;
1331		__be16 tflags = 0;
1332
1333		if (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM)
1334			tflags |= TUNNEL_CSUM;
1335		gre_hdrlen = gre_calc_hlen(tflags);
1336
1337		max_headroom += gre_hdrlen;
1338	}
1339
1340	skb = ip_vs_prepare_tunneled_skb(skb, cp->af, max_headroom,
1341					 &next_protocol, &payload_len,
1342					 &dsfield, &ttl, NULL);
1343	if (IS_ERR(skb))
1344		goto tx_error;
1345
1346	gso_type = __tun_gso_type_mask(AF_INET6, cp->af);
1347	if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
1348		if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM) ||
1349		    (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM))
1350			gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1351		else
1352			gso_type |= SKB_GSO_UDP_TUNNEL;
1353		if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM) &&
1354		    skb->ip_summed == CHECKSUM_PARTIAL) {
1355			gso_type |= SKB_GSO_TUNNEL_REMCSUM;
1356		}
1357	} else if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE) {
1358		if (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM)
1359			gso_type |= SKB_GSO_GRE_CSUM;
1360		else
1361			gso_type |= SKB_GSO_GRE;
1362	}
1363
1364	if (iptunnel_handle_offloads(skb, gso_type))
1365		goto tx_error;
1366
1367	skb->transport_header = skb->network_header;
1368
1369	skb_set_inner_ipproto(skb, next_protocol);
1370
1371	if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) {
1372		bool check = false;
1373
1374		if (ipvs_gue_encap(net, skb, cp, &next_protocol))
1375			goto tx_error;
1376
1377		if ((tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_CSUM) ||
1378		    (tun_flags & IP_VS_TUNNEL_ENCAP_FLAG_REMCSUM))
1379			check = true;
1380
1381		udp6_set_csum(!check, skb, &saddr, &cp->daddr.in6, skb->len);
1382	} else if (tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GRE)
1383		ipvs_gre_encap(net, skb, cp, &next_protocol);
1384
1385	skb_push(skb, sizeof(struct ipv6hdr));
1386	skb_reset_network_header(skb);
1387	memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
1388
1389	/*
1390	 *	Push down and install the IPIP header.
1391	 */
1392	iph			=	ipv6_hdr(skb);
1393	iph->version		=	6;
1394	iph->nexthdr		=	next_protocol;
1395	iph->payload_len	=	htons(payload_len);
1396	memset(&iph->flow_lbl, 0, sizeof(iph->flow_lbl));
1397	ipv6_change_dsfield(iph, 0, dsfield);
1398	iph->daddr = cp->daddr.in6;
1399	iph->saddr = saddr;
1400	iph->hop_limit		=	ttl;
1401
1402	/* Another hack: avoid icmp_send in ip_fragment */
1403	skb->ignore_df = 1;
1404
1405	ret = ip_vs_tunnel_xmit_prepare(skb, cp);
1406	if (ret == NF_ACCEPT)
1407		ip6_local_out(net, skb->sk, skb);
1408	else if (ret == NF_DROP)
1409		kfree_skb(skb);
1410
1411	LeaveFunction(10);
1412
1413	return NF_STOLEN;
1414
1415tx_error:
1416	if (!IS_ERR(skb))
1417		kfree_skb(skb);
1418	LeaveFunction(10);
1419	return NF_STOLEN;
1420}
1421#endif
1422
1423
1424/*
1425 *      Direct Routing transmitter
1426 *      Used for ANY protocol
1427 */
1428int
1429ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
1430	      struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
1431{
1432	int local;
1433
1434	EnterFunction(10);
1435
1436	local = __ip_vs_get_out_rt(cp->ipvs, cp->af, skb, cp->dest, cp->daddr.ip,
1437				   IP_VS_RT_MODE_LOCAL |
1438				   IP_VS_RT_MODE_NON_LOCAL |
1439				   IP_VS_RT_MODE_KNOWN_NH, NULL, ipvsh);
1440	if (local < 0)
1441		goto tx_error;
1442	if (local)
1443		return ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 1);
1444
1445	ip_send_check(ip_hdr(skb));
1446
1447	/* Another hack: avoid icmp_send in ip_fragment */
1448	skb->ignore_df = 1;
1449
1450	ip_vs_send_or_cont(NFPROTO_IPV4, skb, cp, 0);
1451
1452	LeaveFunction(10);
1453	return NF_STOLEN;
1454
1455  tx_error:
1456	kfree_skb(skb);
1457	LeaveFunction(10);
1458	return NF_STOLEN;
1459}
1460
1461#ifdef CONFIG_IP_VS_IPV6
1462int
1463ip_vs_dr_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
1464		 struct ip_vs_protocol *pp, struct ip_vs_iphdr *ipvsh)
1465{
1466	int local;
1467
1468	EnterFunction(10);
1469
1470	local = __ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, cp->dest,
1471				      &cp->daddr.in6,
1472				      NULL, ipvsh, 0,
1473				      IP_VS_RT_MODE_LOCAL |
1474				      IP_VS_RT_MODE_NON_LOCAL |
1475				      IP_VS_RT_MODE_KNOWN_NH);
1476	if (local < 0)
1477		goto tx_error;
1478	if (local)
1479		return ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 1);
1480
1481	/* Another hack: avoid icmp_send in ip_fragment */
1482	skb->ignore_df = 1;
1483
1484	ip_vs_send_or_cont(NFPROTO_IPV6, skb, cp, 0);
1485
1486	LeaveFunction(10);
1487	return NF_STOLEN;
1488
1489tx_error:
1490	kfree_skb(skb);
1491	LeaveFunction(10);
1492	return NF_STOLEN;
1493}
1494#endif
1495
1496
1497/*
1498 *	ICMP packet transmitter
1499 *	called by the ip_vs_in_icmp
1500 */
1501int
1502ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp,
1503		struct ip_vs_protocol *pp, int offset, unsigned int hooknum,
1504		struct ip_vs_iphdr *iph)
1505{
1506	struct rtable	*rt;	/* Route to the other host */
1507	int rc;
1508	int local;
1509	int rt_mode, was_input;
1510
1511	EnterFunction(10);
1512
1513	/* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be
1514	   forwarded directly here, because there is no need to
1515	   translate address/port back */
1516	if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) {
1517		if (cp->packet_xmit)
1518			rc = cp->packet_xmit(skb, cp, pp, iph);
1519		else
1520			rc = NF_ACCEPT;
1521		/* do not touch skb anymore */
1522		atomic_inc(&cp->in_pkts);
1523		goto out;
1524	}
1525
1526	/*
1527	 * mangle and send the packet here (only for VS/NAT)
1528	 */
1529	was_input = rt_is_input_route(skb_rtable(skb));
1530
1531	/* LOCALNODE from FORWARD hook is not supported */
1532	rt_mode = (hooknum != NF_INET_FORWARD) ?
1533		  IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL |
1534		  IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL;
1535	local = __ip_vs_get_out_rt(cp->ipvs, cp->af, skb, cp->dest, cp->daddr.ip, rt_mode,
1536				   NULL, iph);
1537	if (local < 0)
1538		goto tx_error;
1539	rt = skb_rtable(skb);
1540
1541	/*
1542	 * Avoid duplicate tuple in reply direction for NAT traffic
1543	 * to local address when connection is sync-ed
1544	 */
1545#if IS_ENABLED(CONFIG_NF_CONNTRACK)
1546	if (cp->flags & IP_VS_CONN_F_SYNC && local) {
1547		enum ip_conntrack_info ctinfo;
1548		struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
1549
1550		if (ct) {
1551			IP_VS_DBG(10, "%s(): "
1552				  "stopping DNAT to local address %pI4\n",
1553				  __func__, &cp->daddr.ip);
1554			goto tx_error;
1555		}
1556	}
1557#endif
1558
1559	/* From world but DNAT to loopback address? */
1560	if (local && ipv4_is_loopback(cp->daddr.ip) && was_input) {
1561		IP_VS_DBG(1, "%s(): "
1562			  "stopping DNAT to loopback %pI4\n",
1563			  __func__, &cp->daddr.ip);
1564		goto tx_error;
1565	}
1566
1567	/* copy-on-write the packet before mangling it */
1568	if (skb_ensure_writable(skb, offset))
1569		goto tx_error;
1570
1571	if (skb_cow(skb, rt->dst.dev->hard_header_len))
1572		goto tx_error;
1573
1574	ip_vs_nat_icmp(skb, pp, cp, 0);
1575
1576	/* Another hack: avoid icmp_send in ip_fragment */
1577	skb->ignore_df = 1;
1578
1579	rc = ip_vs_nat_send_or_cont(NFPROTO_IPV4, skb, cp, local);
1580	goto out;
1581
1582  tx_error:
1583	kfree_skb(skb);
1584	rc = NF_STOLEN;
1585  out:
1586	LeaveFunction(10);
1587	return rc;
1588}
1589
1590#ifdef CONFIG_IP_VS_IPV6
1591int
1592ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp,
1593		struct ip_vs_protocol *pp, int offset, unsigned int hooknum,
1594		struct ip_vs_iphdr *ipvsh)
1595{
1596	struct rt6_info	*rt;	/* Route to the other host */
1597	int rc;
1598	int local;
1599	int rt_mode;
1600
1601	EnterFunction(10);
1602
1603	/* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be
1604	   forwarded directly here, because there is no need to
1605	   translate address/port back */
1606	if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) {
1607		if (cp->packet_xmit)
1608			rc = cp->packet_xmit(skb, cp, pp, ipvsh);
1609		else
1610			rc = NF_ACCEPT;
1611		/* do not touch skb anymore */
1612		atomic_inc(&cp->in_pkts);
1613		goto out;
1614	}
1615
1616	/*
1617	 * mangle and send the packet here (only for VS/NAT)
1618	 */
1619
1620	/* LOCALNODE from FORWARD hook is not supported */
1621	rt_mode = (hooknum != NF_INET_FORWARD) ?
1622		  IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL |
1623		  IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL;
1624	local = __ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, cp->dest,
1625				      &cp->daddr.in6, NULL, ipvsh, 0, rt_mode);
1626	if (local < 0)
1627		goto tx_error;
1628	rt = (struct rt6_info *) skb_dst(skb);
1629	/*
1630	 * Avoid duplicate tuple in reply direction for NAT traffic
1631	 * to local address when connection is sync-ed
1632	 */
1633#if IS_ENABLED(CONFIG_NF_CONNTRACK)
1634	if (cp->flags & IP_VS_CONN_F_SYNC && local) {
1635		enum ip_conntrack_info ctinfo;
1636		struct nf_conn *ct = nf_ct_get(skb, &ctinfo);
1637
1638		if (ct) {
1639			IP_VS_DBG(10, "%s(): "
1640				  "stopping DNAT to local address %pI6\n",
1641				  __func__, &cp->daddr.in6);
1642			goto tx_error;
1643		}
1644	}
1645#endif
1646
1647	/* From world but DNAT to loopback address? */
1648	if (local && skb->dev && !(skb->dev->flags & IFF_LOOPBACK) &&
1649	    ipv6_addr_type(&cp->daddr.in6) & IPV6_ADDR_LOOPBACK) {
1650		IP_VS_DBG(1, "%s(): "
1651			  "stopping DNAT to loopback %pI6\n",
1652			  __func__, &cp->daddr.in6);
1653		goto tx_error;
1654	}
1655
1656	/* copy-on-write the packet before mangling it */
1657	if (skb_ensure_writable(skb, offset))
1658		goto tx_error;
1659
1660	if (skb_cow(skb, rt->dst.dev->hard_header_len))
1661		goto tx_error;
1662
1663	ip_vs_nat_icmp_v6(skb, pp, cp, 0);
1664
1665	/* Another hack: avoid icmp_send in ip_fragment */
1666	skb->ignore_df = 1;
1667
1668	rc = ip_vs_nat_send_or_cont(NFPROTO_IPV6, skb, cp, local);
1669	goto out;
1670
1671tx_error:
1672	kfree_skb(skb);
1673	rc = NF_STOLEN;
1674out:
1675	LeaveFunction(10);
1676	return rc;
1677}
1678#endif