PageRenderTime 183ms CodeModel.GetById 129ms RepoModel.GetById 18ms app.codeStats 0ms

/src/core/org/apache/hadoop/net/NetUtils.java

https://github.com/dynamicguy/hadoop-20
Java | 550 lines | 267 code | 46 blank | 237 comment | 69 complexity | a070fb52ad6e744f1c8820cb52c2edba MD5 | raw file
Possible License(s): BSD-3-Clause, Apache-2.0
  1. /**
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. package org.apache.hadoop.net;
  19. import java.io.IOException;
  20. import java.io.InputStream;
  21. import java.io.OutputStream;
  22. import java.net.InetAddress;
  23. import java.net.InetSocketAddress;
  24. import java.net.NetworkInterface;
  25. import java.net.Socket;
  26. import java.net.SocketAddress;
  27. import java.net.SocketException;
  28. import java.net.URI;
  29. import java.net.UnknownHostException;
  30. import java.net.ConnectException;
  31. import java.nio.channels.SocketChannel;
  32. import java.util.Map.Entry;
  33. import java.util.*;
  34. import java.util.concurrent.ConcurrentHashMap;
  35. import javax.net.SocketFactory;
  36. import org.apache.commons.logging.Log;
  37. import org.apache.commons.logging.LogFactory;
  38. import org.apache.hadoop.conf.Configuration;
  39. import org.apache.hadoop.fs.Path;
  40. import org.apache.hadoop.ipc.Server;
  41. import org.apache.hadoop.ipc.VersionedProtocol;
  42. import org.apache.hadoop.syscall.LinuxSystemCall;
  43. import org.apache.hadoop.util.ReflectionUtils;
  44. public class NetUtils {
  45. private static final Log LOG = LogFactory.getLog(NetUtils.class);
  46. // Insure that Network Control bits in sock will never be used,
  47. // allow to set them may cause issue on the network, killing network
  48. // control traffic.
  49. // NC bits are:
  50. // in TOS Dec: 192 and up.
  51. public static final int IP_TOS_MAX_VALUE = 191;
  52. public static final int NOT_SET_IP_TOS = -1;
  53. public static final String DFS_CLIENT_TOS_CONF = "dfs.client.tos.value";
  54. private static Map<String, String> hostToResolved =
  55. new HashMap<String, String>();
  56. /**
  57. * Check the tos value is valid, that is, in the -1~191 range
  58. * @param tosValue
  59. * @return
  60. */
  61. public static boolean isValidTOSValue(int tosValue) {
  62. return tosValue >= NOT_SET_IP_TOS && tosValue <= IP_TOS_MAX_VALUE;
  63. }
  64. /**
  65. * Get the socket factory for the given class according to its
  66. * configuration parameter
  67. * <tt>hadoop.rpc.socket.factory.class.&lt;ClassName&gt;</tt>. When no
  68. * such parameter exists then fall back on the default socket factory as
  69. * configured by <tt>hadoop.rpc.socket.factory.class.default</tt>. If
  70. * this default socket factory is not configured, then fall back on the JVM
  71. * default socket factory.
  72. *
  73. * @param conf the configuration
  74. * @param clazz the class (usually a {@link VersionedProtocol})
  75. * @return a socket factory
  76. */
  77. public static SocketFactory getSocketFactory(Configuration conf,
  78. Class<?> clazz) {
  79. SocketFactory factory = null;
  80. String propValue =
  81. conf.get("hadoop.rpc.socket.factory.class." + clazz.getSimpleName());
  82. if ((propValue != null) && (propValue.length() > 0))
  83. factory = getSocketFactoryFromProperty(conf, propValue);
  84. if (factory == null)
  85. factory = getDefaultSocketFactory(conf);
  86. return factory;
  87. }
  88. /**
  89. * Get the default socket factory as specified by the configuration
  90. * parameter <tt>hadoop.rpc.socket.factory.default</tt>
  91. *
  92. * @param conf the configuration
  93. * @return the default socket factory as specified in the configuration or
  94. * the JVM default socket factory if the configuration does not
  95. * contain a default socket factory property.
  96. */
  97. public static SocketFactory getDefaultSocketFactory(Configuration conf) {
  98. String propValue = conf.get("hadoop.rpc.socket.factory.class.default");
  99. if ((propValue == null) || (propValue.length() == 0))
  100. return SocketFactory.getDefault();
  101. return getSocketFactoryFromProperty(conf, propValue);
  102. }
  103. /**
  104. * Get the socket factory corresponding to the given proxy URI. If the
  105. * given proxy URI corresponds to an absence of configuration parameter,
  106. * returns null. If the URI is malformed raises an exception.
  107. *
  108. * @param propValue the property which is the class name of the
  109. * SocketFactory to instantiate; assumed non null and non empty.
  110. * @return a socket factory as defined in the property value.
  111. */
  112. public static SocketFactory getSocketFactoryFromProperty(
  113. Configuration conf, String propValue) {
  114. try {
  115. Class<?> theClass = conf.getClassByName(propValue);
  116. return (SocketFactory) ReflectionUtils.newInstance(theClass, conf);
  117. } catch (ClassNotFoundException cnfe) {
  118. throw new RuntimeException("Socket Factory class not found: " + cnfe);
  119. }
  120. }
  121. /**
  122. * Util method to build socket addr from either:
  123. * <host>:<post>
  124. * <fs>://<host>:<port>/<path>
  125. */
  126. public static InetSocketAddress createSocketAddr(String target) {
  127. return createSocketAddr(target, -1);
  128. }
  129. /**
  130. * Util method to build socket addr from either:
  131. * <host>
  132. * <host>:<post>
  133. * <fs>://<host>:<port>/<path>
  134. */
  135. public static InetSocketAddress createSocketAddr(String target,
  136. int defaultPort) {
  137. int colonIndex = target.indexOf(':');
  138. if (colonIndex < 0 && defaultPort == -1) {
  139. throw new RuntimeException("Not a host:port pair: " + target);
  140. }
  141. String hostname;
  142. int port = -1;
  143. if (!target.contains("/")) {
  144. if (colonIndex == -1) {
  145. hostname = target;
  146. } else {
  147. // must be the old style <host>:<port>
  148. hostname = target.substring(0, colonIndex);
  149. port = Integer.parseInt(target.substring(colonIndex + 1));
  150. }
  151. } else {
  152. // a new uri
  153. URI addr = new Path(target).toUri();
  154. hostname = addr.getHost();
  155. port = addr.getPort();
  156. }
  157. if (port == -1) {
  158. port = defaultPort;
  159. }
  160. if (getStaticResolution(hostname) != null) {
  161. hostname = getStaticResolution(hostname);
  162. }
  163. return InetSocketAddressFactory.createWithResolveRetry(hostname, port);
  164. }
  165. /**
  166. * Handle the transition from pairs of attributes specifying a host and port
  167. * to a single colon separated one.
  168. * @param conf the configuration to check
  169. * @param oldBindAddressName the old address attribute name
  170. * @param oldPortName the old port attribute name
  171. * @param newBindAddressName the new combined name
  172. * @return the complete address from the configuration
  173. */
  174. @Deprecated
  175. public static String getServerAddress(Configuration conf,
  176. String oldBindAddressName,
  177. String oldPortName,
  178. String newBindAddressName) {
  179. String oldAddr = conf.get(oldBindAddressName);
  180. String oldPort = conf.get(oldPortName);
  181. String newAddrPort = conf.get(newBindAddressName);
  182. if (oldAddr == null && oldPort == null) {
  183. return newAddrPort;
  184. }
  185. String[] newAddrPortParts = newAddrPort.split(":",2);
  186. if (newAddrPortParts.length != 2) {
  187. throw new IllegalArgumentException("Invalid address/port: " +
  188. newAddrPort);
  189. }
  190. if (oldAddr == null) {
  191. oldAddr = newAddrPortParts[0];
  192. } else {
  193. LOG.warn("Configuration parameter " + oldBindAddressName +
  194. " is deprecated. Use " + newBindAddressName + " instead.");
  195. }
  196. if (oldPort == null) {
  197. oldPort = newAddrPortParts[1];
  198. } else {
  199. LOG.warn("Configuration parameter " + oldPortName +
  200. " is deprecated. Use " + newBindAddressName + " instead.");
  201. }
  202. return oldAddr + ":" + oldPort;
  203. }
  204. /**
  205. * Adds a static resolution for host. This can be used for setting up
  206. * hostnames with names that are fake to point to a well known host. For e.g.
  207. * in some testcases we require to have daemons with different hostnames
  208. * running on the same machine. In order to create connections to these
  209. * daemons, one can set up mappings from those hostnames to "localhost".
  210. * {@link NetUtils#getStaticResolution(String)} can be used to query for
  211. * the actual hostname.
  212. * @param host
  213. * @param resolvedName
  214. */
  215. public static void addStaticResolution(String host, String resolvedName) {
  216. synchronized (hostToResolved) {
  217. hostToResolved.put(host, resolvedName);
  218. }
  219. }
  220. /**
  221. * Retrieves the resolved name for the passed host. The resolved name must
  222. * have been set earlier using
  223. * {@link NetUtils#addStaticResolution(String, String)}
  224. * @param host
  225. * @return the resolution
  226. */
  227. public static String getStaticResolution(String host) {
  228. synchronized (hostToResolved) {
  229. return hostToResolved.get(host);
  230. }
  231. }
  232. /**
  233. * This is used to get all the resolutions that were added using
  234. * {@link NetUtils#addStaticResolution(String, String)}. The return
  235. * value is a List each element of which contains an array of String
  236. * of the form String[0]=hostname, String[1]=resolved-hostname
  237. * @return the list of resolutions
  238. */
  239. public static List <String[]> getAllStaticResolutions() {
  240. synchronized (hostToResolved) {
  241. Set <Entry <String, String>>entries = hostToResolved.entrySet();
  242. if (entries.size() == 0) {
  243. return null;
  244. }
  245. List <String[]> l = new ArrayList<String[]>(entries.size());
  246. for (Entry<String, String> e : entries) {
  247. l.add(new String[] {e.getKey(), e.getValue()});
  248. }
  249. return l;
  250. }
  251. }
  252. /**
  253. * Returns InetSocketAddress that a client can use to
  254. * connect to the server. Server.getListenerAddress() is not correct when
  255. * the server binds to "0.0.0.0". This returns "127.0.0.1:port" when
  256. * the getListenerAddress() returns "0.0.0.0:port".
  257. *
  258. * @param server
  259. * @return socket address that a client can use to connect to the server.
  260. */
  261. public static InetSocketAddress getConnectAddress(Server server) {
  262. InetSocketAddress addr = server.getListenerAddress();
  263. if (addr.getAddress().getHostAddress().equals("0.0.0.0")) {
  264. addr = new InetSocketAddress("127.0.0.1", addr.getPort());
  265. }
  266. return addr;
  267. }
  268. /**
  269. * Same as getInputStream(socket, socket.getSoTimeout()).<br><br>
  270. *
  271. * From documentation for {@link #getInputStream(Socket, long)}:<br>
  272. * Returns InputStream for the socket. If the socket has an associated
  273. * SocketChannel then it returns a
  274. * {@link SocketInputStream} with the given timeout. If the socket does not
  275. * have a channel, {@link Socket#getInputStream()} is returned. In the later
  276. * case, the timeout argument is ignored and the timeout set with
  277. * {@link Socket#setSoTimeout(int)} applies for reads.<br><br>
  278. *
  279. * Any socket created using socket factories returned by {@link #NetUtils},
  280. * must use this interface instead of {@link Socket#getInputStream()}.
  281. *
  282. * @see #getInputStream(Socket, long)
  283. *
  284. * @param socket
  285. * @return InputStream for reading from the socket.
  286. * @throws IOException
  287. */
  288. public static InputStream getInputStream(Socket socket)
  289. throws IOException {
  290. return getInputStream(socket, socket.getSoTimeout());
  291. }
  292. /**
  293. * Returns InputStream for the socket. If the socket has an associated
  294. * SocketChannel then it returns a
  295. * {@link SocketInputStream} with the given timeout. If the socket does not
  296. * have a channel, {@link Socket#getInputStream()} is returned. In the later
  297. * case, the timeout argument is ignored and the timeout set with
  298. * {@link Socket#setSoTimeout(int)} applies for reads.<br><br>
  299. *
  300. * Any socket created using socket factories returned by {@link #NetUtils},
  301. * must use this interface instead of {@link Socket#getInputStream()}.
  302. *
  303. * @see Socket#getChannel()
  304. *
  305. * @param socket
  306. * @param timeout timeout in milliseconds. This may not always apply. zero
  307. * for waiting as long as necessary.
  308. * @return InputStream for reading from the socket.
  309. * @throws IOException
  310. */
  311. public static InputStream getInputStream(Socket socket, long timeout)
  312. throws IOException {
  313. return (socket.getChannel() == null) ?
  314. socket.getInputStream() : new SocketInputStream(socket, timeout);
  315. }
  316. /**
  317. * Same as getOutputStream(socket, 0). Timeout of zero implies write will
  318. * wait until data is available.<br><br>
  319. *
  320. * From documentation for {@link #getOutputStream(Socket, long)} : <br>
  321. * Returns OutputStream for the socket. If the socket has an associated
  322. * SocketChannel then it returns a
  323. * {@link SocketOutputStream} with the given timeout. If the socket does not
  324. * have a channel, {@link Socket#getOutputStream()} is returned. In the later
  325. * case, the timeout argument is ignored and the write will wait until
  326. * data is available.<br><br>
  327. *
  328. * Any socket created using socket factories returned by {@link #NetUtils},
  329. * must use this interface instead of {@link Socket#getOutputStream()}.
  330. *
  331. * @see #getOutputStream(Socket, long)
  332. *
  333. * @param socket
  334. * @return OutputStream for writing to the socket.
  335. * @throws IOException
  336. */
  337. public static OutputStream getOutputStream(Socket socket)
  338. throws IOException {
  339. return getOutputStream(socket, 0);
  340. }
  341. /**
  342. * Returns OutputStream for the socket. If the socket has an associated
  343. * SocketChannel then it returns a
  344. * {@link SocketOutputStream} with the given timeout. If the socket does not
  345. * have a channel, {@link Socket#getOutputStream()} is returned. In the later
  346. * case, the timeout argument is ignored and the write will wait until
  347. * data is available.<br><br>
  348. *
  349. * Any socket created using socket factories returned by {@link #NetUtils},
  350. * must use this interface instead of {@link Socket#getOutputStream()}.
  351. *
  352. * @see Socket#getChannel()
  353. *
  354. * @param socket
  355. * @param timeout timeout in milliseconds. This may not always apply. zero
  356. * for waiting as long as necessary.
  357. * @return OutputStream for writing to the socket.
  358. * @throws IOException
  359. */
  360. public static OutputStream getOutputStream(Socket socket, long timeout)
  361. throws IOException {
  362. return (socket.getChannel() == null) ?
  363. socket.getOutputStream() : new SocketOutputStream(socket, timeout);
  364. }
  365. /**
  366. * This is a drop-in replacement for
  367. * {@link Socket#connect(SocketAddress, int)}.
  368. * In the case of normal sockets that don't have associated channels, this
  369. * just invokes <code>socket.connect(endpoint, timeout)</code>. If
  370. * <code>socket.getChannel()</code> returns a non-null channel,
  371. * connect is implemented using Hadoop's selectors. This is done mainly
  372. * to avoid Sun's connect implementation from creating thread-local
  373. * selectors, since Hadoop does not have control on when these are closed
  374. * and could end up taking all the available file descriptors.
  375. *
  376. * @see java.net.Socket#connect(java.net.SocketAddress, int)
  377. *
  378. * @param socket
  379. * @param endpoint
  380. * @param timeout - timeout in milliseconds
  381. */
  382. public static void connect(Socket socket,
  383. SocketAddress endpoint,
  384. int timeout) throws IOException {
  385. connect(socket, endpoint, timeout, NOT_SET_IP_TOS);
  386. }
  387. public static void connect(Socket socket,
  388. SocketAddress endpoint,
  389. int timeout,
  390. int ipTosValue) throws IOException {
  391. if (socket == null || endpoint == null || timeout < 0
  392. || ipTosValue < NOT_SET_IP_TOS
  393. || ipTosValue > IP_TOS_MAX_VALUE) {
  394. throw new IllegalArgumentException("Illegal argument for connect()");
  395. }
  396. SocketChannel ch = socket.getChannel();
  397. if (ch == null) {
  398. // let the default implementation handle it.
  399. socket.connect(endpoint, timeout);
  400. } else {
  401. // set the socket IP_TOS value
  402. if (ipTosValue != NOT_SET_IP_TOS) {
  403. LinuxSystemCall.setIPTOSVal(ch, ipTosValue);
  404. }
  405. SocketIOWithTimeout.connect(ch, endpoint, timeout);
  406. }
  407. // There is a very rare case allowed by the TCP specification, such that
  408. // if we are trying to connect to an endpoint on the local machine,
  409. // and we end up choosing an ephemeral port equal to the destination port,
  410. // we will actually end up getting connected to ourself (ie any data we
  411. // send just comes right back). This is only possible if the target
  412. // daemon is down, so we'll treat it like connection refused.
  413. if (socket.getLocalPort() == socket.getPort() &&
  414. socket.getLocalAddress().equals(socket.getInetAddress())) {
  415. LOG.info("Detected a loopback TCP socket, disconnecting it");
  416. socket.close();
  417. throw new ConnectException(
  418. "Localhost targeted connection resulted in a loopback. " +
  419. "No daemon is listening on the target port.");
  420. }
  421. }
  422. /**
  423. * Given a string representation of a host, return its ip address
  424. * in textual presentation.
  425. *
  426. * @param name a string representation of a host:
  427. * either a textual representation its IP address or its host name
  428. * @return its IP address in the string format
  429. */
  430. public static String normalizeHostName(String name) {
  431. if (Character.digit(name.charAt(0), 16) != -1) { // it is an IP
  432. return name;
  433. } else {
  434. try {
  435. InetAddress ipAddress = InetAddress.getByName(name);
  436. return ipAddress.getHostAddress();
  437. } catch (UnknownHostException e) {
  438. return name;
  439. }
  440. }
  441. }
  442. /**
  443. * Given a collection of string representation of hosts, return a list of
  444. * corresponding IP addresses in the textual representation.
  445. *
  446. * @param names a collection of string representations of hosts
  447. * @return a list of corresponding IP addresses in the string format
  448. * @see #normalizeHostName(String)
  449. */
  450. public static List<String> normalizeHostNames(Collection<String> names) {
  451. List<String> hostNames = new ArrayList<String>(names.size());
  452. for (String name : names) {
  453. hostNames.add(normalizeHostName(name));
  454. }
  455. return hostNames;
  456. }
  457. final static Map<InetAddress, Boolean> knownLocalAddrs = new ConcurrentHashMap<InetAddress, Boolean>();
  458. public static boolean isLocalAddressWithCaching(InetAddress addr) {
  459. if (knownLocalAddrs.containsKey(addr)) {
  460. return true;
  461. }
  462. if (isLocalAddress(addr)) {
  463. // add the address to known local address list
  464. knownLocalAddrs.put(addr, Boolean.TRUE);
  465. return true;
  466. } else {
  467. return false;
  468. }
  469. }
  470. public static boolean isLocalAddress(InetAddress addr) {
  471. // Check if the address is any local or loop back
  472. boolean local = addr.isAnyLocalAddress() || addr.isLoopbackAddress();
  473. // Check if the address is defined on any interface
  474. if (!local) {
  475. try {
  476. local = NetworkInterface.getByInetAddress(addr) != null;
  477. } catch (SocketException e) {
  478. local = false;
  479. }
  480. }
  481. return local;
  482. }
  483. public static int getIPTOS(Socket socket) throws IOException {
  484. return LinuxSystemCall.getIPTOSVal(socket);
  485. }
  486. public static int getIPTOS(SocketChannel socketChannel) throws IOException {
  487. return LinuxSystemCall.getIPTOSVal(socketChannel);
  488. }
  489. public static InetSocketAddress resolveAddress(InetSocketAddress address) {
  490. String hostName = address.getHostName() + ":" + address.getPort();
  491. InetSocketAddress newAddr = createSocketAddr(hostName);
  492. if (newAddr.isUnresolved()) {
  493. LOG.warn("Address unresolvable: " + newAddr);
  494. } else if (!newAddr.equals(address)) {
  495. LOG.info("DNS changed from " + address + " to " + newAddr);
  496. return newAddr;
  497. } else {
  498. if (LOG.isDebugEnabled()) {
  499. LOG.debug("DNS Address unchanged: " + address);
  500. }
  501. }
  502. return null;
  503. }
  504. }