PageRenderTime 73ms CodeModel.GetById 45ms app.highlight 21ms RepoModel.GetById 2ms app.codeStats 0ms

/src/core/org/apache/hadoop/net/NetUtils.java

https://github.com/dynamicguy/hadoop-20
Java | 550 lines | 267 code | 46 blank | 237 comment | 69 complexity | a070fb52ad6e744f1c8820cb52c2edba MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18package org.apache.hadoop.net;
 19
 20import java.io.IOException;
 21import java.io.InputStream;
 22import java.io.OutputStream;
 23import java.net.InetAddress;
 24import java.net.InetSocketAddress;
 25import java.net.NetworkInterface;
 26import java.net.Socket;
 27import java.net.SocketAddress;
 28import java.net.SocketException;
 29import java.net.URI;
 30import java.net.UnknownHostException;
 31import java.net.ConnectException;
 32import java.nio.channels.SocketChannel;
 33import java.util.Map.Entry;
 34import java.util.*;
 35import java.util.concurrent.ConcurrentHashMap;
 36
 37import javax.net.SocketFactory;
 38
 39import org.apache.commons.logging.Log;
 40import org.apache.commons.logging.LogFactory;
 41import org.apache.hadoop.conf.Configuration;
 42import org.apache.hadoop.fs.Path;
 43import org.apache.hadoop.ipc.Server;
 44import org.apache.hadoop.ipc.VersionedProtocol;
 45import org.apache.hadoop.syscall.LinuxSystemCall;
 46import org.apache.hadoop.util.ReflectionUtils;
 47
 48public class NetUtils {
 49  private static final Log LOG = LogFactory.getLog(NetUtils.class);
 50  // Insure that Network Control bits in sock will never be used,
 51  // allow to set them may cause issue on the network, killing network
 52  // control traffic.
 53  // NC bits are: 
 54  // in TOS Dec: 192 and up.
 55  public static final int IP_TOS_MAX_VALUE = 191;
 56  public static final int NOT_SET_IP_TOS = -1;
 57  public static final String DFS_CLIENT_TOS_CONF = "dfs.client.tos.value";
 58
 59  private static Map<String, String> hostToResolved = 
 60      new HashMap<String, String>();
 61
 62  /**
 63   * Check the tos value is valid, that is, in the -1~191 range
 64   * @param tosValue
 65   * @return
 66   */
 67  public static boolean isValidTOSValue(int tosValue) {
 68    return tosValue >= NOT_SET_IP_TOS && tosValue <= IP_TOS_MAX_VALUE;
 69  }
 70  
 71  /**
 72   * Get the socket factory for the given class according to its
 73   * configuration parameter
 74   * <tt>hadoop.rpc.socket.factory.class.&lt;ClassName&gt;</tt>. When no
 75   * such parameter exists then fall back on the default socket factory as
 76   * configured by <tt>hadoop.rpc.socket.factory.class.default</tt>. If
 77   * this default socket factory is not configured, then fall back on the JVM
 78   * default socket factory.
 79   * 
 80   * @param conf the configuration
 81   * @param clazz the class (usually a {@link VersionedProtocol})
 82   * @return a socket factory
 83   */
 84  public static SocketFactory getSocketFactory(Configuration conf,
 85      Class<?> clazz) {
 86
 87    SocketFactory factory = null;
 88
 89    String propValue =
 90        conf.get("hadoop.rpc.socket.factory.class." + clazz.getSimpleName());
 91    if ((propValue != null) && (propValue.length() > 0))
 92      factory = getSocketFactoryFromProperty(conf, propValue);
 93
 94    if (factory == null)
 95      factory = getDefaultSocketFactory(conf);
 96
 97    return factory;
 98  }
 99
100  /**
101   * Get the default socket factory as specified by the configuration
102   * parameter <tt>hadoop.rpc.socket.factory.default</tt>
103   * 
104   * @param conf the configuration
105   * @return the default socket factory as specified in the configuration or
106   *         the JVM default socket factory if the configuration does not
107   *         contain a default socket factory property.
108   */
109  public static SocketFactory getDefaultSocketFactory(Configuration conf) {
110
111    String propValue = conf.get("hadoop.rpc.socket.factory.class.default");
112    if ((propValue == null) || (propValue.length() == 0))
113      return SocketFactory.getDefault();
114
115    return getSocketFactoryFromProperty(conf, propValue);
116  }
117
118  /**
119   * Get the socket factory corresponding to the given proxy URI. If the
120   * given proxy URI corresponds to an absence of configuration parameter,
121   * returns null. If the URI is malformed raises an exception.
122   * 
123   * @param propValue the property which is the class name of the
124   *        SocketFactory to instantiate; assumed non null and non empty.
125   * @return a socket factory as defined in the property value.
126   */
127  public static SocketFactory getSocketFactoryFromProperty(
128      Configuration conf, String propValue) {
129
130    try {
131      Class<?> theClass = conf.getClassByName(propValue);
132      return (SocketFactory) ReflectionUtils.newInstance(theClass, conf);
133
134    } catch (ClassNotFoundException cnfe) {
135      throw new RuntimeException("Socket Factory class not found: " + cnfe);
136    }
137  }
138
139  /**
140   * Util method to build socket addr from either:
141   *   <host>:<post>
142   *   <fs>://<host>:<port>/<path>
143   */
144  public static InetSocketAddress createSocketAddr(String target) {
145    return createSocketAddr(target, -1);
146  }
147
148  /**
149   * Util method to build socket addr from either:
150   *   <host>
151   *   <host>:<post>
152   *   <fs>://<host>:<port>/<path>
153   */
154  public static InetSocketAddress createSocketAddr(String target,
155      int defaultPort) {
156    int colonIndex = target.indexOf(':');
157    if (colonIndex < 0 && defaultPort == -1) {
158      throw new RuntimeException("Not a host:port pair: " + target);
159    }
160    String hostname;
161    int port = -1;
162    if (!target.contains("/")) {
163      if (colonIndex == -1) {
164        hostname = target;
165      } else {
166        // must be the old style <host>:<port>
167        hostname = target.substring(0, colonIndex);
168        port = Integer.parseInt(target.substring(colonIndex + 1));
169      }
170    } else {
171      // a new uri
172      URI addr = new Path(target).toUri();
173      hostname = addr.getHost();
174      port = addr.getPort();
175    }
176
177    if (port == -1) {
178      port = defaultPort;
179    }
180
181    if (getStaticResolution(hostname) != null) {
182      hostname = getStaticResolution(hostname);
183    }
184
185    return InetSocketAddressFactory.createWithResolveRetry(hostname, port);
186  }
187
188  /**
189   * Handle the transition from pairs of attributes specifying a host and port
190   * to a single colon separated one.
191   * @param conf the configuration to check
192   * @param oldBindAddressName the old address attribute name
193   * @param oldPortName the old port attribute name
194   * @param newBindAddressName the new combined name
195   * @return the complete address from the configuration
196   */
197  @Deprecated
198  public static String getServerAddress(Configuration conf,
199      String oldBindAddressName,
200      String oldPortName,
201      String newBindAddressName) {
202    String oldAddr = conf.get(oldBindAddressName);
203    String oldPort = conf.get(oldPortName);
204    String newAddrPort = conf.get(newBindAddressName);
205    if (oldAddr == null && oldPort == null) {
206      return newAddrPort;
207    }
208    String[] newAddrPortParts = newAddrPort.split(":",2);
209    if (newAddrPortParts.length != 2) {
210      throw new IllegalArgumentException("Invalid address/port: " + 
211          newAddrPort);
212    }
213    if (oldAddr == null) {
214      oldAddr = newAddrPortParts[0];
215    } else {
216      LOG.warn("Configuration parameter " + oldBindAddressName +
217          " is deprecated. Use " + newBindAddressName + " instead.");
218    }
219    if (oldPort == null) {
220      oldPort = newAddrPortParts[1];
221    } else {
222      LOG.warn("Configuration parameter " + oldPortName +
223          " is deprecated. Use " + newBindAddressName + " instead.");      
224    }
225    return oldAddr + ":" + oldPort;
226  }
227
228  /**
229   * Adds a static resolution for host. This can be used for setting up
230   * hostnames with names that are fake to point to a well known host. For e.g.
231   * in some testcases we require to have daemons with different hostnames
232   * running on the same machine. In order to create connections to these
233   * daemons, one can set up mappings from those hostnames to "localhost".
234   * {@link NetUtils#getStaticResolution(String)} can be used to query for
235   * the actual hostname. 
236   * @param host
237   * @param resolvedName
238   */
239  public static void addStaticResolution(String host, String resolvedName) {
240    synchronized (hostToResolved) {
241      hostToResolved.put(host, resolvedName);
242    }
243  }
244
245  /**
246   * Retrieves the resolved name for the passed host. The resolved name must
247   * have been set earlier using 
248   * {@link NetUtils#addStaticResolution(String, String)}
249   * @param host
250   * @return the resolution
251   */
252  public static String getStaticResolution(String host) {
253    synchronized (hostToResolved) {
254      return hostToResolved.get(host);
255    }
256  }
257
258  /**
259   * This is used to get all the resolutions that were added using
260   * {@link NetUtils#addStaticResolution(String, String)}. The return
261   * value is a List each element of which contains an array of String 
262   * of the form String[0]=hostname, String[1]=resolved-hostname
263   * @return the list of resolutions
264   */
265  public static List <String[]> getAllStaticResolutions() {
266    synchronized (hostToResolved) {
267      Set <Entry <String, String>>entries = hostToResolved.entrySet();
268      if (entries.size() == 0) {
269        return null;
270      }
271      List <String[]> l = new ArrayList<String[]>(entries.size());
272      for (Entry<String, String> e : entries) {
273        l.add(new String[] {e.getKey(), e.getValue()});
274      }
275      return l;
276    }
277  }
278
279  /**
280   * Returns InetSocketAddress that a client can use to 
281   * connect to the server. Server.getListenerAddress() is not correct when
282   * the server binds to "0.0.0.0". This returns "127.0.0.1:port" when
283   * the getListenerAddress() returns "0.0.0.0:port".
284   * 
285   * @param server
286   * @return socket address that a client can use to connect to the server.
287   */
288  public static InetSocketAddress getConnectAddress(Server server) {
289    InetSocketAddress addr = server.getListenerAddress();
290    if (addr.getAddress().getHostAddress().equals("0.0.0.0")) {
291      addr = new InetSocketAddress("127.0.0.1", addr.getPort());
292    }
293    return addr;
294  }
295
296  /**
297   * Same as getInputStream(socket, socket.getSoTimeout()).<br><br>
298   * 
299   * From documentation for {@link #getInputStream(Socket, long)}:<br>
300   * Returns InputStream for the socket. If the socket has an associated
301   * SocketChannel then it returns a 
302   * {@link SocketInputStream} with the given timeout. If the socket does not
303   * have a channel, {@link Socket#getInputStream()} is returned. In the later
304   * case, the timeout argument is ignored and the timeout set with 
305   * {@link Socket#setSoTimeout(int)} applies for reads.<br><br>
306   *
307   * Any socket created using socket factories returned by {@link #NetUtils},
308   * must use this interface instead of {@link Socket#getInputStream()}.
309   *     
310   * @see #getInputStream(Socket, long)
311   * 
312   * @param socket
313   * @return InputStream for reading from the socket.
314   * @throws IOException
315   */
316  public static InputStream getInputStream(Socket socket) 
317      throws IOException {
318    return getInputStream(socket, socket.getSoTimeout());
319  }
320
321  /**
322   * Returns InputStream for the socket. If the socket has an associated
323   * SocketChannel then it returns a 
324   * {@link SocketInputStream} with the given timeout. If the socket does not
325   * have a channel, {@link Socket#getInputStream()} is returned. In the later
326   * case, the timeout argument is ignored and the timeout set with 
327   * {@link Socket#setSoTimeout(int)} applies for reads.<br><br>
328   * 
329   * Any socket created using socket factories returned by {@link #NetUtils},
330   * must use this interface instead of {@link Socket#getInputStream()}.
331   *     
332   * @see Socket#getChannel()
333   * 
334   * @param socket
335   * @param timeout timeout in milliseconds. This may not always apply. zero
336   *        for waiting as long as necessary.
337   * @return InputStream for reading from the socket.
338   * @throws IOException
339   */
340  public static InputStream getInputStream(Socket socket, long timeout) 
341      throws IOException {
342    return (socket.getChannel() == null) ? 
343        socket.getInputStream() : new SocketInputStream(socket, timeout);
344  }
345
346  /**
347   * Same as getOutputStream(socket, 0). Timeout of zero implies write will
348   * wait until data is available.<br><br>
349   * 
350   * From documentation for {@link #getOutputStream(Socket, long)} : <br>
351   * Returns OutputStream for the socket. If the socket has an associated
352   * SocketChannel then it returns a 
353   * {@link SocketOutputStream} with the given timeout. If the socket does not
354   * have a channel, {@link Socket#getOutputStream()} is returned. In the later
355   * case, the timeout argument is ignored and the write will wait until 
356   * data is available.<br><br>
357   * 
358   * Any socket created using socket factories returned by {@link #NetUtils},
359   * must use this interface instead of {@link Socket#getOutputStream()}.
360   * 
361   * @see #getOutputStream(Socket, long)
362   * 
363   * @param socket
364   * @return OutputStream for writing to the socket.
365   * @throws IOException
366   */  
367  public static OutputStream getOutputStream(Socket socket) 
368      throws IOException {
369    return getOutputStream(socket, 0);
370  }
371
372  /**
373   * Returns OutputStream for the socket. If the socket has an associated
374   * SocketChannel then it returns a 
375   * {@link SocketOutputStream} with the given timeout. If the socket does not
376   * have a channel, {@link Socket#getOutputStream()} is returned. In the later
377   * case, the timeout argument is ignored and the write will wait until 
378   * data is available.<br><br>
379   * 
380   * Any socket created using socket factories returned by {@link #NetUtils},
381   * must use this interface instead of {@link Socket#getOutputStream()}.
382   * 
383   * @see Socket#getChannel()
384   * 
385   * @param socket
386   * @param timeout timeout in milliseconds. This may not always apply. zero
387   *        for waiting as long as necessary.
388   * @return OutputStream for writing to the socket.
389   * @throws IOException   
390   */
391  public static OutputStream getOutputStream(Socket socket, long timeout) 
392      throws IOException {
393    return (socket.getChannel() == null) ? 
394        socket.getOutputStream() : new SocketOutputStream(socket, timeout);            
395  }
396
397  /**
398   * This is a drop-in replacement for 
399   * {@link Socket#connect(SocketAddress, int)}.
400   * In the case of normal sockets that don't have associated channels, this 
401   * just invokes <code>socket.connect(endpoint, timeout)</code>. If 
402   * <code>socket.getChannel()</code> returns a non-null channel,
403   * connect is implemented using Hadoop's selectors. This is done mainly
404   * to avoid Sun's connect implementation from creating thread-local 
405   * selectors, since Hadoop does not have control on when these are closed
406   * and could end up taking all the available file descriptors.
407   * 
408   * @see java.net.Socket#connect(java.net.SocketAddress, int)
409   * 
410   * @param socket
411   * @param endpoint 
412   * @param timeout - timeout in milliseconds
413   */
414  public static void connect(Socket socket, 
415      SocketAddress endpoint, 
416      int timeout) throws IOException {
417    connect(socket, endpoint, timeout, NOT_SET_IP_TOS);
418  }
419
420  public static void connect(Socket socket, 
421      SocketAddress endpoint, 
422      int timeout, 
423      int ipTosValue) throws IOException {
424    if (socket == null || endpoint == null || timeout < 0 
425        || ipTosValue < NOT_SET_IP_TOS 
426        || ipTosValue > IP_TOS_MAX_VALUE) {
427      throw new IllegalArgumentException("Illegal argument for connect()");
428    }
429
430    SocketChannel ch = socket.getChannel();
431
432    if (ch == null) {
433      // let the default implementation handle it.
434      socket.connect(endpoint, timeout);
435    } else {
436      // set the socket IP_TOS value
437      if (ipTosValue != NOT_SET_IP_TOS) {
438        LinuxSystemCall.setIPTOSVal(ch, ipTosValue);
439      }
440      SocketIOWithTimeout.connect(ch, endpoint, timeout);
441    }
442
443    // There is a very rare case allowed by the TCP specification, such that
444    // if we are trying to connect to an endpoint on the local machine,
445    // and we end up choosing an ephemeral port equal to the destination port,
446    // we will actually end up getting connected to ourself (ie any data we
447    // send just comes right back). This is only possible if the target
448    // daemon is down, so we'll treat it like connection refused.
449    if (socket.getLocalPort() == socket.getPort() &&
450        socket.getLocalAddress().equals(socket.getInetAddress())) {
451      LOG.info("Detected a loopback TCP socket, disconnecting it");
452      socket.close();
453      throw new ConnectException(
454          "Localhost targeted connection resulted in a loopback. " +
455          "No daemon is listening on the target port.");
456    }
457  }
458
459  /** 
460   * Given a string representation of a host, return its ip address
461   * in textual presentation.
462   * 
463   * @param name a string representation of a host:
464   *             either a textual representation its IP address or its host name
465   * @return its IP address in the string format
466   */
467  public static String normalizeHostName(String name) {
468    if (Character.digit(name.charAt(0), 16) != -1) { // it is an IP
469      return name;
470    } else {
471      try {
472        InetAddress ipAddress = InetAddress.getByName(name);
473        return ipAddress.getHostAddress();
474      } catch (UnknownHostException e) {
475        return name;
476      }
477    }
478  }
479
480  /** 
481   * Given a collection of string representation of hosts, return a list of
482   * corresponding IP addresses in the textual representation.
483   * 
484   * @param names a collection of string representations of hosts
485   * @return a list of corresponding IP addresses in the string format
486   * @see #normalizeHostName(String)
487   */
488  public static List<String> normalizeHostNames(Collection<String> names) {
489    List<String> hostNames = new ArrayList<String>(names.size());
490    for (String name : names) {
491      hostNames.add(normalizeHostName(name));
492    }
493    return hostNames;
494  }
495
496  final static Map<InetAddress, Boolean> knownLocalAddrs = new ConcurrentHashMap<InetAddress, Boolean>();
497
498  public static boolean isLocalAddressWithCaching(InetAddress addr) {
499    if (knownLocalAddrs.containsKey(addr)) {
500      return true;
501    }
502    if (isLocalAddress(addr)) {
503      // add the address to known local address list
504      knownLocalAddrs.put(addr, Boolean.TRUE);
505      return true;
506    } else {
507      return false;
508    }
509  }
510
511  public static boolean isLocalAddress(InetAddress addr) {
512    // Check if the address is any local or loop back
513    boolean local = addr.isAnyLocalAddress() || addr.isLoopbackAddress();
514    // Check if the address is defined on any interface
515    if (!local) {
516      try {
517        local = NetworkInterface.getByInetAddress(addr) != null;
518      } catch (SocketException e) {
519        local = false;
520      }
521    }   
522    return local;
523  }
524
525  public static int getIPTOS(Socket socket) throws IOException {
526    return LinuxSystemCall.getIPTOSVal(socket);
527  }
528
529  public static int getIPTOS(SocketChannel socketChannel) throws IOException {
530    return LinuxSystemCall.getIPTOSVal(socketChannel);
531  }
532
533  public static InetSocketAddress resolveAddress(InetSocketAddress address) {
534    String hostName = address.getHostName() + ":" + address.getPort();
535    InetSocketAddress newAddr = createSocketAddr(hostName);
536    if (newAddr.isUnresolved()) {
537      LOG.warn("Address unresolvable: " + newAddr);
538    } else if (!newAddr.equals(address)) {
539      LOG.info("DNS changed from " + address + " to " + newAddr);
540      return newAddr;
541    } else {
542      if (LOG.isDebugEnabled()) {
543        LOG.debug("DNS Address unchanged: " + address);
544      }
545    }
546    return null;
547  }
548
549
550}