PageRenderTime 28ms CodeModel.GetById 2ms app.highlight 22ms RepoModel.GetById 1ms app.codeStats 0ms

/src/core/org/apache/hadoop/net/NetUtils.java

https://github.com/YaoLin1/hadoop
Java | 443 lines | 185 code | 37 blank | 221 comment | 42 complexity | f5a3c62696fcde26a05ded0592a79c04 MD5 | raw file
  1/**
  2 * Licensed to the Apache Software Foundation (ASF) under one
  3 * or more contributor license agreements.  See the NOTICE file
  4 * distributed with this work for additional information
  5 * regarding copyright ownership.  The ASF licenses this file
  6 * to you under the Apache License, Version 2.0 (the
  7 * "License"); you may not use this file except in compliance
  8 * with the License.  You may obtain a copy of the License at
  9 *
 10 *     http://www.apache.org/licenses/LICENSE-2.0
 11 *
 12 * Unless required by applicable law or agreed to in writing, software
 13 * distributed under the License is distributed on an "AS IS" BASIS,
 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 * See the License for the specific language governing permissions and
 16 * limitations under the License.
 17 */
 18package org.apache.hadoop.net;
 19
 20import java.io.IOException;
 21import java.io.InputStream;
 22import java.io.OutputStream;
 23import java.net.InetAddress;
 24import java.net.InetSocketAddress;
 25import java.net.Socket;
 26import java.net.SocketAddress;
 27import java.net.URI;
 28import java.net.UnknownHostException;
 29import java.nio.channels.SocketChannel;
 30import java.util.Map.Entry;
 31import java.util.regex.Pattern;
 32import java.util.*;
 33
 34import javax.net.SocketFactory;
 35
 36import org.apache.commons.logging.Log;
 37import org.apache.commons.logging.LogFactory;
 38import org.apache.hadoop.conf.Configuration;
 39import org.apache.hadoop.fs.Path;
 40import org.apache.hadoop.ipc.Server;
 41import org.apache.hadoop.ipc.VersionedProtocol;
 42import org.apache.hadoop.util.ReflectionUtils;
 43
 44public class NetUtils {
 45  private static final Log LOG = LogFactory.getLog(NetUtils.class);
 46  
 47  private static Map<String, String> hostToResolved = 
 48                                     new HashMap<String, String>();
 49
 50  /**
 51   * Get the socket factory for the given class according to its
 52   * configuration parameter
 53   * <tt>hadoop.rpc.socket.factory.class.&lt;ClassName&gt;</tt>. When no
 54   * such parameter exists then fall back on the default socket factory as
 55   * configured by <tt>hadoop.rpc.socket.factory.class.default</tt>. If
 56   * this default socket factory is not configured, then fall back on the JVM
 57   * default socket factory.
 58   * 
 59   * @param conf the configuration
 60   * @param clazz the class (usually a {@link VersionedProtocol})
 61   * @return a socket factory
 62   */
 63  public static SocketFactory getSocketFactory(Configuration conf,
 64      Class<?> clazz) {
 65
 66    SocketFactory factory = null;
 67
 68    String propValue =
 69        conf.get("hadoop.rpc.socket.factory.class." + clazz.getSimpleName());
 70    if ((propValue != null) && (propValue.length() > 0))
 71      factory = getSocketFactoryFromProperty(conf, propValue);
 72
 73    if (factory == null)
 74      factory = getDefaultSocketFactory(conf);
 75
 76    return factory;
 77  }
 78
 79  /**
 80   * Get the default socket factory as specified by the configuration
 81   * parameter <tt>hadoop.rpc.socket.factory.default</tt>
 82   * 
 83   * @param conf the configuration
 84   * @return the default socket factory as specified in the configuration or
 85   *         the JVM default socket factory if the configuration does not
 86   *         contain a default socket factory property.
 87   */
 88  public static SocketFactory getDefaultSocketFactory(Configuration conf) {
 89
 90    String propValue = conf.get("hadoop.rpc.socket.factory.class.default");
 91    if ((propValue == null) || (propValue.length() == 0))
 92      return SocketFactory.getDefault();
 93
 94    return getSocketFactoryFromProperty(conf, propValue);
 95  }
 96
 97  /**
 98   * Get the socket factory corresponding to the given proxy URI. If the
 99   * given proxy URI corresponds to an absence of configuration parameter,
100   * returns null. If the URI is malformed raises an exception.
101   * 
102   * @param propValue the property which is the class name of the
103   *        SocketFactory to instantiate; assumed non null and non empty.
104   * @return a socket factory as defined in the property value.
105   */
106  public static SocketFactory getSocketFactoryFromProperty(
107      Configuration conf, String propValue) {
108
109    try {
110      Class<?> theClass = conf.getClassByName(propValue);
111      return (SocketFactory) ReflectionUtils.newInstance(theClass, conf);
112
113    } catch (ClassNotFoundException cnfe) {
114      throw new RuntimeException("Socket Factory class not found: " + cnfe);
115    }
116  }
117
118  /**
119   * Util method to build socket addr from either:
120   *   <host>:<post>
121   *   <fs>://<host>:<port>/<path>
122   */
123  public static InetSocketAddress createSocketAddr(String target) {
124    return createSocketAddr(target, -1);
125  }
126
127  /**
128   * Util method to build socket addr from either:
129   *   <host>
130   *   <host>:<post>
131   *   <fs>://<host>:<port>/<path>
132   */
133  public static InetSocketAddress createSocketAddr(String target,
134                                                   int defaultPort) {
135    if (target == null) {
136      throw new IllegalArgumentException("Target address cannot be null.");
137    }
138    int colonIndex = target.indexOf(':');
139    if (colonIndex < 0 && defaultPort == -1) {
140      throw new RuntimeException("Not a host:port pair: " + target);
141    }
142    String hostname;
143    int port = -1;
144    if (!target.contains("/")) {
145      if (colonIndex == -1) {
146        hostname = target;
147      } else {
148        // must be the old style <host>:<port>
149        hostname = target.substring(0, colonIndex);
150        port = Integer.parseInt(target.substring(colonIndex + 1));
151      }
152    } else {
153      // a new uri
154      URI addr = new Path(target).toUri();
155      hostname = addr.getHost();
156      port = addr.getPort();
157    }
158
159    if (port == -1) {
160      port = defaultPort;
161    }
162  
163    if (getStaticResolution(hostname) != null) {
164      hostname = getStaticResolution(hostname);
165    }
166    return new InetSocketAddress(hostname, port);
167  }
168
169  /**
170   * Adds a static resolution for host. This can be used for setting up
171   * hostnames with names that are fake to point to a well known host. For e.g.
172   * in some testcases we require to have daemons with different hostnames
173   * running on the same machine. In order to create connections to these
174   * daemons, one can set up mappings from those hostnames to "localhost".
175   * {@link NetUtils#getStaticResolution(String)} can be used to query for
176   * the actual hostname. 
177   * @param host
178   * @param resolvedName
179   */
180  public static void addStaticResolution(String host, String resolvedName) {
181    synchronized (hostToResolved) {
182      hostToResolved.put(host, resolvedName);
183    }
184  }
185  
186  /**
187   * Retrieves the resolved name for the passed host. The resolved name must
188   * have been set earlier using 
189   * {@link NetUtils#addStaticResolution(String, String)}
190   * @param host
191   * @return the resolution
192   */
193  public static String getStaticResolution(String host) {
194    synchronized (hostToResolved) {
195      return hostToResolved.get(host);
196    }
197  }
198  
199  /**
200   * This is used to get all the resolutions that were added using
201   * {@link NetUtils#addStaticResolution(String, String)}. The return
202   * value is a List each element of which contains an array of String 
203   * of the form String[0]=hostname, String[1]=resolved-hostname
204   * @return the list of resolutions
205   */
206  public static List <String[]> getAllStaticResolutions() {
207    synchronized (hostToResolved) {
208      Set <Entry <String, String>>entries = hostToResolved.entrySet();
209      if (entries.size() == 0) {
210        return null;
211      }
212      List <String[]> l = new ArrayList<String[]>(entries.size());
213      for (Entry<String, String> e : entries) {
214        l.add(new String[] {e.getKey(), e.getValue()});
215      }
216    return l;
217    }
218  }
219  
220  /**
221   * Returns InetSocketAddress that a client can use to 
222   * connect to the server. Server.getListenerAddress() is not correct when
223   * the server binds to "0.0.0.0". This returns "127.0.0.1:port" when
224   * the getListenerAddress() returns "0.0.0.0:port".
225   * 
226   * @param server
227   * @return socket address that a client can use to connect to the server.
228   */
229  public static InetSocketAddress getConnectAddress(Server server) {
230    InetSocketAddress addr = server.getListenerAddress();
231    if (addr.getAddress().getHostAddress().equals("0.0.0.0")) {
232      addr = new InetSocketAddress("127.0.0.1", addr.getPort());
233    }
234    return addr;
235  }
236  
237  /**
238   * Same as getInputStream(socket, socket.getSoTimeout()).<br><br>
239   * 
240   * From documentation for {@link #getInputStream(Socket, long)}:<br>
241   * Returns InputStream for the socket. If the socket has an associated
242   * SocketChannel then it returns a 
243   * {@link SocketInputStream} with the given timeout. If the socket does not
244   * have a channel, {@link Socket#getInputStream()} is returned. In the later
245   * case, the timeout argument is ignored and the timeout set with 
246   * {@link Socket#setSoTimeout(int)} applies for reads.<br><br>
247   *
248   * Any socket created using socket factories returned by {@link #NetUtils},
249   * must use this interface instead of {@link Socket#getInputStream()}.
250   *     
251   * @see #getInputStream(Socket, long)
252   * 
253   * @param socket
254   * @return InputStream for reading from the socket.
255   * @throws IOException
256   */
257  public static InputStream getInputStream(Socket socket) 
258                                           throws IOException {
259    return getInputStream(socket, socket.getSoTimeout());
260  }
261  
262  /**
263   * Returns InputStream for the socket. If the socket has an associated
264   * SocketChannel then it returns a 
265   * {@link SocketInputStream} with the given timeout. If the socket does not
266   * have a channel, {@link Socket#getInputStream()} is returned. In the later
267   * case, the timeout argument is ignored and the timeout set with 
268   * {@link Socket#setSoTimeout(int)} applies for reads.<br><br>
269   * 
270   * Any socket created using socket factories returned by {@link #NetUtils},
271   * must use this interface instead of {@link Socket#getInputStream()}.
272   *     
273   * @see Socket#getChannel()
274   * 
275   * @param socket
276   * @param timeout timeout in milliseconds. This may not always apply. zero
277   *        for waiting as long as necessary.
278   * @return InputStream for reading from the socket.
279   * @throws IOException
280   */
281  public static InputStream getInputStream(Socket socket, long timeout) 
282                                           throws IOException {
283    return (socket.getChannel() == null) ? 
284          socket.getInputStream() : new SocketInputStream(socket, timeout);
285  }
286  
287  /**
288   * Same as getOutputStream(socket, 0). Timeout of zero implies write will
289   * wait until data is available.<br><br>
290   * 
291   * From documentation for {@link #getOutputStream(Socket, long)} : <br>
292   * Returns OutputStream for the socket. If the socket has an associated
293   * SocketChannel then it returns a 
294   * {@link SocketOutputStream} with the given timeout. If the socket does not
295   * have a channel, {@link Socket#getOutputStream()} is returned. In the later
296   * case, the timeout argument is ignored and the write will wait until 
297   * data is available.<br><br>
298   * 
299   * Any socket created using socket factories returned by {@link #NetUtils},
300   * must use this interface instead of {@link Socket#getOutputStream()}.
301   * 
302   * @see #getOutputStream(Socket, long)
303   * 
304   * @param socket
305   * @return OutputStream for writing to the socket.
306   * @throws IOException
307   */  
308  public static OutputStream getOutputStream(Socket socket) 
309                                             throws IOException {
310    return getOutputStream(socket, 0);
311  }
312  
313  /**
314   * Returns OutputStream for the socket. If the socket has an associated
315   * SocketChannel then it returns a 
316   * {@link SocketOutputStream} with the given timeout. If the socket does not
317   * have a channel, {@link Socket#getOutputStream()} is returned. In the later
318   * case, the timeout argument is ignored and the write will wait until 
319   * data is available.<br><br>
320   * 
321   * Any socket created using socket factories returned by {@link #NetUtils},
322   * must use this interface instead of {@link Socket#getOutputStream()}.
323   * 
324   * @see Socket#getChannel()
325   * 
326   * @param socket
327   * @param timeout timeout in milliseconds. This may not always apply. zero
328   *        for waiting as long as necessary.
329   * @return OutputStream for writing to the socket.
330   * @throws IOException   
331   */
332  public static OutputStream getOutputStream(Socket socket, long timeout) 
333                                             throws IOException {
334    return (socket.getChannel() == null) ? 
335            socket.getOutputStream() : new SocketOutputStream(socket, timeout);            
336  }
337  
338  /**
339   * This is a drop-in replacement for 
340   * {@link Socket#connect(SocketAddress, int)}.
341   * In the case of normal sockets that don't have associated channels, this 
342   * just invokes <code>socket.connect(endpoint, timeout)</code>. If 
343   * <code>socket.getChannel()</code> returns a non-null channel,
344   * connect is implemented using Hadoop's selectors. This is done mainly
345   * to avoid Sun's connect implementation from creating thread-local 
346   * selectors, since Hadoop does not have control on when these are closed
347   * and could end up taking all the available file descriptors.
348   * 
349   * @see java.net.Socket#connect(java.net.SocketAddress, int)
350   * 
351   * @param socket
352   * @param endpoint 
353   * @param timeout - timeout in milliseconds
354   */
355  public static void connect(Socket socket, 
356                             SocketAddress endpoint, 
357                             int timeout) throws IOException {
358    if (socket == null || endpoint == null || timeout < 0) {
359      throw new IllegalArgumentException("Illegal argument for connect()");
360    }
361    
362    SocketChannel ch = socket.getChannel();
363    
364    if (ch == null) {
365      // let the default implementation handle it.
366      socket.connect(endpoint, timeout);
367    } else {
368      SocketIOWithTimeout.connect(ch, endpoint, timeout);
369    }
370  }
371  
372  /** 
373   * Given a string representation of a host, return its ip address
374   * in textual presentation.
375   * 
376   * @param name a string representation of a host:
377   *             either a textual representation its IP address or its host name
378   * @return its IP address in the string format
379   */
380  public static String normalizeHostName(String name) {
381    if (Character.digit(name.charAt(0), 16) != -1) { // it is an IP
382      return name;
383    } else {
384      try {
385        InetAddress ipAddress = InetAddress.getByName(name);
386        return ipAddress.getHostAddress();
387      } catch (UnknownHostException e) {
388        return name;
389      }
390    }
391  }
392  
393  /** 
394   * Given a collection of string representation of hosts, return a list of
395   * corresponding IP addresses in the textual representation.
396   * 
397   * @param names a collection of string representations of hosts
398   * @return a list of corresponding IP addresses in the string format
399   * @see #normalizeHostName(String)
400   */
401  public static List<String> normalizeHostNames(Collection<String> names) {
402    List<String> hostNames = new ArrayList<String>(names.size());
403    for (String name : names) {
404      hostNames.add(normalizeHostName(name));
405    }
406    return hostNames;
407  }
408
409  /**
410   * Attempt to obtain the host name of a name specified by ip address.  
411   * Check that the node name is an ip addr and if so, attempt to determine
412   * its host name.  If the name is not an IP addr, or the actual name cannot
413   * be determined, return null.
414   * 
415   * @return Host name or null
416   */
417  private static final Pattern ipPattern = // Pattern for matching hostname to ip:port
418    Pattern.compile("\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}:?\\d*");
419  public static String getHostNameOfIP(String ip) {
420    // If name is not an ip addr, don't bother looking it up
421    if(!ipPattern.matcher(ip).matches())
422      return null;
423    
424    String hostname = "";
425    try {
426      String n = ip.substring(0, ip.indexOf(':'));
427      hostname = InetAddress.getByName(n).getHostName();
428    } catch (UnknownHostException e) {
429      return null;
430    }
431    
432    return hostname; 
433  }
434
435  /**
436   * Return hostname without throwing exception.
437   * @return hostname
438   */
439  public static String getHostname() {
440    try {return "" + InetAddress.getLocalHost();}
441    catch(UnknownHostException uhe) {return "" + uhe;}
442  }
443}