/interpreter/tags/at2dist110511/src/edu/vub/at/actors/net/comm/CommunicationBus.java
Java | 532 lines | 219 code | 49 blank | 264 comment | 24 complexity | 4d9909411032ac06e843276fcc2e89ed MD5 | raw file
1/** 2 * AmbientTalk/2 Project 3 * CommunicationBus.java created on 2-apr-2007 at 19:30:12 4 * (c) Programming Technology Lab, 2006 - 2007 5 * Authors: Tom Van Cutsem & Stijn Mostinckx 6 * 7 * Permission is hereby granted, free of charge, to any person 8 * obtaining a copy of this software and associated documentation 9 * files (the "Software"), to deal in the Software without 10 * restriction, including without limitation the rights to use, 11 * copy, modify, merge, publish, distribute, sublicense, and/or 12 * sell copies of the Software, and to permit persons to whom the 13 * Software is furnished to do so, subject to the following 14 * conditions: 15 * 16 * The above copyright notice and this permission notice shall be 17 * included in all copies or substantial portions of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 21 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 22 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 23 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 24 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 26 * OTHER DEALINGS IN THE SOFTWARE. 27 */ 28package edu.vub.at.actors.net.comm; 29 30import edu.vub.at.actors.natives.ELVirtualMachine; 31import edu.vub.at.actors.net.cmd.VMCommand; 32import edu.vub.at.util.logging.Logging; 33 34import java.io.BufferedInputStream; 35import java.io.BufferedOutputStream; 36import java.io.IOException; 37import java.io.ObjectInputStream; 38import java.io.ObjectOutputStream; 39import java.net.Socket; 40import java.util.Collection; 41import java.util.HashMap; 42import java.util.Iterator; 43import java.util.Map; 44import java.util.Timer; 45import java.util.TimerTask; 46 47/** 48 * For each AmbientTalk virtual machine, there exists one communication bus instance. 49 * The communication bus can be in two conceptual states. It can be: 50 * <ul> 51 * <li>Connected: when connected, the communication bus's {@link CommunicationBus#networkAddress_} field 52 * is set. 53 * <li>Disconnected: when disconnected, the {@link CommunicationBus#networkAddress_} field is 54 * set to null. 55 * </ul> 56 * 57 * A communication bus is always started in a disconnected state. Toggling between both 58 * states is done using the {@link CommunicationBus#connect()} and 59 * {@link CommunicationBus#disconnect()} methods. 60 * <p> 61 * The communication bus is the central coordinator of the network communication layer. 62 * It encapsulates four important threads: 63 * <ul> 64 * <li>The {@link MulticastServerThread} is responsible for sending 'heartbeat' messages 65 * in a multicast fashion across the network, to notify other VMs in the network that 66 * this VM is still 'alive'. 67 * <li>The {@link MulticastListenerThread} is responsible for listening for the 68 * 'heartbeat' messages of other VMs, and for adding new connections in the 69 * connection table of the communication bus. 70 * <li>The {@link MasterConnectionThread} is responsible for opening a server 71 * socket that accepts incoming connections from so-called <i>slave</i> VMs. 72 * <li>A timer thread which runs a task at a fixed rate to checks the connection 73 * table for timed out VMs. 74 * </ul> 75 * 76 * The most important bookkeeping datastructure of the communication bus is the 77 * {@link CommunicationBus#addressToConnection_} table, also referred to as the 78 * <b>connection table</b>. This table stores all of the currently connected 79 * VMs that the host VM knows about. Whenever a heartbeat is received, a VM's 80 * connection registration in this table is updated. When a heartbeat is first 81 * received, the VM is added. When a heartbeat has not been heard for longer 82 * than the timeout interval, the VM is removed from the connection table. 83 * <p> 84 * Schematically, we have the following situation after establishing a connection 85 * between a master and a slave VM (see the description of {@link MulticastListenerThread}): 86 * 87 * <pre> 88 * Slave Master 89 * cs = new Socket(masterAddress) ms = socket.accept() 90 * cs.in <-------------------------- ms.out 91 * cs.out -------------------------> ms.in 92 * </pre> 93 * 94 * It does not matter which socket is registered in the connection table. Hence, 95 * once both VMs have set up a connection, the concept of 'master' and 'slave' is 96 * no longer useful and both VMs become peers. 97 * 98 * It is always the output stream of either socket that is used for sending messages to the other VM. 99 * It is always the input stream of either socket that is used for receiving messages from the other VM. 100 * 101 * @author tvcutsem 102 */ 103public class CommunicationBus { 104 105 /** the AmbientTalk VM that owns this communication bus */ 106 public final ELVirtualMachine host_; 107 108 /** the name of the overlay network in which to discover AmbientTalk VMs */ 109 private final String groupName_; 110 /** the ip address to which connect or ELVirtualMachine._DEFAULT_IP_ADDRESS_ if not specified*/ 111 private final String ipAddress_; 112 113 /** if non-null, the communication bus is connected to the network */ 114 private volatile Address networkAddress_; 115 116 private MulticastListenerThread mcListener_; 117 private MulticastServerThread mcServer_; 118 private MasterConnectionThread masterConnectionThread_; 119 120 /** 121 * Maps the address of a currently connected VM to connection information 122 * such as the last time it was seen, and the socket connection. 123 * Also known as the "connection table". 124 * 125 * This datastructure is modified by different threads: 126 * <ul> 127 * <li>Addition of master VMs by {@link MulticastListenerThread} 128 * <li>Addition of slave VMs by {@link MasterConnectionThread} 129 * <li>Removal of disconnected VMs by {@link CommandProcessor} 130 * <li>Checking for timed out VMs by {@link CommunicationBus#timeoutDetector_} 131 * <li>Lookup of connections by AmbientTalk event loops for message transmission 132 * </ul> 133 * Hence, access to this datastructure must by <b>synchronized</b>! 134 */ 135 private final HashMap addressToConnection_; 136 137 /** 138 * The timer used for checking 'stale' (i.e. timed out) connections 139 * in the connection table. 140 */ 141 private final Timer timeoutDetectorTimer_; 142 143 /** 144 * The timer task used for removing timed out connections 145 * from the connections table. 146 */ 147 private TimeoutDetectorTask timeoutDetector_; 148 149 /** 150 * Bookkeeping datastructure to store connection-related information 151 * of a connected VM. A connection entry is uniquely identified by 152 * means of the {@link Address} of its VM. 153 */ 154 private static class Connection { 155 public final Socket socket_; 156 /** this value is updated as new heartbeats are received */ 157 public long lastSeenAtTime_; 158 public final ObjectOutputStream outgoing_; 159 public final ObjectInputStream incoming_; 160 public Connection(Socket s, long lastSeenAt) throws IOException { 161 socket_ = s; 162 lastSeenAtTime_ = lastSeenAt; 163 // NOTE: apparently it is highly important that the ObjectOutputStream on the socket.getOutputStream() is 164 // created BEFORE trying to create an ObjectInputStream on the socket.getInputStream() 165 // switching the below two statements causes the master and the slave to deadlock! 166 outgoing_ = new ObjectOutputStream(new BufferedOutputStream(s.getOutputStream())); 167 // The buffered output stream must be EXPLICITLY flushed, otherwise the buffered input stream 168 // that is created below (but by the other VM) will block indefinitely 169 outgoing_.flush(); 170 incoming_ = new ObjectInputStream(new BufferedInputStream(s.getInputStream())); 171 } 172 173 /** 174 * Closes the underlying socket. This will eventually trigger the command processor 175 * tied to this connection, which will cause this connection to be removed 176 * from the table. 177 */ 178 public synchronized void close() { 179 try { 180 socket_.close(); 181 } catch (IOException ioe) { } 182 } 183 184 public synchronized void send(VMCommand msg) throws IOException{ 185 outgoing_.writeObject(msg); 186 outgoing_.flush(); 187 } 188 } 189 190 /** 191 * The timer task used for removing timed out connections 192 * from the connection table. 193 */ 194 private class TimeoutDetectorTask extends TimerTask { 195 196 /** 197 * the maximum amount of time that a remote VM gets to send a new 198 * heartbeat before it is considered as being 'offline' 199 */ 200 private static final int MAX_RESPONSE_DELAY = 10000; // in milliseconds 201 202 /** 203 * The rate at which to schedule this timer task 204 */ 205 public static final int DETECTION_RATE = 4000; // in milliseconds 206 207 public void run() { 208 closeConnectionOfMembersNotSeenIn(MAX_RESPONSE_DELAY); 209 } 210 211 } 212 213 public CommunicationBus(ELVirtualMachine host, String ambientTalkNetworkName, String ipAddress) { 214 host_ = host; 215 groupName_ = ambientTalkNetworkName; 216 ipAddress_ = ipAddress; 217 addressToConnection_ = new HashMap(); 218 timeoutDetectorTimer_ = new Timer(true); // create a daemon timer 219 } 220 221 public String getIpAddress() { 222 return ipAddress_; 223 } 224 225 public String getGroupName(){ 226 return groupName_; 227 } 228 /** 229 * Tries to connect the communication bus to the underlying network. 230 * @throws IOException if no server socket could be created to listen 231 * for incoming connections. If this exception is raised, it is 232 * guaranteed that the communication bus is left disconnected (i.e. 233 * it is not partially connected) 234 */ 235 public Address connect() throws NetworkException { 236 if (networkAddress_ != null) { 237 return networkAddress_; // if the bus is already connected, there is no need to connect it again 238 } 239 240 masterConnectionThread_ = new MasterConnectionThread(this); 241 try { 242 networkAddress_ = masterConnectionThread_.startServing(); 243 } catch (IOException e) { 244 masterConnectionThread_ = null; 245 throw new NetworkException("Could not connect to network:", e); 246 } 247 mcListener_ = new MulticastListenerThread(this, networkAddress_); 248 mcServer_ = new MulticastServerThread(networkAddress_); 249 mcListener_.start(); 250 mcServer_.start(); 251 252 // start detecting timed out VMs 253 timeoutDetector_ = new TimeoutDetectorTask(); 254 timeoutDetectorTimer_.scheduleAtFixedRate(timeoutDetector_, 0, TimeoutDetectorTask.DETECTION_RATE); 255 256 return networkAddress_; 257 } 258 259 /** 260 * Called by the VM when it has disconnected from the underlying channel. 261 * It gracefully shuts down all network threads, sets the network address 262 * to null and removes all current connections from the connection table. 263 */ 264 public void disconnect() { 265 if (networkAddress_ == null) { 266 return; // if the bus is already disconnected, there is no need to take it offline again 267 } 268 269 // once the bus is disconnected, the network address is set to null. 270 // this ensures that no further incoming connections are allowed to be 271 // registered in the addConnection method! 272 networkAddress_ = null; 273 masterConnectionThread_.stopServing(); 274 mcListener_.stopListening(); 275 mcServer_.stopBroadcasting(); 276 masterConnectionThread_ = null; 277 mcListener_ = null; 278 mcServer_ = null; 279 280 // stop detecting timed out VMs 281 timeoutDetector_.cancel(); 282 timeoutDetector_ = null; 283 284 closeConnectionOfAllMembers(); 285 } 286 287 /** 288 * Updates the time that the given virtual machine was last seen online. 289 * This method is invoked frequently by the {@link MulticastListenerThread}. 290 * 291 * @param member the address of the detected virtual machine 292 * @return true if the VM was properly registered, false if the VM is not yet registered 293 * (i.e. it is not yet considered as 'online') 294 */ 295 protected boolean updateTimeLastSeen(Address member) { 296 synchronized (addressToConnection_) { 297 Connection conn = (Connection) addressToConnection_.get(member); 298 if (conn != null) { 299 conn.lastSeenAtTime_ = System.currentTimeMillis(); 300 return true; 301 } else { 302 return false; 303 } 304 } 305 } 306 307 /** 308 * Registers a new virtual machine connection for the given address. 309 * If this VM was a slave in the discovery process, this method is 310 * invoked by the {@link MulticastListenerThread}. If this VM was 311 * a master in the discovery process, this method is invoked by the 312 * {@link MasterConnectionThread}. 313 * 314 * The socket's output stream will be stored in the connection table and is used 315 * for transmitting VM Commands to this VM. The socket's input stream will be 316 * coupled to a dedicated {@link CommandProcessor} which is responsible for 317 * processing incoming VM command objects. 318 * 319 * Calling this method implicitly also triggers a memberJoined event on this VM 320 * 321 * @throws IOException if no ObjectOutputStream can be created for the socket's output stream, 322 * or if no ObjectInputStream can be created for the socket's input stream. 323 * If this exception is raised, it is guaranteed the member is not registered in the connection table. 324 */ 325 protected void addConnection(Address newMember, Socket conn) throws IOException { 326 if (networkAddress_ == null) { 327 Logging.Network_LOG.debug("ignored connection to " + newMember + ": bus disconnected"); 328 return; // the bus has been disconnected, do not accept any new connections 329 } 330 331 // create a new connection object that can be used to send command objects to this member 332 Connection registeredConnection = new Connection(conn, System.currentTimeMillis()); 333 334 // spawn a new command processor dedicated for handling the command objects received from this member 335 CommandProcessor processor = new CommandProcessor(newMember, conn, registeredConnection.incoming_, this); 336 337 synchronized (addressToConnection_) { 338 // first check whether a connection for this member already exists 339 // (may happen when an old connection for this member has not yet been deleted) 340 // This case IS possible when a master receives a new connection from a slave, 341 // but that master has not yet detected the disconnection of the slave's old connection 342 Connection oldConnection = (Connection) addressToConnection_.get(newMember); 343 if (oldConnection != null) { 344 // explicitly close the connection we're about to override 345 // Note: in this case, we should *not* signal to the VM that a new member joined, 346 // because in the corresponding removeConnection, we will also not signal a member left 347 // (i.e. the connection was restored quickly enough such that we can abstract over the disconnection) 348 oldConnection.close(); 349 } else { 350 // notify the VM of a new connection 351 host_.event_memberJoined(newMember); 352 } 353 // register the new connection in the table (this may override an old connection) 354 addressToConnection_.put(newMember, registeredConnection); 355 } 356 357 // only start the processor if the member is properly registered 358 processor.start(); 359 360 Logging.Network_LOG.debug("successfully registered connection to " + newMember); 361 } 362 363 /** 364 * It is the responsibility of the {@link CommandProcessor} tied to the given VM 365 * to invoke this method when its connection has failed. 366 * 367 * This is the <b>only</b> method responsible for removing entries from the 368 * connection table. 369 * 370 * Calling this method implicitly also triggers a memberLeft event on this VM 371 */ 372 protected void removeConnection(Address oldMember, Socket connSocket) { 373 synchronized (addressToConnection_) { 374 Connection conn = (Connection) addressToConnection_.get(oldMember); 375 if (conn != null) { 376 // ONLY delete the connection if that connection was registered for the given 377 // socket. It might be that the connection in the connection table is already 378 // a NEWER connection that has OVERWRITTEN the old one. 379 if (conn.socket_ == connSocket) { 380 conn.close(); 381 addressToConnection_.remove(oldMember); 382 host_.event_memberLeft(oldMember); // notify VM that the member has left 383 } 384 // Note: if the sockets do not match, then the connection to be 385 // removed by this call has already been replaced by a more recent connection to the same address 386 // See the code for addConnection for more details 387 // Because the connection to be removed was already replaced, it is not necessary 388 // to notify the VM that a member has left 389 } 390 } 391 } 392 393 /** 394 * The {@link MulticastListenerThread} invokes this method regularly to 395 * remove connections to VMs which have not responded for longer than the 396 * given timeout period. 397 * 398 * Removal is done implicitly by closing timed out connections. The 399 * {@link CommandProcessor} tied to each connection is responsible for 400 * actually removing it. 401 * 402 * Calling this method might also implicitly trigger one or more memberLeft 403 * events on this VM. 404 */ 405 protected void closeConnectionOfMembersNotSeenIn(long period) { 406 synchronized (addressToConnection_) { 407 long now = System.currentTimeMillis(); 408 Collection connections = addressToConnection_.values(); 409 for (Iterator iter = connections.iterator(); iter.hasNext();) { 410 Connection c = (Connection) iter.next(); 411 if (now - c.lastSeenAtTime_ > period) { 412 c.close(); // the entry will be deleted by the Command Processor 413 Logging.Network_LOG.debug("Detected timed out VM"); 414 } 415 } 416 } 417 } 418 419 /** 420 * When the communication bus is explicitly disconnected from the 421 * network, the VM should be notified that all connected VMs 422 * have disconnected. 423 * 424 * Removal is done implicitly by closing all connections. The 425 * {@link CommandProcessor} tied to each connection is responsible for 426 * actually removing them. 427 */ 428 private void closeConnectionOfAllMembers() { 429 synchronized (addressToConnection_) { 430 Collection connections = addressToConnection_.values(); 431 for (Iterator iter = connections.iterator(); iter.hasNext();) { 432 Connection c = (Connection) iter.next(); 433 c.close(); // ensures the command processor will remove the entry eventually 434 } 435 } 436 } 437 438 /** 439 * Sends a VM Command object asynchronously to the recipient VM. 440 * There are no delivery guarantees for this message. 441 * If the recipient is offline, or the message times out, it is simply discarded 442 */ 443 public void sendAsyncUnicast(VMCommand msg, Address recipientVM) { 444 Logging.Network_LOG.info("sending async unicast cmd " + msg + " to " + recipientVM); 445 Connection conn; 446 synchronized (addressToConnection_) { 447 conn = (Connection) addressToConnection_.get(recipientVM); 448 } 449 if (conn != null) { 450 sendOneAsyncMessage(conn, msg, recipientVM); 451 } 452 } 453 454 /** 455 * Sends a VM Command object asynchronously to all connected VMs. 456 * There are no delivery guarantees for this message, nor is it guaranteed 457 * that all currently connected VMs will receive the message 458 */ 459 public void sendAsyncMulticast(VMCommand msg) { 460 Logging.Network_LOG.debug("sending async multicast cmd: " + msg); 461 462 // first clone the connection table such that we do not need to acquire the 463 // lock for the entire duration of the multicast 464 HashMap clonedConnections; 465 synchronized (addressToConnection_) { 466 clonedConnections = (HashMap) addressToConnection_.clone(); 467 } 468 469 for (Iterator iter = clonedConnections.entrySet().iterator(); iter.hasNext();) { 470 Map.Entry entry = (Map.Entry) iter.next(); 471 Address recipient = (Address) entry.getKey(); 472 Connection conn = (Connection) entry.getValue(); 473 sendOneAsyncMessage(conn, msg, recipient); 474 } 475 } 476 477 /** 478 * Sends the given {@link VMCommand} to the given member. With 'synchronous' transmission, 479 * it is meant that if this method returns gracefully, the caller can rest assured that 480 * the remote VM has correctly received the command object. It does not given any guarantees 481 * that the command object will have been executed remotely. 482 * 483 * @throws NetworkException if either the given address is no longer connected, or 484 * an I/O error occurs during the transmission of the command object. If this exception is 485 * raised, the caller does not know whether the message was correctly received or not. 486 */ 487 public void sendSynchronousUnicast(VMCommand msg, Address recipientVM) throws NetworkException { 488 Logging.Network_LOG.info("sending sync unicast cmd: " + msg + " to " + recipientVM); 489 490 Connection conn; 491 synchronized (addressToConnection_) { 492 conn = (Connection) addressToConnection_.get(recipientVM); 493 } 494 495 if (conn == null) { 496 throw new NetworkException("Recipient " + recipientVM + " is offline"); 497 } 498 499 try { 500 conn.send(msg); 501 } catch (IOException e) { 502 // it is the sender's responsibility to close the connection's socket if something goes wrong 503 // the corresponding CommandProcessor registered on this socket will remove the member from 504 // the connection table 505 conn.close(); 506 Logging.Network_LOG.error("Error while trying to send command " + msg.toString() + " to " + recipientVM, e); 507 throw new NetworkException("Error while trying to transmit message " + msg, e); 508 } 509 } 510 511 /** 512 * Note that an I/O exception during the transmission of a command object is <b>fatal</b> 513 * for the connection. The connection is immediately terminated by closing the socket. 514 * It is the responsibility of the {@link CommandProcessor} registered on the socket 515 * connection's input stream to remove the recipient VM from the connection table. 516 */ 517 private void sendOneAsyncMessage(Connection conn, VMCommand msg, Address recipientVM) { 518 try { 519 conn.send(msg); 520 } catch (IOException e) { 521 // it is the sender's responsibility to close the connection's socket if something goes wrong 522 // the CommandProcessor will remove the member from the connection table 523 conn.close(); 524 Logging.Network_LOG.error("Could not send command " + msg + " to " + recipientVM + ", dropping.", e); 525 } 526 } 527 528 public String toString() { 529 return "communication bus"; 530 } 531 532}