PageRenderTime 42ms CodeModel.GetById 15ms app.highlight 20ms RepoModel.GetById 1ms app.codeStats 1ms

/interpreter/tags/at2dist110511/src/edu/vub/at/actors/net/comm/CommunicationBus.java

http://ambienttalk.googlecode.com/
Java | 532 lines | 219 code | 49 blank | 264 comment | 24 complexity | 4d9909411032ac06e843276fcc2e89ed MD5 | raw file
  1/**
  2 * AmbientTalk/2 Project
  3 * CommunicationBus.java created on 2-apr-2007 at 19:30:12
  4 * (c) Programming Technology Lab, 2006 - 2007
  5 * Authors: Tom Van Cutsem & Stijn Mostinckx
  6 * 
  7 * Permission is hereby granted, free of charge, to any person
  8 * obtaining a copy of this software and associated documentation
  9 * files (the "Software"), to deal in the Software without
 10 * restriction, including without limitation the rights to use,
 11 * copy, modify, merge, publish, distribute, sublicense, and/or
 12 * sell copies of the Software, and to permit persons to whom the
 13 * Software is furnished to do so, subject to the following
 14 * conditions:
 15 *
 16 * The above copyright notice and this permission notice shall be
 17 * included in all copies or substantial portions of the Software.
 18 *
 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 20 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
 21 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 22 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 23 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 24 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 26 * OTHER DEALINGS IN THE SOFTWARE.
 27 */
 28package edu.vub.at.actors.net.comm;
 29
 30import edu.vub.at.actors.natives.ELVirtualMachine;
 31import edu.vub.at.actors.net.cmd.VMCommand;
 32import edu.vub.at.util.logging.Logging;
 33
 34import java.io.BufferedInputStream;
 35import java.io.BufferedOutputStream;
 36import java.io.IOException;
 37import java.io.ObjectInputStream;
 38import java.io.ObjectOutputStream;
 39import java.net.Socket;
 40import java.util.Collection;
 41import java.util.HashMap;
 42import java.util.Iterator;
 43import java.util.Map;
 44import java.util.Timer;
 45import java.util.TimerTask;
 46
 47/**
 48 * For each AmbientTalk virtual machine, there exists one communication bus instance.
 49 * The communication bus can be in two conceptual states. It can be:
 50 * <ul>
 51 *  <li>Connected: when connected, the communication bus's {@link CommunicationBus#networkAddress_} field
 52 *  is set.
 53 *  <li>Disconnected: when disconnected, the {@link CommunicationBus#networkAddress_} field is
 54 *  set to null.
 55 * </ul>
 56 * 
 57 * A communication bus is always started in a disconnected state. Toggling between both
 58 * states is done using the {@link CommunicationBus#connect()} and
 59 * {@link CommunicationBus#disconnect()} methods.
 60 * <p>
 61 * The communication bus is the central coordinator of the network communication layer.
 62 * It encapsulates four important threads:
 63 * <ul>
 64 *  <li>The {@link MulticastServerThread} is responsible for sending 'heartbeat' messages
 65 * in a multicast fashion across the network, to notify other VMs in the network that
 66 * this VM is still 'alive'.
 67 *  <li>The {@link MulticastListenerThread} is responsible for listening for the
 68 *  'heartbeat' messages of other VMs, and for adding new connections in the
 69 *  connection table of the communication bus.
 70 *  <li>The {@link MasterConnectionThread} is responsible for opening a server
 71 *  socket that accepts incoming connections from so-called <i>slave</i> VMs.
 72 *   <li>A timer thread which runs a task at a fixed rate to checks the connection
 73 *   table for timed out VMs.
 74 * </ul>
 75 * 
 76 * The most important bookkeeping datastructure of the communication bus is the
 77 * {@link CommunicationBus#addressToConnection_} table, also referred to as the
 78 * <b>connection table</b>. This table stores all of the currently connected
 79 * VMs that the host VM knows about. Whenever a heartbeat is received, a VM's
 80 * connection registration in this table is updated. When a heartbeat is first
 81 * received, the VM is added. When a heartbeat has not been heard for longer
 82 * than the timeout interval, the VM is removed from the connection table.
 83 * <p>
 84 * Schematically, we have the following situation after establishing a connection
 85 * between a master and a slave VM (see the description of {@link MulticastListenerThread}):
 86 * 
 87 * <pre>
 88 *   Slave                             Master
 89 *    cs = new Socket(masterAddress)     ms = socket.accept()
 90 *       cs.in <-------------------------- ms.out
 91 *       cs.out -------------------------> ms.in
 92 * </pre>
 93 * 
 94 * It does not matter which socket is registered in the connection table. Hence,
 95 * once both VMs have set up a connection, the concept of 'master' and 'slave' is
 96 * no longer useful and both VMs become peers.
 97 * 
 98 * It is always the output stream of either socket that is used for sending messages to the other VM.
 99 * It is always the input stream of either socket that is used for receiving messages from the other VM.
100 * 
101 * @author tvcutsem
102 */
103public class CommunicationBus {
104
105	/** the AmbientTalk VM that owns this communication bus */
106	public final ELVirtualMachine host_;
107	
108	/** the name of the overlay network in which to discover AmbientTalk VMs */
109	private final String groupName_;
110	/** the ip address to which connect or ELVirtualMachine._DEFAULT_IP_ADDRESS_ if not specified*/
111	private final String ipAddress_;
112	
113	/** if non-null, the communication bus is connected to the network */
114	private volatile Address networkAddress_;
115	
116	private MulticastListenerThread mcListener_;
117	private MulticastServerThread mcServer_;
118	private MasterConnectionThread masterConnectionThread_;
119	
120	/**
121	 * Maps the address of a currently connected VM to connection information
122	 * such as the last time it was seen, and the socket connection.
123	 * Also known as the "connection table".
124	 * 
125	 * This datastructure is modified by different threads:
126	 *  <ul>
127	 *   <li>Addition of master VMs by {@link MulticastListenerThread}
128	 *   <li>Addition of slave VMs by {@link MasterConnectionThread}
129	 *   <li>Removal of disconnected VMs by {@link CommandProcessor}
130	 *   <li>Checking for timed out VMs by {@link CommunicationBus#timeoutDetector_}
131	 *   <li>Lookup of connections by AmbientTalk event loops for message transmission
132	 *  </ul>
133	 * Hence, access to this datastructure must by <b>synchronized</b>!
134	 */
135	private final HashMap addressToConnection_;
136	
137	/**
138	 * The timer used for checking 'stale' (i.e. timed out) connections
139	 * in the connection table.
140	 */
141	private final Timer timeoutDetectorTimer_;
142	
143	/**
144	 * The timer task used for removing timed out connections
145	 * from the connections table.
146	 */
147	private TimeoutDetectorTask timeoutDetector_;
148	
149	/**
150	 * Bookkeeping datastructure to store connection-related information
151	 * of a connected VM. A connection entry is uniquely identified by
152	 * means of the {@link Address} of its VM.
153	 */
154	private static class Connection {
155		public final Socket socket_;
156		/** this value is updated as new heartbeats are received */
157		public long lastSeenAtTime_;
158		public final ObjectOutputStream outgoing_;
159		public final ObjectInputStream incoming_;
160		public Connection(Socket s, long lastSeenAt) throws IOException {
161			socket_ = s;
162			lastSeenAtTime_ = lastSeenAt;
163			// NOTE: apparently it is highly important that the ObjectOutputStream on the socket.getOutputStream() is
164			// created BEFORE trying to create an ObjectInputStream on the socket.getInputStream()
165			// switching the below two statements causes the master and the slave to deadlock!
166			outgoing_ = new ObjectOutputStream(new BufferedOutputStream(s.getOutputStream()));
167			// The buffered output stream must be EXPLICITLY flushed, otherwise the buffered input stream
168			// that is created below (but by the other VM) will block indefinitely
169			outgoing_.flush();
170			incoming_ = new ObjectInputStream(new BufferedInputStream(s.getInputStream()));
171		}
172		
173		/**
174		 * Closes the underlying socket. This will eventually trigger the command processor
175		 * tied to this connection, which will cause this connection to be removed
176		 * from the table.
177		 */
178		public synchronized void close() {
179			try {
180				socket_.close();
181			} catch (IOException ioe) { }
182		}
183		
184		public synchronized void send(VMCommand msg) throws IOException{
185			outgoing_.writeObject(msg);
186			outgoing_.flush();
187		}
188	}
189	
190	/**
191	 * The timer task used for removing timed out connections
192	 * from the connection table.
193	 */
194	private class TimeoutDetectorTask extends TimerTask {
195		
196		/**
197		 * the maximum amount of time that a remote VM gets to send a new
198		 * heartbeat before it is considered as being 'offline'
199		 */
200		private static final int MAX_RESPONSE_DELAY = 10000; // in milliseconds
201		
202		/**
203		 * The rate at which to schedule this timer task
204		 */
205		public static final int DETECTION_RATE = 4000; // in milliseconds
206		
207		public void run() {
208			closeConnectionOfMembersNotSeenIn(MAX_RESPONSE_DELAY);
209		}
210		
211	}
212	
213	public CommunicationBus(ELVirtualMachine host, String ambientTalkNetworkName, String ipAddress) {
214		host_ = host;
215		groupName_ = ambientTalkNetworkName;
216		ipAddress_ = ipAddress;
217		addressToConnection_ = new HashMap();
218		timeoutDetectorTimer_ = new Timer(true); // create a daemon timer
219	}
220	
221	public String getIpAddress() {
222		return ipAddress_;
223	}
224	
225	public String getGroupName(){
226		return groupName_;
227	}
228	/**
229	 * Tries to connect the communication bus to the underlying network.
230	 * @throws IOException if no server socket could be created to listen
231	 * for incoming connections. If this exception is raised, it is
232	 * guaranteed that the communication bus is left disconnected (i.e.
233	 * it is not partially connected)
234	 */
235	public Address connect() throws NetworkException {
236		if (networkAddress_ != null) {
237			return networkAddress_; // if the bus is already connected, there is no need to connect it again
238		}
239		
240		masterConnectionThread_ = new MasterConnectionThread(this);
241		try {
242			networkAddress_ = masterConnectionThread_.startServing();
243		} catch (IOException e) {
244			masterConnectionThread_ = null;
245			throw new NetworkException("Could not connect to network:", e);
246		}
247		mcListener_ = new MulticastListenerThread(this, networkAddress_);
248		mcServer_ = new MulticastServerThread(networkAddress_);
249		mcListener_.start();
250		mcServer_.start();
251		
252		// start detecting timed out VMs
253		timeoutDetector_ = new TimeoutDetectorTask();
254		timeoutDetectorTimer_.scheduleAtFixedRate(timeoutDetector_, 0, TimeoutDetectorTask.DETECTION_RATE);
255		
256		return networkAddress_;
257	}
258	
259    /**
260     * Called by the VM when it has disconnected from the underlying channel.
261     * It gracefully shuts down all network threads, sets the network address
262     * to null and removes all current connections from the connection table.
263     */
264	public void disconnect() {
265		if (networkAddress_ == null) {
266			return; // if the bus is already disconnected, there is no need to take it offline again
267		}
268		
269		// once the bus is disconnected, the network address is set to null.
270		// this ensures that no further incoming connections are allowed to be
271		// registered in the addConnection method!
272		networkAddress_ = null;
273		masterConnectionThread_.stopServing();
274		mcListener_.stopListening();
275		mcServer_.stopBroadcasting();
276		masterConnectionThread_ = null;
277		mcListener_ = null;
278		mcServer_ = null;
279		
280		// stop detecting timed out VMs
281		timeoutDetector_.cancel();
282		timeoutDetector_ = null;
283		
284		closeConnectionOfAllMembers();
285	}
286	
287	/**
288	 * Updates the time that the given virtual machine was last seen online.
289	 * This method is invoked frequently by the {@link MulticastListenerThread}.
290	 * 
291	 * @param member the address of the detected virtual machine
292	 * @return true if the VM was properly registered, false if the VM is not yet registered
293	 * (i.e. it is not yet considered as 'online')
294	 */
295	protected boolean updateTimeLastSeen(Address member) {
296		synchronized (addressToConnection_) {
297			Connection conn = (Connection) addressToConnection_.get(member);
298			if (conn != null) {
299				conn.lastSeenAtTime_ = System.currentTimeMillis();
300				return true;
301			} else {
302				return false;
303			}
304		}
305	}
306	
307	/**
308	 * Registers a new virtual machine connection for the given address.
309	 * If this VM was a slave in the discovery process, this method is
310	 * invoked by the {@link MulticastListenerThread}. If this VM was
311	 * a master in the discovery process, this method is invoked by the
312	 * {@link MasterConnectionThread}.
313	 * 
314	 * The socket's output stream will be stored in the connection table and is used
315	 * for transmitting VM Commands to this VM. The socket's input stream will be
316	 * coupled to a dedicated {@link CommandProcessor} which is responsible for
317	 * processing incoming VM command objects.
318	 * 
319	 * Calling this method implicitly also triggers a memberJoined event on this VM
320	 * 
321	 * @throws IOException if no ObjectOutputStream can be created for the socket's output stream,
322	 * or if no ObjectInputStream can be created for the socket's input stream.
323	 * If this exception is raised, it is guaranteed the member is not registered in the connection table.
324	 */
325	protected void addConnection(Address newMember, Socket conn) throws IOException {
326		if (networkAddress_ == null) {
327			Logging.Network_LOG.debug("ignored connection to " + newMember + ": bus disconnected");
328			return; // the bus has been disconnected, do not accept any new connections
329		}
330		
331		// create a new connection object that can be used to send command objects to this member
332		Connection registeredConnection = new Connection(conn, System.currentTimeMillis());
333		
334		// spawn a new command processor dedicated for handling the command objects received from this member
335		CommandProcessor processor = new CommandProcessor(newMember, conn, registeredConnection.incoming_, this);
336
337		synchronized (addressToConnection_) {
338			// first check whether a connection for this member already exists
339			// (may happen when an old connection for this member has not yet been deleted)
340			// This case IS possible when a master receives a new connection from a slave,
341			// but that master has not yet detected the disconnection of the slave's old connection
342			Connection oldConnection = (Connection) addressToConnection_.get(newMember);
343			if (oldConnection != null) {
344				// explicitly close the connection we're about to override
345				// Note: in this case, we should *not* signal to the VM that a new member joined,
346				// because in the corresponding removeConnection, we will also not signal a member left
347				// (i.e. the connection was restored quickly enough such that we can abstract over the disconnection)
348				oldConnection.close();
349			} else {
350				// notify the VM of a new connection
351				host_.event_memberJoined(newMember);
352			}
353			// register the new connection in the table (this may override an old connection)
354			addressToConnection_.put(newMember, registeredConnection);
355		}
356		
357		// only start the processor if the member is properly registered
358		processor.start();
359		
360		Logging.Network_LOG.debug("successfully registered connection to " + newMember);
361	}
362	
363	/**
364	 * It is the responsibility of the {@link CommandProcessor} tied to the given VM
365	 * to invoke this method when its connection has failed.
366	 * 
367	 * This is the <b>only</b> method responsible for removing entries from the
368	 * connection table.
369	 * 
370	 * Calling this method implicitly also triggers a memberLeft event on this VM
371	 */
372	protected void removeConnection(Address oldMember, Socket connSocket) {
373		synchronized (addressToConnection_) {
374			Connection conn = (Connection) addressToConnection_.get(oldMember);
375			if (conn != null) {
376				// ONLY delete the connection if that connection was registered for the given
377				// socket. It might be that the connection in the connection table is already
378				// a NEWER connection that has OVERWRITTEN the old one.
379				if (conn.socket_ == connSocket) {
380					conn.close();
381					addressToConnection_.remove(oldMember);
382					host_.event_memberLeft(oldMember); // notify VM that the member has left
383				}
384				// Note: if the sockets do not match, then the connection to be
385				// removed by this call has already been replaced by a more recent connection to the same address
386				// See the code for addConnection for more details
387				// Because the connection to be removed was already replaced, it is not necessary
388				// to notify the VM that a member has left
389			}
390		}
391	}
392	
393	/**
394	 * The {@link MulticastListenerThread} invokes this method regularly to
395	 * remove connections to VMs which have not responded for longer than the
396	 * given timeout period.
397	 * 
398	 * Removal is done implicitly by closing timed out connections. The
399	 * {@link CommandProcessor} tied to each connection is responsible for
400	 * actually removing it.
401	 * 
402	 * Calling this method might also implicitly trigger one or more memberLeft
403	 * events on this VM.
404	 */
405	protected void closeConnectionOfMembersNotSeenIn(long period) {
406		synchronized (addressToConnection_) {
407			long now = System.currentTimeMillis();
408			Collection connections = addressToConnection_.values();
409			for (Iterator iter = connections.iterator(); iter.hasNext();) {
410				Connection c = (Connection) iter.next();
411				if (now - c.lastSeenAtTime_ > period) {
412					c.close(); // the entry will be deleted by the Command Processor
413					Logging.Network_LOG.debug("Detected timed out VM");
414				}
415			}
416		}
417	}
418	
419	/**
420	 * When the communication bus is explicitly disconnected from the
421	 * network, the VM should be notified that all connected VMs
422	 * have disconnected.
423	 * 
424	 * Removal is done implicitly by closing all connections. The
425	 * {@link CommandProcessor} tied to each connection is responsible for
426	 * actually removing them.
427	 */
428	private void closeConnectionOfAllMembers() {
429		synchronized (addressToConnection_) {
430			Collection connections = addressToConnection_.values();
431			for (Iterator iter = connections.iterator(); iter.hasNext();) {
432				Connection c = (Connection) iter.next();
433				c.close(); // ensures the command processor will remove the entry eventually
434			}
435		}
436	}
437	
438	/**
439	 * Sends a VM Command object asynchronously to the recipient VM.
440	 * There are no delivery guarantees for this message.
441	 * If the recipient is offline, or the message times out, it is simply discarded
442	 */
443	public void sendAsyncUnicast(VMCommand msg, Address recipientVM) {
444		Logging.Network_LOG.info("sending async unicast cmd " + msg + " to " + recipientVM);
445		Connection conn;
446		synchronized (addressToConnection_) {
447			conn = (Connection) addressToConnection_.get(recipientVM);
448		}
449		if (conn != null) {
450			sendOneAsyncMessage(conn, msg, recipientVM);
451		}
452	}
453	
454	/**
455	 * Sends a VM Command object asynchronously to all connected VMs.
456	 * There are no delivery guarantees for this message, nor is it guaranteed
457	 * that all currently connected VMs will receive the message
458	 */
459	public void sendAsyncMulticast(VMCommand msg) {
460		Logging.Network_LOG.debug("sending async multicast cmd: " + msg);
461		
462		// first clone the connection table such that we do not need to acquire the
463		// lock for the entire duration of the multicast
464		HashMap clonedConnections;
465		synchronized (addressToConnection_) {
466			clonedConnections  = (HashMap) addressToConnection_.clone();
467		}
468		
469		for (Iterator iter = clonedConnections.entrySet().iterator(); iter.hasNext();) {
470			Map.Entry entry = (Map.Entry) iter.next();
471			Address recipient = (Address) entry.getKey();
472			Connection conn = (Connection) entry.getValue();
473			sendOneAsyncMessage(conn, msg, recipient);
474		}
475	}
476	
477	/**
478	 * Sends the given {@link VMCommand} to the given member. With 'synchronous' transmission,
479	 * it is meant that if this method returns gracefully, the caller can rest assured that
480	 * the remote VM has correctly received the command object. It does not given any guarantees
481	 * that the command object will have been executed remotely.
482	 * 
483	 * @throws NetworkException if either the given address is no longer connected, or
484	 * an I/O error occurs during the transmission of the command object. If this exception is
485	 * raised, the caller does not know whether the message was correctly received or not.
486	 */
487	public void sendSynchronousUnicast(VMCommand msg, Address recipientVM) throws NetworkException {
488		Logging.Network_LOG.info("sending sync unicast cmd: " + msg + " to " + recipientVM);
489		
490		Connection conn;
491		synchronized (addressToConnection_) {
492			conn = (Connection) addressToConnection_.get(recipientVM);
493		}
494		
495		if (conn == null) {
496			throw new NetworkException("Recipient " + recipientVM + " is offline");
497		}
498		
499		try {
500			conn.send(msg);
501		} catch (IOException e) {
502			// it is the sender's responsibility to close the connection's socket if something goes wrong
503			// the corresponding CommandProcessor registered on this socket will remove the member from
504			// the connection table
505			conn.close();
506			Logging.Network_LOG.error("Error while trying to send command " + msg.toString() + " to " + recipientVM, e);
507			throw new NetworkException("Error while trying to transmit message " + msg, e);
508		} 
509	}
510	
511	/**
512	 * Note that an I/O exception during the transmission of a command object is <b>fatal</b>
513	 * for the connection. The connection is immediately terminated by closing the socket.
514	 * It is the responsibility of the {@link CommandProcessor} registered on the socket
515	 * connection's input stream to remove the recipient VM from the connection table.
516	 */
517	private void sendOneAsyncMessage(Connection conn, VMCommand msg, Address recipientVM) {
518		try {
519			conn.send(msg);
520		} catch (IOException e) {
521			// it is the sender's responsibility to close the connection's socket if something goes wrong
522			// the CommandProcessor will remove the member from the connection table
523			conn.close();
524			Logging.Network_LOG.error("Could not send command " + msg + " to " + recipientVM + ", dropping.", e);
525		}
526	}
527	
528	public String toString() {
529		return "communication bus";
530	}
531	
532}