PageRenderTime 50ms CodeModel.GetById 18ms RepoModel.GetById 0ms app.codeStats 0ms

/projects/tomcat-7.0.2/java/org/apache/catalina/tribes/membership/McastServiceImpl.java

https://gitlab.com/essere.lab.public/qualitas.class-corpus
Java | 653 lines | 464 code | 46 blank | 143 comment | 84 complexity | 9c6e5a334d2a3698052af86123810bd6 MD5 | raw file
  1. /*
  2. * Licensed to the Apache Software Foundation (ASF) under one or more
  3. * contributor license agreements. See the NOTICE file distributed with
  4. * this work for additional information regarding copyright ownership.
  5. * The ASF licenses this file to You under the Apache License, Version 2.0
  6. * (the "License"); you may not use this file except in compliance with
  7. * the License. You may obtain a copy of the License at
  8. *
  9. * http://www.apache.org/licenses/LICENSE-2.0
  10. *
  11. * Unless required by applicable law or agreed to in writing, software
  12. * distributed under the License is distributed on an "AS IS" BASIS,
  13. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. * See the License for the specific language governing permissions and
  15. * limitations under the License.
  16. */
  17. package org.apache.catalina.tribes.membership;
  18. import java.io.IOException;
  19. import java.net.BindException;
  20. import java.net.DatagramPacket;
  21. import java.net.InetAddress;
  22. import java.net.InetSocketAddress;
  23. import java.net.MulticastSocket;
  24. import java.net.SocketTimeoutException;
  25. import java.util.Arrays;
  26. import java.util.concurrent.ExecutorService;
  27. import java.util.concurrent.TimeUnit;
  28. import org.apache.catalina.tribes.Channel;
  29. import org.apache.catalina.tribes.Member;
  30. import org.apache.catalina.tribes.MembershipListener;
  31. import org.apache.catalina.tribes.MessageListener;
  32. import org.apache.catalina.tribes.io.ChannelData;
  33. import org.apache.catalina.tribes.io.XByteBuffer;
  34. import org.apache.catalina.tribes.util.ExecutorFactory;
  35. /**
  36. * A <b>membership</b> implementation using simple multicast.
  37. * This is the representation of a multicast membership service.
  38. * This class is responsible for maintaining a list of active cluster nodes in the cluster.
  39. * If a node fails to send out a heartbeat, the node will be dismissed.
  40. * This is the low level implementation that handles the multicasting sockets.
  41. * Need to fix this, could use java.nio and only need one thread to send and receive, or
  42. * just use a timeout on the receive
  43. * @author Filip Hanik
  44. * @version $Id: McastServiceImpl.java 978840 2010-07-24 10:17:45Z markt $
  45. */
  46. public class McastServiceImpl
  47. {
  48. private static final org.apache.juli.logging.Log log =
  49. org.apache.juli.logging.LogFactory.getLog( McastService.class );
  50. protected static int MAX_PACKET_SIZE = 65535;
  51. /**
  52. * Internal flag used for the listen thread that listens to the multicasting socket.
  53. */
  54. protected volatile boolean doRunSender = false;
  55. protected volatile boolean doRunReceiver = false;
  56. protected int startLevel = 0;
  57. /**
  58. * Socket that we intend to listen to
  59. */
  60. protected MulticastSocket socket;
  61. /**
  62. * The local member that we intend to broad cast over and over again
  63. */
  64. protected MemberImpl member;
  65. /**
  66. * The multicast address
  67. */
  68. protected InetAddress address;
  69. /**
  70. * The multicast port
  71. */
  72. protected int port;
  73. /**
  74. * The time it takes for a member to expire.
  75. */
  76. protected long timeToExpiration;
  77. /**
  78. * How often to we send out a broadcast saying we are alive, must be smaller than timeToExpiration
  79. */
  80. protected long sendFrequency;
  81. /**
  82. * Reuse the sendPacket, no need to create a new one everytime
  83. */
  84. protected DatagramPacket sendPacket;
  85. /**
  86. * Reuse the receivePacket, no need to create a new one everytime
  87. */
  88. protected DatagramPacket receivePacket;
  89. /**
  90. * The membership, used so that we calculate memberships when they arrive or don't arrive
  91. */
  92. protected Membership membership;
  93. /**
  94. * The actual listener, for callback when stuff goes down
  95. */
  96. protected MembershipListener service;
  97. /**
  98. * The actual listener for broadcast callbacks
  99. */
  100. protected MessageListener msgservice;
  101. /**
  102. * Thread to listen for pings
  103. */
  104. protected ReceiverThread receiver;
  105. /**
  106. * Thread to send pings
  107. */
  108. protected SenderThread sender;
  109. /**
  110. * Time to live for the multicast packets that are being sent out
  111. */
  112. protected int mcastTTL = -1;
  113. /**
  114. * Read timeout on the mcast socket
  115. */
  116. protected int mcastSoTimeout = -1;
  117. /**
  118. * bind address
  119. */
  120. protected InetAddress mcastBindAddress = null;
  121. /**
  122. * nr of times the system has to fail before a recovery is initiated
  123. */
  124. protected int recoveryCounter = 10;
  125. /**
  126. * The time the recovery thread sleeps between recovery attempts
  127. */
  128. protected long recoverySleepTime = 5000;
  129. /**
  130. * Add the ability to turn on/off recovery
  131. */
  132. protected boolean recoveryEnabled = true;
  133. /**
  134. * Dont interrupt the sender/receiver thread, but pass off to an executor
  135. */
  136. protected ExecutorService executor = ExecutorFactory.newThreadPool(0, 2, 2, TimeUnit.SECONDS);
  137. /**
  138. * disable/enable local loopback message
  139. */
  140. protected boolean localLoopbackDisabled = false;
  141. /**
  142. * Create a new mcast service impl
  143. * @param member - the local member
  144. * @param sendFrequency - the time (ms) in between pings sent out
  145. * @param expireTime - the time (ms) for a member to expire
  146. * @param port - the mcast port
  147. * @param bind - the bind address (not sure this is used yet)
  148. * @param mcastAddress - the mcast address
  149. * @param service - the callback service
  150. * @param localLoopbackDisabled - disable loopbackMode
  151. * @throws IOException
  152. */
  153. public McastServiceImpl(
  154. MemberImpl member,
  155. long sendFrequency,
  156. long expireTime,
  157. int port,
  158. InetAddress bind,
  159. InetAddress mcastAddress,
  160. int ttl,
  161. int soTimeout,
  162. MembershipListener service,
  163. MessageListener msgservice,
  164. boolean localLoopbackDisabled)
  165. throws IOException {
  166. this.member = member;
  167. this.address = mcastAddress;
  168. this.port = port;
  169. this.mcastSoTimeout = soTimeout;
  170. this.mcastTTL = ttl;
  171. this.mcastBindAddress = bind;
  172. this.timeToExpiration = expireTime;
  173. this.service = service;
  174. this.msgservice = msgservice;
  175. this.sendFrequency = sendFrequency;
  176. this.localLoopbackDisabled = localLoopbackDisabled;
  177. init();
  178. }
  179. public void init() throws IOException {
  180. setupSocket();
  181. sendPacket = new DatagramPacket(new byte[MAX_PACKET_SIZE],MAX_PACKET_SIZE);
  182. sendPacket.setAddress(address);
  183. sendPacket.setPort(port);
  184. receivePacket = new DatagramPacket(new byte[MAX_PACKET_SIZE],MAX_PACKET_SIZE);
  185. receivePacket.setAddress(address);
  186. receivePacket.setPort(port);
  187. member.setCommand(new byte[0]);
  188. member.getData(true, true);
  189. if ( membership == null ) membership = new Membership(member);
  190. }
  191. protected void setupSocket() throws IOException {
  192. if (mcastBindAddress != null) {
  193. try {
  194. log.info("Attempting to bind the multicast socket to "+address+":"+port);
  195. socket = new MulticastSocket(new InetSocketAddress(address,port));
  196. } catch (BindException e) {
  197. /*
  198. * On some platforms (e.g. Linux) it is not possible to bind
  199. * to the multicast address. In this case only bind to the
  200. * port.
  201. */
  202. log.info("Binding to multicast address, failed. Binding to port only.");
  203. socket = new MulticastSocket(port);
  204. }
  205. } else {
  206. socket = new MulticastSocket(port);
  207. }
  208. socket.setLoopbackMode(localLoopbackDisabled); //hint if we want disable loop back(local machine) messages
  209. if (mcastBindAddress != null) {
  210. if(log.isInfoEnabled())
  211. log.info("Setting multihome multicast interface to:" +mcastBindAddress);
  212. socket.setInterface(mcastBindAddress);
  213. } //end if
  214. //force a so timeout so that we don't block forever
  215. if ( mcastSoTimeout <= 0 ) mcastSoTimeout = (int)sendFrequency;
  216. if(log.isInfoEnabled())
  217. log.info("Setting cluster mcast soTimeout to "+mcastSoTimeout);
  218. socket.setSoTimeout(mcastSoTimeout);
  219. if ( mcastTTL >= 0 ) {
  220. if(log.isInfoEnabled())
  221. log.info("Setting cluster mcast TTL to " + mcastTTL);
  222. socket.setTimeToLive(mcastTTL);
  223. }
  224. }
  225. /**
  226. * Start the service
  227. * @param level 1 starts the receiver, level 2 starts the sender
  228. * @throws IOException if the service fails to start
  229. * @throws IllegalStateException if the service is already started
  230. */
  231. public synchronized void start(int level) throws IOException {
  232. boolean valid = false;
  233. if ( (level & Channel.MBR_RX_SEQ)==Channel.MBR_RX_SEQ ) {
  234. if ( receiver != null ) throw new IllegalStateException("McastService.receive already running.");
  235. try {
  236. if ( sender == null ) socket.joinGroup(address);
  237. }catch (IOException iox) {
  238. log.error("Unable to join multicast group, make sure your system has multicasting enabled.");
  239. throw iox;
  240. }
  241. doRunReceiver = true;
  242. receiver = new ReceiverThread();
  243. receiver.setDaemon(true);
  244. receiver.start();
  245. valid = true;
  246. }
  247. if ( (level & Channel.MBR_TX_SEQ)==Channel.MBR_TX_SEQ ) {
  248. if ( sender != null ) throw new IllegalStateException("McastService.send already running.");
  249. if ( receiver == null ) socket.joinGroup(address);
  250. //make sure at least one packet gets out there
  251. send(false);
  252. doRunSender = true;
  253. sender = new SenderThread(sendFrequency);
  254. sender.setDaemon(true);
  255. sender.start();
  256. //we have started the receiver, but not yet waited for membership to establish
  257. valid = true;
  258. }
  259. if (!valid) {
  260. throw new IllegalArgumentException("Invalid start level. Only acceptable levels are Channel.MBR_RX_SEQ and Channel.MBR_TX_SEQ");
  261. }
  262. //pause, once or twice
  263. waitForMembers(level);
  264. startLevel = (startLevel | level);
  265. }
  266. private void waitForMembers(int level) {
  267. long memberwait = sendFrequency*2;
  268. if(log.isInfoEnabled())
  269. log.info("Sleeping for "+memberwait+" milliseconds to establish cluster membership, start level:"+level);
  270. try {Thread.sleep(memberwait);}catch (InterruptedException ignore){}
  271. if(log.isInfoEnabled())
  272. log.info("Done sleeping, membership established, start level:"+level);
  273. }
  274. /**
  275. * Stops the service
  276. * @throws IOException if the service fails to disconnect from the sockets
  277. */
  278. public synchronized boolean stop(int level) throws IOException {
  279. boolean valid = false;
  280. if ( (level & Channel.MBR_RX_SEQ)==Channel.MBR_RX_SEQ ) {
  281. valid = true;
  282. doRunReceiver = false;
  283. if ( receiver !=null ) receiver.interrupt();
  284. receiver = null;
  285. }
  286. if ( (level & Channel.MBR_TX_SEQ)==Channel.MBR_TX_SEQ ) {
  287. valid = true;
  288. doRunSender = false;
  289. if ( sender != null )sender.interrupt();
  290. sender = null;
  291. }
  292. if (!valid) {
  293. throw new IllegalArgumentException("Invalid stop level. Only acceptable levels are Channel.MBR_RX_SEQ and Channel.MBR_TX_SEQ");
  294. }
  295. startLevel = (startLevel & (~level));
  296. //we're shutting down, send a shutdown message and close the socket
  297. if ( startLevel == 0 ) {
  298. //send a stop message
  299. member.setCommand(Member.SHUTDOWN_PAYLOAD);
  300. member.getData(true, true);
  301. send(false);
  302. //leave mcast group
  303. try {socket.leaveGroup(address);}catch ( Exception ignore){}
  304. try {socket.close();}catch ( Exception ignore){}
  305. member.setServiceStartTime(-1);
  306. }
  307. return (startLevel == 0);
  308. }
  309. /**
  310. * Receive a datagram packet, locking wait
  311. * @throws IOException
  312. */
  313. public void receive() throws IOException {
  314. boolean checkexpired = true;
  315. try {
  316. socket.receive(receivePacket);
  317. if(receivePacket.getLength() > MAX_PACKET_SIZE) {
  318. log.error("Multicast packet received was too long, dropping package:"+receivePacket.getLength());
  319. } else {
  320. byte[] data = new byte[receivePacket.getLength()];
  321. System.arraycopy(receivePacket.getData(), receivePacket.getOffset(), data, 0, data.length);
  322. if (XByteBuffer.firstIndexOf(data,0,MemberImpl.TRIBES_MBR_BEGIN)==0) {
  323. memberDataReceived(data);
  324. } else {
  325. memberBroadcastsReceived(data);
  326. }
  327. }
  328. } catch (SocketTimeoutException x ) {
  329. //do nothing, this is normal, we don't want to block forever
  330. //since the receive thread is the same thread
  331. //that does membership expiration
  332. }
  333. if (checkexpired) checkExpired();
  334. }
  335. private void memberDataReceived(byte[] data) {
  336. final MemberImpl m = MemberImpl.getMember(data);
  337. if (log.isTraceEnabled()) log.trace("Mcast receive ping from member " + m);
  338. Runnable t = null;
  339. if (Arrays.equals(m.getCommand(), Member.SHUTDOWN_PAYLOAD)) {
  340. if (log.isDebugEnabled()) log.debug("Member has shutdown:" + m);
  341. membership.removeMember(m);
  342. t = new Runnable() {
  343. public void run() {
  344. String name = Thread.currentThread().getName();
  345. try {
  346. Thread.currentThread().setName("Membership-MemberDisappeared.");
  347. service.memberDisappeared(m);
  348. }finally {
  349. Thread.currentThread().setName(name);
  350. }
  351. }
  352. };
  353. } else if (membership.memberAlive(m)) {
  354. if (log.isDebugEnabled()) log.debug("Mcast add member " + m);
  355. t = new Runnable() {
  356. public void run() {
  357. String name = Thread.currentThread().getName();
  358. try {
  359. Thread.currentThread().setName("Membership-MemberAdded.");
  360. service.memberAdded(m);
  361. }finally {
  362. Thread.currentThread().setName(name);
  363. }
  364. }
  365. };
  366. } //end if
  367. if ( t != null ) {
  368. executor.execute(t);
  369. }
  370. }
  371. private void memberBroadcastsReceived(final byte[] b) {
  372. if (log.isTraceEnabled()) log.trace("Mcast received broadcasts.");
  373. XByteBuffer buffer = new XByteBuffer(b,true);
  374. if (buffer.countPackages(true)>0) {
  375. int count = buffer.countPackages();
  376. final ChannelData[] data = new ChannelData[count];
  377. for (int i=0; i<count; i++) {
  378. try {
  379. data[i] = buffer.extractPackage(true);
  380. }catch (IllegalStateException ise) {
  381. log.debug("Unable to decode message.",ise);
  382. }catch (IOException x) {
  383. log.debug("Unable to decode message.",x);
  384. }
  385. }
  386. Runnable t = new Runnable() {
  387. public void run() {
  388. String name = Thread.currentThread().getName();
  389. try {
  390. Thread.currentThread().setName("Membership-MemberAdded.");
  391. for (int i=0; i<data.length; i++ ) {
  392. try {
  393. if (data[i]!=null && !member.equals(data[i].getAddress())) {
  394. msgservice.messageReceived(data[i]);
  395. }
  396. }catch (Throwable t) {
  397. log.error("Unable to receive broadcast message.",t);
  398. }
  399. }
  400. }finally {
  401. Thread.currentThread().setName(name);
  402. }
  403. }
  404. };
  405. executor.execute(t);
  406. }
  407. }
  408. protected final Object expiredMutex = new Object();
  409. protected void checkExpired() {
  410. synchronized (expiredMutex) {
  411. MemberImpl[] expired = membership.expire(timeToExpiration);
  412. for (int i = 0; i < expired.length; i++) {
  413. final MemberImpl member = expired[i];
  414. if (log.isDebugEnabled())
  415. log.debug("Mcast exipre member " + expired[i]);
  416. try {
  417. Runnable t = new Runnable() {
  418. public void run() {
  419. String name = Thread.currentThread().getName();
  420. try {
  421. Thread.currentThread().setName("Membership-MemberExpired.");
  422. service.memberDisappeared(member);
  423. }finally {
  424. Thread.currentThread().setName(name);
  425. }
  426. }
  427. };
  428. executor.execute(t);
  429. } catch (Exception x) {
  430. log.error("Unable to process member disappeared message.", x);
  431. }
  432. }
  433. }
  434. }
  435. /**
  436. * Send a ping
  437. * @throws IOException
  438. */
  439. public void send(boolean checkexpired) throws IOException{
  440. send(checkexpired,null);
  441. }
  442. private final Object sendLock = new Object();
  443. public void send(boolean checkexpired, DatagramPacket packet) throws IOException{
  444. checkexpired = (checkexpired && (packet==null));
  445. //ignore if we haven't started the sender
  446. //if ( (startLevel&Channel.MBR_TX_SEQ) != Channel.MBR_TX_SEQ ) return;
  447. if (packet==null) {
  448. member.inc();
  449. if(log.isTraceEnabled()) {
  450. log.trace("Mcast send ping from member " + member);
  451. }
  452. byte[] data = member.getData();
  453. packet = new DatagramPacket(data,data.length);
  454. } else if (log.isTraceEnabled()) {
  455. log.trace("Sending message broadcast "+packet.getLength()+ " bytes from "+ member);
  456. }
  457. packet.setAddress(address);
  458. packet.setPort(port);
  459. //TODO this operation is not thread safe
  460. synchronized (sendLock) {
  461. socket.send(packet);
  462. }
  463. if ( checkexpired ) checkExpired();
  464. }
  465. public long getServiceStartTime() {
  466. return (member!=null) ? member.getServiceStartTime() : -1l;
  467. }
  468. public int getRecoveryCounter() {
  469. return recoveryCounter;
  470. }
  471. public boolean isRecoveryEnabled() {
  472. return recoveryEnabled;
  473. }
  474. public long getRecoverySleepTime() {
  475. return recoverySleepTime;
  476. }
  477. public class ReceiverThread extends Thread {
  478. int errorCounter = 0;
  479. public ReceiverThread() {
  480. super();
  481. setName("Tribes-MembershipReceiver");
  482. }
  483. @Override
  484. public void run() {
  485. while ( doRunReceiver ) {
  486. try {
  487. receive();
  488. errorCounter=0;
  489. } catch ( ArrayIndexOutOfBoundsException ax ) {
  490. //we can ignore this, as it means we have an invalid package
  491. //but we will log it to debug
  492. if ( log.isDebugEnabled() )
  493. log.debug("Invalid member mcast package.",ax);
  494. } catch ( Exception x ) {
  495. if (x instanceof InterruptedException) interrupted();
  496. else {
  497. if (errorCounter==0 && doRunReceiver) log.warn("Error receiving mcast package. Sleeping 500ms",x);
  498. else if (log.isDebugEnabled()) log.debug("Error receiving mcast package"+(doRunReceiver?". Sleeping 500ms":"."),x);
  499. if (doRunReceiver) {
  500. try { Thread.sleep(500); } catch ( Exception ignore ){}
  501. if ( (++errorCounter)>=recoveryCounter ) {
  502. errorCounter=0;
  503. new RecoveryThread(McastServiceImpl.this);
  504. }
  505. }
  506. }
  507. }
  508. }
  509. }
  510. }//class ReceiverThread
  511. public class SenderThread extends Thread {
  512. long time;
  513. int errorCounter=0;
  514. public SenderThread(long time) {
  515. this.time = time;
  516. setName("Tribes-MembershipSender");
  517. }
  518. @Override
  519. public void run() {
  520. while ( doRunSender ) {
  521. try {
  522. send(true);
  523. errorCounter = 0;
  524. } catch ( Exception x ) {
  525. if (errorCounter==0) log.warn("Unable to send mcast message.",x);
  526. else log.debug("Unable to send mcast message.",x);
  527. if ( (++errorCounter)>=recoveryCounter ) {
  528. errorCounter=0;
  529. new RecoveryThread(McastServiceImpl.this);
  530. }
  531. }
  532. try { Thread.sleep(time); } catch ( Exception ignore ) {}
  533. }
  534. }
  535. }//class SenderThread
  536. protected static class RecoveryThread extends Thread {
  537. static volatile boolean running = false;
  538. McastServiceImpl parent = null;
  539. public RecoveryThread(McastServiceImpl parent) {
  540. this.parent = parent;
  541. if (!init(this)) parent = null;
  542. }
  543. public static synchronized boolean init(RecoveryThread t) {
  544. if ( running ) return false;
  545. if ( !t.parent.isRecoveryEnabled()) return false;
  546. running = true;
  547. t.setName("Tribes-MembershipRecovery");
  548. t.setDaemon(true);
  549. t.start();
  550. return true;
  551. }
  552. public boolean stopService() {
  553. try {
  554. parent.stop(Channel.MBR_RX_SEQ | Channel.MBR_TX_SEQ);
  555. return true;
  556. } catch (Exception x) {
  557. log.warn("Recovery thread failed to stop membership service.", x);
  558. return false;
  559. }
  560. }
  561. public boolean startService() {
  562. try {
  563. parent.init();
  564. parent.start(Channel.MBR_RX_SEQ | Channel.MBR_TX_SEQ);
  565. return true;
  566. } catch (Exception x) {
  567. log.warn("Recovery thread failed to start membership service.", x);
  568. return false;
  569. }
  570. }
  571. @Override
  572. public void run() {
  573. boolean success = false;
  574. int attempt = 0;
  575. try {
  576. while (!success) {
  577. if(log.isInfoEnabled())
  578. log.info("Tribes membership, running recovery thread, multicasting is not functional.");
  579. if (stopService() & startService()) {
  580. success = true;
  581. if(log.isInfoEnabled())
  582. log.info("Membership recovery was successful.");
  583. }
  584. try {
  585. if (!success) {
  586. if(log.isInfoEnabled())
  587. log.info("Recovery attempt "+(++attempt)+" failed, trying again in " +parent.recoverySleepTime+ " seconds");
  588. Thread.sleep(parent.recoverySleepTime);
  589. }
  590. }catch (InterruptedException ignore) {
  591. }
  592. }
  593. }finally {
  594. running = false;
  595. }
  596. }
  597. }
  598. public void setRecoveryCounter(int recoveryCounter) {
  599. this.recoveryCounter = recoveryCounter;
  600. }
  601. public void setRecoveryEnabled(boolean recoveryEnabled) {
  602. this.recoveryEnabled = recoveryEnabled;
  603. }
  604. public void setRecoverySleepTime(long recoverySleepTime) {
  605. this.recoverySleepTime = recoverySleepTime;
  606. }
  607. }