PageRenderTime 96ms CodeModel.GetById 55ms RepoModel.GetById 0ms app.codeStats 0ms

/projects/jgroups-2.10.0/src/org/jgroups/protocols/pbcast/STREAMING_STATE_TRANSFER.java

https://gitlab.com/essere.lab.public/qualitas.class-corpus
Java | 1016 lines | 770 code | 147 blank | 99 comment | 120 complexity | d6a0ec6d5ff17afc394fe3a763c2eb9a MD5 | raw file
  1. package org.jgroups.protocols.pbcast;
  2. import org.jgroups.*;
  3. import org.jgroups.annotations.*;
  4. import org.jgroups.conf.PropertyConverters;
  5. import org.jgroups.stack.IpAddress;
  6. import org.jgroups.stack.Protocol;
  7. import org.jgroups.stack.StateTransferInfo;
  8. import org.jgroups.util.Digest;
  9. import org.jgroups.util.ShutdownRejectedExecutionHandler;
  10. import org.jgroups.util.Util;
  11. import java.io.*;
  12. import java.net.InetAddress;
  13. import java.net.InetSocketAddress;
  14. import java.net.ServerSocket;
  15. import java.net.Socket;
  16. import java.util.HashMap;
  17. import java.util.Map;
  18. import java.util.Vector;
  19. import java.util.concurrent.*;
  20. import java.util.concurrent.atomic.AtomicBoolean;
  21. import java.util.concurrent.atomic.AtomicInteger;
  22. import java.util.concurrent.atomic.AtomicLong;
  23. /**
  24. * <code>STREAMING_STATE_TRANSFER</code>, as its name implies, allows a
  25. * streaming state transfer between two channel instances.
  26. * <p>
  27. *
  28. * Major advantage of this approach is that transferring application state to a
  29. * joining member of a group does not entail loading of the complete application
  30. * state into memory. Application state, for example, might be located entirely
  31. * on some form of disk based storage. The default <code>STATE_TRANSFER</code>
  32. * requires this state to be loaded entirely into memory before being
  33. * transferred to a group member while <code>STREAMING_STATE_TRANSFER</code>
  34. * does not. Thus <code>STREAMING_STATE_TRANSFER</code> protocol is able to
  35. * transfer application state that is very large (>1Gb) without a likelihood of
  36. * such transfer resulting in OutOfMemoryException.
  37. * <p>
  38. *
  39. * <code>STREAMING_STATE_TRANSFER</code> allows use of either default channel
  40. * transport or separate tcp sockets for state transfer. If firewalls are not a
  41. * concern then separate tcp sockets should be used as they offer faster state
  42. * transfer. Transport for state transfer is selected using
  43. * <code>use_default_transport<code> boolean property.
  44. * <p>
  45. *
  46. *
  47. * Channel instance can be configured with either
  48. * <code>STREAMING_STATE_TRANSFER</code> or <code>STATE_TRANSFER</code> but not
  49. * both protocols at the same time.
  50. *
  51. * <p>
  52. *
  53. * In order to process streaming state transfer an application has to implement
  54. * <code>ExtendedMessageListener</code> if it is using channel in a push style
  55. * mode or it has to process <code>StreamingSetStateEvent</code> and
  56. * <code>StreamingGetStateEvent</code> if it is using channel in a pull style
  57. * mode.
  58. *
  59. *
  60. * @author Vladimir Blagojevic
  61. * @see org.jgroups.ExtendedMessageListener
  62. * @see org.jgroups.StreamingGetStateEvent
  63. * @see org.jgroups.StreamingSetStateEvent
  64. * @see org.jgroups.protocols.pbcast.STATE_TRANSFER
  65. * @since 2.4
  66. *
  67. * @version $Id$
  68. *
  69. */
  70. @MBean(description = "State trasnfer protocol based on streaming state transfer")
  71. @DeprecatedProperty(names = {"use_flush", "flush_timeout", "use_reading_thread"})
  72. public class STREAMING_STATE_TRANSFER extends Protocol {
  73. /*
  74. * ----------------------------------------------Properties -----------------------------------
  75. *
  76. */
  77. @Property(description = "The interface (NIC) used to accept state requests. " +
  78. "The following special values are also recognized: GLOBAL, SITE_LOCAL, LINK_LOCAL and NON_LOOPBACK",
  79. systemProperty={Global.BIND_ADDR, Global.BIND_ADDR_OLD},
  80. defaultValueIPv4=Global.NON_LOOPBACK_ADDRESS, defaultValueIPv6=Global.NON_LOOPBACK_ADDRESS)
  81. private InetAddress bind_addr;
  82. @Property(name="bind_interface", converter=PropertyConverters.BindInterface.class,
  83. description="The interface (NIC) which should be used by this transport", dependsUpon="bind_addr")
  84. protected String bind_interface_str=null;
  85. @Property(description = "The port listening for state requests. Default value of 0 binds to any (ephemeral) port")
  86. private int bind_port = 0;
  87. @Property(description = "Maximum number of pool threads serving state requests. Default is 5")
  88. private int max_pool = 5;
  89. @Property(description = "Keep alive for pool threads serving state requests. Default is 20000 msec")
  90. private long pool_thread_keep_alive = 20 * 1000;
  91. @Property(description = "Buffer size for state transfer. Default is 8192 bytes")
  92. private int socket_buffer_size = 8 * 1024;
  93. @Property(description = "If default transport is used the total state buffer size before state producer is blocked. Default is 81920 bytes")
  94. private int buffer_queue_size = 81920;
  95. @Property(description = "If true default transport is used for state transfer rather than seperate TCP sockets. Default is false")
  96. boolean use_default_transport = false;
  97. /*
  98. * --------------------------------------------- JMX statistics -------------------------------
  99. */
  100. private final AtomicInteger num_state_reqs = new AtomicInteger(0);
  101. private final AtomicLong num_bytes_sent = new AtomicLong(0);
  102. private volatile double avg_state_size = 0;
  103. /*
  104. * --------------------------------------------- Fields ---------------------------------------
  105. */
  106. private Address local_addr = null;
  107. @GuardedBy("members")
  108. private final Vector<Address> members;
  109. /*
  110. * BlockingQueue to accept state transfer Message(s) if default transport
  111. * is used. Only state recipient uses this queue
  112. */
  113. private BlockingQueue<Message> stateQueue;
  114. /*
  115. * Runnable that listens for state requests and spawns threads to serve
  116. * those requests if socket transport is used
  117. */
  118. private StateProviderThreadSpawner spawner;
  119. /*
  120. * Set to true if FLUSH protocol is detected in protocol stack
  121. */
  122. private AtomicBoolean flushProtocolInStack = new AtomicBoolean(false);
  123. public STREAMING_STATE_TRANSFER() {
  124. members = new Vector<Address>();
  125. }
  126. @ManagedAttribute
  127. public int getNumberOfStateRequests() {
  128. return num_state_reqs.get();
  129. }
  130. @ManagedAttribute
  131. public long getNumberOfStateBytesSent() {
  132. return num_bytes_sent.get();
  133. }
  134. @ManagedAttribute
  135. public double getAverageStateSize() {
  136. return avg_state_size;
  137. }
  138. public Vector<Integer> requiredDownServices() {
  139. Vector<Integer> retval = new Vector<Integer>();
  140. retval.addElement(new Integer(Event.GET_DIGEST));
  141. retval.addElement(new Integer(Event.OVERWRITE_DIGEST));
  142. return retval;
  143. }
  144. public void resetStats() {
  145. super.resetStats();
  146. num_state_reqs.set(0);
  147. num_bytes_sent.set(0);
  148. avg_state_size = 0;
  149. }
  150. public void init() throws Exception {
  151. }
  152. public void start() throws Exception {
  153. Map<String, Object> map = new HashMap<String, Object>();
  154. map.put("state_transfer", Boolean.TRUE);
  155. map.put("protocol_class", getClass().getName());
  156. up_prot.up(new Event(Event.CONFIG, map));
  157. if (use_default_transport) {
  158. int size = buffer_queue_size / socket_buffer_size;
  159. // idiot proof it
  160. if (size <= 1) {
  161. size = 10;
  162. }
  163. if (log.isDebugEnabled()) {
  164. log.debug("buffer_queue_size=" + buffer_queue_size + ", socket_buffer_size="
  165. + socket_buffer_size + ", creating queue of size " + size);
  166. }
  167. stateQueue = new ArrayBlockingQueue<Message>(size);
  168. }
  169. }
  170. public void stop() {
  171. super.stop();
  172. if (spawner != null) {
  173. spawner.stop();
  174. }
  175. }
  176. public Object up(Event evt) {
  177. switch (evt.getType()) {
  178. case Event.MSG :
  179. Message msg = (Message) evt.getArg();
  180. StateHeader hdr = (StateHeader) msg.getHeader(this.id);
  181. if (hdr != null) {
  182. switch (hdr.type) {
  183. case StateHeader.STATE_REQ :
  184. handleStateReq(hdr);
  185. break;
  186. case StateHeader.STATE_RSP :
  187. // fix for https://jira.jboss.org/jira/browse/JGRP-1013
  188. if(isDigestNeeded())
  189. down_prot.down(new Event(Event.CLOSE_BARRIER));
  190. try {
  191. handleStateRsp(hdr);
  192. }
  193. finally {
  194. if(isDigestNeeded())
  195. down_prot.down(new Event(Event.OPEN_BARRIER));
  196. }
  197. break;
  198. case StateHeader.STATE_PART :
  199. try {
  200. stateQueue.put(msg);
  201. } catch (InterruptedException e) {
  202. Thread.currentThread().interrupt();
  203. }
  204. break;
  205. default :
  206. if (log.isErrorEnabled())
  207. log.error("type " + hdr.type + " not known in StateHeader");
  208. break;
  209. }
  210. return null;
  211. }
  212. break;
  213. case Event.TMP_VIEW :
  214. case Event.VIEW_CHANGE :
  215. handleViewChange((View) evt.getArg());
  216. break;
  217. case Event.CONFIG :
  218. Map<String, Object> config = (Map<String, Object>) evt.getArg();
  219. if (bind_addr == null && (config != null && config.containsKey("bind_addr"))) {
  220. bind_addr = (InetAddress) config.get("bind_addr");
  221. if (log.isDebugEnabled())
  222. log.debug("using bind_addr from CONFIG event " + bind_addr);
  223. }
  224. if (config != null && config.containsKey("state_transfer")) {
  225. log.error("Protocol stack must have only one state transfer protocol");
  226. }
  227. break;
  228. }
  229. return up_prot.up(evt);
  230. }
  231. public Object down(Event evt) {
  232. switch (evt.getType()) {
  233. case Event.TMP_VIEW :
  234. case Event.VIEW_CHANGE :
  235. handleViewChange((View) evt.getArg());
  236. break;
  237. case Event.GET_STATE :
  238. StateTransferInfo info = (StateTransferInfo) evt.getArg();
  239. Address target;
  240. if (info.target == null) {
  241. target = determineCoordinator();
  242. } else {
  243. target = info.target;
  244. if (target.equals(local_addr)) {
  245. if (log.isErrorEnabled())
  246. log.error("GET_STATE: cannot fetch state from myself !");
  247. target = null;
  248. }
  249. }
  250. if (target == null) {
  251. if (log.isDebugEnabled())
  252. log.debug("GET_STATE: first member (no state)");
  253. up_prot.up(new Event(Event.GET_STATE_OK, new StateTransferInfo()));
  254. } else {
  255. Message state_req = new Message(target, null, null);
  256. state_req.putHeader(this.id, new StateHeader(StateHeader.STATE_REQ,
  257. local_addr, info.state_id));
  258. if (log.isDebugEnabled())
  259. log.debug("GET_STATE: asking " + target
  260. + " for state, passing down a SUSPEND_STABLE event, timeout="
  261. + info.timeout);
  262. down_prot.down(new Event(Event.SUSPEND_STABLE, new Long(info.timeout)));
  263. down_prot.down(new Event(Event.MSG, state_req));
  264. }
  265. return null; // don't pass down any further !
  266. case Event.STATE_TRANSFER_INPUTSTREAM_CLOSED :
  267. if (log.isDebugEnabled())
  268. log.debug("STATE_TRANSFER_INPUTSTREAM_CLOSED received");
  269. down_prot.down(new Event(Event.RESUME_STABLE));
  270. return null;
  271. case Event.CONFIG :
  272. Map<String, Object> config = (Map<String, Object>) evt.getArg();
  273. if (config != null && config.containsKey("flush_supported")) {
  274. flushProtocolInStack.set(true);
  275. }
  276. break;
  277. case Event.SET_LOCAL_ADDRESS:
  278. local_addr=(Address)evt.getArg();
  279. break;
  280. }
  281. return down_prot.down(evt); // pass on to the layer below us
  282. }
  283. /*
  284. * --------------------------- Private Methods ------------------------------------------------
  285. */
  286. /**
  287. * When FLUSH is used we do not need to pass digests between members
  288. *
  289. * see JGroups/doc/design/PArtialStateTransfer.txt see
  290. * JGroups/doc/design/FLUSH.txt
  291. *
  292. * @return true if use of digests is required, false otherwise
  293. */
  294. private boolean isDigestNeeded() {
  295. return !flushProtocolInStack.get();
  296. }
  297. private void respondToStateRequester(String id, Address stateRequester, boolean open_barrier) {
  298. // setup socket plumbing if needed
  299. if (spawner == null && !use_default_transport) {
  300. spawner = new StateProviderThreadSpawner(setupThreadPool(),
  301. Util.createServerSocket(getSocketFactory(),
  302. Global.STREAMING_STATE_TRANSFER_SERVER_SOCK,
  303. bind_addr, bind_port));
  304. Thread t = getThreadFactory().newThread(spawner, "STREAMING_STATE_TRANSFER server socket acceptor");
  305. t.start();
  306. }
  307. Digest digest = isDigestNeeded() ? (Digest) down_prot.down(Event.GET_DIGEST_EVT) : null;
  308. Message state_rsp = new Message(stateRequester);
  309. StateHeader hdr = new StateHeader(StateHeader.STATE_RSP, local_addr, use_default_transport
  310. ? null
  311. : spawner.getServerSocketAddress(), digest, id);
  312. state_rsp.putHeader(this.id, hdr);
  313. if (log.isDebugEnabled())
  314. log.debug("Responding to state requester " + state_rsp.getDest() + " with address "
  315. + (use_default_transport ? null : spawner.getServerSocketAddress())
  316. + " and digest " + digest);
  317. down_prot.down(new Event(Event.MSG, state_rsp));
  318. if (stats) {
  319. num_state_reqs.incrementAndGet();
  320. }
  321. if (open_barrier)
  322. down_prot.down(new Event(Event.OPEN_BARRIER));
  323. if (use_default_transport) {
  324. openAndProvideOutputStreamToStateProvider(stateRequester, id);
  325. }
  326. }
  327. private ThreadPoolExecutor setupThreadPool() {
  328. ThreadPoolExecutor threadPool = new ThreadPoolExecutor(0, max_pool, pool_thread_keep_alive,
  329. TimeUnit.MILLISECONDS, new SynchronousQueue<Runnable>());
  330. ThreadFactory factory = new ThreadFactory() {
  331. private final AtomicInteger threadNumber = new AtomicInteger(1);
  332. public Thread newThread(final Runnable command) {
  333. return getThreadFactory().newThread(command, "STREAMING_STATE_TRANSFER-sender-"
  334. + threadNumber.getAndIncrement());
  335. }
  336. };
  337. threadPool.setRejectedExecutionHandler(new ShutdownRejectedExecutionHandler(threadPool
  338. .getRejectedExecutionHandler()));
  339. threadPool.setThreadFactory(factory);
  340. return threadPool;
  341. }
  342. private Address determineCoordinator() {
  343. synchronized (members) {
  344. for (Address member : members) {
  345. if (!local_addr.equals(member)) {
  346. return member;
  347. }
  348. }
  349. }
  350. return null;
  351. }
  352. private void handleViewChange(View v) {
  353. Vector<Address> new_members = v.getMembers();
  354. synchronized (members) {
  355. members.clear();
  356. members.addAll(new_members);
  357. }
  358. }
  359. private void handleStateReq(StateHeader hdr) {
  360. Address sender = hdr.sender;
  361. String id = hdr.state_id;
  362. if (sender == null) {
  363. if (log.isErrorEnabled())
  364. log.error("sender is null !");
  365. return;
  366. }
  367. if (isDigestNeeded()) {
  368. down_prot.down(new Event(Event.CLOSE_BARRIER));
  369. // drain (and block) incoming msgs until after state has been
  370. // returned
  371. }
  372. try {
  373. respondToStateRequester(id, sender, isDigestNeeded());
  374. } catch (Throwable t) {
  375. if (log.isErrorEnabled())
  376. log.error("failed fetching state from application", t);
  377. if (isDigestNeeded())
  378. down_prot.down(new Event(Event.OPEN_BARRIER));
  379. }
  380. }
  381. void handleStateRsp(final StateHeader hdr) {
  382. Digest tmp_digest = hdr.my_digest;
  383. if (isDigestNeeded()) {
  384. if (tmp_digest == null) {
  385. if (log.isWarnEnabled())
  386. log.warn("digest received from " + hdr.sender
  387. + " is null, skipping setting digest !");
  388. } else {
  389. down_prot.down(new Event(Event.OVERWRITE_DIGEST, tmp_digest));
  390. }
  391. }
  392. if (use_default_transport) {
  393. // have to use another thread to read state while state recipient
  394. // has to accept state messages from state provider
  395. Thread t = getThreadFactory().newThread(new Runnable() {
  396. public void run() {
  397. openAndProvideInputStreamToStateReceiver(hdr.sender, hdr.getStateId());
  398. }
  399. }, "STREAMING_STATE_TRANSFER state reader");
  400. t.start();
  401. } else {
  402. connectToStateProvider(hdr);
  403. }
  404. }
  405. private void openAndProvideInputStreamToStateReceiver(Address stateProvider, String state_id) {
  406. BufferedInputStream bis = null;
  407. try {
  408. bis = new BufferedInputStream(new StateInputStream(), socket_buffer_size);
  409. up_prot.up(new Event(Event.STATE_TRANSFER_INPUTSTREAM, new StateTransferInfo(stateProvider, bis, state_id)));
  410. } catch (IOException e) {
  411. // pass null stream up so that JChannel.getState() returns false
  412. log.error("Could not provide state recipient with appropriate stream", e);
  413. InputStream is = null;
  414. up_prot.up(new Event(Event.STATE_TRANSFER_INPUTSTREAM, new StateTransferInfo(stateProvider, is, state_id)));
  415. } finally {
  416. Util.close(bis);
  417. }
  418. }
  419. private void openAndProvideOutputStreamToStateProvider(Address stateRequester, String state_id) {
  420. BufferedOutputStream bos = null;
  421. try {
  422. bos = new BufferedOutputStream(new StateOutputStream(stateRequester, state_id),socket_buffer_size);
  423. up_prot.up(new Event(Event.STATE_TRANSFER_OUTPUTSTREAM, new StateTransferInfo(stateRequester, bos, state_id)));
  424. } catch (IOException e) {
  425. if (log.isWarnEnabled()) {
  426. log.warn("StateOutputStream could not be given to application", e);
  427. }
  428. } finally {
  429. Util.close(bos);
  430. }
  431. }
  432. private void connectToStateProvider(StateHeader hdr) {
  433. IpAddress address = hdr.bind_addr;
  434. String tmp_state_id = hdr.getStateId();
  435. InputStream bis = null;
  436. StateTransferInfo sti = null;
  437. Socket socket = new Socket();
  438. try {
  439. socket.bind(new InetSocketAddress(bind_addr, 0));
  440. int bufferSize = socket.getReceiveBufferSize();
  441. socket.setReceiveBufferSize(socket_buffer_size);
  442. if (log.isDebugEnabled())
  443. log.debug("Connecting to state provider " + address.getIpAddress() + ":"
  444. + address.getPort() + ", original buffer size was " + bufferSize
  445. + " and was reset to " + socket.getReceiveBufferSize());
  446. Util.connect(socket, new InetSocketAddress(address.getIpAddress(), address.getPort()), 0);
  447. if (log.isDebugEnabled())
  448. log.debug("Connected to state provider, my end of the socket is "
  449. + socket.getLocalAddress() + ":" + socket.getLocalPort()
  450. + " passing inputstream up...");
  451. // write out our state_id and address
  452. ObjectOutputStream out = new ObjectOutputStream(socket.getOutputStream());
  453. out.writeObject(tmp_state_id);
  454. out.writeObject(local_addr);
  455. bis = new BufferedInputStream(new StreamingInputStreamWrapper(socket),socket_buffer_size);
  456. sti = new StateTransferInfo(hdr.sender, bis, tmp_state_id);
  457. up_prot.up(new Event(Event.STATE_TRANSFER_INPUTSTREAM, sti));
  458. } catch (IOException e) {
  459. if (log.isWarnEnabled()) {
  460. log.warn("State reader socket thread spawned abnormaly", e);
  461. }
  462. // pass null stream up so that JChannel.getState() returns false
  463. InputStream is = null;
  464. sti = new StateTransferInfo(hdr.sender, is, tmp_state_id);
  465. up_prot.up(new Event(Event.STATE_TRANSFER_INPUTSTREAM, sti));
  466. } finally {
  467. if (!socket.isConnected()) {
  468. if (log.isWarnEnabled())
  469. log.warn("Could not connect to state provider. Closing socket...");
  470. }
  471. Util.close(bis);
  472. Util.close(socket);
  473. }
  474. }
  475. /*
  476. * ------------------------ End of Private Methods --------------------------------------------
  477. */
  478. private class StateProviderThreadSpawner implements Runnable {
  479. private final ExecutorService pool;
  480. private final ServerSocket serverSocket;
  481. private final IpAddress address;
  482. Thread runner;
  483. private volatile boolean running = true;
  484. public StateProviderThreadSpawner(ExecutorService pool, ServerSocket stateServingSocket) {
  485. super();
  486. this.pool = pool;
  487. this.serverSocket = stateServingSocket;
  488. this.address = new IpAddress(STREAMING_STATE_TRANSFER.this.bind_addr, serverSocket.getLocalPort());
  489. }
  490. public void run() {
  491. runner = Thread.currentThread();
  492. for (; running;) {
  493. try {
  494. if (log.isDebugEnabled())
  495. log.debug("StateProviderThreadSpawner listening at "
  496. + getServerSocketAddress() + "...");
  497. final Socket socket = serverSocket.accept();
  498. pool.execute(new Runnable() {
  499. public void run() {
  500. if (log.isDebugEnabled())
  501. log.debug("Accepted request for state transfer from "
  502. + socket.getInetAddress() + ":" + socket.getPort()
  503. + " handing of to PooledExecutor thread");
  504. new StateProviderHandler().process(socket);
  505. }
  506. });
  507. } catch (IOException e) {
  508. if (log.isWarnEnabled()) {
  509. // we get this exception when we close server socket
  510. // exclude that case
  511. if (serverSocket != null && !serverSocket.isClosed()) {
  512. log.warn("Spawning socket from server socket finished abnormaly", e);
  513. }
  514. }
  515. }
  516. }
  517. }
  518. public IpAddress getServerSocketAddress() {
  519. return address;
  520. }
  521. public void stop() {
  522. running = false;
  523. try {
  524. getSocketFactory().close(serverSocket);
  525. } catch (Exception ignored) {
  526. } finally {
  527. if (log.isDebugEnabled())
  528. log.debug("Waiting for StateProviderThreadSpawner to die ... ");
  529. if (runner != null) {
  530. try {
  531. runner.join(Global.THREAD_SHUTDOWN_WAIT_TIME);
  532. } catch (InterruptedException ignored) {
  533. Thread.currentThread().interrupt();
  534. }
  535. }
  536. if (log.isDebugEnabled())
  537. log.debug("Shutting the thread pool down... ");
  538. pool.shutdownNow();
  539. try {
  540. pool.awaitTermination(Global.THREADPOOL_SHUTDOWN_WAIT_TIME,
  541. TimeUnit.MILLISECONDS);
  542. } catch (InterruptedException ignored) {
  543. Thread.currentThread().interrupt();
  544. }
  545. }
  546. if (log.isDebugEnabled())
  547. log.debug("Thread pool is shutdown. All pool threads are cleaned up.");
  548. }
  549. }
  550. private class StateProviderHandler {
  551. public void process(Socket socket) {
  552. OutputStream bos = null;
  553. ObjectInputStream ois = null;
  554. try {
  555. int bufferSize = socket.getSendBufferSize();
  556. socket.setSendBufferSize(socket_buffer_size);
  557. if (log.isDebugEnabled())
  558. log.debug("Running on " + Thread.currentThread()
  559. + ". Accepted request for state transfer from "
  560. + socket.getInetAddress() + ":" + socket.getPort()
  561. + ", original buffer size was " + bufferSize + " and was reset to "
  562. + socket.getSendBufferSize() + ", passing outputstream up... ");
  563. ois = new ObjectInputStream(socket.getInputStream());
  564. String state_id = (String) ois.readObject();
  565. Address stateRequester = (Address) ois.readObject();
  566. bos = new BufferedOutputStream(new StreamingOutputStreamWrapper(socket),socket_buffer_size);
  567. up_prot.up(new Event(Event.STATE_TRANSFER_OUTPUTSTREAM, new StateTransferInfo(stateRequester, bos, state_id)));
  568. } catch (IOException e) {
  569. if (log.isWarnEnabled()) {
  570. log.warn("State writer socket thread spawned abnormaly", e);
  571. }
  572. } catch (ClassNotFoundException e) {
  573. // thrown by ois.readObject()
  574. // should never happen since String/Address are core classes
  575. } finally {
  576. if (!socket.isConnected()) {
  577. if (log.isWarnEnabled())
  578. log.warn("Could not receive connection from state receiver. Closing socket...");
  579. }
  580. Util.close(bos);
  581. Util.close(socket);
  582. }
  583. }
  584. }
  585. private class StreamingInputStreamWrapper extends FilterInputStream {
  586. private final Socket inputStreamOwner;
  587. private final AtomicBoolean closed = new AtomicBoolean(false);
  588. public StreamingInputStreamWrapper(Socket inputStreamOwner) throws IOException {
  589. super(inputStreamOwner.getInputStream());
  590. this.inputStreamOwner = inputStreamOwner;
  591. }
  592. public void close() throws IOException {
  593. if (closed.compareAndSet(false, true)) {
  594. if (log.isDebugEnabled()) {
  595. log.debug("State reader is closing the socket ");
  596. }
  597. Util.close(inputStreamOwner);
  598. up(new Event(Event.STATE_TRANSFER_INPUTSTREAM_CLOSED));
  599. down(new Event(Event.STATE_TRANSFER_INPUTSTREAM_CLOSED));
  600. }
  601. super.close();
  602. }
  603. }
  604. private class StreamingOutputStreamWrapper extends FilterOutputStream {
  605. private final Socket outputStreamOwner;
  606. private final AtomicBoolean closed = new AtomicBoolean(false);
  607. private long bytesWrittenCounter = 0;
  608. public StreamingOutputStreamWrapper(Socket outputStreamOwner) throws IOException {
  609. super(outputStreamOwner.getOutputStream());
  610. this.outputStreamOwner = outputStreamOwner;
  611. }
  612. public void close() throws IOException {
  613. if (closed.compareAndSet(false, true)) {
  614. if (log.isDebugEnabled()) {
  615. log.debug("State writer is closing the socket ");
  616. }
  617. Util.close(outputStreamOwner);
  618. up(new Event(Event.STATE_TRANSFER_OUTPUTSTREAM_CLOSED));
  619. down(new Event(Event.STATE_TRANSFER_OUTPUTSTREAM_CLOSED));
  620. if (stats) {
  621. avg_state_size = num_bytes_sent.addAndGet(bytesWrittenCounter)
  622. / num_state_reqs.doubleValue();
  623. }
  624. super.close();
  625. }
  626. }
  627. public void write(byte[] b, int off, int len) throws IOException {
  628. //Thanks Kornelius Elstner
  629. //https://jira.jboss.org/browse/JGRP-1223
  630. out.write(b, off, len);
  631. bytesWrittenCounter += len;
  632. }
  633. public void write(byte[] b) throws IOException {
  634. super.write(b);
  635. bytesWrittenCounter += b.length;
  636. }
  637. public void write(int b) throws IOException {
  638. super.write(b);
  639. bytesWrittenCounter += 1;
  640. }
  641. }
  642. private class StateInputStream extends InputStream {
  643. private final AtomicBoolean closed;
  644. public StateInputStream() throws IOException {
  645. super();
  646. this.closed = new AtomicBoolean(false);
  647. }
  648. public void close() throws IOException {
  649. if (closed.compareAndSet(false, true)) {
  650. if (log.isDebugEnabled()) {
  651. log.debug("State reader is closing the stream");
  652. }
  653. stateQueue.clear();
  654. up(new Event(Event.STATE_TRANSFER_INPUTSTREAM_CLOSED));
  655. down(new Event(Event.STATE_TRANSFER_INPUTSTREAM_CLOSED));
  656. super.close();
  657. }
  658. }
  659. public int read() throws IOException {
  660. if (closed.get())
  661. return -1;
  662. final byte[] array = new byte[1];
  663. return read(array) == -1 ? -1 : array[0];
  664. }
  665. public int read(byte[] b, int off, int len) throws IOException {
  666. if (closed.get())
  667. return -1;
  668. Message m = null;
  669. try {
  670. m = stateQueue.take();
  671. StateHeader hdr = (StateHeader) m.getHeader(id);
  672. if (hdr.type == StateHeader.STATE_PART) {
  673. return readAndTransferPayload(m, b, off, len);
  674. }
  675. } catch (InterruptedException e) {
  676. Thread.currentThread().interrupt();
  677. throw new InterruptedIOException();
  678. }
  679. return -1;
  680. }
  681. private int readAndTransferPayload(Message m, byte[] b, int off, int len) {
  682. byte[] buffer = m.getBuffer();
  683. if (log.isDebugEnabled()) {
  684. log.debug(local_addr + " reading chunk of state " + "byte[] b=" + b.length
  685. + ", off=" + off + ", buffer.length=" + buffer.length);
  686. }
  687. System.arraycopy(buffer, 0, b, off, buffer.length);
  688. return buffer.length;
  689. }
  690. public int read(byte[] b) throws IOException {
  691. if (closed.get())
  692. return -1;
  693. return read(b, 0, b.length);
  694. }
  695. }
  696. private class StateOutputStream extends OutputStream {
  697. private final Address stateRequester;
  698. private final String state_id;
  699. private final AtomicBoolean closed;
  700. private long bytesWrittenCounter = 0;
  701. public StateOutputStream(Address stateRequester, String state_id) throws IOException {
  702. super();
  703. this.stateRequester = stateRequester;
  704. this.state_id = state_id;
  705. this.closed = new AtomicBoolean(false);
  706. }
  707. public void close() throws IOException {
  708. if (closed.compareAndSet(false, true)) {
  709. if (log.isDebugEnabled()) {
  710. log.debug("State writer " + local_addr
  711. + " is closing the output stream for state_id " + state_id);
  712. }
  713. up(new Event(Event.STATE_TRANSFER_OUTPUTSTREAM_CLOSED));
  714. down(new Event(Event.STATE_TRANSFER_OUTPUTSTREAM_CLOSED));
  715. if (stats) {
  716. avg_state_size = num_bytes_sent.addAndGet(bytesWrittenCounter)
  717. / num_state_reqs.doubleValue();
  718. }
  719. super.close();
  720. }
  721. }
  722. public void write(byte[] b, int off, int len) throws IOException {
  723. if (closed.get())
  724. throw closed();
  725. sendMessage(b, off, len);
  726. }
  727. public void write(byte[] b) throws IOException {
  728. if (closed.get())
  729. throw closed();
  730. sendMessage(b, 0, b.length);
  731. }
  732. public void write(int b) throws IOException {
  733. if (closed.get())
  734. throw closed();
  735. byte buf[] = new byte[]{(byte) b};
  736. write(buf);
  737. }
  738. private void sendMessage(byte[] b, int off, int len) throws IOException {
  739. Message m = new Message(stateRequester);
  740. m.putHeader(id, new StateHeader(StateHeader.STATE_PART, local_addr, state_id));
  741. m.setBuffer(b, off, len);
  742. bytesWrittenCounter += (len - off);
  743. if (Thread.interrupted()) {
  744. throw interrupted((int) bytesWrittenCounter);
  745. }
  746. down_prot.down(new Event(Event.MSG, m));
  747. if (log.isDebugEnabled()) {
  748. log.debug(local_addr + " sent chunk of state to " + stateRequester + "byte[] b="
  749. + b.length + ", off=" + off + ", len=" + len);
  750. }
  751. }
  752. private IOException closed() {
  753. return new IOException("The output stream is closed");
  754. }
  755. private InterruptedIOException interrupted(int cnt) {
  756. final InterruptedIOException ex = new InterruptedIOException();
  757. ex.bytesTransferred = cnt;
  758. return ex;
  759. }
  760. }
  761. public static class StateHeader extends Header {
  762. public static final byte STATE_REQ = 1;
  763. public static final byte STATE_RSP = 2;
  764. public static final byte STATE_PART = 3;
  765. long id = 0; // state transfer ID (to separate multiple state transfers
  766. // at the same time)
  767. byte type = 0;
  768. Address sender; // sender of state STATE_REQ or STATE_RSP
  769. Digest my_digest = null; // digest of sender (if type is STATE_RSP)
  770. IpAddress bind_addr = null;
  771. String state_id = null; // for partial state transfer
  772. public StateHeader() {
  773. } // for externalization
  774. public StateHeader(byte type, Address sender, String state_id) {
  775. this.type = type;
  776. this.sender = sender;
  777. this.state_id = state_id;
  778. }
  779. public StateHeader(byte type, Address sender, long id, Digest digest) {
  780. this.type = type;
  781. this.sender = sender;
  782. this.id = id;
  783. this.my_digest = digest;
  784. }
  785. public StateHeader(byte type, Address sender, IpAddress bind_addr, Digest digest,
  786. String state_id) {
  787. this.type = type;
  788. this.sender = sender;
  789. this.my_digest = digest;
  790. this.bind_addr = bind_addr;
  791. this.state_id = state_id;
  792. }
  793. public int getType() {
  794. return type;
  795. }
  796. public Digest getDigest() {
  797. return my_digest;
  798. }
  799. public String getStateId() {
  800. return state_id;
  801. }
  802. public boolean equals(Object o) {
  803. StateHeader other;
  804. if (sender != null && o != null) {
  805. if (!(o instanceof StateHeader))
  806. return false;
  807. other = (StateHeader) o;
  808. return sender.equals(other.sender) && id == other.id;
  809. }
  810. return false;
  811. }
  812. public int hashCode() {
  813. if (sender != null)
  814. return sender.hashCode() + (int) id;
  815. else
  816. return (int) id;
  817. }
  818. public String toString() {
  819. StringBuilder sb = new StringBuilder();
  820. sb.append("type=").append(type2Str(type));
  821. if (sender != null)
  822. sb.append(", sender=").append(sender).append(" id=").append(id);
  823. if (my_digest != null)
  824. sb.append(", digest=").append(my_digest);
  825. return sb.toString();
  826. }
  827. static String type2Str(int t) {
  828. switch (t) {
  829. case STATE_REQ :
  830. return "STATE_REQ";
  831. case STATE_RSP :
  832. return "STATE_RSP";
  833. case STATE_PART :
  834. return "STATE_PART";
  835. default :
  836. return "<unknown>";
  837. }
  838. }
  839. public void writeTo(DataOutputStream out) throws IOException {
  840. out.writeByte(type);
  841. out.writeLong(id);
  842. Util.writeAddress(sender, out);
  843. Util.writeStreamable(my_digest, out);
  844. Util.writeStreamable(bind_addr, out);
  845. Util.writeString(state_id, out);
  846. }
  847. public void readFrom(DataInputStream in) throws IOException, IllegalAccessException,
  848. InstantiationException {
  849. type = in.readByte();
  850. id = in.readLong();
  851. sender = Util.readAddress(in);
  852. my_digest = (Digest) Util.readStreamable(Digest.class, in);
  853. bind_addr = (IpAddress) Util.readStreamable(IpAddress.class, in);
  854. state_id = Util.readString(in);
  855. }
  856. public int size() {
  857. int retval = Global.LONG_SIZE + Global.BYTE_SIZE; // id and type
  858. retval += Util.size(sender);
  859. retval += Global.BYTE_SIZE; // presence byte for my_digest
  860. if (my_digest != null)
  861. retval += my_digest.serializedSize();
  862. retval += Util.size(bind_addr);
  863. retval += Global.BYTE_SIZE; // presence byte for state_id
  864. if (state_id != null)
  865. retval += state_id.length() + 2;
  866. return retval;
  867. }
  868. }
  869. }