/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestFailoverProxy.java

https://github.com/Beckham007/hadoop-common · Java · 267 lines · 197 code · 40 blank · 30 comment · 3 complexity · b979c213120f794ea942e121c444e97f MD5 · raw file

  1. /*
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. package org.apache.hadoop.io.retry;
  19. import static org.junit.Assert.*;
  20. import java.io.IOException;
  21. import java.util.concurrent.CountDownLatch;
  22. import org.apache.hadoop.io.retry.UnreliableImplementation.TypeOfExceptionToFailWith;
  23. import org.apache.hadoop.io.retry.UnreliableInterface.UnreliableException;
  24. import org.apache.hadoop.ipc.StandbyException;
  25. import org.junit.Test;
  26. public class TestFailoverProxy {
  27. public static class FlipFlopProxyProvider implements FailoverProxyProvider {
  28. private Class<?> iface;
  29. private Object currentlyActive;
  30. private Object impl1;
  31. private Object impl2;
  32. private boolean latchEnabled = false;
  33. private CountDownLatch getProxyLatch;
  34. private int failoversOccurred = 0;
  35. public FlipFlopProxyProvider(Class<?> iface, Object activeImpl,
  36. Object standbyImpl, int getProxyCountDown) {
  37. this.iface = iface;
  38. this.impl1 = activeImpl;
  39. this.impl2 = standbyImpl;
  40. currentlyActive = impl1;
  41. getProxyLatch = new CountDownLatch(getProxyCountDown);
  42. }
  43. public FlipFlopProxyProvider(Class<?> iface, Object activeImpl,
  44. Object standbyImpl) {
  45. this(iface, activeImpl, standbyImpl, 0);
  46. }
  47. @Override
  48. public Object getProxy() {
  49. if (latchEnabled) {
  50. getProxyLatch.countDown();
  51. try {
  52. getProxyLatch.await();
  53. } catch (InterruptedException e) {
  54. throw new RuntimeException(e);
  55. }
  56. }
  57. return currentlyActive;
  58. }
  59. @Override
  60. public synchronized void performFailover(Object currentProxy) {
  61. currentlyActive = impl1 == currentProxy ? impl2 : impl1;
  62. failoversOccurred++;
  63. }
  64. @Override
  65. public Class<?> getInterface() {
  66. return iface;
  67. }
  68. @Override
  69. public void close() throws IOException {
  70. // Nothing to do.
  71. }
  72. public void setLatchEnabled(boolean latchEnabled) {
  73. this.latchEnabled = latchEnabled;
  74. }
  75. public int getFailoversOccurred() {
  76. return failoversOccurred;
  77. }
  78. }
  79. public static class FailOverOnceOnAnyExceptionPolicy implements RetryPolicy {
  80. @Override
  81. public RetryAction shouldRetry(Exception e, int retries, int failovers,
  82. boolean isMethodIdempotent) {
  83. return failovers < 1 ? RetryAction.FAILOVER_AND_RETRY : RetryAction.FAIL;
  84. }
  85. }
  86. @Test
  87. public void testSuccedsOnceThenFailOver() throws UnreliableException,
  88. IOException, StandbyException {
  89. UnreliableInterface unreliable = (UnreliableInterface)RetryProxy
  90. .create(UnreliableInterface.class,
  91. new FlipFlopProxyProvider(UnreliableInterface.class,
  92. new UnreliableImplementation("impl1"),
  93. new UnreliableImplementation("impl2")),
  94. new FailOverOnceOnAnyExceptionPolicy());
  95. assertEquals("impl1", unreliable.succeedsOnceThenFailsReturningString());
  96. assertEquals("impl2", unreliable.succeedsOnceThenFailsReturningString());
  97. try {
  98. unreliable.succeedsOnceThenFailsReturningString();
  99. fail("should not have succeeded more than twice");
  100. } catch (UnreliableException e) {
  101. // expected
  102. }
  103. }
  104. @Test
  105. public void testSucceedsTenTimesThenFailOver() throws UnreliableException,
  106. IOException, StandbyException {
  107. UnreliableInterface unreliable = (UnreliableInterface)RetryProxy
  108. .create(UnreliableInterface.class,
  109. new FlipFlopProxyProvider(UnreliableInterface.class,
  110. new UnreliableImplementation("impl1"),
  111. new UnreliableImplementation("impl2")),
  112. new FailOverOnceOnAnyExceptionPolicy());
  113. for (int i = 0; i < 10; i++) {
  114. assertEquals("impl1", unreliable.succeedsTenTimesThenFailsReturningString());
  115. }
  116. assertEquals("impl2", unreliable.succeedsTenTimesThenFailsReturningString());
  117. }
  118. @Test
  119. public void testNeverFailOver() throws UnreliableException,
  120. IOException, StandbyException {
  121. UnreliableInterface unreliable = (UnreliableInterface)RetryProxy
  122. .create(UnreliableInterface.class,
  123. new FlipFlopProxyProvider(UnreliableInterface.class,
  124. new UnreliableImplementation("impl1"),
  125. new UnreliableImplementation("impl2")),
  126. RetryPolicies.TRY_ONCE_DONT_FAIL);
  127. unreliable.succeedsOnceThenFailsReturningString();
  128. try {
  129. unreliable.succeedsOnceThenFailsReturningString();
  130. fail("should not have succeeded twice");
  131. } catch (UnreliableException e) {
  132. assertEquals("impl1", e.getMessage());
  133. }
  134. }
  135. @Test
  136. public void testFailoverOnStandbyException()
  137. throws UnreliableException, IOException, StandbyException {
  138. UnreliableInterface unreliable = (UnreliableInterface)RetryProxy
  139. .create(UnreliableInterface.class,
  140. new FlipFlopProxyProvider(UnreliableInterface.class,
  141. new UnreliableImplementation("impl1"),
  142. new UnreliableImplementation("impl2")),
  143. RetryPolicies.failoverOnNetworkException(1));
  144. assertEquals("impl1", unreliable.succeedsOnceThenFailsReturningString());
  145. try {
  146. unreliable.succeedsOnceThenFailsReturningString();
  147. fail("should not have succeeded twice");
  148. } catch (UnreliableException e) {
  149. // Make sure there was no failover on normal exception.
  150. assertEquals("impl1", e.getMessage());
  151. }
  152. unreliable = (UnreliableInterface)RetryProxy
  153. .create(UnreliableInterface.class,
  154. new FlipFlopProxyProvider(UnreliableInterface.class,
  155. new UnreliableImplementation("impl1", TypeOfExceptionToFailWith.STANDBY_EXCEPTION),
  156. new UnreliableImplementation("impl2", TypeOfExceptionToFailWith.UNRELIABLE_EXCEPTION)),
  157. RetryPolicies.failoverOnNetworkException(1));
  158. assertEquals("impl1", unreliable.succeedsOnceThenFailsReturningString());
  159. // Make sure we fail over since the first implementation threw a StandbyException
  160. assertEquals("impl2", unreliable.succeedsOnceThenFailsReturningString());
  161. }
  162. @Test
  163. public void testFailoverOnNetworkExceptionIdempotentOperation()
  164. throws UnreliableException, IOException, StandbyException {
  165. UnreliableInterface unreliable = (UnreliableInterface)RetryProxy
  166. .create(UnreliableInterface.class,
  167. new FlipFlopProxyProvider(UnreliableInterface.class,
  168. new UnreliableImplementation("impl1", TypeOfExceptionToFailWith.IO_EXCEPTION),
  169. new UnreliableImplementation("impl2", TypeOfExceptionToFailWith.UNRELIABLE_EXCEPTION)),
  170. RetryPolicies.failoverOnNetworkException(1));
  171. assertEquals("impl1", unreliable.succeedsOnceThenFailsReturningString());
  172. try {
  173. assertEquals("impl2", unreliable.succeedsOnceThenFailsReturningString());
  174. fail("should not have succeeded twice");
  175. } catch (IOException e) {
  176. // Make sure we *don't* fail over since the first implementation threw an
  177. // IOException and this method is not idempotent
  178. assertEquals("impl1", e.getMessage());
  179. }
  180. assertEquals("impl1", unreliable.succeedsOnceThenFailsReturningStringIdempotent());
  181. // Make sure we fail over since the first implementation threw an
  182. // IOException and this method is idempotent.
  183. assertEquals("impl2", unreliable.succeedsOnceThenFailsReturningStringIdempotent());
  184. }
  185. private static class ConcurrentMethodThread extends Thread {
  186. private UnreliableInterface unreliable;
  187. public String result;
  188. public ConcurrentMethodThread(UnreliableInterface unreliable) {
  189. this.unreliable = unreliable;
  190. }
  191. public void run() {
  192. try {
  193. result = unreliable.failsIfIdentifierDoesntMatch("impl2");
  194. } catch (Exception e) {
  195. throw new RuntimeException(e);
  196. }
  197. }
  198. }
  199. /**
  200. * Test that concurrent failed method invocations only result in a single
  201. * failover.
  202. */
  203. @Test
  204. public void testConcurrentMethodFailures() throws InterruptedException {
  205. FlipFlopProxyProvider proxyProvider = new FlipFlopProxyProvider(
  206. UnreliableInterface.class,
  207. new UnreliableImplementation("impl1",
  208. TypeOfExceptionToFailWith.STANDBY_EXCEPTION),
  209. new UnreliableImplementation("impl2",
  210. TypeOfExceptionToFailWith.STANDBY_EXCEPTION),
  211. 2);
  212. final UnreliableInterface unreliable = (UnreliableInterface)RetryProxy
  213. .create(UnreliableInterface.class, proxyProvider,
  214. RetryPolicies.failoverOnNetworkException(10));
  215. ConcurrentMethodThread t1 = new ConcurrentMethodThread(unreliable);
  216. ConcurrentMethodThread t2 = new ConcurrentMethodThread(unreliable);
  217. // Getting a proxy will now wait on a latch.
  218. proxyProvider.setLatchEnabled(true);
  219. t1.start();
  220. t2.start();
  221. t1.join();
  222. t2.join();
  223. assertEquals("impl2", t1.result);
  224. assertEquals("impl2", t2.result);
  225. assertEquals(1, proxyProvider.getFailoversOccurred());
  226. }
  227. }