PageRenderTime 110ms CodeModel.GetById 43ms app.highlight 55ms RepoModel.GetById 1ms app.codeStats 0ms

/jboss-as-7.1.1.Final/clustering/api/src/test/java/org/jboss/as/clustering/lock/ReadWriteClusteredLockManagerUnitTestCase.java

#
Java | 351 lines | 194 code | 78 blank | 79 comment | 6 complexity | 869ae2762eb89da391e37c04f7dd1742 MD5 | raw file
Possible License(s): LGPL-2.1, Apache-2.0
  1/*
  2 * JBoss, Home of Professional Open Source.
  3 * Copyright 2009, Red Hat Middleware LLC, and individual contributors
  4 * as indicated by the @author tags. See the copyright.txt file in the
  5 * distribution for a full listing of individual contributors.
  6 *
  7 * This is free software; you can redistribute it and/or modify it
  8 * under the terms of the GNU Lesser General Public License as
  9 * published by the Free Software Foundation; either version 2.1 of
 10 * the License, or (at your option) any later version.
 11 *
 12 * This software is distributed in the hope that it will be useful,
 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 15 * Lesser General Public License for more details.
 16 *
 17 * You should have received a copy of the GNU Lesser General Public
 18 * License along with this software; if not, write to the Free
 19 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 20 * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 21 */
 22
 23package org.jboss.as.clustering.lock;
 24
 25import static org.mockito.Mockito.*;
 26import static org.mockito.AdditionalMatchers.*;
 27import static org.junit.Assert.*;
 28import static org.jboss.as.clustering.lock.LockParamsMatcher.*;
 29
 30import java.util.ArrayList;
 31import java.util.List;
 32import java.util.Vector;
 33import java.util.concurrent.CountDownLatch;
 34import java.util.concurrent.TimeUnit;
 35
 36import org.jboss.as.clustering.ClusterNode;
 37import org.jboss.as.clustering.GroupMembershipNotifier;
 38import org.jboss.as.clustering.GroupRpcDispatcher;
 39import org.jboss.as.clustering.lock.AbstractClusterLockSupport.RpcTarget;
 40import org.junit.Test;
 41
 42/**
 43 * Unit test of ClusteredLockManagerImpl
 44 *
 45 * @author Brian Stansberry
 46 *
 47 */
 48public class ReadWriteClusteredLockManagerUnitTestCase extends ClusteredLockManagerTestBase<NonGloballyExclusiveClusterLockSupport> {
 49    @Override
 50    protected NonGloballyExclusiveClusterLockSupport createClusteredLockManager(String serviceHAName, GroupRpcDispatcher rpcDispatcher, GroupMembershipNotifier notifier, LocalLockHandler handler) {
 51        return new NonGloballyExclusiveClusterLockSupport(serviceHAName, rpcDispatcher, notifier, handler);
 52    }
 53
 54    @Test
 55    public void testBasicRemoteLock() throws Exception {
 56        TesteeSet<NonGloballyExclusiveClusterLockSupport> testeeSet = getTesteeSet(node1, 1, 2);
 57        NonGloballyExclusiveClusterLockSupport testee = testeeSet.impl;
 58        LocalLockHandler handler = testee.getLocalHandler();
 59        RpcTarget target = testeeSet.target;
 60
 61        ClusterNode caller = testee.getCurrentView().get(0);
 62        assertFalse(node1.equals(caller));
 63
 64//        resetToStrict(handler);
 65        RemoteLockResponse rsp = target.remoteLock("test", caller, 1000);
 66
 67        assertEquals(RemoteLockResponse.Flag.OK, rsp.flag);
 68        assertNull(rsp.holder);
 69
 70        verify(handler).lockFromCluster("test", caller, 1000);
 71
 72        // Do it again; should fail as another thread from caller already
 73        // acquired the lock
 74//        resetToStrict(handler); // fail if we call the local handler
 75//        replay(handler);
 76
 77        rsp = target.remoteLock("test", caller, 1000);
 78
 79        assertEquals(RemoteLockResponse.Flag.REJECT, rsp.flag);
 80        assertEquals(caller, rsp.holder);
 81
 82        verifyZeroInteractions(handler);
 83    }
 84
 85    @Test
 86    public void testContestedRemoteLock() throws Exception {
 87        TesteeSet<NonGloballyExclusiveClusterLockSupport> testeeSet = getTesteeSet(node1, 1, 3);
 88        NonGloballyExclusiveClusterLockSupport testee = testeeSet.impl;
 89        LocalLockHandler handler = testee.getLocalHandler();
 90        RpcTarget target = testeeSet.target;
 91
 92        ClusterNode caller1 = testee.getCurrentView().get(0);
 93        assertFalse(node1.equals(caller1));
 94
 95        ClusterNode caller2 = testee.getCurrentView().get(2);
 96        assertFalse(node1.equals(caller2));
 97
 98        RemoteLockResponse rsp = target.remoteLock("test", caller1, 1000);
 99
100        assertEquals(RemoteLockResponse.Flag.OK, rsp.flag);
101        assertNull(rsp.holder);
102
103        verify(handler).lockFromCluster("test", caller1, 1000);
104
105        // A call from a different caller should be rejected without need
106        // to go to the LocalLockHandler
107//        resetToStrict(handler);
108//        replay(handler);
109
110        rsp = target.remoteLock("test", caller2, 1000);
111
112        assertEquals(RemoteLockResponse.Flag.REJECT, rsp.flag);
113        assertEquals(caller1, rsp.holder);
114
115        verifyNoMoreInteractions(handler);
116    }
117
118    /**
119     * Test for handling concurrent calls to remoteLock when the lock is in UNLOCKED state. Calls should get passed to the local
120     * lock handler, which allows one to succeed and the other to throw a TimeoutException; testee should react correctly.
121     *
122     * FIXME We are using a MockObject for the LocalLockHandler impl, and with that approach we can't really get concurrent
123     * calls to it. Effect is sometimes the thread that acquires the lock has already done so before the other thread even
124     * invokes remoteLock, defeating the purpose of this test and turning it into a variant of testContestedRemoteLock. Need to
125     * redo this test with a true multithreaded local lock handler, updating the latches such that both threads are in
126     * BlockingAnswer.answer at the same time.
127     *
128     * @throws Exception
129     */
130    @Test
131    public void testConcurrentRemoteLock() throws Exception {
132        TesteeSet<NonGloballyExclusiveClusterLockSupport> testeeSet = getTesteeSet(node1, 1, 3);
133        NonGloballyExclusiveClusterLockSupport testee = testeeSet.impl;
134        LocalLockHandler handler = testee.getLocalHandler();
135        final RpcTarget target = testeeSet.target;
136
137        ClusterNode caller1 = testee.getCurrentView().get(0);
138        assertFalse(node1.equals(caller1));
139
140        ClusterNode caller2 = testee.getCurrentView().get(2);
141        assertFalse(node1.equals(caller2));
142
143//        resetToStrict(handler);
144//        makeThreadSafe(handler, true);
145        // When caller 1 invokes, block before giving response
146        CountDownLatch answerStartLatch = new CountDownLatch(1);
147        CountDownLatch answerDoneLatch = new CountDownLatch(1);
148        BlockingAnswer<Boolean> caller1Answer = new BlockingAnswer<Boolean>(Boolean.TRUE, answerStartLatch, null,
149                answerDoneLatch);
150        BlockingAnswer<Boolean> caller2Answer = new BlockingAnswer<Boolean>(new TimeoutException(caller1), answerDoneLatch, 0,
151                null, null);
152        
153        doAnswer(caller1Answer).when(handler).lockFromCluster("test", caller1, 1000);
154        doAnswer(caller2Answer).when(handler).lockFromCluster("test", caller2, 1000);
155
156        // There is a race where t1 may have already marked the lock as LOCKED in
157        // which case t2 will not call handler.lockFromCluster("test", caller2, 1000);
158        // See FIXME in method javadoc. So, we use times(0, 1) to specify no
159        // calls are OK
160/*        
161        expectLastCall().andAnswer(caller2Answer).times(0, 1);
162        replay(handler);
163*/
164        CountDownLatch startLatch1 = new CountDownLatch(1);
165        CountDownLatch startLatch2 = new CountDownLatch(1);
166        CountDownLatch finishedLatch = new CountDownLatch(2);
167
168        RemoteLockCaller winner = new RemoteLockCaller(target, caller1, startLatch1, null, finishedLatch);
169        RemoteLockCaller loser = new RemoteLockCaller(target, caller2, startLatch2, null, finishedLatch);
170
171        Thread t1 = new Thread(winner);
172        t1.setDaemon(true);
173        Thread t2 = new Thread(loser);
174        t2.setDaemon(true);
175
176        try {
177            t1.start();
178            assertTrue(startLatch1.await(1, TimeUnit.SECONDS));
179            // t1 should now be blocking in caller1Answer
180
181            t2.start();
182            assertTrue(startLatch2.await(1, TimeUnit.SECONDS));
183            // t2 should now be blocking due to t1
184
185            // release t1
186            answerStartLatch.countDown();
187
188            // wait for both to complete
189            assertTrue(finishedLatch.await(1, TimeUnit.SECONDS));
190
191//            verifyNoMoreInteractions(handler);
192
193            rethrow("winner had an exception", winner.getException());
194            rethrow("loser had an exception", loser.getException());
195
196            RemoteLockResponse rsp = winner.getResult();
197            assertEquals(RemoteLockResponse.Flag.OK, rsp.flag);
198            assertNull(rsp.holder);
199
200            rsp = loser.getResult();
201            if (rsp.flag != RemoteLockResponse.Flag.REJECT) {
202                assertEquals(RemoteLockResponse.Flag.FAIL, rsp.flag);
203            }
204            assertEquals(caller1, rsp.holder);
205        } finally {
206            if (t1.isAlive()) {
207                t1.interrupt();
208            }
209            if (t2.isAlive()) {
210                t2.interrupt();
211            }
212        }
213    }
214
215    @Test
216    public void testRemoteLockFailsAgainstLocalLock() throws Exception {
217        TesteeSet<NonGloballyExclusiveClusterLockSupport> testeeSet = getTesteeSet(node1, 1, 2);
218        NonGloballyExclusiveClusterLockSupport testee = testeeSet.impl;
219        LocalLockHandler handler = testee.getLocalHandler();
220        RpcTarget target = testeeSet.target;
221
222        ClusterNode caller1 = testee.getCurrentView().get(0);
223        assertFalse(node1.equals(caller1));
224
225        // We throw TimeoutException to indicate "node1" holds the lock
226        // A second attempt should succeed if the local lock is released
227        // We return normally to indicate success
228        doThrow(new TimeoutException(node1)).doNothing().when(handler).lockFromCluster("test", caller1, 1000);
229
230        RemoteLockResponse rsp = target.remoteLock("test", caller1, 1000);
231
232        assertEquals(RemoteLockResponse.Flag.FAIL, rsp.flag);
233        assertEquals(node1, rsp.holder);
234
235        rsp = target.remoteLock("test", caller1, 1000);
236        
237        assertEquals(RemoteLockResponse.Flag.OK, rsp.flag);
238        assertNull(rsp.holder);
239    }
240
241    @Test
242    public void testBasicClusterLockFailsAgainstLocalLock() throws Exception {
243        basicClusterLockFailsAgainstLocalLockTest(2);
244    }
245
246    @Test
247    public void testStandaloneClusterLockFailsAgainstLocalLock() throws Exception {
248        basicClusterLockFailsAgainstLocalLockTest(2);
249    }
250
251    private void basicClusterLockFailsAgainstLocalLockTest(int viewSize) throws Exception {
252        int viewPos = viewSize == 1 ? 0 : 1;
253        TesteeSet<NonGloballyExclusiveClusterLockSupport> testeeSet = getTesteeSet(node1, viewPos, viewSize);
254        NonGloballyExclusiveClusterLockSupport testee = testeeSet.impl;
255        GroupRpcDispatcher rpcDispatcher = testee.getGroupRpcDispatcher();
256        LocalLockHandler handler = testee.getLocalHandler();
257
258        List<RemoteLockResponse> rspList = new ArrayList<RemoteLockResponse>();
259        for (int i = 0; i < viewSize - 1; i++) {
260            rspList.add(new RemoteLockResponse(null, RemoteLockResponse.Flag.OK));
261        }
262
263        when(rpcDispatcher.getMethodCallTimeout()).thenReturn(60000l);
264        when(rpcDispatcher.<RemoteLockResponse>callMethodOnCluster(eq("test"), eq("remoteLock"), eqLockParams(node1, 2000000),
265                        aryEq(AbstractClusterLockSupport.REMOTE_LOCK_TYPES), eq(true), eq(NULL_FILTER), anyInt(), eq(false))).thenReturn(rspList);
266
267        doThrow(new TimeoutException(node1)).when(handler).lockFromCluster(eq("test"), eq(node1), anyLong());
268
269        when((List<Object>) rpcDispatcher.callMethodOnCluster(eq("test"), eq("releaseRemoteLock"), aryEq(new Object[] { "test", node1 }), 
270                        aryEq(AbstractClusterLockSupport.RELEASE_REMOTE_LOCK_TYPES), eq(true))).thenReturn(new ArrayList<Object>());
271        
272        assertFalse(testee.lock("test", 10));
273    }
274
275    /**
276     * Test that if a member holds a lock but is then removed from the view, another remote member can obtain the lock.
277     *
278     * @throws Exception
279     */
280    @Test
281    public void testDeadMemberCleanupAllowsRemoteLock() throws Exception {
282        TesteeSet<NonGloballyExclusiveClusterLockSupport> testeeSet = getTesteeSet(node1, 1, 3);
283        NonGloballyExclusiveClusterLockSupport testee = testeeSet.impl;
284        LocalLockHandler handler = testee.getLocalHandler();
285        RpcTarget target = testeeSet.target;
286
287        List<ClusterNode> members = testee.getCurrentView();
288        ClusterNode caller1 = members.get(0);
289        assertFalse(node1.equals(caller1));
290
291        ClusterNode caller2 = members.get(2);
292        assertFalse(node1.equals(caller2));
293
294        RemoteLockResponse rsp = target.remoteLock("test", caller1, 1000);
295
296        verify(handler).lockFromCluster("test", caller1, 1000);
297
298        assertEquals(RemoteLockResponse.Flag.OK, rsp.flag);
299        assertNull(rsp.holder);
300
301        // Change the view
302        Vector<ClusterNode> dead = new Vector<ClusterNode>();
303        dead.add(caller1);
304
305        Vector<ClusterNode> all = new Vector<ClusterNode>(members);
306        all.remove(caller1);
307
308        when(handler.getLockHolder("test")).thenReturn(caller1);
309
310        testee.membershipChanged(dead, new Vector<ClusterNode>(), all);
311
312        verify(handler).unlockFromCluster("test", caller1);
313        
314        // A call from a different caller should work
315        rsp = target.remoteLock("test", caller2, 1000);
316
317        verify(handler).lockFromCluster("test", caller2, 1000);
318
319        assertEquals(RemoteLockResponse.Flag.OK, rsp.flag);
320        assertNull(rsp.holder);
321    }
322
323    /**
324     * Remote node acquires a lock; different remote node tries to release which is ignored.
325     *
326     * @throws Exception
327     */
328    @Test
329    public void testSpuriousLockReleaseIgnored2() throws Exception {
330        TesteeSet<NonGloballyExclusiveClusterLockSupport> testeeSet = getTesteeSet(node1, 1, 3);
331        NonGloballyExclusiveClusterLockSupport testee = testeeSet.impl;
332        LocalLockHandler handler = testee.getLocalHandler();
333        RpcTarget target = testeeSet.target;
334
335        ClusterNode caller1 = testee.getCurrentView().get(0);
336        ClusterNode caller2 = testee.getCurrentView().get(2);
337
338        when(handler.getLockHolder("test")).thenReturn(caller1);
339
340        RemoteLockResponse rsp = target.remoteLock("test", caller1, 1);
341        
342        verify(handler).lockFromCluster(eq("test"), eq(caller1), anyLong());
343
344        assertEquals(RemoteLockResponse.Flag.OK, rsp.flag);
345
346        target.releaseRemoteLock("test", caller2);
347        
348        verify(handler, never()).unlockFromCluster("test", caller2);
349    }
350
351}