PageRenderTime 32ms CodeModel.GetById 6ms app.highlight 22ms RepoModel.GetById 1ms app.codeStats 0ms

/hazelcast/src/test/java/com/hazelcast/cluster/MemberListTest.java

https://bitbucket.org/gabral6_gmailcom/hazelcast
Java | 348 lines | 243 code | 64 blank | 41 comment | 3 complexity | b047893f3b370a66c8b54db6839f7371 MD5 | raw file
  1/*
  2 * Copyright (c) 2008-2013, Hazelcast, Inc. All Rights Reserved.
  3 *
  4 * Licensed under the Apache License, Version 2.0 (the "License");
  5 * you may not use this file except in compliance with the License.
  6 * You may obtain a copy of the License at
  7 *
  8 * http://www.apache.org/licenses/LICENSE-2.0
  9 *
 10 * Unless required by applicable law or agreed to in writing, software
 11 * distributed under the License is distributed on an "AS IS" BASIS,
 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 * See the License for the specific language governing permissions and
 14 * limitations under the License.
 15 */
 16
 17package com.hazelcast.cluster;
 18
 19import com.hazelcast.config.Config;
 20import com.hazelcast.config.NetworkConfig;
 21import com.hazelcast.core.Hazelcast;
 22import com.hazelcast.core.HazelcastInstance;
 23import com.hazelcast.core.Member;
 24import com.hazelcast.core.MultiTask;
 25import com.hazelcast.impl.*;
 26import org.junit.After;
 27import org.junit.BeforeClass;
 28import org.junit.Test;
 29import org.junit.runner.RunWith;
 30
 31import java.io.Serializable;
 32import java.util.*;
 33import java.util.concurrent.Callable;
 34import java.util.concurrent.atomic.AtomicBoolean;
 35
 36import static junit.framework.Assert.assertEquals;
 37
 38@RunWith(com.hazelcast.util.RandomBlockJUnit4ClassRunner.class)
 39public class MemberListTest {
 40
 41    @BeforeClass
 42    public static void init() throws Exception {
 43        Hazelcast.shutdownAll();
 44    }
 45
 46    @After
 47    public void cleanup() throws Exception {
 48        Hazelcast.shutdownAll();
 49    }
 50
 51    /*
 52     * Sets up a situation where node3 removes the master and sets node2 as the
 53     * master but none of the other nodes do. This means that node3 thinks node2
 54     * is master but node2 thinks node1 is master.
 55     */
 56    @Test
 57    public void testOutOfSyncMemberListIssue274() throws Exception {
 58        Config c1 = buildConfig(false);
 59        Config c2 = buildConfig(false);
 60        Config c3 = buildConfig(false);
 61        
 62        c1.getNetworkConfig().setPort(35701);
 63        c2.getNetworkConfig().setPort(35702);
 64        c3.getNetworkConfig().setPort(35703);
 65        
 66        List<String> allMembers = Arrays.asList("127.0.0.1:35701, 127.0.0.1:35702, 127.0.0.1:35703");
 67        c1.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
 68        c2.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
 69        c3.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
 70        
 71        final HazelcastInstance h1 = Hazelcast.newHazelcastInstance(c1);
 72        final HazelcastInstance h2 = Hazelcast.newHazelcastInstance(c2);
 73        final HazelcastInstance h3 = Hazelcast.newHazelcastInstance(c3);
 74        
 75        // All three nodes join into one cluster
 76        assertEquals(3, h1.getCluster().getMembers().size());
 77        assertEquals(3, h2.getCluster().getMembers().size());
 78        assertEquals(3, h3.getCluster().getMembers().size());
 79       
 80        // This simulates each node reading from the other nodes in the list at regular intervals
 81        // This prevents the heart beat code from timing out
 82        final HazelcastInstance[] instances = new HazelcastInstance[] {h1, h2, h3};
 83        final AtomicBoolean doingWork = new AtomicBoolean(true);
 84        Thread[] workThreads = new Thread[instances.length];
 85        for (int i = 0; i < instances.length; i++) {
 86            final int threadNum = i;
 87            workThreads[threadNum] = new Thread(new Runnable() {
 88
 89                public void run() {
 90                    while (doingWork.get()) {
 91                        final HazelcastInstance hz = instances[threadNum];
 92                        
 93                        Set<Member> members = new HashSet<Member>(hz.getCluster().getMembers());
 94                        members.remove(hz.getCluster().getLocalMember());
 95                        
 96                        MultiTask<String> task = new MultiTask<String>(new PingCallable(), members);
 97                        hz.getExecutorService().execute(task);
 98
 99                        try {
100                            task.get();
101                        } catch (Exception e) {
102                            e.printStackTrace();
103                        }
104
105                        try {
106                            Thread.sleep(2000);
107                        } catch (InterruptedException e) {
108                            e.printStackTrace();
109                        }
110                    }
111                }
112            });
113            workThreads[threadNum].start();
114        }
115        
116        
117        final Node n3 = TestUtil.getNode(h3);
118        n3.clusterManager.enqueueAndWait(new Processable() {
119            public void process() {
120                
121                // Simulates node3's heartbeat code choosing to remove node1 
122                n3.clusterManager.doRemoveAddress(((MemberImpl) h1.getCluster().getLocalMember()).getAddress());
123                assertEquals(2, n3.clusterManager.getMembers().size());
124            }
125        }, 5);
126        
127        // Give the cluster some time to figure things out. The merge and heartbeat code should have kicked in by this point
128        Thread.sleep(30 * 1000);
129        
130        doingWork.set(false);
131        for (Thread t : workThreads) {
132            t.join();
133        }
134        
135        assertEquals(3, h1.getCluster().getMembers().size());
136        assertEquals(3, h2.getCluster().getMembers().size());
137        assertEquals(3, h3.getCluster().getMembers().size());
138    }
139    
140    private static class PingCallable implements Callable<String>, Serializable {
141        public String call() throws Exception {
142            return "ping response";
143        }
144    }
145    
146    /*
147     * Sets up a situation where the member list is out of order on node2. Both
148     * node2 and node1 think they are masters and both think each other are in
149     * their clusters.
150     */
151    @Test
152    public void testOutOfSyncMemberListTwoMastersIssue274() throws Exception {
153        Config c1 = buildConfig(false);
154        Config c2 = buildConfig(false);
155        Config c3 = buildConfig(false);
156        
157        c1.getNetworkConfig().setPort(45701);
158        c2.getNetworkConfig().setPort(45702);
159        c3.getNetworkConfig().setPort(45703);
160        
161        List<String> allMembers = Arrays.asList("127.0.0.1:45701, 127.0.0.1:45702, 127.0.0.1:45703");
162        c1.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
163        c2.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
164        c3.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
165        
166        final HazelcastInstance h1 = Hazelcast.newHazelcastInstance(c1);
167        final HazelcastInstance h2 = Hazelcast.newHazelcastInstance(c2);
168        final HazelcastInstance h3 = Hazelcast.newHazelcastInstance(c3);
169        
170        final MemberImpl m1 = (MemberImpl) h1.getCluster().getLocalMember();
171        final MemberImpl m2 = (MemberImpl) h2.getCluster().getLocalMember();
172        final MemberImpl m3 = (MemberImpl) h3.getCluster().getLocalMember();
173
174        // All three nodes join into one cluster
175        assertEquals(3, h1.getCluster().getMembers().size());
176        assertEquals(3, h2.getCluster().getMembers().size());
177        assertEquals(3, h3.getCluster().getMembers().size());
178        
179        final Node n2 = TestUtil.getNode(h2);
180        
181        n2.clusterManager.enqueueAndWait(new Processable() {
182            public void process() {
183                
184                // Simulates node2 getting an out of order member list. That causes node2 to think it's the master.
185                List<MemberInfo> members = new ArrayList<MemberInfo>();
186                members.add(new MemberInfo(m2.getAddress(), m2.getNodeType(), m2.getUuid()));
187                members.add(new MemberInfo(m3.getAddress(), m3.getNodeType(), m3.getUuid()));
188                members.add(new MemberInfo(m1.getAddress(), m1.getNodeType(), m1.getUuid()));
189                n2.clusterManager.updateMembers(members);
190                n2.setMasterAddress(m2.getAddress());
191            }
192        }, 5);
193        
194        // Give the cluster some time to figure things out. The merge and heartbeat code should have kicked in by this point
195        Thread.sleep(30 * 1000);
196        
197        assertEquals(m1, h1.getCluster().getMembers().iterator().next());
198        assertEquals(m1, h2.getCluster().getMembers().iterator().next());
199        assertEquals(m1, h3.getCluster().getMembers().iterator().next());
200        
201        assertEquals(3, h1.getCluster().getMembers().size());
202        assertEquals(3, h2.getCluster().getMembers().size());
203        assertEquals(3, h3.getCluster().getMembers().size());
204    }
205    
206    /*
207     * Sets up situation where all nodes have the same master, but node 2's list
208     * doesn't contain node 3.
209     */
210    @Test
211    public void testSameMasterDifferentMemberListIssue274() throws Exception {
212        Config c1 = buildConfig(false);
213        Config c2 = buildConfig(false);
214        Config c3 = buildConfig(false);
215        
216        c1.getNetworkConfig().setPort(55701);
217        c2.getNetworkConfig().setPort(55702);
218        c3.getNetworkConfig().setPort(55703);
219        
220        List<String> allMembers = Arrays.asList("127.0.0.1:55701, 127.0.0.1:55702, 127.0.0.1:55703");
221        c1.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
222        c2.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
223        c3.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
224        
225        final HazelcastInstance h1 = Hazelcast.newHazelcastInstance(c1);
226        final HazelcastInstance h2 = Hazelcast.newHazelcastInstance(c2);
227        final HazelcastInstance h3 = Hazelcast.newHazelcastInstance(c3);
228        
229        final MemberImpl m1 = (MemberImpl) h1.getCluster().getLocalMember();
230        final MemberImpl m2 = (MemberImpl) h2.getCluster().getLocalMember();
231
232        // All three nodes join into one cluster
233        assertEquals(3, h1.getCluster().getMembers().size());
234        assertEquals(3, h2.getCluster().getMembers().size());
235        assertEquals(3, h3.getCluster().getMembers().size());
236        
237        final Node n2 = TestUtil.getNode(h2);
238        
239        n2.clusterManager.enqueueAndWait(new Processable() {
240            public void process() {
241                
242                // Simulates node2 getting an out of order member list. That causes node2 to think it's the master.
243                List<MemberInfo> members = new ArrayList<MemberInfo>();
244                members.add(new MemberInfo(m1.getAddress(), m1.getNodeType(), m1.getUuid()));
245                members.add(new MemberInfo(m2.getAddress(), m2.getNodeType(), m2.getUuid()));
246                n2.clusterManager.updateMembers(members);
247            }
248        }, 5);
249        
250        // Give the cluster some time to figure things out. The merge and heartbeat code should have kicked in by this point
251        Thread.sleep(30 * 1000);
252        
253        assertEquals(m1, h1.getCluster().getMembers().iterator().next());
254        assertEquals(m1, h2.getCluster().getMembers().iterator().next());
255        assertEquals(m1, h3.getCluster().getMembers().iterator().next());
256        
257        assertEquals(3, h1.getCluster().getMembers().size());
258        assertEquals(3, h2.getCluster().getMembers().size());
259        assertEquals(3, h3.getCluster().getMembers().size());
260    }
261    
262    @Test
263    public void testSwitchingMastersIssue274() throws Exception {
264        Config c1 = buildConfig(false);
265        Config c2 = buildConfig(false);
266        Config c3 = buildConfig(false);
267        Config c4 = buildConfig(false);
268        Config c5 = buildConfig(false);
269        
270        c1.setProperty(GroupProperties.PROP_MASTER_CONFIRMATION_INTERVAL_SECONDS, "15");
271        c2.setProperty(GroupProperties.PROP_MASTER_CONFIRMATION_INTERVAL_SECONDS, "15");
272        c3.setProperty(GroupProperties.PROP_MASTER_CONFIRMATION_INTERVAL_SECONDS, "15");
273        c4.setProperty(GroupProperties.PROP_MASTER_CONFIRMATION_INTERVAL_SECONDS, "15");
274        c5.setProperty(GroupProperties.PROP_MASTER_CONFIRMATION_INTERVAL_SECONDS, "15");
275        
276        c1.setProperty(GroupProperties.PROP_MAX_NO_MASTER_CONFIRMATION_SECONDS, "45");
277        c2.setProperty(GroupProperties.PROP_MAX_NO_MASTER_CONFIRMATION_SECONDS, "45");
278        c3.setProperty(GroupProperties.PROP_MAX_NO_MASTER_CONFIRMATION_SECONDS, "45");
279        c4.setProperty(GroupProperties.PROP_MAX_NO_MASTER_CONFIRMATION_SECONDS, "45");
280        c5.setProperty(GroupProperties.PROP_MAX_NO_MASTER_CONFIRMATION_SECONDS, "45");
281        
282        c1.setProperty(GroupProperties.PROP_MEMBER_LIST_PUBLISH_INTERVAL_SECONDS, "120");
283        c2.setProperty(GroupProperties.PROP_MEMBER_LIST_PUBLISH_INTERVAL_SECONDS, "120");
284        c3.setProperty(GroupProperties.PROP_MEMBER_LIST_PUBLISH_INTERVAL_SECONDS, "120");
285        c4.setProperty(GroupProperties.PROP_MEMBER_LIST_PUBLISH_INTERVAL_SECONDS, "120");
286        c5.setProperty(GroupProperties.PROP_MEMBER_LIST_PUBLISH_INTERVAL_SECONDS, "120");
287        
288        c1.getNetworkConfig().setPort(55701);
289        c2.getNetworkConfig().setPort(55702);
290        c3.getNetworkConfig().setPort(55703);
291        c4.getNetworkConfig().setPort(55704);
292        c5.getNetworkConfig().setPort(55705);
293        
294        List<String> allMembers = Arrays.asList("127.0.0.1:55701", "127.0.0.1:55702", "127.0.0.1:55703", "127.0.0.1:55704", "127.0.0.1:55705");
295        c1.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
296        c2.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
297        c3.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
298        c4.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
299        c5.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers);
300        
301        final HazelcastInstance h1 = Hazelcast.newHazelcastInstance(c1);
302        final HazelcastInstance h2 = Hazelcast.newHazelcastInstance(c2);
303        final HazelcastInstance h3 = Hazelcast.newHazelcastInstance(c3);
304        final HazelcastInstance h4 = Hazelcast.newHazelcastInstance(c4);
305        final HazelcastInstance h5 = Hazelcast.newHazelcastInstance(c5);
306
307        assertEquals(5, h1.getCluster().getMembers().size());
308        assertEquals(5, h2.getCluster().getMembers().size());
309        assertEquals(5, h3.getCluster().getMembers().size());
310        assertEquals(5, h4.getCluster().getMembers().size());
311        assertEquals(5, h5.getCluster().getMembers().size());
312        
313        // Need to wait for at least as long as PROP_MAX_NO_MASTER_CONFIRMATION_SECONDS
314        Thread.sleep(60 * 1000);
315        
316        h1.getLifecycleService().shutdown();
317        
318        Thread.sleep(10 * 1000);
319        
320        assertEquals(4, h2.getCluster().getMembers().size());
321        assertEquals(4, h3.getCluster().getMembers().size());
322        assertEquals(4, h4.getCluster().getMembers().size());
323        assertEquals(4, h5.getCluster().getMembers().size());
324        
325        Thread.sleep(20 * 1000);
326        
327        assertEquals(4, h2.getCluster().getMembers().size());
328        assertEquals(4, h3.getCluster().getMembers().size());
329        assertEquals(4, h4.getCluster().getMembers().size());
330        assertEquals(4, h5.getCluster().getMembers().size());
331    }
332    
333    private static Config buildConfig(boolean multicastEnabled) {
334        Config c = new Config();
335        c.getGroupConfig().setName("group").setPassword("pass");
336        c.setProperty(GroupProperties.PROP_MERGE_FIRST_RUN_DELAY_SECONDS, "10");
337        c.setProperty(GroupProperties.PROP_MERGE_NEXT_RUN_DELAY_SECONDS, "5");
338        c.setProperty(GroupProperties.PROP_MAX_NO_HEARTBEAT_SECONDS, "10");
339        c.setProperty(GroupProperties.PROP_MASTER_CONFIRMATION_INTERVAL_SECONDS, "2");
340        c.setProperty(GroupProperties.PROP_MAX_NO_MASTER_CONFIRMATION_SECONDS, "10");
341        c.setProperty(GroupProperties.PROP_MEMBER_LIST_PUBLISH_INTERVAL_SECONDS, "10");
342        final NetworkConfig networkConfig = c.getNetworkConfig();
343        networkConfig.getJoin().getMulticastConfig().setEnabled(multicastEnabled);
344        networkConfig.getJoin().getTcpIpConfig().setEnabled(!multicastEnabled);
345        networkConfig.setPortAutoIncrement(false);
346        return c;
347    }
348}