/hazelcast/src/test/java/com/hazelcast/cluster/MemberListTest.java
Java | 348 lines | 243 code | 64 blank | 41 comment | 3 complexity | b047893f3b370a66c8b54db6839f7371 MD5 | raw file
1/* 2 * Copyright (c) 2008-2013, Hazelcast, Inc. All Rights Reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package com.hazelcast.cluster; 18 19import com.hazelcast.config.Config; 20import com.hazelcast.config.NetworkConfig; 21import com.hazelcast.core.Hazelcast; 22import com.hazelcast.core.HazelcastInstance; 23import com.hazelcast.core.Member; 24import com.hazelcast.core.MultiTask; 25import com.hazelcast.impl.*; 26import org.junit.After; 27import org.junit.BeforeClass; 28import org.junit.Test; 29import org.junit.runner.RunWith; 30 31import java.io.Serializable; 32import java.util.*; 33import java.util.concurrent.Callable; 34import java.util.concurrent.atomic.AtomicBoolean; 35 36import static junit.framework.Assert.assertEquals; 37 38@RunWith(com.hazelcast.util.RandomBlockJUnit4ClassRunner.class) 39public class MemberListTest { 40 41 @BeforeClass 42 public static void init() throws Exception { 43 Hazelcast.shutdownAll(); 44 } 45 46 @After 47 public void cleanup() throws Exception { 48 Hazelcast.shutdownAll(); 49 } 50 51 /* 52 * Sets up a situation where node3 removes the master and sets node2 as the 53 * master but none of the other nodes do. This means that node3 thinks node2 54 * is master but node2 thinks node1 is master. 55 */ 56 @Test 57 public void testOutOfSyncMemberListIssue274() throws Exception { 58 Config c1 = buildConfig(false); 59 Config c2 = buildConfig(false); 60 Config c3 = buildConfig(false); 61 62 c1.getNetworkConfig().setPort(35701); 63 c2.getNetworkConfig().setPort(35702); 64 c3.getNetworkConfig().setPort(35703); 65 66 List<String> allMembers = Arrays.asList("127.0.0.1:35701, 127.0.0.1:35702, 127.0.0.1:35703"); 67 c1.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers); 68 c2.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers); 69 c3.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers); 70 71 final HazelcastInstance h1 = Hazelcast.newHazelcastInstance(c1); 72 final HazelcastInstance h2 = Hazelcast.newHazelcastInstance(c2); 73 final HazelcastInstance h3 = Hazelcast.newHazelcastInstance(c3); 74 75 // All three nodes join into one cluster 76 assertEquals(3, h1.getCluster().getMembers().size()); 77 assertEquals(3, h2.getCluster().getMembers().size()); 78 assertEquals(3, h3.getCluster().getMembers().size()); 79 80 // This simulates each node reading from the other nodes in the list at regular intervals 81 // This prevents the heart beat code from timing out 82 final HazelcastInstance[] instances = new HazelcastInstance[] {h1, h2, h3}; 83 final AtomicBoolean doingWork = new AtomicBoolean(true); 84 Thread[] workThreads = new Thread[instances.length]; 85 for (int i = 0; i < instances.length; i++) { 86 final int threadNum = i; 87 workThreads[threadNum] = new Thread(new Runnable() { 88 89 public void run() { 90 while (doingWork.get()) { 91 final HazelcastInstance hz = instances[threadNum]; 92 93 Set<Member> members = new HashSet<Member>(hz.getCluster().getMembers()); 94 members.remove(hz.getCluster().getLocalMember()); 95 96 MultiTask<String> task = new MultiTask<String>(new PingCallable(), members); 97 hz.getExecutorService().execute(task); 98 99 try { 100 task.get(); 101 } catch (Exception e) { 102 e.printStackTrace(); 103 } 104 105 try { 106 Thread.sleep(2000); 107 } catch (InterruptedException e) { 108 e.printStackTrace(); 109 } 110 } 111 } 112 }); 113 workThreads[threadNum].start(); 114 } 115 116 117 final Node n3 = TestUtil.getNode(h3); 118 n3.clusterManager.enqueueAndWait(new Processable() { 119 public void process() { 120 121 // Simulates node3's heartbeat code choosing to remove node1 122 n3.clusterManager.doRemoveAddress(((MemberImpl) h1.getCluster().getLocalMember()).getAddress()); 123 assertEquals(2, n3.clusterManager.getMembers().size()); 124 } 125 }, 5); 126 127 // Give the cluster some time to figure things out. The merge and heartbeat code should have kicked in by this point 128 Thread.sleep(30 * 1000); 129 130 doingWork.set(false); 131 for (Thread t : workThreads) { 132 t.join(); 133 } 134 135 assertEquals(3, h1.getCluster().getMembers().size()); 136 assertEquals(3, h2.getCluster().getMembers().size()); 137 assertEquals(3, h3.getCluster().getMembers().size()); 138 } 139 140 private static class PingCallable implements Callable<String>, Serializable { 141 public String call() throws Exception { 142 return "ping response"; 143 } 144 } 145 146 /* 147 * Sets up a situation where the member list is out of order on node2. Both 148 * node2 and node1 think they are masters and both think each other are in 149 * their clusters. 150 */ 151 @Test 152 public void testOutOfSyncMemberListTwoMastersIssue274() throws Exception { 153 Config c1 = buildConfig(false); 154 Config c2 = buildConfig(false); 155 Config c3 = buildConfig(false); 156 157 c1.getNetworkConfig().setPort(45701); 158 c2.getNetworkConfig().setPort(45702); 159 c3.getNetworkConfig().setPort(45703); 160 161 List<String> allMembers = Arrays.asList("127.0.0.1:45701, 127.0.0.1:45702, 127.0.0.1:45703"); 162 c1.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers); 163 c2.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers); 164 c3.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers); 165 166 final HazelcastInstance h1 = Hazelcast.newHazelcastInstance(c1); 167 final HazelcastInstance h2 = Hazelcast.newHazelcastInstance(c2); 168 final HazelcastInstance h3 = Hazelcast.newHazelcastInstance(c3); 169 170 final MemberImpl m1 = (MemberImpl) h1.getCluster().getLocalMember(); 171 final MemberImpl m2 = (MemberImpl) h2.getCluster().getLocalMember(); 172 final MemberImpl m3 = (MemberImpl) h3.getCluster().getLocalMember(); 173 174 // All three nodes join into one cluster 175 assertEquals(3, h1.getCluster().getMembers().size()); 176 assertEquals(3, h2.getCluster().getMembers().size()); 177 assertEquals(3, h3.getCluster().getMembers().size()); 178 179 final Node n2 = TestUtil.getNode(h2); 180 181 n2.clusterManager.enqueueAndWait(new Processable() { 182 public void process() { 183 184 // Simulates node2 getting an out of order member list. That causes node2 to think it's the master. 185 List<MemberInfo> members = new ArrayList<MemberInfo>(); 186 members.add(new MemberInfo(m2.getAddress(), m2.getNodeType(), m2.getUuid())); 187 members.add(new MemberInfo(m3.getAddress(), m3.getNodeType(), m3.getUuid())); 188 members.add(new MemberInfo(m1.getAddress(), m1.getNodeType(), m1.getUuid())); 189 n2.clusterManager.updateMembers(members); 190 n2.setMasterAddress(m2.getAddress()); 191 } 192 }, 5); 193 194 // Give the cluster some time to figure things out. The merge and heartbeat code should have kicked in by this point 195 Thread.sleep(30 * 1000); 196 197 assertEquals(m1, h1.getCluster().getMembers().iterator().next()); 198 assertEquals(m1, h2.getCluster().getMembers().iterator().next()); 199 assertEquals(m1, h3.getCluster().getMembers().iterator().next()); 200 201 assertEquals(3, h1.getCluster().getMembers().size()); 202 assertEquals(3, h2.getCluster().getMembers().size()); 203 assertEquals(3, h3.getCluster().getMembers().size()); 204 } 205 206 /* 207 * Sets up situation where all nodes have the same master, but node 2's list 208 * doesn't contain node 3. 209 */ 210 @Test 211 public void testSameMasterDifferentMemberListIssue274() throws Exception { 212 Config c1 = buildConfig(false); 213 Config c2 = buildConfig(false); 214 Config c3 = buildConfig(false); 215 216 c1.getNetworkConfig().setPort(55701); 217 c2.getNetworkConfig().setPort(55702); 218 c3.getNetworkConfig().setPort(55703); 219 220 List<String> allMembers = Arrays.asList("127.0.0.1:55701, 127.0.0.1:55702, 127.0.0.1:55703"); 221 c1.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers); 222 c2.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers); 223 c3.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers); 224 225 final HazelcastInstance h1 = Hazelcast.newHazelcastInstance(c1); 226 final HazelcastInstance h2 = Hazelcast.newHazelcastInstance(c2); 227 final HazelcastInstance h3 = Hazelcast.newHazelcastInstance(c3); 228 229 final MemberImpl m1 = (MemberImpl) h1.getCluster().getLocalMember(); 230 final MemberImpl m2 = (MemberImpl) h2.getCluster().getLocalMember(); 231 232 // All three nodes join into one cluster 233 assertEquals(3, h1.getCluster().getMembers().size()); 234 assertEquals(3, h2.getCluster().getMembers().size()); 235 assertEquals(3, h3.getCluster().getMembers().size()); 236 237 final Node n2 = TestUtil.getNode(h2); 238 239 n2.clusterManager.enqueueAndWait(new Processable() { 240 public void process() { 241 242 // Simulates node2 getting an out of order member list. That causes node2 to think it's the master. 243 List<MemberInfo> members = new ArrayList<MemberInfo>(); 244 members.add(new MemberInfo(m1.getAddress(), m1.getNodeType(), m1.getUuid())); 245 members.add(new MemberInfo(m2.getAddress(), m2.getNodeType(), m2.getUuid())); 246 n2.clusterManager.updateMembers(members); 247 } 248 }, 5); 249 250 // Give the cluster some time to figure things out. The merge and heartbeat code should have kicked in by this point 251 Thread.sleep(30 * 1000); 252 253 assertEquals(m1, h1.getCluster().getMembers().iterator().next()); 254 assertEquals(m1, h2.getCluster().getMembers().iterator().next()); 255 assertEquals(m1, h3.getCluster().getMembers().iterator().next()); 256 257 assertEquals(3, h1.getCluster().getMembers().size()); 258 assertEquals(3, h2.getCluster().getMembers().size()); 259 assertEquals(3, h3.getCluster().getMembers().size()); 260 } 261 262 @Test 263 public void testSwitchingMastersIssue274() throws Exception { 264 Config c1 = buildConfig(false); 265 Config c2 = buildConfig(false); 266 Config c3 = buildConfig(false); 267 Config c4 = buildConfig(false); 268 Config c5 = buildConfig(false); 269 270 c1.setProperty(GroupProperties.PROP_MASTER_CONFIRMATION_INTERVAL_SECONDS, "15"); 271 c2.setProperty(GroupProperties.PROP_MASTER_CONFIRMATION_INTERVAL_SECONDS, "15"); 272 c3.setProperty(GroupProperties.PROP_MASTER_CONFIRMATION_INTERVAL_SECONDS, "15"); 273 c4.setProperty(GroupProperties.PROP_MASTER_CONFIRMATION_INTERVAL_SECONDS, "15"); 274 c5.setProperty(GroupProperties.PROP_MASTER_CONFIRMATION_INTERVAL_SECONDS, "15"); 275 276 c1.setProperty(GroupProperties.PROP_MAX_NO_MASTER_CONFIRMATION_SECONDS, "45"); 277 c2.setProperty(GroupProperties.PROP_MAX_NO_MASTER_CONFIRMATION_SECONDS, "45"); 278 c3.setProperty(GroupProperties.PROP_MAX_NO_MASTER_CONFIRMATION_SECONDS, "45"); 279 c4.setProperty(GroupProperties.PROP_MAX_NO_MASTER_CONFIRMATION_SECONDS, "45"); 280 c5.setProperty(GroupProperties.PROP_MAX_NO_MASTER_CONFIRMATION_SECONDS, "45"); 281 282 c1.setProperty(GroupProperties.PROP_MEMBER_LIST_PUBLISH_INTERVAL_SECONDS, "120"); 283 c2.setProperty(GroupProperties.PROP_MEMBER_LIST_PUBLISH_INTERVAL_SECONDS, "120"); 284 c3.setProperty(GroupProperties.PROP_MEMBER_LIST_PUBLISH_INTERVAL_SECONDS, "120"); 285 c4.setProperty(GroupProperties.PROP_MEMBER_LIST_PUBLISH_INTERVAL_SECONDS, "120"); 286 c5.setProperty(GroupProperties.PROP_MEMBER_LIST_PUBLISH_INTERVAL_SECONDS, "120"); 287 288 c1.getNetworkConfig().setPort(55701); 289 c2.getNetworkConfig().setPort(55702); 290 c3.getNetworkConfig().setPort(55703); 291 c4.getNetworkConfig().setPort(55704); 292 c5.getNetworkConfig().setPort(55705); 293 294 List<String> allMembers = Arrays.asList("127.0.0.1:55701", "127.0.0.1:55702", "127.0.0.1:55703", "127.0.0.1:55704", "127.0.0.1:55705"); 295 c1.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers); 296 c2.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers); 297 c3.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers); 298 c4.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers); 299 c5.getNetworkConfig().getJoin().getTcpIpConfig().setMembers(allMembers); 300 301 final HazelcastInstance h1 = Hazelcast.newHazelcastInstance(c1); 302 final HazelcastInstance h2 = Hazelcast.newHazelcastInstance(c2); 303 final HazelcastInstance h3 = Hazelcast.newHazelcastInstance(c3); 304 final HazelcastInstance h4 = Hazelcast.newHazelcastInstance(c4); 305 final HazelcastInstance h5 = Hazelcast.newHazelcastInstance(c5); 306 307 assertEquals(5, h1.getCluster().getMembers().size()); 308 assertEquals(5, h2.getCluster().getMembers().size()); 309 assertEquals(5, h3.getCluster().getMembers().size()); 310 assertEquals(5, h4.getCluster().getMembers().size()); 311 assertEquals(5, h5.getCluster().getMembers().size()); 312 313 // Need to wait for at least as long as PROP_MAX_NO_MASTER_CONFIRMATION_SECONDS 314 Thread.sleep(60 * 1000); 315 316 h1.getLifecycleService().shutdown(); 317 318 Thread.sleep(10 * 1000); 319 320 assertEquals(4, h2.getCluster().getMembers().size()); 321 assertEquals(4, h3.getCluster().getMembers().size()); 322 assertEquals(4, h4.getCluster().getMembers().size()); 323 assertEquals(4, h5.getCluster().getMembers().size()); 324 325 Thread.sleep(20 * 1000); 326 327 assertEquals(4, h2.getCluster().getMembers().size()); 328 assertEquals(4, h3.getCluster().getMembers().size()); 329 assertEquals(4, h4.getCluster().getMembers().size()); 330 assertEquals(4, h5.getCluster().getMembers().size()); 331 } 332 333 private static Config buildConfig(boolean multicastEnabled) { 334 Config c = new Config(); 335 c.getGroupConfig().setName("group").setPassword("pass"); 336 c.setProperty(GroupProperties.PROP_MERGE_FIRST_RUN_DELAY_SECONDS, "10"); 337 c.setProperty(GroupProperties.PROP_MERGE_NEXT_RUN_DELAY_SECONDS, "5"); 338 c.setProperty(GroupProperties.PROP_MAX_NO_HEARTBEAT_SECONDS, "10"); 339 c.setProperty(GroupProperties.PROP_MASTER_CONFIRMATION_INTERVAL_SECONDS, "2"); 340 c.setProperty(GroupProperties.PROP_MAX_NO_MASTER_CONFIRMATION_SECONDS, "10"); 341 c.setProperty(GroupProperties.PROP_MEMBER_LIST_PUBLISH_INTERVAL_SECONDS, "10"); 342 final NetworkConfig networkConfig = c.getNetworkConfig(); 343 networkConfig.getJoin().getMulticastConfig().setEnabled(multicastEnabled); 344 networkConfig.getJoin().getTcpIpConfig().setEnabled(!multicastEnabled); 345 networkConfig.setPortAutoIncrement(false); 346 return c; 347 } 348}