/hazelcast/src/main/java/com/hazelcast/cluster/ClusterManager.java

   1/*
   2 * Copyright (c) 2008-2013, Hazelcast, Inc. All Rights Reserved.
   3 *
   4 * Licensed under the Apache License, Version 2.0 (the "License");
   5 * you may not use this file except in compliance with the License.
   6 * You may obtain a copy of the License at
   7 *
   8 * http://www.apache.org/licenses/LICENSE-2.0
   9 *
  10 * Unless required by applicable law or agreed to in writing, software
  11 * distributed under the License is distributed on an "AS IS" BASIS,
  12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 * See the License for the specific language governing permissions and
  14 * limitations under the License.
  15 */
  16
  17package com.hazelcast.cluster;
  18
  19import com.hazelcast.core.Member;
  20import com.hazelcast.impl.*;
  21import com.hazelcast.impl.base.Call;
  22import com.hazelcast.impl.base.PacketProcessor;
  23import com.hazelcast.impl.base.ScheduledAction;
  24import com.hazelcast.logging.ILogger;
  25import com.hazelcast.nio.*;
  26import com.hazelcast.security.Credentials;
  27import com.hazelcast.util.Clock;
  28import com.hazelcast.util.Prioritized;
  29
  30import javax.security.auth.login.LoginContext;
  31import javax.security.auth.login.LoginException;
  32import java.net.ConnectException;
  33import java.util.*;
  34import java.util.concurrent.TimeUnit;
  35import java.util.logging.Level;
  36
  37import static com.hazelcast.nio.IOUtil.toData;
  38import static com.hazelcast.nio.IOUtil.toObject;
  39
  40public final class ClusterManager extends BaseManager implements ConnectionListener {
  41
  42    private final long WAIT_MILLIS_BEFORE_JOIN;
  43
  44    private final long MAX_WAIT_SECONDS_BEFORE_JOIN;
  45
  46    private final long MAX_NO_HEARTBEAT_MILLIS;
  47
  48    private final long MAX_NO_MASTER_CONFIRMATION_MILLIS;
  49
  50    private final boolean ICMP_ENABLED;
  51
  52    private final int ICMP_TTL;
  53
  54    private final int ICMP_TIMEOUT;
  55
  56    private final Set<ScheduledAction> setScheduledActions = new LinkedHashSet<ScheduledAction>(1000);
  57
  58    private final Set<MemberInfo> setJoins = new LinkedHashSet<MemberInfo>(100);
  59
  60    private boolean joinInProgress = false;
  61
  62    private long timeToStartJoin = 0;
  63
  64    private long firstJoinRequest = 0;
  65
  66    private final Map<MemberImpl, Long> memberMasterConfirmationTimes = new HashMap<MemberImpl, Long>();
  67    
  68    public ClusterManager(final Node node) {
  69        super(node);
  70        WAIT_MILLIS_BEFORE_JOIN = node.groupProperties.WAIT_SECONDS_BEFORE_JOIN.getInteger() * 1000L;
  71        MAX_WAIT_SECONDS_BEFORE_JOIN = node.groupProperties.MAX_WAIT_SECONDS_BEFORE_JOIN.getInteger();
  72        MAX_NO_HEARTBEAT_MILLIS = node.groupProperties.MAX_NO_HEARTBEAT_SECONDS.getInteger() * 1000L;
  73        long heartbeatIntervalMillis = node.groupProperties.HEARTBEAT_INTERVAL_SECONDS.getInteger() * 1000L;
  74        MAX_NO_MASTER_CONFIRMATION_MILLIS = node.groupProperties.MAX_NO_MASTER_CONFIRMATION_SECONDS.getInteger() * 1000L;
  75        long masterConfirmationIntervalMillis = node.groupProperties.MASTER_CONFIRMATION_INTERVAL_SECONDS.getInteger() * 1000L;
  76        long memberListPublishIntervalMillis = node.groupProperties.MEMBER_LIST_PUBLISH_INTERVAL_SECONDS.getInteger() * 1000L;
  77        ICMP_ENABLED = node.groupProperties.ICMP_ENABLED.getBoolean();
  78        ICMP_TTL = node.groupProperties.ICMP_TTL.getInteger();
  79        ICMP_TIMEOUT = node.groupProperties.ICMP_TIMEOUT.getInteger();
  80
  81        registerPeriodicProcessable(new Processable() {
  82            public void process() {
  83                heartBeater();
  84            }
  85        }, heartbeatIntervalMillis, heartbeatIntervalMillis);
  86
  87        registerPeriodicProcessable(new Processable() {
  88            public void process() {
  89                sendMasterConfirmation();
  90            }
  91        }, masterConfirmationIntervalMillis, masterConfirmationIntervalMillis);
  92
  93        registerPeriodicProcessable(new Processable() {
  94            public void process() {
  95                sendMemberListToOthers();
  96            }
  97        }, memberListPublishIntervalMillis, memberListPublishIntervalMillis);
  98
  99        registerPeriodicProcessable(new Processable() {
 100            public void process() {
 101                checkScheduledActions();
 102            }
 103        }, 0, 1000);
 104
 105        final SplitBrainHandler splitBrainHandler = new SplitBrainHandler(node);
 106        registerPeriodicProcessable(splitBrainHandler,
 107                splitBrainHandler.getFirstRunDelayMillis(), splitBrainHandler.getNextRunDelayMillis());
 108
 109        registerPeriodicProcessable(new Processable() {
 110            public void process() {
 111                node.clusterService.checkIdle();
 112            }
 113        }, 0, 1000);
 114
 115        node.connectionManager.addConnectionListener(this);
 116        registerPacketProcessor(ClusterOperation.RESPONSE, new PacketProcessor() {
 117            public void process(Packet packet) {
 118                handleResponse(packet);
 119            }
 120        });
 121        registerPacketProcessor(ClusterOperation.HEARTBEAT, new PacketProcessor() {
 122            public void process(Packet packet) {
 123                releasePacket(packet);
 124            }
 125        });
 126        registerPacketProcessor(ClusterOperation.LOG, new PacketProcessor() {
 127            public void process(Packet packet) {
 128                logger.log(Level.parse(packet.name), toObject(packet.getValueData()).toString());
 129                releasePacket(packet);
 130            }
 131        });
 132        registerPacketProcessor(ClusterOperation.JOIN_CHECK, new PacketProcessor() {
 133            public void process(Packet packet) {
 134                Connection conn = packet.conn;
 135                Request request = Request.copyFromPacket(packet);
 136                Data value = request.value;
 137                request.clearForResponse();
 138                if (node.isMaster() && node.joined() && node.isActive()) {
 139                    JoinInfo joinInfo = (JoinInfo) toObject(value);
 140                    if (joinInfo != null) {
 141                        try {
 142                            node.validateJoinRequest(joinInfo);
 143                            request.response = toData(node.createJoinInfo());
 144                        } catch (Exception e) {
 145                            request.response = toData(e);
 146                        }
 147                    }
 148                }
 149                returnResponse(request, conn);
 150                releasePacket(packet);
 151            }
 152        });
 153        registerPacketProcessor(ClusterOperation.REMOTELY_PROCESS_AND_RESPOND,
 154                new PacketProcessor() {
 155                    public void process(Packet packet) {
 156                        Data data = packet.getValueData();
 157                        RemotelyProcessable rp = (RemotelyProcessable) toObject(data);
 158                        rp.setConnection(packet.conn);
 159                        rp.setNode(node);
 160                        rp.process();
 161                        sendResponse(packet);
 162                    }
 163                });
 164        registerPacketProcessor(ClusterOperation.REMOTELY_PROCESS,
 165                new PacketProcessor() {
 166                    public void process(Packet packet) {
 167                        Data data = packet.getValueData();
 168                        RemotelyProcessable rp = (RemotelyProcessable) toObject(data);
 169                        rp.setConnection(packet.conn);
 170                        rp.setNode(node);
 171                        rp.process();
 172                        releasePacket(packet);
 173                    }
 174                });
 175        registerPacketProcessor(ClusterOperation.REMOTELY_CALLABLE_BOOLEAN,
 176                new PacketProcessor() {
 177                    public void process(Packet packet) {
 178                        Boolean result;
 179                        AbstractRemotelyCallable<Boolean> callable = null;
 180                        try {
 181                            Data data = packet.getValueData();
 182                            callable = (AbstractRemotelyCallable<Boolean>) toObject(data);
 183                            callable.setConnection(packet.conn);
 184                            callable.setNode(node);
 185                            result = callable.call();
 186                        } catch (Exception e) {
 187                            logger.log(Level.SEVERE, "Error processing " + callable, e);
 188                            result = Boolean.FALSE;
 189                        }
 190                        if (result == Boolean.TRUE) {
 191                            sendResponse(packet);
 192                        } else {
 193                            sendResponseFailure(packet);
 194                        }
 195                    }
 196                });
 197        registerPacketProcessor(ClusterOperation.REMOTELY_CALLABLE_OBJECT,
 198                new PacketProcessor() {
 199                    public void process(Packet packet) {
 200                        Object result;
 201                        AbstractRemotelyCallable<Boolean> callable = null;
 202                        try {
 203                            Data data = packet.getValueData();
 204                            callable = (AbstractRemotelyCallable) toObject(data);
 205                            callable.setConnection(packet.conn);
 206                            callable.setNode(node);
 207                            result = callable.call();
 208                        } catch (Exception e) {
 209                            logger.log(Level.SEVERE, "Error processing " + callable, e);
 210                            result = null;
 211                        }
 212                        if (result != null) {
 213                            Data value;
 214                            if (result instanceof Data) {
 215                                value = (Data) result;
 216                            } else {
 217                                value = toData(result);
 218                            }
 219                            packet.setValue(value);
 220                        }
 221                        sendResponse(packet);
 222                    }
 223                });
 224    }
 225
 226    private void registerPeriodicProcessable(final Processable p, long delay, long period) {
 227        delay = delay < 0 ? 0 : delay;
 228        period = period <= 0 ? 1: period;
 229        node.executorManager.getScheduledExecutorService().scheduleAtFixedRate(new Runnable() {
 230            public void run() {
 231                node.clusterService.enqueuePriorityAndReturn(p);
 232            }
 233        }, delay, period, TimeUnit.MILLISECONDS);
 234    }
 235
 236    public boolean shouldTryMerge() {
 237        return !joinInProgress && setJoins.size() == 0;
 238    }
 239
 240    public JoinInfo checkJoin(Connection conn) {
 241        return new JoinCall(conn).checkJoin();
 242    }
 243
 244    public void appendState(StringBuffer sbState) {
 245        sbState.append("\nClusterManager {");
 246        for (ScheduledAction sa : setScheduledActions) {
 247            sbState.append("\n\t" + sa + ", from:" + sa.getRequest().caller);
 248        }
 249        sbState.append("\n}");
 250    }
 251
 252    class JoinCall extends ConnectionAwareOp {
 253        JoinCall(Connection target) {
 254            super(target);
 255        }
 256
 257        JoinInfo checkJoin() {
 258            setLocal(ClusterOperation.JOIN_CHECK, "join", null, node.createJoinInfo(), -1, 0);
 259            doOp();
 260            return (JoinInfo) getResultAsObject();
 261        }
 262    }
 263
 264    void logMissingConnection(Address address) {
 265        String msg = thisMember + " has no connection to " + address;
 266        logAtMaster(Level.WARNING, msg);
 267        logger.log(Level.WARNING, msg);
 268    }
 269
 270    public void logAtMaster(Level level, String msg) {
 271        Address master = getMasterAddress();
 272        if (!isMaster() && master != null) {
 273            Connection connMaster = node.connectionManager.getOrConnect(getMasterAddress());
 274            if (connMaster != null) {
 275                Packet packet = obtainPacket(level.toString(), null, toData(msg), ClusterOperation.LOG, 0);
 276                sendOrReleasePacket(packet, connMaster);
 277            }
 278        } else {
 279            logger.log(level, msg);
 280        }
 281    }
 282
 283    public final void heartBeater() {
 284        if (!node.joined() || !node.isActive()) return;
 285        long now = Clock.currentTimeMillis();
 286        if (isMaster()) {
 287            List<Address> lsDeadAddresses = null;
 288            for (MemberImpl memberImpl : lsMembers) {
 289                final Address address = memberImpl.getAddress();
 290                if (!thisAddress.equals(address)) {
 291                    try {
 292                        Connection conn = node.connectionManager.getOrConnect(address);
 293                        if (conn != null && conn.live()) {
 294                            if ((now - memberImpl.getLastRead()) >= (MAX_NO_HEARTBEAT_MILLIS)) {
 295                                conn = null;
 296                                if (lsDeadAddresses == null) {
 297                                    lsDeadAddresses = new ArrayList<Address>();
 298                                }
 299                                logger.log(Level.WARNING, "Added " + address
 300                                                          + " to list of dead addresses because of timeout since last read");
 301                                lsDeadAddresses.add(address);
 302                            } else if ((now - memberImpl.getLastRead()) >= 5000 && (now - memberImpl.getLastPing()) >= 5000) {
 303                                ping(memberImpl);
 304                            }
 305                            if ((now - memberImpl.getLastWrite()) > 500) {
 306                                sendHeartbeat(conn);
 307                            }
 308                            Long lastConfirmation = memberMasterConfirmationTimes.get(memberImpl);
 309                            if (lastConfirmation == null || (now - lastConfirmation > MAX_NO_MASTER_CONFIRMATION_MILLIS)) {
 310                                if (lsDeadAddresses == null) {
 311                                    lsDeadAddresses = new ArrayList<Address>();
 312                                }
 313                                logger.log(Level.WARNING, "Added " + address
 314                                                          + " to list of dead addresses because it has not sent a master confirmation recently");
 315                                lsDeadAddresses.add(address);
 316                            }
 317                        } else if (conn == null && (now - memberImpl.getLastRead()) > 5000) {
 318                            logMissingConnection(address);
 319                            memberImpl.didRead();
 320                        }
 321                    } catch (Exception e) {
 322                        logger.log(Level.SEVERE, e.getMessage(), e);
 323                    }
 324                }
 325            }
 326            if (lsDeadAddresses != null) {
 327                for (Address address : lsDeadAddresses) {
 328                    logger.log(Level.FINEST, "No heartbeat should remove " + address);
 329                    doRemoveAddress(address);
 330                }
 331            }
 332        } else {
 333            // send heartbeat to master
 334            Address masterAddress = getMasterAddress();
 335            if (masterAddress != null) {
 336                final Connection connMaster = node.connectionManager.getOrConnect(masterAddress);
 337                MemberImpl masterMember = getMember(masterAddress);
 338                boolean removed = false;
 339                if (masterMember != null) {
 340                    if ((now - masterMember.getLastRead()) >= (MAX_NO_HEARTBEAT_MILLIS)) {
 341                        logger.log(Level.WARNING, "Master node has timed out its heartbeat and will be removed");
 342                        doRemoveAddress(masterAddress);
 343                        removed = true;
 344                    } else if ((now - masterMember.getLastRead()) >= 5000 && (now - masterMember.getLastPing()) >= 5000) {
 345                        ping(masterMember);
 346                    }
 347                }
 348                if (!removed) {
 349                    sendHeartbeat(connMaster);
 350                }
 351            }
 352            for (MemberImpl member : lsMembers) {
 353                if (!member.localMember()) {
 354                    Address address = member.getAddress();
 355                    Connection conn = node.connectionManager.getOrConnect(address);
 356                    if (conn != null) {
 357                        sendHeartbeat(conn);
 358                    } else {
 359                        logger.log(Level.FINEST, "Could not connect to " + address + " to send heartbeat");
 360                    }
 361                }
 362            }
 363        }
 364    }
 365    
 366    private void sendMasterConfirmation() {
 367        if (!node.joined() || !node.isActive() || isMaster()) {
 368            return;
 369        }
 370        
 371        final Address masterAddress = getMasterAddress();
 372        if (masterAddress == null) {
 373            logger.log(Level.FINEST, "Could not send MasterConfirmation, master is null!");
 374            return;
 375        }
 376        
 377        final MemberImpl masterMember = getMember(masterAddress);
 378        if (masterMember == null) {
 379            logger.log(Level.FINEST, "Could not send MasterConfirmation, master is null!");
 380            return;
 381        }
 382        if (logger.isLoggable(Level.FINEST)) {
 383            logger.log(Level.FINEST, "Sending MasterConfirmation to " + masterMember);
 384        }
 385        sendProcessableTo(new MasterConfirmation(), masterAddress);
 386    }
 387
 388    public static class MasterConfirmation extends AbstractRemotelyProcessable {
 389
 390        public void process() {
 391            final Address endPoint = conn.getEndPoint();
 392            if (endPoint == null) {
 393                return;
 394            }
 395            final ILogger logger = node.getLogger(MasterConfirmation.class.getName());
 396            final ClusterManager clusterManager = node.clusterManager;
 397            final MemberImpl member = clusterManager.getMember(endPoint);
 398            if (member != null) {
 399                if (getNode().isMaster()) {
 400                    if (logger.isLoggable(Level.FINEST)) {
 401                        logger.log(Level.FINEST, "MasterConfirmation has been received from " + member);
 402                    }
 403                    clusterManager.memberMasterConfirmationTimes.put(member, Clock.currentTimeMillis());
 404                } else {
 405                    logger.log(Level.WARNING, endPoint + " has sent MasterConfirmation, but this node is not master!");
 406                }
 407            } else {
 408                logger.log(Level.WARNING, "MasterConfirmation has been received from " + endPoint
 409                          + ", but it is not a member of this cluster!");
 410                clusterManager.sendProcessableTo(new MemberRemover(clusterManager.thisAddress), conn);
 411            }
 412        }
 413    }
 414
 415    private void ping(final MemberImpl memberImpl) {
 416        memberImpl.didPing();
 417        if (!ICMP_ENABLED) return;
 418        node.executorManager.executeNow(new Runnable() {
 419            public void run() {
 420                try {
 421                    final Address address = memberImpl.getAddress();
 422                    logger.log(Level.WARNING, thisAddress + " will ping " + address);
 423                    for (int i = 0; i < 5; i++) {
 424                        try {
 425                            if (address.getInetAddress().isReachable(null, ICMP_TTL, ICMP_TIMEOUT)) {
 426                                logger.log(Level.INFO, thisAddress + " pings successfully. Target: " + address);
 427                                return;
 428                            }
 429                        } catch (ConnectException ignored) {
 430                            // no route to host
 431                            // means we cannot connect anymore
 432                        }
 433                            }
 434                    logger.log(Level.WARNING, thisAddress + " couldn't ping " + address);
 435                            // not reachable.
 436                            enqueueAndReturn(new Processable() {
 437                                public void process() {
 438                                    doRemoveAddress(address);
 439                                }
 440                            });
 441                } catch (Throwable ignored) {
 442                }
 443            }
 444        });
 445    }
 446
 447    void sendHeartbeat(Connection conn) {
 448        if (conn == null) return;
 449        Packet packet = obtainPacket("heartbeat", null, null, ClusterOperation.HEARTBEAT, 0);
 450        sendOrReleasePacket(packet, conn);
 451    }
 452    
 453    private void sendRemoveMemberToOthers(final Address deadAddress) {
 454        for (MemberImpl member : lsMembers) {
 455            Address address = member.getAddress();
 456            if (!thisAddress.equals(address) && !address.equals(deadAddress)) {
 457                sendProcessableTo(new MemberRemover(deadAddress), address);
 458            }
 459        }
 460    }
 461    
 462    // Will be called just before this node becomes the master
 463    private void resetMemberMasterConfirmations() {
 464        checkServiceThread();
 465        for (MemberImpl member : lsMembers) {
 466            memberMasterConfirmationTimes.put(member, Clock.currentTimeMillis());
 467        }
 468    }
 469    
 470    private void sendMemberListToOthers() {
 471        checkServiceThread();
 472        if (!isMaster()) {
 473            return;
 474        }
 475        sendProcessableToAll(new MembersUpdateCall(lsMembers, node.getClusterImpl().getClusterTime()), false);
 476    }
 477    
 478    public void sendClusterMergeToOthers(final Address newTargetAddress) {
 479    	sendProcessableToAll(new MergeClusters(newTargetAddress), false);
 480    }
 481
 482    public void handleMaster(Master master) {
 483        if (!node.joined() && !thisAddress.equals(master.address)) {
 484            logger.log(Level.FINEST, "Handling master response: " + master);
 485            final Address currentMaster = node.getMasterAddress();
 486            if (currentMaster != null && !currentMaster.equals(master.address)) {
 487                final Connection conn = node.connectionManager.getConnection(currentMaster);
 488                if (conn != null && conn.live()) {
 489                    logger.log(Level.FINEST, "Ignoring master response " + master +
 490                              " since node has an active master: " + currentMaster);
 491                    return;
 492                }
 493            }
 494            node.setMasterAddress(master.address);
 495            node.connectionManager.getOrConnect(master.address);
 496            if (!sendJoinRequest(master.address, true)) {
 497                logger.log(Level.WARNING, "Could not create connection to possible master " + master.address);
 498            }
 499        }
 500    }
 501
 502    public void handleAddRemoveConnection(final AddOrRemoveConnection connection) {
 503        if (connection.add) { // Just connect to the new address if not connected already.
 504            if (!connection.address.equals(thisAddress)) {
 505                node.connectionManager.getOrConnect(connection.address);
 506            }
 507        } else { // Remove dead member
 508            if (connection.address != null) {
 509                logger.log(Level.FINEST, "Disconnected from " + connection.address + "... will be removed!");
 510                doRemoveAddress(connection.address);
 511            }
 512        } // end of REMOVE CONNECTION
 513    }
 514
 515    void doRemoveAddress(Address deadAddress) {
 516        doRemoveAddress(deadAddress, true);
 517    }
 518
 519    void doRemoveAddress(Address deadAddress, boolean destroyConnection) {
 520        checkServiceThread();
 521        logger.log(Level.INFO, "Removing Address " + deadAddress);
 522        if (!node.joined()) {
 523            node.failedConnection(deadAddress);
 524            return;
 525        }
 526        if (deadAddress.equals(thisAddress))
 527            return;
 528        if (deadAddress.equals(getMasterAddress())) {
 529            if (node.joined()) {
 530                MemberImpl newMaster = getNextMemberAfter(deadAddress, false, 1);
 531                if (newMaster != null)
 532                    node.setMasterAddress(newMaster.getAddress());
 533                else
 534                    node.setMasterAddress(null);
 535            } else {
 536                node.setMasterAddress(null);
 537            }
 538            logger.log(Level.FINEST, "Now Master " + node.getMasterAddress());
 539        }
 540        if (isMaster()) {
 541            setJoins.remove(new MemberInfo(deadAddress));
 542            resetMemberMasterConfirmations();
 543        }
 544        final Connection conn = node.connectionManager.getConnection(deadAddress);
 545        if (destroyConnection && conn != null) {
 546            node.connectionManager.destroyConnection(conn);
 547        }
 548        MemberImpl deadMember = getMember(deadAddress);
 549        if (deadMember != null) {
 550            removeMember(deadMember);
 551            node.getClusterImpl().setMembers(lsMembers);
 552            node.concurrentMapManager.syncForDead(deadMember);
 553            node.blockingQueueManager.syncForDead(deadAddress);
 554            node.listenerManager.syncForDead(deadAddress);
 555            node.topicManager.syncForDead(deadAddress);
 556            // node.getClusterImpl().setMembers(lsMembers); // shifted up to get members in syncForDead methods
 557            disconnectExistingCalls(deadAddress);
 558            if (isMaster()) {
 559                logger.log(Level.FINEST, deadAddress + " is dead. Sending remove to all other members.");
 560                sendRemoveMemberToOthers(deadAddress);
 561            }
 562            logger.log(Level.INFO, this.toString());
 563        }
 564    }
 565
 566    public void disconnectExistingCalls(Address deadAddress) {
 567        Object[] calls = mapCalls.values().toArray();
 568        for (Object call : calls) {
 569            ((Call) call).onDisconnect(deadAddress);
 570        }
 571    }
 572
 573    void handleJoinRequest(JoinRequest joinRequest) {
 574        final long now = Clock.currentTimeMillis();
 575        String msg = "Handling join from " + joinRequest.address + ", inProgress: " + joinInProgress
 576                     + (timeToStartJoin > 0 ? ", timeToStart: " + (timeToStartJoin - now) : "");
 577        logger.log(Level.FINEST, msg);
 578        boolean validJoinRequest;
 579        try {
 580            validJoinRequest = node.validateJoinRequest(joinRequest);
 581        } catch (Exception e) {
 582            validJoinRequest = false;
 583        }
 584        final Connection conn = joinRequest.getConnection();
 585        if (validJoinRequest) {
 586            final MemberImpl member = getMember(joinRequest.address);
 587            if (member != null) {
 588                if (joinRequest.getUuid().equals(member.getUuid())) {
 589                    String message = "Ignoring join request, member already exists.. => " + joinRequest;
 590                    logger.log(Level.FINEST, message);
 591
 592                    // send members update back to node trying to join again...
 593                    final long clusterTime = node.getClusterImpl().getClusterTime();
 594                    sendProcessableTo(new MembersUpdateCall(lsMembers, clusterTime), conn);
 595                    sendProcessableTo(new SyncProcess(), conn);
 596                    return;
 597                }
 598                // If this node is master then remove old member and process join request.
 599                // If requesting address is equal to master node's address, that means master node
 600                // somehow disconnected and wants to join back.
 601                // So drop old member and process join request if this node becomes master.
 602                if (isMaster() || member.getAddress().equals(getMasterAddress())) {
 603                    logger.log(Level.WARNING, "New join request has been received from an existing endpoint! => " + member
 604                                              + " Removing old member and processing join request...");
 605                    // If existing connection of endpoint is different from current connection
 606                    // destroy it, otherwise keep it.
 607//                    final Connection existingConnection = node.connectionManager.getConnection(joinRequest.address);
 608//                    final boolean destroyExistingConnection = existingConnection != conn;
 609                    doRemoveAddress(member.getAddress(), false);
 610                }
 611            }
 612            if (!node.getConfig().getNetworkConfig().getJoin().getMulticastConfig().isEnabled()) {
 613                if (node.isActive() && node.joined() && node.getMasterAddress() != null && !isMaster()) {
 614                    sendProcessableTo(new Master(node.getMasterAddress()), conn);
 615                }
 616            }
 617            if (isMaster() && node.joined() && node.isActive()) {
 618                final MemberInfo newMemberInfo = new MemberInfo(joinRequest.address, joinRequest.nodeType, joinRequest.getUuid());
 619                if (node.securityContext != null && !setJoins.contains(newMemberInfo)) {
 620                    final ILogger securityLogger = node.loggingService.getLogger("com.hazelcast.security");
 621                    final Credentials cr = joinRequest.getCredentials();
 622                    if (cr == null) {
 623                        securityLogger.log(Level.SEVERE, "Expecting security credentials " +
 624                                "but credentials could not be found in JoinRequest!");
 625                        sendAuthFail(conn);
 626                        return;
 627                    } else {
 628                        try {
 629                            LoginContext lc = node.securityContext.createMemberLoginContext(cr);
 630                            lc.login();
 631                        } catch (LoginException e) {
 632                            securityLogger.log(Level.SEVERE, "Authentication has failed for " + cr.getPrincipal()
 633                                    + '@' + cr.getEndpoint() + " => (" + e.getMessage() + ")");
 634                            securityLogger.log(Level.FINEST, e.getMessage(), e);
 635                            sendAuthFail(conn);
 636                            return;
 637                        }
 638                    }
 639                }
 640                if (joinRequest.to != null && !joinRequest.to.equals(thisAddress)) {
 641                    sendProcessableTo(new Master(node.getMasterAddress()), conn);
 642                    return;
 643                }
 644                if (!joinInProgress) {
 645                    if (firstJoinRequest != 0 && now - firstJoinRequest >= MAX_WAIT_SECONDS_BEFORE_JOIN * 1000) {
 646                        startJoin();
 647                    } else {
 648                        if (setJoins.add(newMemberInfo)) {
 649                            sendProcessableTo(new Master(node.getMasterAddress()), conn);
 650                            if (firstJoinRequest == 0) {
 651                                firstJoinRequest = now;
 652                            }
 653                            if (now - firstJoinRequest < MAX_WAIT_SECONDS_BEFORE_JOIN * 1000) {
 654                                timeToStartJoin = now + WAIT_MILLIS_BEFORE_JOIN;
 655                            }
 656                        }
 657                        if (now > timeToStartJoin) {
 658                            startJoin();
 659                        }
 660                    }
 661                }
 662            }
 663        } else {
 664            conn.close();
 665        }
 666    }
 667
 668    public static class AuthenticationFailureProcessable extends AbstractRemotelyProcessable implements RemotelyProcessable {
 669        public void process() {
 670            node.executorManager.executeNow(new Runnable() {
 671                public void run() {
 672                    final ILogger logger = node.loggingService.getLogger("com.hazelcast.security");
 673                    logger.log(Level.SEVERE, "Authentication failed on master node! Node is going to shutdown now!");
 674                    node.shutdown(true, true);
 675                }
 676            });
 677        }
 678    }
 679
 680    private void sendAuthFail(Connection conn) {
 681        sendProcessableTo(new AuthenticationFailureProcessable(), conn);
 682    }
 683
 684    @Override
 685    public String toString() {
 686        StringBuilder sb = new StringBuilder("\n\nMembers [");
 687        sb.append(lsMembers.size());
 688        sb.append("] {");
 689        for (MemberImpl member : lsMembers) {
 690            sb.append("\n\t").append(member);
 691        }
 692        sb.append("\n}\n");
 693        return sb.toString();
 694    }
 695
 696    void joinReset() {
 697        joinInProgress = false;
 698        setJoins.clear();
 699        timeToStartJoin = Clock.currentTimeMillis() + WAIT_MILLIS_BEFORE_JOIN;
 700        firstJoinRequest = 0;
 701    }
 702
 703    public void onRestart() {
 704        enqueueAndWait(new Processable() {
 705            public void process() {
 706                joinReset();
 707                lsMembers.clear();
 708                mapMembers.clear();
 709                dataMemberCount.reset();
 710                memberMasterConfirmationTimes.clear();
 711            }
 712        }, 5);
 713    }
 714
 715    public boolean checkAuthorization(String groupName, String groupPassword, Address target) {
 716        AbstractRemotelyCallable<Boolean> authorizationCall = new AuthorizationCall(groupName, groupPassword);
 717        AsyncRemotelyBooleanOp op = new NoneMemberAsyncRemotelyBooleanOp(authorizationCall, target, true);
 718        op.execute();
 719        return op.getResultAsBoolean();
 720    }
 721
 722    public class NoneMemberAsyncRemotelyBooleanOp extends AsyncRemotelyBooleanOp {
 723
 724        public NoneMemberAsyncRemotelyBooleanOp(final AbstractRemotelyCallable<Boolean> arp,
 725                                                final Address target, final boolean canTimeout) {
 726            super(arp, target, canTimeout);
 727        }
 728
 729        @Override
 730        protected boolean memberOnly() {
 731            return false;
 732        }
 733    }
 734
 735    public class AsyncRemotelyBooleanOp extends TargetAwareOp {
 736        private final AbstractRemotelyCallable<Boolean> arp;
 737        private final boolean canTimeout;
 738
 739        public AsyncRemotelyBooleanOp(final AbstractRemotelyCallable<Boolean> arp,
 740                                      final Address target, final boolean canTimeout) {
 741            this.arp = arp;
 742            this.target = target;
 743            this.canTimeout = canTimeout;
 744        }
 745
 746        public void execute() {
 747            arp.setNode(node);
 748            setLocal(ClusterOperation.REMOTELY_CALLABLE_BOOLEAN, "call", null, arp, 0, -1);
 749            request.setBooleanRequest();
 750            doOp();
 751        }
 752
 753        @Override
 754        public Address getTarget() {
 755            return target;
 756        }
 757
 758        @Override
 759        public void onDisconnect(final Address dead) {
 760            if (dead.equals(target)) {
 761                removeRemoteCall(getCallId());
 762                setResult(Boolean.FALSE);
 763            }
 764        }
 765
 766        @Override
 767        public void doLocalOp() {
 768            Boolean result;
 769            try {
 770                result = arp.call();
 771                setResult(result);
 772            } catch (Exception e) {
 773                logger.log(Level.FINEST, e.getMessage(), e);
 774            }
 775        }
 776
 777        @Override
 778        public void setTarget() {
 779        }
 780
 781        @Override
 782        public void redo(int redoTypeCode) {
 783            removeRemoteCall(getCallId());
 784            setResult(Boolean.FALSE);
 785        }
 786
 787        @Override
 788        protected void memberDoesNotExist() {
 789            setResult(Boolean.FALSE);
 790        }
 791
 792        @Override
 793        protected void packetNotSent() {
 794            setResult(Boolean.FALSE);
 795        }
 796
 797        @Override
 798        protected final boolean canTimeout() {
 799            return canTimeout;
 800        }
 801    }
 802
 803    public void finalizeJoin() {
 804        Set<Member> members = node.getClusterImpl().getMembers();
 805        List<AsyncRemotelyBooleanOp> calls = new ArrayList<AsyncRemotelyBooleanOp>();
 806        for (Member m : members) {
 807            MemberImpl member = (MemberImpl) m;
 808            if (!member.localMember()) {
 809                AsyncRemotelyBooleanOp op = new AsyncRemotelyBooleanOp(
 810                        new FinalizeJoin(), member.getAddress(), false);
 811                op.execute();
 812                calls.add(op);
 813            }
 814        }
 815        for (AsyncRemotelyBooleanOp call : calls) {
 816            call.getResultAsBoolean();
 817        }
 818    }
 819
 820    class JoinRunnable extends FallThroughRunnable implements Prioritized {
 821
 822        final MembersUpdateCall membersUpdate;
 823
 824        JoinRunnable(MembersUpdateCall membersUpdate) {
 825            this.membersUpdate = membersUpdate;
 826        }
 827
 828        @Override
 829        public void doRun() {
 830            Collection<MemberInfo> lsMemberInfos = membersUpdate.getMemberInfos();
 831            List<Address> newMemberList = new ArrayList<Address>(lsMemberInfos.size());
 832            for (final MemberInfo memberInfo : lsMemberInfos) {
 833                newMemberList.add(memberInfo.address);
 834            }
 835            doCall(membersUpdate, newMemberList, true);
 836            systemLogService.logJoin("JoinRunnable update members done.");
 837            doCall(new SyncProcess(), newMemberList, false);
 838            systemLogService.logJoin("JoinRunnable sync done.");
 839            doCall(new ConnectionCheckCall(), newMemberList, false);
 840            systemLogService.logJoin("JoinRunnable connection check done.");
 841        }
 842
 843        void doCall(AbstractRemotelyCallable callable, List<Address> targets, boolean ignoreThis) {
 844            List<AsyncRemotelyBooleanOp> calls = new ArrayList<AsyncRemotelyBooleanOp>(targets.size());
 845            for (final Address target : targets) {
 846                boolean skip = ignoreThis && thisAddress.equals(target);
 847                if (!skip) {
 848                    AsyncRemotelyBooleanOp op = new AsyncRemotelyBooleanOp(callable, target, false);
 849                    op.execute();
 850                    calls.add(op);
 851                }
 852            }
 853            for (AsyncRemotelyBooleanOp call : calls) {
 854                if (!call.getResultAsBoolean(5)) {
 855                    targets.remove(call.getTarget());
 856                }
 857            }
 858        }
 859    }
 860
 861    void startJoin() {
 862        logger.log(Level.FINEST, "Starting Join.");
 863        joinInProgress = true;
 864        final MembersUpdateCall membersUpdate = new MembersUpdateCall(lsMembers, node.getClusterImpl().getClusterTime());
 865        if (setJoins != null && setJoins.size() > 0) {
 866            for (MemberInfo memberJoined : setJoins) {
 867                membersUpdate.addMemberInfo(memberJoined);
 868            }
 869        }
 870        membersUpdate.setNode(node);
 871        membersUpdate.call();
 872        node.executorManager.executeNow(new JoinRunnable(membersUpdate));
 873    }
 874
 875    void updateMembers(Collection<MemberInfo> lsMemberInfos) {
 876        checkServiceThread();
 877        final Map<Address, MemberImpl> mapOldMembers = new HashMap<Address, MemberImpl>();
 878        for (MemberImpl member : lsMembers) {
 879            mapOldMembers.put(member.getAddress(), member);
 880        }
 881        if (mapOldMembers.size() == lsMemberInfos.size()) {
 882            boolean same = true;
 883            for (MemberInfo memberInfo : lsMemberInfos) {
 884                MemberImpl member = mapOldMembers.get(memberInfo.getAddress());
 885                if (member == null || !member.getUuid().equals(memberInfo.uuid)) {
 886                    same = false;
 887                    break;
 888                }
 889            }
 890            if (same) {
 891                logger.log(Level.FINEST, "No need to process member update...");
 892                return;
 893            }
 894        }
 895
 896        logger.log(Level.FINEST, "Updating Members");
 897        lsMembers.clear();
 898        dataMemberCount.reset();
 899        mapMembers.clear();
 900        memberMasterConfirmationTimes.clear();
 901        for (MemberInfo memberInfo : lsMemberInfos) {
 902            MemberImpl member = mapOldMembers.get(memberInfo.address);
 903            if (member == null) {
 904                member = addMember(memberInfo.address, memberInfo.nodeType, memberInfo.uuid);
 905            } else {
 906                addMember(member);
 907            }
 908            member.didRead();
 909        }
 910        if (!lsMembers.contains(thisMember)) {
 911            throw new RuntimeException("Member list doesn't contain local member!");
 912        }
 913        heartBeater();
 914        node.getClusterImpl().setMembers(lsMembers);
 915        node.setJoined();
 916        logger.log(Level.INFO, this.toString());
 917    }
 918
 919    public boolean sendJoinRequest(Address toAddress, boolean withCredentials) {
 920        if (toAddress == null) {
 921            toAddress = node.getMasterAddress();
 922        }
 923        logger.log(Level.FINEST, "Sending join request to " + toAddress);
 924        final boolean send = sendProcessableTo(node.createJoinInfo(withCredentials), toAddress);
 925        if (!send) {
 926            logger.log(Level.WARNING, "Could not send join request to " + toAddress);
 927        }
 928        return send;
 929    }
 930
 931    public void registerScheduledAction(ScheduledAction scheduledAction) {
 932        setScheduledActions.add(scheduledAction);
 933    }
 934
 935    public void deregisterScheduledAction(ScheduledAction scheduledAction) {
 936        setScheduledActions.remove(scheduledAction);
 937    }
 938
 939    public void checkScheduledActions() {
 940        if (!node.joined() || !node.isActive()) return;
 941        if (setScheduledActions.size() > 0) {
 942            Iterator<ScheduledAction> it = setScheduledActions.iterator();
 943            while (it.hasNext()) {
 944                ScheduledAction sa = it.next();
 945                if (sa.expired() && sa.isValid()) {
 946                    sa.onExpire();
 947                    it.remove();
 948                } else if (!sa.isValid()) {
 949                    it.remove();
 950                }
 951            }
 952        }
 953    }
 954
 955    public void invalidateScheduledActionsFor(Address endpoint, Set<Integer> threadIds) {
 956        if (!node.joined() || !node.isActive()) return;
 957        if (setScheduledActions.size() > 0) {
 958            Iterator<ScheduledAction> it = setScheduledActions.iterator();
 959            while (it.hasNext()) {
 960                ScheduledAction sa = it.next();
 961                Request request = sa.getRequest();
 962                if (endpoint.equals(request.caller) && threadIds.contains(request.lockThreadId)) {
 963                    sa.setValid(false);
 964                    it.remove();
 965                }
 966            }
 967        }
 968    }
 969
 970    public void connectionAdded(final Connection connection) {
 971        enqueueAndReturn(new Processable() {
 972            public void process() {
 973                MemberImpl member = getMember(connection.getEndPoint());
 974                if (member != null) {
 975                    member.didRead();
 976                }
 977            }
 978        });
 979    }
 980
 981    public void connectionRemoved(Connection connection) {
 982        logger.log(Level.FINEST, "Connection is removed " + connection.getEndPoint());
 983        if (!node.joined()) {
 984            if (getMasterAddress() != null) {
 985                if (getMasterAddress().equals(connection.getEndPoint())) {
 986                    node.setMasterAddress(null);
 987                }
 988            }
 989        }
 990    }
 991
 992    public Member addMember(MemberImpl member) {
 993        return addMember(true, member);
 994    }
 995
 996    public Member addMember(boolean checkServiceThread, MemberImpl member) {
 997        if (checkServiceThread) {
 998            checkServiceThread();
 999        }
1000        logger.log(Level.FINEST, "ClusterManager adding " + member);
1001        if (lsMembers.contains(member)) {
1002            for (MemberImpl m : lsMembers) {
1003                if (m.equals(member)) {
1004                    member = m;
1005                }
1006            }
1007            mapMembers.put(member.getAddress(), member);
1008        } else {
1009            lsMembers.add(member);
1010            mapMembers.put(member.getAddress(), member);
1011            if (!member.isLiteMember()) {
1012                dataMemberCount.increment();
1013            }
1014        }
1015       
1016        memberMasterConfirmationTimes.put(member, Clock.currentTimeMillis());
1017        
1018        return member;
1019    }
1020
1021    public void removeMember(MemberImpl member) {
1022        checkServiceThread();
1023        logger.log(Level.FINEST, "ClusterManager removing  " + member);
1024        mapMembers.remove(member.getAddress());
1025        lsMembers.remove(member);
1026        memberMasterConfirmationTimes.remove(member);
1027        if (!member.isLiteMember()) {
1028            dataMemberCount.decrement();
1029        }
1030    }
1031
1032    protected MemberImpl createMember(Address address, NodeType nodeType, String nodeUuid, String ipV6ScopeId) {
1033        address.setScopeId(ipV6ScopeId);
1034        return new MemberImpl(address, thisAddress.equals(address), nodeType, nodeUuid);
1035    }
1036
1037    public MemberImpl getMember(Address address) {
1038        if (address == null) {
1039            return null;
1040        }
1041        return mapMembers.get(address);
1042    }
1043
1044    final public MemberImpl addMember(Address address, NodeType nodeType, String nodeUuid) {
1045        checkServiceThread();
1046        if (address == null) {
1047            logger.log(Level.FINEST, "Address cannot be null");
1048            return null;
1049        }
1050        MemberImpl member = getMember(address);
1051        if (member == null) {
1052            member = createMember(address, nodeType, nodeUuid, thisAddress.getScopeId());
1053        }
1054        addMember(member);
1055        return member;
1056    }
1057
1058    public void stop() {
1059        if (setJoins != null) {
1060            setJoins.clear();
1061        }
1062        timeToStartJoin = 0;
1063        if (lsMembers != null) {
1064            lsMembers.clear();
1065        }
1066        dataMemberCount.reset();
1067        if (mapMembers != null) {
1068            mapMembers.clear();
1069        }
1070        if (mapCalls != null) {
1071            mapCalls.clear();
1072        }
1073        if (memberMasterConfirmationTimes != null) {
1074            memberMasterConfirmationTimes.clear();
1075        }
1076    }
1077}