1/*2 * Copyright (c) 2009-Present, Redis Ltd.3 * All rights reserved.4 *5 * Copyright (c) 2024-present, Valkey contributors.6 * All rights reserved.7 *8 * Licensed under your choice of (a) the Redis Source Available License 2.09 * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the10 * GNU Affero General Public License v3 (AGPLv3).11 *12 * Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.13 */1415/*16 * cluster_legacy.c contains the implementation of the cluster API that is17 * specific to the standard, Redis cluster-bus based clustering mechanism.18 */1920#include "server.h"21#include "cluster.h"22#include "cluster_legacy.h"23#include "cluster_asm.h"24#include "cluster_slot_stats.h"25#include "endianconv.h"26#include "connection.h"2728#include <sys/types.h>29#include <sys/socket.h>30#include <arpa/inet.h>31#include <fcntl.h>32#include <unistd.h>33#include <sys/stat.h>34#include <math.h>35#include <sys/file.h>3637/* A global reference to myself is handy to make code more clear.38 * Myself always points to server.cluster->myself, that is, the clusterNode39 * that represents this node. */40clusterNode *myself = NULL;4142clusterNode *createClusterNode(char *nodename, int flags);43void clusterAddNode(clusterNode *node);44void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask);45void clusterReadHandler(connection *conn);46void clusterSendPing(clusterLink *link, int type);47void clusterSendFail(char *nodename);48void clusterSendFailoverAuthIfNeeded(clusterNode *node, clusterMsg *request);49void clusterUpdateState(void);50int clusterNodeCoversSlot(clusterNode *n, int slot);51list *clusterGetNodesInMyShard(clusterNode *node);52int clusterNodeAddSlave(clusterNode *master, clusterNode *slave);53int clusterAddSlot(clusterNode *n, int slot);54int clusterDelSlot(int slot);55int clusterMoveNodeSlots(clusterNode *from_node, clusterNode *to_node);56int clusterDelNodeSlots(clusterNode *node);57int clusterNodeSetSlotBit(clusterNode *n, int slot);58void clusterSetMaster(clusterNode *n);59void clusterHandleSlaveFailover(void);60void clusterHandleSlaveMigration(int max_slaves);61int bitmapTestBit(unsigned char *bitmap, int pos);62void bitmapSetBit(unsigned char *bitmap, int pos);63void bitmapClearBit(unsigned char *bitmap, int pos);64void clusterDoBeforeSleep(int flags);65void clusterSendUpdate(clusterLink *link, clusterNode *node);66void resetManualFailover(void);67void clusterCloseAllSlots(void);68void clusterSetNodeAsMaster(clusterNode *n);69void clusterDelNode(clusterNode *delnode);70sds representClusterNodeFlags(sds ci, uint16_t flags);71sds representSlotInfo(sds ci, uint16_t *slot_info_pairs, int slot_info_pairs_count);72void clusterFreeNodesSlotsInfo(clusterNode *n);73uint64_t clusterGetMaxEpoch(void);74int clusterBumpConfigEpochWithoutConsensus(void);75void moduleCallClusterReceivers(const char *sender_id, uint64_t module_id, uint8_t type, const unsigned char *payload, uint32_t len);76const char *clusterGetMessageTypeString(int type);77void removeChannelsInSlot(unsigned int slot);78unsigned int countKeysInSlot(unsigned int hashslot);79unsigned int countChannelsInSlot(unsigned int hashslot);80unsigned int clusterDelKeysInSlot(unsigned int hashslot, int flags);81void clusterAddNodeToShard(const char *shard_id, clusterNode *node);82list *clusterLookupNodeListByShardId(const char *shard_id);83void clusterRemoveNodeFromShard(clusterNode *node);84int auxShardIdSetter(clusterNode *n, void *value, int length);85sds auxShardIdGetter(clusterNode *n, sds s);86int auxShardIdPresent(clusterNode *n);87int auxHumanNodenameSetter(clusterNode *n, void *value, int length);88sds auxHumanNodenameGetter(clusterNode *n, sds s);89int auxHumanNodenamePresent(clusterNode *n);90int auxTcpPortSetter(clusterNode *n, void *value, int length);91sds auxTcpPortGetter(clusterNode *n, sds s);92int auxTcpPortPresent(clusterNode *n);93int auxTlsPortSetter(clusterNode *n, void *value, int length);94sds auxTlsPortGetter(clusterNode *n, sds s);95int auxTlsPortPresent(clusterNode *n);96static void clusterBuildMessageHdr(clusterMsg *hdr, int type, size_t msglen);97void freeClusterLink(clusterLink *link);98int verifyClusterNodeId(const char *name, int length);99static void updateShardId(clusterNode *node, const char *shard_id);100101int getNodeDefaultClientPort(clusterNode *n) {102 return clusterDefaultClientPortIsTLS() ? n->tls_port : n->tcp_port;103}104105static inline int getNodeDefaultReplicationPort(clusterNode *n) {106 return server.tls_replication ? n->tls_port : n->tcp_port;107}108109int clusterNodeClientPort(clusterNode *n, int use_tls) {110 return use_tls ? n->tls_port : n->tcp_port;111}112113static inline int defaultClientPort(void) {114 return clusterDefaultClientPortIsTLS() ? server.tls_port : server.port;115}116117#define isSlotUnclaimed(slot) \118 (server.cluster->slots[slot] == NULL || \119 bitmapTestBit(server.cluster->owner_not_claiming_slot, slot))120121#define RCVBUF_INIT_LEN 1024122#define RCVBUF_MAX_PREALLOC (1<<20) /* 1MB */123124/* Cluster nodes hash table, mapping nodes addresses 1.2.3.4:6379 to125 * clusterNode structures. */126dictType clusterNodesDictType = {127 dictSdsHash, /* hash function */128 NULL, /* key dup */129 NULL, /* val dup */130 dictSdsKeyCompare, /* key compare */131 dictSdsDestructor, /* key destructor */132 NULL, /* val destructor */133 NULL /* allow to expand */134};135136/* Cluster re-addition blacklist. This maps node IDs to the time137 * we can re-add this node. The goal is to avoid reading a removed138 * node for some time. */139dictType clusterNodesBlackListDictType = {140 dictSdsCaseHash, /* hash function */141 NULL, /* key dup */142 NULL, /* val dup */143 dictSdsKeyCaseCompare, /* key compare */144 dictSdsDestructor, /* key destructor */145 NULL, /* val destructor */146 NULL /* allow to expand */147};148149/* Cluster shards hash table, mapping shard id to list of nodes */150dictType clusterSdsToListType = {151 dictSdsHash, /* hash function */152 NULL, /* key dup */153 NULL, /* val dup */154 dictSdsKeyCompare, /* key compare */155 dictSdsDestructor, /* key destructor */156 dictListDestructor, /* val destructor */157 NULL /* allow to expand */158};159160/* Aux fields are introduced in Redis 7.2 to support the persistence161 * of various important node properties, such as shard id, in nodes.conf.162 * Aux fields take an explicit format of name=value pairs and have no163 * intrinsic order among them. Aux fields are always grouped together164 * at the end of the second column of each row after the node's IP165 * address/port/cluster_port and the optional hostname. Aux fields166 * are separated by ','. */167168/* Aux field setter function prototype169 * return C_OK when the update is successful; C_ERR otherwise */170typedef int (aux_value_setter) (clusterNode* n, void *value, int length);171/* Aux field getter function prototype172 * return an sds that is a concatenation of the input sds string and173 * the aux value */174typedef sds (aux_value_getter) (clusterNode* n, sds s);175176typedef int (aux_value_present) (clusterNode* n);177178typedef struct {179 char *field;180 aux_value_setter *setter;181 aux_value_getter *getter;182 aux_value_present *isPresent;183} auxFieldHandler;184185/* Assign index to each aux field */186typedef enum {187 af_shard_id,188 af_human_nodename,189 af_tcp_port,190 af_tls_port,191 af_count,192} auxFieldIndex;193194/* Note that195 * 1. the order of the elements below must match that of their196 * indices as defined in auxFieldIndex197 * 2. aux name can contain characters that pass the isValidAuxChar check only */198auxFieldHandler auxFieldHandlers[] = {199 {"shard-id", auxShardIdSetter, auxShardIdGetter, auxShardIdPresent},200 {"nodename", auxHumanNodenameSetter, auxHumanNodenameGetter, auxHumanNodenamePresent},201 {"tcp-port", auxTcpPortSetter, auxTcpPortGetter, auxTcpPortPresent},202 {"tls-port", auxTlsPortSetter, auxTlsPortGetter, auxTlsPortPresent},203};204205int auxShardIdSetter(clusterNode *n, void *value, int length) {206 if (verifyClusterNodeId(value, length) == C_ERR) {207 return C_ERR;208 }209 memcpy(n->shard_id, value, CLUSTER_NAMELEN);210 /* if n already has replicas, make sure they all use211 * the primary shard id */212 for (int i = 0; i < n->numslaves; i++) {213 if (memcmp(n->slaves[i]->shard_id, n->shard_id, CLUSTER_NAMELEN) != 0)214 updateShardId(n->slaves[i], n->shard_id);215 }216 clusterAddNodeToShard(value, n);217 return C_OK;218}219220sds auxShardIdGetter(clusterNode *n, sds s) {221 return sdscatprintf(s, "%.40s", n->shard_id);222}223224int auxShardIdPresent(clusterNode *n) {225 return strnlen(n->shard_id, CLUSTER_NAMELEN);226}227228int auxHumanNodenameSetter(clusterNode *n, void *value, int length) {229 if (n && !strncmp(value, n->human_nodename, length)) {230 return C_OK;231 } else if (!n && (length == 0)) {232 return C_OK;233 }234 if (n) {235 n->human_nodename = sdscpylen(n->human_nodename, value, length);236 } else if (sdslen(n->human_nodename) != 0) {237 sdsclear(n->human_nodename);238 } else {239 return C_ERR;240 }241 return C_OK;242}243244sds auxHumanNodenameGetter(clusterNode *n, sds s) {245 return sdscatprintf(s, "%s", n->human_nodename);246}247248int auxHumanNodenamePresent(clusterNode *n) {249 return sdslen(n->human_nodename);250}251252int auxTcpPortSetter(clusterNode *n, void *value, int length) {253 if (length > 5 || length < 1) {254 return C_ERR;255 }256 char buf[length + 1];257 memcpy(buf, (char*)value, length);258 buf[length] = '\0';259 n->tcp_port = atoi(buf);260 return (n->tcp_port < 0 || n->tcp_port >= 65536) ? C_ERR : C_OK;261}262263sds auxTcpPortGetter(clusterNode *n, sds s) {264 return sdscatprintf(s, "%d", n->tcp_port);265}266267int auxTcpPortPresent(clusterNode *n) {268 return n->tcp_port >= 0 && n->tcp_port < 65536;269}270271int auxTlsPortSetter(clusterNode *n, void *value, int length) {272 if (length > 5 || length < 1) {273 return C_ERR;274 }275 char buf[length + 1];276 memcpy(buf, (char*)value, length);277 buf[length] = '\0';278 n->tls_port = atoi(buf);279 return (n->tls_port < 0 || n->tls_port >= 65536) ? C_ERR : C_OK;280}281282sds auxTlsPortGetter(clusterNode *n, sds s) {283 return sdscatprintf(s, "%d", n->tls_port);284}285286int auxTlsPortPresent(clusterNode *n) {287 return n->tls_port >= 0 && n->tls_port < 65536;288}289290/* clusterLink send queue blocks */291typedef struct {292 size_t totlen; /* Total length of this block including the message */293 int refcount; /* Number of cluster link send msg queues containing the message */294 clusterMsg msg[];295} clusterMsgSendBlock;296297/* Helper function to extract a normal message from a send block. */298static clusterMsg *getMessageFromSendBlock(clusterMsgSendBlock *msgblock) {299 return &msgblock->msg[0];300}301302/* -----------------------------------------------------------------------------303 * Initialization304 * -------------------------------------------------------------------------- */305306/* Load the cluster config from 'filename'.307 *308 * If the file does not exist or is zero-length (this may happen because309 * when we lock the nodes.conf file, we create a zero-length one for the310 * sake of locking if it does not already exist), C_ERR is returned.311 * If the configuration was loaded from the file, C_OK is returned. */312int clusterLoadConfig(char *filename) {313 FILE *fp = fopen(filename,"r");314 struct stat sb;315 char *line;316 int maxline, j;317318 if (fp == NULL) {319 if (errno == ENOENT) {320 return C_ERR;321 } else {322 serverLog(LL_WARNING,323 "Loading the cluster node config from %s: %s",324 filename, strerror(errno));325 exit(1);326 }327 }328329 if (redis_fstat(fileno(fp),&sb) == -1) {330 serverLog(LL_WARNING,331 "Unable to obtain the cluster node config file stat %s: %s",332 filename, strerror(errno));333 exit(1);334 }335 /* Check if the file is zero-length: if so return C_ERR to signal336 * we have to write the config. */337 if (sb.st_size == 0) {338 fclose(fp);339 return C_ERR;340 }341342 /* Parse the file. Note that single lines of the cluster config file can343 * be really long as they include all the hash slots of the node.344 * This means in the worst possible case, half of the Redis slots will be345 * present in a single line, possibly in importing or migrating state, so346 * together with the node ID of the sender/receiver.347 *348 * To simplify we allocate 1024+CLUSTER_SLOTS*128 bytes per line. */349 maxline = 1024+CLUSTER_SLOTS*128;350 line = zmalloc(maxline);351 while(fgets(line,maxline,fp) != NULL) {352 int argc, aux_argc;353 sds *argv, *aux_argv;354 clusterNode *n, *master;355 char *p, *s;356357 /* Skip blank lines, they can be created either by users manually358 * editing nodes.conf or by the config writing process if stopped359 * before the truncate() call. */360 if (line[0] == '\n' || line[0] == '\0') continue;361362 /* Split the line into arguments for processing. */363 argv = sdssplitargs(line,&argc);364 if (argv == NULL) goto fmterr;365366 /* Handle the special "vars" line. Don't pretend it is the last367 * line even if it actually is when generated by Redis. */368 if (strcasecmp(argv[0],"vars") == 0) {369 if (!(argc % 2)) goto fmterr;370 for (j = 1; j < argc; j += 2) {371 if (strcasecmp(argv[j],"currentEpoch") == 0) {372 server.cluster->currentEpoch =373 strtoull(argv[j+1],NULL,10);374 } else if (strcasecmp(argv[j],"lastVoteEpoch") == 0) {375 server.cluster->lastVoteEpoch =376 strtoull(argv[j+1],NULL,10);377 } else {378 serverLog(LL_NOTICE,379 "Skipping unknown cluster config variable '%s'",380 argv[j]);381 }382 }383 sdsfreesplitres(argv,argc);384 continue;385 }386387 /* Regular config lines have at least eight fields */388 if (argc < 8) {389 sdsfreesplitres(argv,argc);390 goto fmterr;391 }392393 /* Create this node if it does not exist */394 if (verifyClusterNodeId(argv[0], sdslen(argv[0])) == C_ERR) {395 sdsfreesplitres(argv, argc);396 goto fmterr;397 }398 n = clusterLookupNode(argv[0], sdslen(argv[0]));399 if (!n) {400 n = createClusterNode(argv[0],0);401 clusterAddNode(n);402 }403 /* Format for the node address and auxiliary argument information:404 * ip:port[@cport][,hostname][,aux=val]*] */405406 aux_argv = sdssplitlen(argv[1], sdslen(argv[1]), ",", 1, &aux_argc);407 if (aux_argv == NULL) {408 sdsfreesplitres(argv,argc);409 goto fmterr;410 }411412 /* Hostname is an optional argument that defines the endpoint413 * that can be reported to clients instead of IP. */414 if (aux_argc > 1 && sdslen(aux_argv[1]) > 0) {415 n->hostname = sdscpy(n->hostname, aux_argv[1]);416 } else if (sdslen(n->hostname) != 0) {417 sdsclear(n->hostname);418 }419420 /* All fields after hostname are auxiliary and they take on421 * the format of "aux=val" where both aux and val can contain422 * characters that pass the isValidAuxChar check only. The order423 * of the aux fields is insignificant. */424 int aux_tcp_port = 0;425 int aux_tls_port = 0;426 int aux_shard_id = 0;427 for (int i = 2; i < aux_argc; i++) {428 int field_argc;429 sds *field_argv;430 field_argv = sdssplitlen(aux_argv[i], sdslen(aux_argv[i]), "=", 1, &field_argc);431 if (field_argv == NULL || field_argc != 2) {432 /* Invalid aux field format */433 if (field_argv != NULL) sdsfreesplitres(field_argv, field_argc);434 sdsfreesplitres(aux_argv, aux_argc);435 sdsfreesplitres(argv,argc);436 goto fmterr;437 }438439 /* Validate that both aux and value contain valid characters only */440 for (unsigned j = 0; j < 2; j++) {441 if (!isValidAuxString(field_argv[j],sdslen(field_argv[j]))){442 /* Invalid aux field format */443 sdsfreesplitres(field_argv, field_argc);444 sdsfreesplitres(aux_argv, aux_argc);445 sdsfreesplitres(argv,argc);446 goto fmterr;447 }448 }449450 /* Note that we don't expect lots of aux fields in the foreseeable451 * future so a linear search is completely fine. */452 int field_found = 0;453 for (unsigned j = 0; j < numElements(auxFieldHandlers); j++) {454 if (sdslen(field_argv[0]) != strlen(auxFieldHandlers[j].field) ||455 memcmp(field_argv[0], auxFieldHandlers[j].field, sdslen(field_argv[0])) != 0) {456 continue;457 }458 field_found = 1;459 aux_shard_id |= j == af_shard_id;460 aux_tcp_port |= j == af_tcp_port;461 aux_tls_port |= j == af_tls_port;462 if (auxFieldHandlers[j].setter(n, field_argv[1], sdslen(field_argv[1])) != C_OK) {463 /* Invalid aux field format */464 sdsfreesplitres(field_argv, field_argc);465 sdsfreesplitres(aux_argv, aux_argc);466 sdsfreesplitres(argv,argc);467 goto fmterr;468 }469 }470471 if (field_found == 0) {472 /* Invalid aux field format */473 sdsfreesplitres(field_argv, field_argc);474 sdsfreesplitres(aux_argv, aux_argc);475 sdsfreesplitres(argv,argc);476 goto fmterr;477 }478479 sdsfreesplitres(field_argv, field_argc);480 }481 /* Address and port */482 if ((p = strrchr(aux_argv[0],':')) == NULL) {483 sdsfreesplitres(aux_argv, aux_argc);484 sdsfreesplitres(argv,argc);485 goto fmterr;486 }487 *p = '\0';488 memcpy(n->ip,aux_argv[0],strlen(aux_argv[0])+1);489 char *port = p+1;490 char *busp = strchr(port,'@');491 if (busp) {492 *busp = '\0';493 busp++;494 }495 /* If neither TCP or TLS port is found in aux field, it is considered496 * an old version of nodes.conf file.*/497 if (!aux_tcp_port && !aux_tls_port) {498 if (clusterDefaultClientPortIsTLS()) {499 n->tls_port = atoi(port);500 } else {501 n->tcp_port = atoi(port);502 }503 } else if (!aux_tcp_port) {504 n->tcp_port = atoi(port);505 } else if (!aux_tls_port) {506 n->tls_port = atoi(port);507 }508 /* In older versions of nodes.conf the "@busport" part is missing.509 * In this case we set it to the default offset of 10000 from the510 * base port. */511 n->cport = busp ? atoi(busp) : (getNodeDefaultClientPort(n) + CLUSTER_PORT_INCR);512513 /* The plaintext port for client in a TLS cluster (n->pport) is not514 * stored in nodes.conf. It is received later over the bus protocol. */515516 sdsfreesplitres(aux_argv, aux_argc);517518 /* Parse flags */519 p = s = argv[2];520 while(p) {521 p = strchr(s,',');522 if (p) *p = '\0';523 if (!strcasecmp(s,"myself")) {524 serverAssert(server.cluster->myself == NULL);525 myself = server.cluster->myself = n;526 n->flags |= CLUSTER_NODE_MYSELF;527 } else if (!strcasecmp(s,"master")) {528 n->flags |= CLUSTER_NODE_MASTER;529 } else if (!strcasecmp(s,"slave")) {530 n->flags |= CLUSTER_NODE_SLAVE;531 } else if (!strcasecmp(s,"fail?")) {532 n->flags |= CLUSTER_NODE_PFAIL;533 } else if (!strcasecmp(s,"fail")) {534 n->flags |= CLUSTER_NODE_FAIL;535 n->fail_time = mstime();536 } else if (!strcasecmp(s,"handshake")) {537 n->flags |= CLUSTER_NODE_HANDSHAKE;538 } else if (!strcasecmp(s,"noaddr")) {539 n->flags |= CLUSTER_NODE_NOADDR;540 } else if (!strcasecmp(s,"nofailover")) {541 n->flags |= CLUSTER_NODE_NOFAILOVER;542 } else if (!strcasecmp(s,"noflags")) {543 /* nothing to do */544 } else {545 serverPanic("Unknown flag in redis cluster config file");546 }547 if (p) s = p+1;548 }549550 /* Get master if any. Set the master and populate master's551 * slave list. */552 if (argv[3][0] != '-') {553 if (verifyClusterNodeId(argv[3], sdslen(argv[3])) == C_ERR) {554 sdsfreesplitres(argv, argc);555 goto fmterr;556 }557 master = clusterLookupNode(argv[3], sdslen(argv[3]));558 if (!master) {559 master = createClusterNode(argv[3],0);560 clusterAddNode(master);561 }562 /* shard_id can be absent if we are loading a nodes.conf generated563 * by an older version of Redis; 564 * ignore replica's shard_id in the file, only use the primary's.565 * If replica precedes primary in file, it will be corrected566 * later by the auxShardIdSetter.567 * Remove node from its old shard before adding it to the new one. */568 if (aux_shard_id == 1) clusterRemoveNodeFromShard(n);569 memcpy(n->shard_id, master->shard_id, CLUSTER_NAMELEN);570 clusterAddNodeToShard(master->shard_id, n);571 n->slaveof = master;572 clusterNodeAddSlave(master,n);573 } else if (aux_shard_id == 0) {574 /* n is a primary but it does not have a persisted shard_id.575 * This happens if we are loading a nodes.conf generated by576 * an older version of Redis. We should manually update the577 * shard membership in this case */578 clusterAddNodeToShard(n->shard_id, n);579 }580581 /* Set ping sent / pong received timestamps */582 if (atoi(argv[4])) n->ping_sent = mstime();583 if (atoi(argv[5])) n->pong_received = mstime();584585 /* Set configEpoch for this node.586 * If the node is a replica, set its config epoch to 0.587 * If it's a primary, load the config epoch from the configuration file. */588 n->configEpoch = (nodeIsSlave(n) && n->slaveof) ? 0 : strtoull(argv[6],NULL,10);589590 /* Populate hash slots served by this instance. */591 for (j = 8; j < argc; j++) {592 int start, stop;593594 if (argv[j][0] == '[') {595 /* Here we handle migrating / importing slots */596 int slot;597 char direction;598 clusterNode *cn;599600 p = strchr(argv[j],'-');601 serverAssert(p != NULL);602 *p = '\0';603 direction = p[1]; /* Either '>' or '<' */604 slot = atoi(argv[j]+1);605 if (slot < 0 || slot >= CLUSTER_SLOTS) {606 sdsfreesplitres(argv,argc);607 goto fmterr;608 }609 p += 3;610611 char *pr = strchr(p, ']');612 size_t node_len = pr - p;613 if (pr == NULL || verifyClusterNodeId(p, node_len) == C_ERR) {614 sdsfreesplitres(argv, argc);615 goto fmterr;616 }617 cn = clusterLookupNode(p, CLUSTER_NAMELEN);618 if (!cn) {619 cn = createClusterNode(p,0);620 clusterAddNode(cn);621 }622 if (direction == '>') {623 server.cluster->migrating_slots_to[slot] = cn;624 } else {625 server.cluster->importing_slots_from[slot] = cn;626 }627 continue;628 } else if ((p = strchr(argv[j],'-')) != NULL) {629 *p = '\0';630 start = atoi(argv[j]);631 stop = atoi(p+1);632 } else {633 start = stop = atoi(argv[j]);634 }635 if (start < 0 || start >= CLUSTER_SLOTS ||636 stop < 0 || stop >= CLUSTER_SLOTS)637 {638 sdsfreesplitres(argv,argc);639 goto fmterr;640 }641 while(start <= stop) clusterAddSlot(n, start++);642 }643644 sdsfreesplitres(argv,argc);645 }646 /* Config sanity check */647 if (server.cluster->myself == NULL) goto fmterr;648 if (!(myself->flags & (CLUSTER_NODE_MASTER | CLUSTER_NODE_SLAVE))) goto fmterr;649 if (nodeIsSlave(myself) && myself->slaveof == NULL) goto fmterr;650651 zfree(line);652 fclose(fp);653654 serverLog(LL_NOTICE,"Node configuration loaded, I'm %.40s", myself->name);655656 /* Something that should never happen: currentEpoch smaller than657 * the max epoch found in the nodes configuration. However we handle this658 * as some form of protection against manual editing of critical files. */659 if (clusterGetMaxEpoch() > server.cluster->currentEpoch) {660 server.cluster->currentEpoch = clusterGetMaxEpoch();661 }662 return C_OK;663664fmterr:665 serverLog(LL_WARNING,666 "Unrecoverable error: corrupted cluster config file \"%s\".", line);667 zfree(line);668 if (fp) fclose(fp);669 exit(1);670}671672/* Cluster node configuration is exactly the same as CLUSTER NODES output.673 *674 * This function writes the node config and returns 0, on error -1675 * is returned.676 *677 * Note: we need to write the file in an atomic way from the point of view678 * of the POSIX filesystem semantics, so that if the server is stopped679 * or crashes during the write, we'll end with either the old file or the680 * new one. Since we have the full payload to write available we can use681 * a single write to write the whole file. If the pre-existing file was682 * bigger we pad our payload with newlines that are anyway ignored and truncate683 * the file afterward. */684int clusterSaveConfig(int do_fsync) {685 sds ci,tmpfilename;686 size_t content_size,offset = 0;687 ssize_t written_bytes;688 int fd = -1;689 int retval = C_ERR;690691 server.cluster->todo_before_sleep &= ~CLUSTER_TODO_SAVE_CONFIG;692693 /* Get the nodes description and concatenate our "vars" directive to694 * save currentEpoch and lastVoteEpoch. */695 ci = clusterGenNodesDescription(NULL, CLUSTER_NODE_HANDSHAKE, 0);696 ci = sdscatprintf(ci,"vars currentEpoch %llu lastVoteEpoch %llu\n",697 (unsigned long long) server.cluster->currentEpoch,698 (unsigned long long) server.cluster->lastVoteEpoch);699 content_size = sdslen(ci);700701 /* Create a temp file with the new content. */702 tmpfilename = sdscatfmt(sdsempty(),"%s.tmp-%i-%I",703 server.cluster_configfile,(int) getpid(),mstime());704 if ((fd = open(tmpfilename,O_WRONLY|O_CREAT,0644)) == -1) {705 serverLog(LL_WARNING,"Could not open temp cluster config file: %s",strerror(errno));706 goto cleanup;707 }708709 while (offset < content_size) {710 written_bytes = write(fd,ci + offset,content_size - offset);711 if (written_bytes <= 0) {712 if (errno == EINTR) continue;713 serverLog(LL_WARNING,"Failed after writing (%zd) bytes to tmp cluster config file: %s",714 offset,strerror(errno));715 goto cleanup;716 }717 offset += written_bytes;718 }719720 if (do_fsync) {721 server.cluster->todo_before_sleep &= ~CLUSTER_TODO_FSYNC_CONFIG;722 if (redis_fsync(fd) == -1) {723 serverLog(LL_WARNING,"Could not sync tmp cluster config file: %s",strerror(errno));724 goto cleanup;725 }726 }727728 if (rename(tmpfilename, server.cluster_configfile) == -1) {729 serverLog(LL_WARNING,"Could not rename tmp cluster config file: %s",strerror(errno));730 goto cleanup;731 }732733 if (do_fsync) {734 if (fsyncFileDir(server.cluster_configfile) == -1) {735 serverLog(LL_WARNING,"Could not sync cluster config file dir: %s",strerror(errno));736 goto cleanup;737 }738 }739 retval = C_OK; /* If we reached this point, everything is fine. */740741cleanup:742 if (fd != -1) close(fd);743 if (retval) unlink(tmpfilename);744 sdsfree(tmpfilename);745 sdsfree(ci);746 return retval;747}748749void clusterSaveConfigOrDie(int do_fsync) {750 if (clusterSaveConfig(do_fsync) == -1) {751 serverLog(LL_WARNING,"Fatal: can't update cluster config file.");752 exit(1);753 }754}755756/* Lock the cluster config using flock(), and retain the file descriptor used to757 * acquire the lock so that the file will be locked as long as the process is up.758 *759 * This works because we always update nodes.conf with a new version760 * in-place, reopening the file, and writing to it in place (later adjusting761 * the length with ftruncate()).762 *763 * On success C_OK is returned, otherwise an error is logged and764 * the function returns C_ERR to signal a lock was not acquired. */765int clusterLockConfig(char *filename) {766/* flock() does not exist on Solaris767 * and a fcntl-based solution won't help, as we constantly re-open that file,768 * which will release _all_ locks anyway769 */770#if !defined(__sun)771 /* To lock it, we need to open the file in a way it is created if772 * it does not exist, otherwise there is a race condition with other773 * processes. */774 int fd = open(filename,O_WRONLY|O_CREAT|O_CLOEXEC,0644);775 if (fd == -1) {776 serverLog(LL_WARNING,777 "Can't open %s in order to acquire a lock: %s",778 filename, strerror(errno));779 return C_ERR;780 }781782 if (flock(fd,LOCK_EX|LOCK_NB) == -1) {783 if (errno == EWOULDBLOCK) {784 serverLog(LL_WARNING,785 "Sorry, the cluster configuration file %s is already used "786 "by a different Redis Cluster node. Please make sure that "787 "different nodes use different cluster configuration "788 "files.", filename);789 } else {790 serverLog(LL_WARNING,791 "Impossible to lock %s: %s", filename, strerror(errno));792 }793 close(fd);794 return C_ERR;795 }796 /* Lock acquired: leak the 'fd' by not closing it until shutdown time, so that797 * we'll retain the lock to the file as long as the process exists.798 *799 * After fork, the child process will get the fd opened by the parent process,800 * we need save `fd` to `cluster_config_file_lock_fd`, so that in redisFork(),801 * it will be closed in the child process.802 * If it is not closed, when the main process is killed -9, but the child process803 * (redis-aof-rewrite) is still alive, the fd(lock) will still be held by the804 * child process, and the main process will fail to get lock, means fail to start. */805 server.cluster_config_file_lock_fd = fd;806#else807 UNUSED(filename);808#endif /* __sun */809810 return C_OK;811}812813/* Derives our ports to be announced in the cluster bus. */814void deriveAnnouncedPorts(int *announced_tcp_port, int *announced_tls_port,815 int *announced_cport) {816 /* Config overriding announced ports. */817 *announced_tcp_port = server.cluster_announce_port ? 818 server.cluster_announce_port : server.port;819 *announced_tls_port = server.cluster_announce_tls_port ? 820 server.cluster_announce_tls_port : server.tls_port;821 /* Derive cluster bus port. */822 if (server.cluster_announce_bus_port) {823 *announced_cport = server.cluster_announce_bus_port;824 } else if (server.cluster_port) {825 *announced_cport = server.cluster_port;826 } else {827 *announced_cport = defaultClientPort() + CLUSTER_PORT_INCR;828 }829}830831/* Some flags (currently just the NOFAILOVER flag) may need to be updated832 * in the "myself" node based on the current configuration of the node,833 * that may change at runtime via CONFIG SET. This function changes the834 * set of flags in myself->flags accordingly. */835void clusterUpdateMyselfFlags(void) {836 if (!myself) return;837 int oldflags = myself->flags;838 int nofailover = server.cluster_slave_no_failover ?839 CLUSTER_NODE_NOFAILOVER : 0;840 myself->flags &= ~CLUSTER_NODE_NOFAILOVER;841 myself->flags |= nofailover;842 if (myself->flags != oldflags) {843 clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|844 CLUSTER_TODO_UPDATE_STATE);845 }846}847848849/* We want to take myself->port/cport/pport in sync with the850* cluster-announce-port/cluster-announce-bus-port/cluster-announce-tls-port option.851* The option can be set at runtime via CONFIG SET. */852void clusterUpdateMyselfAnnouncedPorts(void) {853 if (!myself) return;854 int old_tcp_port = myself->tcp_port;855 int old_tls_port = myself->tls_port;856857 deriveAnnouncedPorts(&myself->tcp_port,&myself->tls_port,&myself->cport);858 if (myself->tcp_port != old_tcp_port || myself->tls_port != old_tls_port) {859 clusterNotifyTopologyChanged(CLUSTER_TOPOLOGY_CHANGE_FLAG_NODE, NULL);860 }861}862863/* We want to take myself->ip in sync with the cluster-announce-ip option.864* The option can be set at runtime via CONFIG SET. */865void clusterUpdateMyselfIp(void) {866 if (!myself) return;867 static char *prev_ip = NULL;868 char *curr_ip = server.cluster_announce_ip;869 int changed = 0;870871 if (prev_ip == NULL && curr_ip != NULL) changed = 1;872 else if (prev_ip != NULL && curr_ip == NULL) changed = 1;873 else if (prev_ip && curr_ip && strcmp(prev_ip,curr_ip)) changed = 1;874875 if (changed) {876 if (prev_ip) zfree(prev_ip);877 prev_ip = curr_ip;878879 if (curr_ip) {880 /* We always take a copy of the previous IP address, by881 * duplicating the string. This way later we can check if882 * the address really changed. */883 prev_ip = zstrdup(prev_ip);884 redis_strlcpy(myself->ip,server.cluster_announce_ip,NET_IP_STR_LEN);885 } else {886 myself->ip[0] = '\0'; /* Force autodetection. */887 }888 clusterNotifyTopologyChanged(CLUSTER_TOPOLOGY_CHANGE_FLAG_NODE, NULL);889 }890}891892/* Update the hostname for the specified node with the provided C string. */893static void updateAnnouncedHostname(clusterNode *node, char *new) {894 /* Previous and new hostname are the same, no need to update. */895 if (new && !strcmp(new, node->hostname)) {896 return;897 } else if (!new && (sdslen(node->hostname) == 0)) {898 return;899 }900901 if (new) {902 node->hostname = sdscpy(node->hostname, new);903 } else if (sdslen(node->hostname) != 0) {904 sdsclear(node->hostname);905 }906 clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG);907 clusterNotifyTopologyChanged(CLUSTER_TOPOLOGY_CHANGE_FLAG_NODE, NULL);908}909910static void updateAnnouncedHumanNodename(clusterNode *node, char *new) {911 if (new && !strcmp(new, node->human_nodename)) {912 return;913 } else if (!new && (sdslen(node->human_nodename) == 0)) {914 return;915 }916 917 if (new) {918 node->human_nodename = sdscpy(node->human_nodename, new);919 } else if (sdslen(node->human_nodename) != 0) {920 sdsclear(node->human_nodename);921 }922 clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG);923}924925static void assignShardIdToNode(clusterNode *node, const char *shard_id, int flag) {926 clusterRemoveNodeFromShard(node);927 memcpy(node->shard_id, shard_id, CLUSTER_NAMELEN);928 clusterAddNodeToShard(shard_id, node);929 clusterDoBeforeSleep(flag); 930}931932static void updateShardId(clusterNode *node, const char *shard_id) {933 if (shard_id && memcmp(node->shard_id, shard_id, CLUSTER_NAMELEN) != 0) {934 /* We always make our best effort to keep the shard-id consistent935 * between the master and its replicas:936 *937 * 1. When updating the master's shard-id, we simultaneously update the938 * shard-id of all its replicas to ensure consistency.939 * 2. When updating replica's shard-id, if it differs from its master's shard-id,940 * we discard this replica's shard-id and continue using master's shard-id.941 * This applies even if the master does not support shard-id, in which942 * case we rely on the master's randomly generated shard-id. */943 if (node->slaveof == NULL) {944 assignShardIdToNode(node, shard_id, CLUSTER_TODO_SAVE_CONFIG);945 for (int i = 0; i < clusterNodeNumSlaves(node); i++) {946 clusterNode *slavenode = clusterNodeGetSlave(node, i);947 if (memcmp(slavenode->shard_id, shard_id, CLUSTER_NAMELEN) != 0)948 assignShardIdToNode(slavenode, shard_id, CLUSTER_TODO_SAVE_CONFIG|CLUSTER_TODO_FSYNC_CONFIG);949 }950 } else if (memcmp(node->slaveof->shard_id, shard_id, CLUSTER_NAMELEN) == 0) {951 assignShardIdToNode(node, shard_id, CLUSTER_TODO_SAVE_CONFIG);952 }953 }954}955956/* Update my hostname based on server configuration values */957void clusterUpdateMyselfHostname(void) {958 if (!myself) return;959 updateAnnouncedHostname(myself, server.cluster_announce_hostname);960}961962void clusterUpdateMyselfHumanNodename(void) {963 if (!myself) return;964 updateAnnouncedHumanNodename(myself, server.cluster_announce_human_nodename);965}966967void clusterInit(void) {968 int saveconf = 0;969970 server.cluster = zmalloc(sizeof(struct clusterState));971 server.cluster->myself = NULL;972 server.cluster->currentEpoch = 0;973 server.cluster->state = CLUSTER_FAIL;974 server.cluster->size = 0;975 server.cluster->todo_before_sleep = 0;976 server.cluster->nodes = dictCreate(&clusterNodesDictType);977 server.cluster->shards = dictCreate(&clusterSdsToListType);978 server.cluster->nodes_black_list =979 dictCreate(&clusterNodesBlackListDictType);980 server.cluster->failover_auth_time = 0;981 server.cluster->failover_auth_count = 0;982 server.cluster->failover_auth_rank = 0;983 server.cluster->failover_auth_epoch = 0;984 server.cluster->cant_failover_reason = CLUSTER_CANT_FAILOVER_NONE;985 server.cluster->lastVoteEpoch = 0;986987 /* Initialize stats */988 for (int i = 0; i < CLUSTERMSG_TYPE_COUNT; i++) {989 server.cluster->stats_bus_messages_sent[i] = 0;990 server.cluster->stats_bus_messages_received[i] = 0;991 }992 server.cluster->stats_pfail_nodes = 0;993 server.cluster->stat_cluster_links_buffer_limit_exceeded = 0;994995 memset(server.cluster->slots,0, sizeof(server.cluster->slots));996 clusterCloseAllSlots();997998 memset(server.cluster->owner_not_claiming_slot, 0, sizeof(server.cluster->owner_not_claiming_slot));9991000 /* Lock the cluster config file to make sure every node uses1001 * its own nodes.conf. */1002 server.cluster_config_file_lock_fd = -1;1003 if (clusterLockConfig(server.cluster_configfile) == C_ERR)1004 exit(1);10051006 /* Load or create a new nodes configuration. */1007 if (clusterLoadConfig(server.cluster_configfile) == C_ERR) {1008 /* No configuration found. We will just use the random name provided1009 * by the createClusterNode() function. */1010 myself = server.cluster->myself =1011 createClusterNode(NULL,CLUSTER_NODE_MYSELF|CLUSTER_NODE_MASTER);1012 serverLog(LL_NOTICE,"No cluster configuration found, I'm %.40s",1013 myself->name);1014 clusterAddNode(myself);1015 clusterAddNodeToShard(myself->shard_id, myself);1016 saveconf = 1;1017 }1018 if (saveconf) clusterSaveConfigOrDie(1);10191020 /* Port sanity check II1021 * The other handshake port check is triggered too late to stop1022 * us from trying to use a too-high cluster port number. */1023 int port = defaultClientPort();1024 if (!server.cluster_port && port > (65535-CLUSTER_PORT_INCR)) {1025 serverLog(LL_WARNING, "Redis port number too high. "1026 "Cluster communication port is 10,000 port "1027 "numbers higher than your Redis port. "1028 "Your Redis port number must be 55535 or less.");1029 exit(1);1030 }1031 if (!server.bindaddr_count) {1032 serverLog(LL_WARNING, "No bind address is configured, but it is required for the Cluster bus.");1033 exit(1);1034 }10351036 /* Set myself->port/cport/pport to my listening ports, we'll just need to1037 * discover the IP address via MEET messages. */1038 deriveAnnouncedPorts(&myself->tcp_port, &myself->tls_port, &myself->cport);10391040 server.cluster->mf_end = 0;1041 server.cluster->mf_slave = NULL;1042 resetManualFailover();1043 clusterUpdateMyselfFlags();1044 clusterUpdateMyselfIp();1045 clusterUpdateMyselfHostname();1046 clusterUpdateMyselfHumanNodename();10471048 getRandomHexChars(server.cluster->internal_secret, CLUSTER_INTERNALSECRETLEN);1049}10501051void clusterInitLast(void) {1052 if (connectionIndexByType(connTypeOfCluster()->get_type(NULL)) < 0) {1053 serverLog(LL_WARNING, "Missing connection type %s, but it is required for the Cluster bus.", connTypeOfCluster()->get_type(NULL));1054 exit(1);1055 }10561057 int port = defaultClientPort();1058 connListener *listener = &server.clistener;1059 listener->count = 0;1060 listener->bindaddr = server.bindaddr;1061 listener->bindaddr_count = server.bindaddr_count;1062 listener->port = server.cluster_port ? server.cluster_port : port + CLUSTER_PORT_INCR;1063 listener->ct = connTypeOfCluster();1064 if (connListen(listener) == C_ERR ) {1065 /* Note: the following log text is matched by the test suite. */1066 serverLog(LL_WARNING, "Failed listening on port %u (cluster), aborting.", listener->port);1067 exit(1);1068 }1069 1070 if (createSocketAcceptHandler(&server.clistener, clusterAcceptHandler) != C_OK) {1071 serverPanic("Unrecoverable error creating Redis Cluster socket accept handler.");1072 }1073}10741075/* Reset a node performing a soft or hard reset:1076 *1077 * 1) All other nodes are forgotten.1078 * 2) All the assigned / open slots are released.1079 * 3) If the node is a slave, it turns into a master.1080 * 4) Only for hard reset: a new Node ID is generated.1081 * 5) Only for hard reset: currentEpoch and configEpoch are set to 0.1082 * 6) The new configuration is saved and the cluster state updated.1083 * 7) If the node was a slave, the whole data set is flushed away. */1084void clusterReset(int hard) {1085 dictIterator di;1086 dictEntry *de;1087 int j;10881089 /* Turn into master. */1090 if (nodeIsSlave(myself)) {1091 asmFinalizeMasterTask();1092 clusterSetNodeAsMaster(myself);1093 replicationUnsetMaster();1094 emptyData(-1,EMPTYDB_NO_FLAGS,NULL);1095 }10961097 /* Close slots, reset manual failover state. */1098 clusterCloseAllSlots();1099 resetManualFailover();11001101 /* Cancel all ASM tasks */1102 clusterAsmCancel(NULL, "CLUSTER RESET");1103 asmCancelTrimJobs();11041105 /* Unassign all the slots. */1106 for (j = 0; j < CLUSTER_SLOTS; j++) clusterDelSlot(j);11071108 /* Recreate shards dict */1109 dictEmpty(server.cluster->shards, NULL);11101111 /* Forget all the nodes, but myself. */1112 dictInitSafeIterator(&di, server.cluster->nodes);1113 while((de = dictNext(&di)) != NULL) {1114 clusterNode *node = dictGetVal(de);11151116 if (node == myself) continue;1117 clusterDelNode(node);1118 }1119 dictResetIterator(&di);11201121 /* Empty the nodes blacklist. */1122 dictEmpty(server.cluster->nodes_black_list, NULL);11231124 /* Hard reset only: set epochs to 0, change node ID. */1125 if (hard) {1126 sds oldname;11271128 server.cluster->currentEpoch = 0;1129 server.cluster->lastVoteEpoch = 0;1130 myself->configEpoch = 0;1131 serverLog(LL_NOTICE, "configEpoch set to 0 via CLUSTER RESET HARD");11321133 /* To change the Node ID we need to remove the old name from the1134 * nodes table, change the ID, and re-add back with new name. */1135 oldname = sdsnewlen(myself->name, CLUSTER_NAMELEN);1136 dictDelete(server.cluster->nodes,oldname);1137 sdsfree(oldname);1138 getRandomHexChars(myself->name, CLUSTER_NAMELEN);1139 getRandomHexChars(myself->shard_id, CLUSTER_NAMELEN);1140 clusterAddNode(myself);1141 serverLog(LL_NOTICE,"Node hard reset, now I'm %.40s", myself->name);1142 }11431144 /* Re-populate shards */1145 clusterAddNodeToShard(myself->shard_id, myself);11461147 /* Make sure to persist the new config and update the state. */1148 clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|1149 CLUSTER_TODO_UPDATE_STATE|1150 CLUSTER_TODO_FSYNC_CONFIG);1151}11521153/* -----------------------------------------------------------------------------1154 * CLUSTER communication link1155 * -------------------------------------------------------------------------- */1156static clusterMsgSendBlock *createClusterMsgSendBlock(int type, uint32_t msglen) {1157 uint32_t blocklen = msglen + sizeof(clusterMsgSendBlock);1158 clusterMsgSendBlock *msgblock = zcalloc(blocklen);1159 msgblock->refcount = 1;1160 msgblock->totlen = blocklen;1161 server.stat_cluster_links_memory += blocklen;1162 clusterBuildMessageHdr(getMessageFromSendBlock(msgblock),type,msglen);1163 return msgblock;1164}11651166static void clusterMsgSendBlockDecrRefCount(void *node) {1167 clusterMsgSendBlock *msgblock = (clusterMsgSendBlock*)node;1168 msgblock->refcount--;1169 serverAssert(msgblock->refcount >= 0);1170 if (msgblock->refcount == 0) {1171 server.stat_cluster_links_memory -= msgblock->totlen;1172 zfree(msgblock);1173 }1174}11751176clusterLink *createClusterLink(clusterNode *node) {1177 clusterLink *link = zmalloc(sizeof(*link));1178 link->ctime = mstime();1179 link->send_msg_queue = listCreate();1180 listSetFreeMethod(link->send_msg_queue, clusterMsgSendBlockDecrRefCount);1181 link->head_msg_send_offset = 0;1182 link->send_msg_queue_mem = sizeof(list);1183 link->rcvbuf = zmalloc(link->rcvbuf_alloc = RCVBUF_INIT_LEN);1184 link->rcvbuf_len = 0;1185 server.stat_cluster_links_memory += link->rcvbuf_alloc + link->send_msg_queue_mem;1186 link->conn = NULL;1187 link->node = node;1188 /* Related node can only possibly be known at link creation time if this is an outbound link */1189 link->inbound = (node == NULL);1190 if (!link->inbound) {1191 node->link = link;1192 }1193 return link;1194}11951196/* Free a cluster link, but does not free the associated node of course.1197 * This function will just make sure that the original node associated1198 * with this link will have the 'link' field set to NULL. */1199void freeClusterLink(clusterLink *link) {1200 if (link->conn) {1201 connClose(link->conn);1202 link->conn = NULL;1203 }1204 server.stat_cluster_links_memory -= sizeof(list) + listLength(link->send_msg_queue)*sizeof(listNode);1205 listRelease(link->send_msg_queue);1206 server.stat_cluster_links_memory -= link->rcvbuf_alloc;1207 zfree(link->rcvbuf);1208 if (link->node) {1209 if (link->node->link == link) {1210 serverAssert(!link->inbound);1211 link->node->link = NULL;1212 } else if (link->node->inbound_link == link) {1213 serverAssert(link->inbound);1214 link->node->inbound_link = NULL;1215 }1216 }1217 zfree(link);1218}12191220void setClusterNodeToInboundClusterLink(clusterNode *node, clusterLink *link) {1221 serverAssert(!link->node);1222 serverAssert(link->inbound);1223 if (node->inbound_link) {1224 /* A peer may disconnect and then reconnect with us, and it's not guaranteed that1225 * we would always process the disconnection of the existing inbound link before1226 * accepting a new existing inbound link. Therefore, it's possible to have more than1227 * one inbound link from the same node at the same time. Our cleanup logic assumes1228 * a one to one relationship between nodes and inbound links, so we need to kill1229 * one of the links. The existing link is more likely the outdated one, but it's1230 * possible the other node may need to open another link. */1231 serverLog(LL_DEBUG, "Replacing inbound link fd %d from node %.40s with fd %d",1232 node->inbound_link->conn->fd, node->name, link->conn->fd);1233 freeClusterLink(node->inbound_link);1234 }1235 serverAssert(!node->inbound_link);1236 node->inbound_link = link;1237 link->node = node;1238}12391240static void clusterConnAcceptHandler(connection *conn) {1241 clusterLink *link;12421243 if (connGetState(conn) != CONN_STATE_CONNECTED) {1244 serverLog(LL_VERBOSE,1245 "Error accepting cluster node connection: %s", connGetLastError(conn));1246 connClose(conn);1247 return;1248 }12491250 /* Create a link object we use to handle the connection.1251 * It gets passed to the readable handler when data is available.1252 * Initially the link->node pointer is set to NULL as we don't know1253 * which node is, but the right node is references once we know the1254 * node identity. */1255 link = createClusterLink(NULL);1256 link->conn = conn;1257 connSetPrivateData(conn, link);12581259 /* Register read handler */1260 connSetReadHandler(conn, clusterReadHandler);1261}12621263#define MAX_CLUSTER_ACCEPTS_PER_CALL 10001264void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) {1265 int cport, cfd;1266 int max = MAX_CLUSTER_ACCEPTS_PER_CALL;1267 char cip[NET_IP_STR_LEN];1268 int require_auth = TLS_CLIENT_AUTH_YES;1269 UNUSED(el);1270 UNUSED(mask);1271 UNUSED(privdata);12721273 /* If the server is starting up, don't accept cluster connections:1274 * UPDATE messages may interact with the database content. */1275 if (server.masterhost == NULL && server.loading) return;12761277 while(max--) {1278 cfd = anetTcpAccept(server.neterr, fd, cip, sizeof(cip), &cport);1279 if (cfd == ANET_ERR) {1280 if (anetAcceptFailureNeedsRetry(errno))1281 continue;1282 if (errno != EWOULDBLOCK)1283 serverLog(LL_VERBOSE,1284 "Error accepting cluster node: %s", server.neterr);1285 return;1286 }12871288 connection *conn = connCreateAccepted(server.el, connTypeOfCluster(), cfd, &require_auth);12891290 /* Make sure connection is not in an error state */1291 if (connGetState(conn) != CONN_STATE_ACCEPTING) {1292 serverLog(LL_VERBOSE,1293 "Error creating an accepting connection for cluster node: %s",1294 connGetLastError(conn));1295 connClose(conn);1296 return;1297 }1298 connEnableTcpNoDelay(conn);1299 connKeepAlive(conn,server.cluster_node_timeout / 1000 * 2);13001301 /* When tls-expected-peer-name is configured, verify the connecting peer's1302 * certificate SAN/CN against it before completing the TLS handshake. This1303 * closes the cluster-bus node-ID impersonation vector: a certificate that1304 * chains to the CA but lacks the cluster identity (e.g. a sibling cert from1305 * a shared CA) cannot open a bus link and inject forged messages. No-op for1306 * non-TLS connections. */1307 if (server.tls_ctx_config.expected_peer_name != NULL &&1308 connSetVerifyName(conn, server.tls_ctx_config.expected_peer_name) == C_ERR)1309 {1310 serverLog(LL_VERBOSE,1311 "Error setting expected peer name on cluster node connection from %s:%d", cip, cport);1312 connClose(conn);1313 continue;1314 }13151316 /* Use non-blocking I/O for cluster messages. */1317 serverLog(LL_VERBOSE,"Accepting cluster node connection from %s:%d", cip, cport);13181319 /* Accept the connection now. connAccept() may call our handler directly1320 * or schedule it for later depending on connection implementation.1321 */1322 if (connAccept(conn, clusterConnAcceptHandler) == C_ERR) {1323 if (connGetState(conn) == CONN_STATE_ERROR)1324 serverLog(LL_VERBOSE,1325 "Error accepting cluster node connection: %s",1326 connGetLastError(conn));1327 connClose(conn);1328 return;1329 }1330 }1331}13321333/* Return the approximated number of sockets we are using in order to1334 * take the cluster bus connections. */1335unsigned long getClusterConnectionsCount(void) {1336 /* We decrement the number of nodes by one, since there is the1337 * "myself" node too in the list. Each node uses two file descriptors,1338 * one incoming and one outgoing, thus the multiplication by 2. */1339 return server.cluster_enabled ?1340 ((dictSize(server.cluster->nodes)-1)*2) : 0;1341}13421343/* -----------------------------------------------------------------------------1344 * CLUSTER node API1345 * -------------------------------------------------------------------------- */13461347/* Create a new cluster node, with the specified flags.1348 * If "nodename" is NULL this is considered a first handshake and a random1349 * node name is assigned to this node (it will be fixed later when we'll1350 * receive the first pong).1351 *1352 * The node is created and returned to the user, but it is not automatically1353 * added to the nodes hash table. */1354clusterNode *createClusterNode(char *nodename, int flags) {1355 clusterNode *node = zmalloc(sizeof(*node));13561357 if (nodename)1358 memcpy(node->name, nodename, CLUSTER_NAMELEN);1359 else1360 getRandomHexChars(node->name, CLUSTER_NAMELEN);1361 getRandomHexChars(node->shard_id, CLUSTER_NAMELEN);1362 node->ctime = mstime();1363 node->configEpoch = 0;1364 node->flags = flags;1365 memset(node->slots,0,sizeof(node->slots));1366 node->slot_info_pairs = NULL;1367 node->slot_info_pairs_count = 0;1368 node->numslots = 0;1369 node->numslaves = 0;1370 node->slaves = NULL;1371 node->slaveof = NULL;1372 node->last_in_ping_gossip = 0;1373 node->ping_sent = node->pong_received = 0;1374 node->data_received = 0;1375 node->fail_time = 0;1376 node->link = NULL;1377 node->inbound_link = NULL;1378 memset(node->ip,0,sizeof(node->ip));1379 node->hostname = sdsempty();1380 node->human_nodename = sdsempty();1381 node->tcp_port = 0;1382 node->cport = 0;1383 node->tls_port = 0;1384 node->fail_reports = listCreate();1385 node->voted_time = 0;1386 node->orphaned_time = 0;1387 node->repl_offset_time = 0;1388 node->repl_offset = 0;1389 listSetFreeMethod(node->fail_reports,zfree);1390 return node;1391}13921393/* This function is called every time we get a failure report from a node.1394 * The side effect is to populate the fail_reports list (or to update1395 * the timestamp of an existing report).1396 *1397 * 'failing' is the node that is in failure state according to the1398 * 'sender' node.1399 *1400 * The function returns 0 if it just updates a timestamp of an existing1401 * failure report from the same sender. 1 is returned if a new failure1402 * report is created. */1403int clusterNodeAddFailureReport(clusterNode *failing, clusterNode *sender) {1404 list *l = failing->fail_reports;1405 listNode *ln;1406 listIter li;1407 clusterNodeFailReport *fr;14081409 /* If a failure report from the same sender already exists, just update1410 * the timestamp. */1411 listRewind(l,&li);1412 while ((ln = listNext(&li)) != NULL) {1413 fr = ln->value;1414 if (fr->node == sender) {1415 fr->time = mstime();1416 return 0;1417 }1418 }14191420 /* Otherwise create a new report. */1421 fr = zmalloc(sizeof(*fr));1422 fr->node = sender;1423 fr->time = mstime();1424 listAddNodeTail(l,fr);1425 return 1;1426}14271428/* Remove failure reports that are too old, where too old means reasonably1429 * older than the global node timeout. Note that anyway for a node to be1430 * flagged as FAIL we need to have a local PFAIL state that is at least1431 * older than the global node timeout, so we don't just trust the number1432 * of failure reports from other nodes. */1433void clusterNodeCleanupFailureReports(clusterNode *node) {1434 list *l = node->fail_reports;1435 listNode *ln;1436 listIter li;1437 clusterNodeFailReport *fr;1438 mstime_t maxtime = server.cluster_node_timeout *1439 CLUSTER_FAIL_REPORT_VALIDITY_MULT;1440 mstime_t now = mstime();14411442 listRewind(l,&li);1443 while ((ln = listNext(&li)) != NULL) {1444 fr = ln->value;1445 if (now - fr->time > maxtime) listDelNode(l,ln);1446 }1447}14481449/* Remove the failing report for 'node' if it was previously considered1450 * failing by 'sender'. This function is called when a node informs us via1451 * gossip that a node is OK from its point of view (no FAIL or PFAIL flags).1452 *1453 * Note that this function is called relatively often as it gets called even1454 * when there are no nodes failing, and is O(N), however when the cluster is1455 * fine the failure reports list is empty so the function runs in constant1456 * time.1457 *1458 * The function returns 1 if the failure report was found and removed.1459 * Otherwise 0 is returned. */1460int clusterNodeDelFailureReport(clusterNode *node, clusterNode *sender) {1461 list *l = node->fail_reports;1462 listNode *ln;1463 listIter li;1464 clusterNodeFailReport *fr;14651466 /* Search for a failure report from this sender. */1467 listRewind(l,&li);1468 while ((ln = listNext(&li)) != NULL) {1469 fr = ln->value;1470 if (fr->node == sender) break;1471 }1472 if (!ln) return 0; /* No failure report from this sender. */14731474 /* Remove the failure report. */1475 listDelNode(l,ln);1476 clusterNodeCleanupFailureReports(node);1477 return 1;1478}14791480/* Return the number of external nodes that believe 'node' is failing,1481 * not including this node, that may have a PFAIL or FAIL state for this1482 * node as well. */1483int clusterNodeFailureReportsCount(clusterNode *node) {1484 clusterNodeCleanupFailureReports(node);1485 return listLength(node->fail_reports);1486}14871488int clusterNodeRemoveSlave(clusterNode *master, clusterNode *slave) {1489 int j;14901491 for (j = 0; j < master->numslaves; j++) {1492 if (master->slaves[j] == slave) {1493 if ((j+1) < master->numslaves) {1494 int remaining_slaves = (master->numslaves - j) - 1;1495 memmove(master->slaves+j,master->slaves+(j+1),1496 (sizeof(*master->slaves) * remaining_slaves));1497 }1498 master->numslaves--;1499 if (master->numslaves == 0)1500 master->flags &= ~CLUSTER_NODE_MIGRATE_TO;1501 return C_OK;1502 }1503 }1504 return C_ERR;1505}15061507int clusterNodeAddSlave(clusterNode *master, clusterNode *slave) {1508 int j;15091510 /* If it's already a slave, don't add it again. */1511 for (j = 0; j < master->numslaves; j++)1512 if (master->slaves[j] == slave) return C_ERR;1513 master->slaves = zrealloc(master->slaves,1514 sizeof(clusterNode*)*(master->numslaves+1));1515 master->slaves[master->numslaves] = slave;1516 master->numslaves++;1517 master->flags |= CLUSTER_NODE_MIGRATE_TO;1518 return C_OK;1519}15201521int clusterCountNonFailingSlaves(clusterNode *n) {1522 int j, okslaves = 0;15231524 for (j = 0; j < n->numslaves; j++)1525 if (!nodeFailed(n->slaves[j])) okslaves++;1526 return okslaves;1527}15281529/* Low level cleanup of the node structure. Only called by clusterDelNode(). */1530void freeClusterNode(clusterNode *n) {1531 sds nodename;1532 int j;15331534 /* If the node has associated slaves, we have to set1535 * all the slaves->slaveof fields to NULL (unknown). */1536 for (j = 0; j < n->numslaves; j++)1537 n->slaves[j]->slaveof = NULL;15381539 /* Remove this node from the list of slaves of its master. */1540 if (nodeIsSlave(n) && n->slaveof) clusterNodeRemoveSlave(n->slaveof,n);15411542 /* Unlink from the set of nodes. */1543 nodename = sdsnewlen(n->name, CLUSTER_NAMELEN);1544 serverAssert(dictDelete(server.cluster->nodes,nodename) == DICT_OK);1545 sdsfree(nodename);1546 sdsfree(n->hostname);1547 sdsfree(n->human_nodename);15481549 /* Release links and associated data structures. */1550 if (n->link) freeClusterLink(n->link);1551 if (n->inbound_link) freeClusterLink(n->inbound_link);1552 listRelease(n->fail_reports);1553 zfree(n->slaves);1554 zfree(n);1555}15561557/* Add a node to the nodes hash table */1558void clusterAddNode(clusterNode *node) {1559 int retval;15601561 retval = dictAdd(server.cluster->nodes,1562 sdsnewlen(node->name,CLUSTER_NAMELEN), node);1563 serverAssert(retval == DICT_OK);1564 clusterNotifyTopologyChanged(CLUSTER_TOPOLOGY_CHANGE_FLAG_NODE, NULL);1565}15661567/* Remove a node from the cluster. The function performs the high level1568 * cleanup, calling freeClusterNode() for the low level cleanup.1569 * Here we do the following:1570 *1571 * 1) Mark all the slots handled by it as unassigned.1572 * 2) Remove all the failure reports sent by this node and referenced by1573 * other nodes.1574 * 3) Remove the node from the owning shard1575 * 4) Free the node with freeClusterNode() that will in turn remove it1576 * from the hash table and from the list of slaves of its master, if1577 * it is a slave node.1578 * 5) Notify Redis of the topology change.1579 */1580void clusterDelNode(clusterNode *delnode) {1581 int j;1582 dictIterator di;1583 dictEntry *de;15841585 /* 1) Mark slots as unassigned. */1586 for (j = 0; j < CLUSTER_SLOTS; j++) {1587 if (server.cluster->importing_slots_from[j] == delnode)1588 server.cluster->importing_slots_from[j] = NULL;1589 if (server.cluster->migrating_slots_to[j] == delnode)1590 server.cluster->migrating_slots_to[j] = NULL;1591 if (server.cluster->slots[j] == delnode)1592 clusterDelSlot(j);1593 }15941595 /* 2) Remove failure reports. */1596 dictInitSafeIterator(&di, server.cluster->nodes);1597 while((de = dictNext(&di)) != NULL) {1598 clusterNode *node = dictGetVal(de);15991600 if (node == delnode) continue;1601 clusterNodeDelFailureReport(node,delnode);1602 }1603 dictResetIterator(&di);16041605 /* 3) Remove the node from the owning shard */1606 clusterRemoveNodeFromShard(delnode);16071608 /* 4) Free the node, unlinking it from the cluster. */1609 freeClusterNode(delnode);16101611 /* 5) Notify Redis of the topology change. */1612 clusterNotifyTopologyChanged(CLUSTER_TOPOLOGY_CHANGE_FLAG_NODE, NULL);1613}16141615/* Node lookup by name */1616clusterNode *clusterLookupNode(const char *name, int length) {1617 if (verifyClusterNodeId(name, length) != C_OK) return NULL;1618 sds s = sdsnewlen(name, length);1619 dictEntry *de = dictFind(server.cluster->nodes, s);1620 sdsfree(s);1621 if (de == NULL) return NULL;1622 return dictGetVal(de);1623}16241625const char *clusterGetSecret(size_t *len) {1626 if (!server.cluster) {1627 return NULL;1628 }1629 *len = CLUSTER_INTERNALSECRETLEN;1630 return server.cluster->internal_secret;1631}16321633/* Get all the nodes in my shard.1634 * Note that the list returned is not computed on the fly1635 * via slaveof; rather, it is maintained permanently to1636 * track the shard membership and its life cycle is tied1637 * to this Redis process. Therefore, the caller must not1638 * release the list. */1639list *clusterGetNodesInMyShard(clusterNode *node) {1640 sds s = sdsnewlen(node->shard_id, CLUSTER_NAMELEN);1641 dictEntry *de = dictFind(server.cluster->shards,s);1642 sdsfree(s);1643 return (de != NULL) ? dictGetVal(de) : NULL;1644}16451646/* This is only used after the handshake. When we connect a given IP/PORT1647 * as a result of CLUSTER MEET we don't have the node name yet, so we1648 * pick a random one, and will fix it when we receive the PONG request using1649 * this function. */1650void clusterRenameNode(clusterNode *node, char *newname) {1651 int retval;1652 sds s = sdsnewlen(node->name, CLUSTER_NAMELEN);16531654 serverLog(LL_DEBUG,"Renaming node %.40s into %.40s",1655 node->name, newname);1656 retval = dictDelete(server.cluster->nodes, s);1657 sdsfree(s);1658 serverAssert(retval == DICT_OK);1659 memcpy(node->name, newname, CLUSTER_NAMELEN);1660 clusterAddNode(node);1661 clusterAddNodeToShard(node->shard_id, node);1662}16631664void clusterAddNodeToShard(const char *shard_id, clusterNode *node) {1665 sds s = sdsnewlen(shard_id, CLUSTER_NAMELEN);1666 dictEntry *de = dictFind(server.cluster->shards,s);1667 if (de == NULL) {1668 list *l = listCreate();1669 listAddNodeTail(l, node);1670 serverAssert(dictAdd(server.cluster->shards, s, l) == DICT_OK);1671 } else {1672 list *l = dictGetVal(de);1673 if (listSearchKey(l, node) == NULL) {1674 listAddNodeTail(l, node);1675 }1676 sdsfree(s);1677 }1678}16791680void clusterRemoveNodeFromShard(clusterNode *node) {1681 sds s = sdsnewlen(node->shard_id, CLUSTER_NAMELEN);1682 dictEntry *de = dictFind(server.cluster->shards, s);1683 if (de != NULL) {1684 list *l = dictGetVal(de);1685 listNode *ln = listSearchKey(l, node);1686 if (ln != NULL) {1687 listDelNode(l, ln);1688 }1689 if (listLength(l) == 0) {1690 dictDelete(server.cluster->shards, s);1691 }1692 }1693 sdsfree(s);1694}16951696/* -----------------------------------------------------------------------------1697 * CLUSTER config epoch handling1698 * -------------------------------------------------------------------------- */16991700/* Return the greatest configEpoch found in the cluster, or the current1701 * epoch if greater than any node configEpoch. */1702uint64_t clusterGetMaxEpoch(void) {1703 uint64_t max = 0;1704 dictIterator di;1705 dictEntry *de;17061707 dictInitSafeIterator(&di, server.cluster->nodes);1708 while((de = dictNext(&di)) != NULL) {1709 clusterNode *node = dictGetVal(de);1710 if (node->configEpoch > max) max = node->configEpoch;1711 }1712 dictResetIterator(&di);1713 if (max < server.cluster->currentEpoch) max = server.cluster->currentEpoch;1714 return max;1715}17161717/* If this node epoch is zero or is not already the greatest across the1718 * cluster (from the POV of the local configuration), this function will:1719 *1720 * 1) Generate a new config epoch, incrementing the current epoch.1721 * 2) Assign the new epoch to this node, WITHOUT any consensus.1722 * 3) Persist the configuration on disk before sending packets with the1723 * new configuration.1724 *1725 * If the new config epoch is generated and assigned, C_OK is returned,1726 * otherwise C_ERR is returned (since the node has already the greatest1727 * configuration around) and no operation is performed.1728 *1729 * Important note: this function violates the principle that config epochs1730 * should be generated with consensus and should be unique across the cluster.1731 * However Redis Cluster uses this auto-generated new config epochs in two1732 * cases:1733 *1734 * 1) When slots are closed after importing. Otherwise resharding would be1735 * too expensive.1736 * 2) When CLUSTER FAILOVER is called with options that force a slave to1737 * failover its master even if there is not master majority able to1738 * create a new configuration epoch.1739 *1740 * Redis Cluster will not explode using this function, even in the case of1741 * a collision between this node and another node, generating the same1742 * configuration epoch unilaterally, because the config epoch conflict1743 * resolution algorithm will eventually move colliding nodes to different1744 * config epochs. However using this function may violate the "last failover1745 * wins" rule, so should only be used with care. */1746int clusterBumpConfigEpochWithoutConsensus(void) {1747 uint64_t maxEpoch = clusterGetMaxEpoch();17481749 if (myself->configEpoch == 0 ||1750 myself->configEpoch != maxEpoch)1751 {1752 server.cluster->currentEpoch++;1753 myself->configEpoch = server.cluster->currentEpoch;1754 /* Save the new config epoch and broadcast it to the other nodes. */1755 clusterDoBeforeSleep(CLUSTER_TODO_SAVE_CONFIG|1756 CLUSTER_TODO_FSYNC_CONFIG|1757 CLUSTER_TODO_BROADCAST_PONG);1758 serverLog(LL_NOTICE,1759 "New configEpoch set to %llu",1760 (unsigned long long) myself->configEpoch);1761 return C_OK;1762 } else {1763 return C_ERR;1764 }1765}17661767/* This function is called when this node is a master, and we receive from1768 * another master a configuration epoch that is equal to our configuration1769 * epoch.1770 *1771 * BACKGROUND1772 *1773 * It is not possible that different slaves get the same config1774 * epoch during a failover election, because the slaves need to get voted1775 * by a majority. However when we perform a manual resharding of the cluster1776 * the node will assign a configuration epoch to itself without to ask1777 * for agreement. Usually resharding happens when the cluster is working well1778 * and is supervised by the sysadmin, however it is possible for a failover1779 * to happen exactly while the node we are resharding a slot to assigns itself1780 * a new configuration epoch, but before it is able to propagate it.1781 *1782 * So technically it is possible in this condition that two nodes end with1783 * the same configuration epoch.1784 *1785 * Another possibility is that there are bugs in the implementation causing1786 * this to happen.1787 *1788 * Moreover when a new cluster is created, all the nodes start with the same1789 * configEpoch. This collision resolution code allows nodes to automatically1790 * end with a different configEpoch at startup automatically.1791 *1792 * In all the cases, we want a mechanism that resolves this issue automatically1793 * as a safeguard. The same configuration epoch for masters serving different1794 * set of slots is not harmful, but it is if the nodes end serving the same1795 * slots for some reason (manual errors or software bugs) without a proper1796 * failover procedure.1797 *1798 * In general we want a system that eventually always ends with different1799 * masters having different configuration epochs whatever happened, since1800 * nothing is worse than a split-brain condition in a distributed system.1801 *1802 * BEHAVIOR1803 *1804 * When this function gets called, what happens is that if this node1805 * has the lexicographically smaller Node ID compared to the other node1806 * with the conflicting epoch (the 'sender' node), it will assign itself1807 * the greatest configuration epoch currently detected among nodes plus 1.1808 *1809 * This means that even if there are multiple nodes colliding, the node1810 * with the greatest Node ID never moves forward, so eventually all the nodes1811 * end with a different configuration epoch.1812 */1813void clusterHandleConfigEpochCollision(clusterNode *sender) {1814 /* Prerequisites: nodes have the same configEpoch and are both masters. */1815 if (sender->configEpoch != myself->configEpoch ||1816 !clusterNodeIsMaster(sender) || !clusterNodeIsMaster(myself)) return;1817 /* Don't act if the colliding node has a smaller Node ID. */1818 if (memcmp(sender->name,myself->name,CLUSTER_NAMELEN) <= 0) return;1819 /* Get the next ID available at the best of this node knowledge. */1820 server.cluster->currentEpoch++;1821 myself->configEpoch = server.cluster->currentEpoch;1822 clusterSaveConfigOrDie(1);1823 /* Broadcast new config epoch to the other nodes. */1824 clusterDoBeforeSleep(CLUSTER_TODO_BROADCAST_PONG);1825 serverLog(LL_VERBOSE,1826 "WARNING: configEpoch collision with node %.40s (%s)."1827 " configEpoch set to %llu",1828 sender->name,sender->human_nodename,1829 (unsigned long long) myself->configEpoch);1830}18311832/* -----------------------------------------------------------------------------1833 * CLUSTER nodes blacklist1834 *1835 * The nodes blacklist is just a way to ensure that a given node with a given1836 * Node ID is not re-added before some time elapsed (this time is specified1837 * in seconds in CLUSTER_BLACKLIST_TTL).1838 *1839 * This is useful when we want to remove a node from the cluster completely:1840 * when CLUSTER FORGET is called, it also puts the node into the blacklist so1841 * that even if we receive gossip messages from other nodes that still remember1842 * about the node we want to remove, we don't re-add it before some time.1843 *1844 * Currently the CLUSTER_BLACKLIST_TTL is set to 1 minute, this means1845 * that redis-cli has 60 seconds to send CLUSTER FORGET messages to nodes1846 * in the cluster without dealing with the problem of other nodes re-adding1847 * back the node to nodes we already sent the FORGET command to.1848 *1849 * The data structure used is a hash table with an sds string representing1850 * the node ID as key, and the time when it is ok to re-add the node as1851 * value.1852 * -------------------------------------------------------------------------- */18531854#define CLUSTER_BLACKLIST_TTL 60 /* 1 minute. */185518561857/* Before of the addNode() or Exists() operations we always remove expired1858 * entries from the black list. This is an O(N) operation but it is not a1859 * problem since add / exists operations are called very infrequently and1860 * the hash table is supposed to contain very little elements at max.1861 * However without the cleanup during long uptime and with some automated1862 * node add/removal procedures, entries could accumulate. */1863void clusterBlacklistCleanup(void) {1864 dictIterator di;1865 dictEntry *de;18661867 dictInitSafeIterator(&di, server.cluster->nodes_black_list);1868 while((de = dictNext(&di)) != NULL) {1869 int64_t expire = dictGetUnsignedIntegerVal(de);18701871 if (expire < server.unixtime)1872 dictDelete(server.cluster->nodes_black_list,dictGetKey(de));1873 }1874 dictResetIterator(&di);1875}18761877/* Cleanup the blacklist and add a new node ID to the black list. */1878void clusterBlacklistAddNode(clusterNode *node) {1879 dictEntry *de;1880 sds id = sdsnewlen(node->name,CLUSTER_NAMELEN);18811882 clusterBlacklistCleanup();1883 if (dictAdd(server.cluster->nodes_black_list,id,NULL) == DICT_OK) {1884 /* If the key was added, duplicate the sds string representation of1885 * the key for the next lookup. We'll free it at the end. */1886 id = sdsdup(id);1887 }1888 de = dictFind(server.cluster->nodes_black_list,id);1889 dictSetUnsignedIntegerVal(de,time(NULL)+CLUSTER_BLACKLIST_TTL);1890 sdsfree(id);1891}18921893/* Return non-zero if the specified node ID exists in the blacklist.1894 * You don't need to pass an sds string here, any pointer to 40 bytes1895 * will work. */1896int clusterBlacklistExists(char *nodeid, size_t len) {1897 sds id = sdsnewlen(nodeid,len);1898 int retval;18991900 clusterBlacklistCleanup();1901 retval = dictFind(server.cluster->nodes_black_list,id) != NULL;1902 sdsfree(id);1903 return retval;1904}19051906/* -----------------------------------------------------------------------------1907 * CLUSTER messages exchange - PING/PONG and gossip1908 * -------------------------------------------------------------------------- */19091910/* This function checks if a given node should be marked as FAIL.1911 * It happens if the following conditions are met:1912 *1913 * 1) We received enough failure reports from other master nodes via gossip.1914 * Enough means that the majority of the masters signaled the node is1915 * down recently.1916 * 2) We believe this node is in PFAIL state.1917 *1918 * If a failure is detected we also inform the whole cluster about this1919 * event trying to force every other node to set the FAIL flag for the node.1920 *1921 * Note that the form of agreement used here is weak, as we collect the majority1922 * of masters state during some time, and even if we force agreement by1923 * propagating the FAIL message, because of partitions we may not reach every1924 * node. However:1925 *1926 * 1) Either we reach the majority and eventually the FAIL state will propagate1927 * to all the cluster.1928 * 2) Or there is no majority so no slave promotion will be authorized and the1929 * FAIL flag will be cleared after some time.1930 */1931void markNodeAsFailingIfNeeded(clusterNode *node) {1932 int failures;1933 int needed_quorum = (server.cluster->size / 2) + 1;19341935 if (!nodeTimedOut(node)) return; /* We can reach it. */1936 if (nodeFailed(node)) return; /* Already FAILing. */19371938 failures = clusterNodeFailureReportsCount(node);1939 /* Also count myself as a voter if I'm a master. */1940 if (clusterNodeIsMaster(myself)) failures++;1941 if (failures < needed_quorum) return; /* No weak agreement from masters. */19421943 serverLog(LL_NOTICE,1944 "Marking node %.40s (%s) as failing (quorum reached).", node->name, node->human_nodename);19451946 /* Mark the node as failing. */1947 node->flags &= ~CLUSTER_NODE_PFAIL;1948 node->flags |= CLUSTER_NODE_FAIL;1949 node->fail_time = mstime();19501951 /* Broadcast the failing node name to everybody, forcing all the other1952 * reachable nodes to flag the node as FAIL.1953 * We do that even if this node is a replica and not a master: anyway1954 * the failing state is triggered collecting failure reports from masters,1955 * so here the replica is only helping propagating this status. */1956 clusterSendFail(node->name);1957 clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG);1958}19591960/* This function is called only if a node is marked as FAIL, but we are able1961 * to reach it again. It checks if there are the conditions to undo the FAIL1962 * state. */1963void clearNodeFailureIfNeeded(clusterNode *node) {1964 mstime_t now = mstime();19651966 serverAssert(nodeFailed(node));19671968 /* For slaves we always clear the FAIL flag if we can contact the1969 * node again. */1970 if (nodeIsSlave(node) || node->numslots == 0) {1971 serverLog(LL_NOTICE,1972 "Clear FAIL state for node %.40s (%s):%s is reachable again.",1973 node->name,node->human_nodename,1974 nodeIsSlave(node) ? "replica" : "master without slots");1975 node->flags &= ~CLUSTER_NODE_FAIL;1976 clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG);1977 }19781979 /* If it is a master and...1980 * 1) The FAIL state is old enough.1981 * 2) It is yet serving slots from our point of view (not failed over).1982 * Apparently no one is going to fix these slots, clear the FAIL flag. */1983 if (clusterNodeIsMaster(node) && node->numslots > 0 &&1984 (now - node->fail_time) >1985 (server.cluster_node_timeout * CLUSTER_FAIL_UNDO_TIME_MULT))1986 {1987 serverLog(LL_NOTICE,1988 "Clear FAIL state for node %.40s (%s): is reachable again and nobody is serving its slots after some time.",1989 node->name, node->human_nodename);1990 node->flags &= ~CLUSTER_NODE_FAIL;1991 clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG);1992 }1993}19941995/* Return true if we already have a node in HANDSHAKE state matching the1996 * specified ip address and port number. This function is used in order to1997 * avoid adding a new handshake node for the same address multiple times. */1998int clusterHandshakeInProgress(char *ip, int port, int cport) {1999 dictIterator di;2000 dictEntry *de;
Findings
✓ No findings reported for this file.