1/* Redis Sentinel implementation2 *3 * Copyright (c) 2009-Present, Redis Ltd.4 * All rights reserved.5 *6 * Licensed under your choice of (a) the Redis Source Available License 2.07 * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the8 * GNU Affero General Public License v3 (AGPLv3).9 */1011#include "server.h"12#include "hiredis.h"13#if USE_OPENSSL == 1 /* BUILD_YES */14#include "openssl/ssl.h"15#include "hiredis_ssl.h"16#endif17#include "async.h"1819#include <ctype.h>20#include <arpa/inet.h>21#include <sys/socket.h>22#include <sys/wait.h>23#include <fcntl.h>2425extern char **environ;2627#if USE_OPENSSL == 1 /* BUILD_YES */28extern SSL_CTX *redis_tls_ctx;29extern SSL_CTX *redis_tls_client_ctx;30#endif3132#define REDIS_SENTINEL_PORT 263793334/* ======================== Sentinel global state =========================== */3536/* Address object, used to describe an ip:port pair. */37typedef struct sentinelAddr {38    char *hostname;         /* Hostname OR address, as specified */39    char *ip;               /* Always a resolved address */40    int port;41} sentinelAddr;4243/* A Sentinel Redis Instance object is monitoring. */44#define SRI_MASTER  (1<<0)45#define SRI_SLAVE   (1<<1)46#define SRI_SENTINEL (1<<2)47#define SRI_S_DOWN (1<<3)   /* Subjectively down (no quorum). */48#define SRI_O_DOWN (1<<4)   /* Objectively down (confirmed by others). */49#define SRI_MASTER_DOWN (1<<5) /* A Sentinel with this flag set thinks that50                                   its master is down. */51#define SRI_FAILOVER_IN_PROGRESS (1<<6) /* Failover is in progress for52                                           this master. */53#define SRI_PROMOTED (1<<7)            /* Slave selected for promotion. */54#define SRI_RECONF_SENT (1<<8)     /* SLAVEOF <newmaster> sent. */55#define SRI_RECONF_INPROG (1<<9)   /* Slave synchronization in progress. */56#define SRI_RECONF_DONE (1<<10)     /* Slave synchronized with new master. */57#define SRI_FORCE_FAILOVER (1<<11)  /* Force failover with master up. */58#define SRI_SCRIPT_KILL_SENT (1<<12) /* SCRIPT KILL already sent on -BUSY */59#define SRI_MASTER_REBOOT  (1<<13)   /* Master was detected as rebooting */60/* Note: when adding new flags, please check the flags section in addReplySentinelRedisInstance. */6162/* Note: times are in milliseconds. */63#define SENTINEL_PING_PERIOD 10006465static mstime_t sentinel_info_period = 10000;66static mstime_t sentinel_ping_period = SENTINEL_PING_PERIOD;67static mstime_t sentinel_ask_period = 1000;68static mstime_t sentinel_publish_period = 2000;69static mstime_t sentinel_default_down_after = 30000;70static mstime_t sentinel_tilt_trigger = 2000;71static mstime_t sentinel_tilt_period = SENTINEL_PING_PERIOD * 30;72static mstime_t sentinel_slave_reconf_timeout = 10000;73static mstime_t sentinel_min_link_reconnect_period = 15000;74static mstime_t sentinel_election_timeout = 10000;75static mstime_t sentinel_script_max_runtime = 60000;  /* 60 seconds max exec time. */76static mstime_t sentinel_script_retry_delay = 30000;  /* 30 seconds between retries. */77static mstime_t sentinel_default_failover_timeout = 60*3*1000;7879#define SENTINEL_HELLO_CHANNEL "__sentinel__:hello"80#define SENTINEL_DEFAULT_SLAVE_PRIORITY 10081#define SENTINEL_DEFAULT_PARALLEL_SYNCS 182#define SENTINEL_MAX_PENDING_COMMANDS 1008384#define SENTINEL_MAX_DESYNC 100085#define SENTINEL_DEFAULT_DENY_SCRIPTS_RECONFIG 186#define SENTINEL_DEFAULT_RESOLVE_HOSTNAMES 087#define SENTINEL_DEFAULT_ANNOUNCE_HOSTNAMES 08889/* Failover machine different states. */90#define SENTINEL_FAILOVER_STATE_NONE 0  /* No failover in progress. */91#define SENTINEL_FAILOVER_STATE_WAIT_START 1  /* Wait for failover_start_time*/92#define SENTINEL_FAILOVER_STATE_SELECT_SLAVE 2 /* Select slave to promote */93#define SENTINEL_FAILOVER_STATE_SEND_SLAVEOF_NOONE 3 /* Slave -> Master */94#define SENTINEL_FAILOVER_STATE_WAIT_PROMOTION 4 /* Wait slave to change role */95#define SENTINEL_FAILOVER_STATE_RECONF_SLAVES 5 /* SLAVEOF newmaster */96#define SENTINEL_FAILOVER_STATE_UPDATE_CONFIG 6 /* Monitor promoted slave. */9798#define SENTINEL_MASTER_LINK_STATUS_UP 099#define SENTINEL_MASTER_LINK_STATUS_DOWN 1100101/* Generic flags that can be used with different functions.102 * They use higher bits to avoid colliding with the function specific103 * flags. */104#define SENTINEL_NO_FLAGS 0105#define SENTINEL_GENERATE_EVENT (1<<16)106#define SENTINEL_LEADER (1<<17)107#define SENTINEL_OBSERVER (1<<18)108109/* Script execution flags and limits. */110#define SENTINEL_SCRIPT_NONE 0111#define SENTINEL_SCRIPT_RUNNING 1112#define SENTINEL_SCRIPT_MAX_QUEUE 256113#define SENTINEL_SCRIPT_MAX_RUNNING 16114#define SENTINEL_SCRIPT_MAX_RETRY 10115116/* SENTINEL SIMULATE-FAILURE command flags. */117#define SENTINEL_SIMFAILURE_NONE 0118#define SENTINEL_SIMFAILURE_CRASH_AFTER_ELECTION (1<<0)119#define SENTINEL_SIMFAILURE_CRASH_AFTER_PROMOTION (1<<1)120121/* The link to a sentinelRedisInstance. When we have the same set of Sentinels122 * monitoring many masters, we have different instances representing the123 * same Sentinels, one per master, and we need to share the hiredis connections124 * among them. Otherwise if 5 Sentinels are monitoring 100 masters we create125 * 500 outgoing connections instead of 5.126 *127 * So this structure represents a reference counted link in terms of the two128 * hiredis connections for commands and Pub/Sub, and the fields needed for129 * failure detection, since the ping/pong time are now local to the link: if130 * the link is available, the instance is available. This way we don't just131 * have 5 connections instead of 500, we also send 5 pings instead of 500.132 *133 * Links are shared only for Sentinels: master and slave instances have134 * a link with refcount = 1, always. */135typedef struct instanceLink {136    int refcount;          /* Number of sentinelRedisInstance owners. */137    int disconnected;      /* Non-zero if we need to reconnect cc or pc. */138    int pending_commands;  /* Number of commands sent waiting for a reply. */139    redisAsyncContext *cc; /* Hiredis context for commands. */140    redisAsyncContext *pc; /* Hiredis context for Pub / Sub. */141    mstime_t cc_conn_time; /* cc connection time. */142    mstime_t pc_conn_time; /* pc connection time. */143    mstime_t pc_last_activity; /* Last time we received any message. */144    mstime_t last_avail_time; /* Last time the instance replied to ping with145                                 a reply we consider valid. */146    mstime_t act_ping_time;   /* Time at which the last pending ping (no pong147                                 received after it) was sent. This field is148                                 set to 0 when a pong is received, and set again149                                 to the current time if the value is 0 and a new150                                 ping is sent. */151    mstime_t last_ping_time;  /* Time at which we sent the last ping. This is152                                 only used to avoid sending too many pings153                                 during failure. Idle time is computed using154                                 the act_ping_time field. */155    mstime_t last_pong_time;  /* Last time the instance replied to ping,156                                 whatever the reply was. That's used to check157                                 if the link is idle and must be reconnected. */158    mstime_t last_reconn_time;  /* Last reconnection attempt performed when159                                   the link was down. */160} instanceLink;161162typedef struct sentinelRedisInstance {163    int flags;      /* See SRI_... defines */164    char *name;     /* Master name from the point of view of this sentinel. */165    char *runid;    /* Run ID of this instance, or unique ID if is a Sentinel.*/166    uint64_t config_epoch;  /* Configuration epoch. */167    sentinelAddr *addr; /* Master host. */168    instanceLink *link; /* Link to the instance, may be shared for Sentinels. */169    mstime_t last_pub_time;   /* Last time we sent hello via Pub/Sub. */170    mstime_t last_hello_time; /* Only used if SRI_SENTINEL is set. Last time171                                 we received a hello from this Sentinel172                                 via Pub/Sub. */173    mstime_t last_master_down_reply_time; /* Time of last reply to174                                             SENTINEL is-master-down command. */175    mstime_t s_down_since_time; /* Subjectively down since time. */176    mstime_t o_down_since_time; /* Objectively down since time. */177    mstime_t down_after_period; /* Consider it down after that period. */178    mstime_t master_reboot_down_after_period; /* Consider master down after that period. */179    mstime_t master_reboot_since_time; /* master reboot time since time. */180    mstime_t info_refresh;  /* Time at which we received INFO output from it. */181    dict *renamed_commands;     /* Commands renamed in this instance:182                                   Sentinel will use the alternative commands183                                   mapped on this table to send things like184                                   SLAVEOF, CONFIG, INFO, ... */185186    /* Role and the first time we observed it.187     * This is useful in order to delay replacing what the instance reports188     * with our own configuration. We need to always wait some time in order189     * to give a chance to the leader to report the new configuration before190     * we do silly things. */191    int role_reported;192    mstime_t role_reported_time;193    mstime_t slave_conf_change_time; /* Last time slave master addr changed. */194195    /* Master specific. */196    dict *sentinels;    /* Other sentinels monitoring the same master. */197    dict *slaves;       /* Slaves for this master instance. */198    unsigned int quorum;/* Number of sentinels that need to agree on failure. */199    int parallel_syncs; /* How many slaves to reconfigure at same time. */200    char *auth_pass;    /* Password to use for AUTH against master & replica. */201    char *auth_user;    /* Username for ACLs AUTH against master & replica. */202203    /* Slave specific. */204    mstime_t master_link_down_time; /* Slave replication link down time. */205    int slave_priority; /* Slave priority according to its INFO output. */206    int replica_announced; /* Replica announcing according to its INFO output. */207    mstime_t slave_reconf_sent_time; /* Time at which we sent SLAVE OF <new> */208    struct sentinelRedisInstance *master; /* Master instance if it's slave. */209    char *slave_master_host;    /* Master host as reported by INFO */210    int slave_master_port;      /* Master port as reported by INFO */211    int slave_master_link_status; /* Master link status as reported by INFO */212    unsigned long long slave_repl_offset; /* Slave replication offset. */213    /* Failover */214    char *leader;       /* If this is a master instance, this is the runid of215                           the Sentinel that should perform the failover. If216                           this is a Sentinel, this is the runid of the Sentinel217                           that this Sentinel voted as leader. */218    uint64_t leader_epoch; /* Epoch of the 'leader' field. */219    uint64_t failover_epoch; /* Epoch of the currently started failover. */220    int failover_state; /* See SENTINEL_FAILOVER_STATE_* defines. */221    mstime_t failover_state_change_time;222    mstime_t failover_start_time;   /* Last failover attempt start time. */223    mstime_t failover_timeout;      /* Max time to refresh failover state. */224    mstime_t failover_delay_logged; /* For what failover_start_time value we225                                       logged the failover delay. */226    struct sentinelRedisInstance *promoted_slave; /* Promoted slave instance. */227    /* Scripts executed to notify admin or reconfigure clients: when they228     * are set to NULL no script is executed. */229    char *notification_script;230    char *client_reconfig_script;231    sds info; /* cached INFO output */232} sentinelRedisInstance;233234/* Main state. */235struct sentinelState {236    char myid[CONFIG_RUN_ID_SIZE+1]; /* This sentinel ID. */237    uint64_t current_epoch;         /* Current epoch. */238    dict *masters;      /* Dictionary of master sentinelRedisInstances.239                           Key is the instance name, value is the240                           sentinelRedisInstance structure pointer. */241    int tilt;           /* Are we in TILT mode? */242    int total_tilt;   /* Number of tilt. */243    int running_scripts;    /* Number of scripts in execution right now. */244    mstime_t tilt_start_time;       /* When TITL started. */245    mstime_t previous_time;         /* Last time we ran the time handler. */246    list *scripts_queue;            /* Queue of user scripts to execute. */247    char *announce_ip;  /* IP addr that is gossiped to other sentinels if248                           not NULL. */249    int announce_port;  /* Port that is gossiped to other sentinels if250                           non zero. */251    unsigned long simfailure_flags; /* Failures simulation. */252    int deny_scripts_reconfig; /* Allow SENTINEL SET ... to change script253                                  paths at runtime? */254    char *sentinel_auth_pass;    /* Password to use for AUTH against other sentinel */255    char *sentinel_auth_user;    /* Username for ACLs AUTH against other sentinel. */256    int resolve_hostnames;       /* Support use of hostnames, assuming DNS is well configured. */257    int announce_hostnames;      /* Announce hostnames instead of IPs when we have them. */258} sentinel;259260/* A script execution job. */261typedef struct sentinelScriptJob {262    int flags;              /* Script job flags: SENTINEL_SCRIPT_* */263    int retry_num;          /* Number of times we tried to execute it. */264    char **argv;            /* Arguments to call the script. */265    mstime_t start_time;    /* Script execution time if the script is running,266                               otherwise 0 if we are allowed to retry the267                               execution at any time. If the script is not268                               running and it's not 0, it means: do not run269                               before the specified time. */270    pid_t pid;              /* Script execution pid. */271} sentinelScriptJob;272273/* ======================= hiredis ae.c adapters =============================274 * Note: this implementation is taken from hiredis/adapters/ae.h, however275 * we have our modified copy for Sentinel in order to use our allocator276 * and to have full control over how the adapter works. */277278typedef struct redisAeEvents {279    redisAsyncContext *context;280    aeEventLoop *loop;281    int fd;282    int reading, writing;283} redisAeEvents;284285static void redisAeReadEvent(aeEventLoop *el, int fd, void *privdata, int mask) {286    ((void)el); ((void)fd); ((void)mask);287288    redisAeEvents *e = (redisAeEvents*)privdata;289    redisAsyncHandleRead(e->context);290}291292static void redisAeWriteEvent(aeEventLoop *el, int fd, void *privdata, int mask) {293    ((void)el); ((void)fd); ((void)mask);294295    redisAeEvents *e = (redisAeEvents*)privdata;296    redisAsyncHandleWrite(e->context);297}298299static void redisAeAddRead(void *privdata) {300    redisAeEvents *e = (redisAeEvents*)privdata;301    aeEventLoop *loop = e->loop;302    if (!e->reading) {303        e->reading = 1;304        aeCreateFileEvent(loop,e->fd,AE_READABLE,redisAeReadEvent,e);305    }306}307308static void redisAeDelRead(void *privdata) {309    redisAeEvents *e = (redisAeEvents*)privdata;310    aeEventLoop *loop = e->loop;311    if (e->reading) {312        e->reading = 0;313        aeDeleteFileEvent(loop,e->fd,AE_READABLE);314    }315}316317static void redisAeAddWrite(void *privdata) {318    redisAeEvents *e = (redisAeEvents*)privdata;319    aeEventLoop *loop = e->loop;320    if (!e->writing) {321        e->writing = 1;322        aeCreateFileEvent(loop,e->fd,AE_WRITABLE,redisAeWriteEvent,e);323    }324}325326static void redisAeDelWrite(void *privdata) {327    redisAeEvents *e = (redisAeEvents*)privdata;328    aeEventLoop *loop = e->loop;329    if (e->writing) {330        e->writing = 0;331        aeDeleteFileEvent(loop,e->fd,AE_WRITABLE);332    }333}334335static void redisAeCleanup(void *privdata) {336    redisAeEvents *e = (redisAeEvents*)privdata;337    redisAeDelRead(privdata);338    redisAeDelWrite(privdata);339    zfree(e);340}341342static int redisAeAttach(aeEventLoop *loop, redisAsyncContext *ac) {343    redisContext *c = &(ac->c);344    redisAeEvents *e;345346    /* Nothing should be attached when something is already attached */347    if (ac->ev.data != NULL)348        return C_ERR;349350    /* Create container for context and r/w events */351    e = (redisAeEvents*)zmalloc(sizeof(*e));352    e->context = ac;353    e->loop = loop;354    e->fd = c->fd;355    e->reading = e->writing = 0;356357    /* Register functions to start/stop listening for events */358    ac->ev.addRead = redisAeAddRead;359    ac->ev.delRead = redisAeDelRead;360    ac->ev.addWrite = redisAeAddWrite;361    ac->ev.delWrite = redisAeDelWrite;362    ac->ev.cleanup = redisAeCleanup;363    ac->ev.data = e;364365    return C_OK;366}367368/* ============================= Prototypes ================================= */369370void sentinelLinkEstablishedCallback(const redisAsyncContext *c, int status);371void sentinelDisconnectCallback(const redisAsyncContext *c, int status);372void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privdata);373sentinelRedisInstance *sentinelGetMasterByName(char *name);374char *sentinelGetSubjectiveLeader(sentinelRedisInstance *master);375char *sentinelGetObjectiveLeader(sentinelRedisInstance *master);376void instanceLinkConnectionError(const redisAsyncContext *c);377const char *sentinelRedisInstanceTypeStr(sentinelRedisInstance *ri);378void sentinelAbortFailover(sentinelRedisInstance *ri);379void sentinelEvent(int level, char *type, sentinelRedisInstance *ri, const char *fmt, ...);380sentinelRedisInstance *sentinelSelectSlave(sentinelRedisInstance *master);381void sentinelScheduleScriptExecution(char *path, ...);382void sentinelStartFailover(sentinelRedisInstance *master);383void sentinelDiscardReplyCallback(redisAsyncContext *c, void *reply, void *privdata);384int sentinelSendSlaveOf(sentinelRedisInstance *ri, const sentinelAddr *addr);385char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char *req_runid, uint64_t *leader_epoch);386int sentinelFlushConfig(void);387void sentinelGenerateInitialMonitorEvents(void);388int sentinelSendPing(sentinelRedisInstance *ri);389int sentinelForceHelloUpdateForMaster(sentinelRedisInstance *master);390sentinelRedisInstance *getSentinelRedisInstanceByAddrAndRunID(dict *instances, char *ip, int port, char *runid);391void sentinelSimFailureCrash(void);392393/* ========================= Dictionary types =============================== */394395void releaseSentinelRedisInstance(sentinelRedisInstance *ri);396397void dictInstancesValDestructor (dict *d, void *obj) {398    UNUSED(d);399    releaseSentinelRedisInstance(obj);400}401402/* Instance name (sds) -> instance (sentinelRedisInstance pointer)403 *404 * also used for: sentinelRedisInstance->sentinels dictionary that maps405 * sentinels ip:port to last seen time in Pub/Sub hello message. */406dictType instancesDictType = {407    dictSdsHash,               /* hash function */408    NULL,                      /* key dup */409    NULL,                      /* val dup */410    dictSdsKeyCompare,         /* key compare */411    NULL,                      /* key destructor */412    dictInstancesValDestructor,/* val destructor */413    NULL                       /* allow to expand */414};415416/* Instance runid (sds) -> votes (long casted to void*)417 *418 * This is useful into sentinelGetObjectiveLeader() function in order to419 * count the votes and understand who is the leader. */420dictType leaderVotesDictType = {421    dictSdsHash,               /* hash function */422    NULL,                      /* key dup */423    NULL,                      /* val dup */424    dictSdsKeyCompare,         /* key compare */425    NULL,                      /* key destructor */426    NULL,                      /* val destructor */427    NULL                       /* allow to expand */428};429430/* Instance renamed commands table. */431dictType renamedCommandsDictType = {432    dictSdsCaseHash,           /* hash function */433    NULL,                      /* key dup */434    NULL,                      /* val dup */435    dictSdsKeyCaseCompare,     /* key compare */436    dictSdsDestructor,         /* key destructor */437    dictSdsDestructor,         /* val destructor */438    NULL                       /* allow to expand */439};440441/* =========================== Initialization =============================== */442443void sentinelSetCommand(client *c);444void sentinelConfigGetCommand(client *c);445void sentinelConfigSetCommand(client *c);446447/* this array is used for sentinel config lookup, which need to be loaded448 * before monitoring masters config to avoid dependency issues */449const char *preMonitorCfgName[] = { 450    "announce-ip",451    "announce-port",452    "deny-scripts-reconfig",453    "sentinel-user",454    "sentinel-pass",455    "current-epoch",456    "myid",457    "resolve-hostnames",458    "announce-hostnames"459};460461/* Returns 1 if the string contains control characters (0x00-0x1F or 0x7F),462 * which must be rejected to prevent config injection via newlines/etc. */463int sentinelStringContainsControlChars(sds s) {464    for (size_t i = 0; i < sdslen(s); i++) {465        unsigned char c = (unsigned char)s[i];466        if (c < 0x20 || c == 0x7F) return 1;467    }468    return 0;469}470471/* Append an sds value to dest, quoting it with sdscatrepr only if the value472 * contains characters that need escaping (spaces, quotes, control chars, etc.).473 * Simple values are appended as-is, preserving the traditional config format. */474static sds sentinelSdscatConfigArg(sds dest, sds value) {475    if (sdsneedsrepr(value))476        return sdscatrepr(dest, value, sdslen(value));477    return sdscatsds(dest, value);478}479480/* This function overwrites a few normal Redis config default with Sentinel481 * specific defaults. */482void initSentinelConfig(void) {483    server.port = REDIS_SENTINEL_PORT;484    server.protected_mode = 0; /* Sentinel must be exposed. */485}486487void freeSentinelLoadQueueEntry(void *item);488489/* Perform the Sentinel mode initialization. */490void initSentinel(void) {491    /* Initialize various data structures. */492    sentinel.current_epoch = 0;493    sentinel.masters = dictCreate(&instancesDictType);494    sentinel.tilt = 0;495    sentinel.tilt_start_time = 0;496    sentinel.total_tilt = 0;497    sentinel.previous_time = mstime();498    sentinel.running_scripts = 0;499    sentinel.scripts_queue = listCreate();500    sentinel.announce_ip = NULL;501    sentinel.announce_port = 0;502    sentinel.simfailure_flags = SENTINEL_SIMFAILURE_NONE;503    sentinel.deny_scripts_reconfig = SENTINEL_DEFAULT_DENY_SCRIPTS_RECONFIG;504    sentinel.sentinel_auth_pass = NULL;505    sentinel.sentinel_auth_user = NULL;506    sentinel.resolve_hostnames = SENTINEL_DEFAULT_RESOLVE_HOSTNAMES;507    sentinel.announce_hostnames = SENTINEL_DEFAULT_ANNOUNCE_HOSTNAMES;508    memset(sentinel.myid,0,sizeof(sentinel.myid));509    server.sentinel_config = NULL;510}511512/* This function is for checking whether sentinel config file has been set,513 * also checking whether we have write permissions. */514void sentinelCheckConfigFile(void) {515    if (server.configfile == NULL) {516        serverLog(LL_WARNING,517            "Sentinel needs config file on disk to save state. Exiting...");518        exit(1);519    } else if (access(server.configfile,W_OK) == -1) {520        serverLog(LL_WARNING,521            "Sentinel config file %s is not writable: %s. Exiting...",522            server.configfile,strerror(errno));523        exit(1);524    }525}526527/* This function gets called when the server is in Sentinel mode, started,528 * loaded the configuration, and is ready for normal operations. */529void sentinelIsRunning(void) {530    int j;531532    /* If this Sentinel has yet no ID set in the configuration file, we533     * pick a random one and persist the config on disk. From now on this534     * will be this Sentinel ID across restarts. */535    for (j = 0; j < CONFIG_RUN_ID_SIZE; j++)536        if (sentinel.myid[j] != 0) break;537538    if (j == CONFIG_RUN_ID_SIZE) {539        /* Pick ID and persist the config. */540        getRandomHexChars(sentinel.myid,CONFIG_RUN_ID_SIZE);541        sentinelFlushConfig();542    }543544    /* Log its ID to make debugging of issues simpler. */545    serverLog(LL_NOTICE,"Sentinel ID is %s", sentinel.myid);546547    /* We want to generate a +monitor event for every configured master548     * at startup. */549    sentinelGenerateInitialMonitorEvents();550}551552/* ============================== sentinelAddr ============================== */553554/* Create a sentinelAddr object and return it on success.555 * On error NULL is returned and errno is set to:556 *  ENOENT: Can't resolve the hostname, unless accept_unresolved is non-zero.557 *  EINVAL: Invalid port number.558 */559sentinelAddr *createSentinelAddr(char *hostname, int port, int is_accept_unresolved) {560    char ip[NET_IP_STR_LEN];561    sentinelAddr *sa;562563    if (port < 0 || port > 65535) {564        errno = EINVAL;565        return NULL;566    }567    if (anetResolve(NULL,hostname,ip,sizeof(ip),568                    sentinel.resolve_hostnames ? ANET_NONE : ANET_IP_ONLY) == ANET_ERR) {569        serverLog(LL_WARNING, "Failed to resolve hostname '%s'", hostname);570        if (sentinel.resolve_hostnames && is_accept_unresolved) {571            ip[0] = '\0';572        }573        else {574            errno = ENOENT;575            return NULL;576        }577    }578    sa = zmalloc(sizeof(*sa));579    sa->hostname = sdsnew(hostname);580    sa->ip = sdsnew(ip);581    sa->port = port;582    return sa;583}584585/* Return a duplicate of the source address. */586sentinelAddr *dupSentinelAddr(sentinelAddr *src) {587    sentinelAddr *sa;588589    sa = zmalloc(sizeof(*sa));590    sa->hostname = sdsnew(src->hostname);591    sa->ip = sdsnew(src->ip);592    sa->port = src->port;593    return sa;594}595596/* Free a Sentinel address. Can't fail. */597void releaseSentinelAddr(sentinelAddr *sa) {598    sdsfree(sa->hostname);599    sdsfree(sa->ip);600    zfree(sa);601}602603/* Return non-zero if the two addresses are equal, either by address604 * or by hostname if they could not have been resolved.605 */606int sentinelAddrOrHostnameEqual(sentinelAddr *a, sentinelAddr *b) {607    return a->port == b->port &&608            (!strcmp(a->ip, b->ip)  ||609            !strcasecmp(a->hostname, b->hostname));610}611612/* Return non-zero if a hostname matches an address. */613int sentinelAddrEqualsHostname(sentinelAddr *a, char *hostname) {614    char ip[NET_IP_STR_LEN];615616    /* Try resolve the hostname and compare it to the address */617    if (anetResolve(NULL, hostname, ip, sizeof(ip),618                    sentinel.resolve_hostnames ? ANET_NONE : ANET_IP_ONLY) == ANET_ERR) {619620        /* If failed resolve then compare based on hostnames. That is our best effort as621         * long as the server is unavailable for some reason. It is fine since Redis 622         * instance cannot have multiple hostnames for a given setup */623        return !strcasecmp(sentinel.resolve_hostnames ? a->hostname : a->ip, hostname);624    }625    /* Compare based on address */626    return !strcasecmp(a->ip, ip);627}628629const char *announceSentinelAddr(const sentinelAddr *a) {630    return sentinel.announce_hostnames ? a->hostname : a->ip;631}632633/* Return an allocated sds with hostname/address:port. IPv6634 * addresses are bracketed the same way anetFormatAddr() does.635 */636sds announceSentinelAddrAndPort(const sentinelAddr *a) {637    const char *addr = announceSentinelAddr(a);638    if (strchr(addr, ':') != NULL)639        return sdscatprintf(sdsempty(), "[%s]:%d", addr, a->port);640    else641        return sdscatprintf(sdsempty(), "%s:%d", addr, a->port);642}643644/* =========================== Events notification ========================== */645646/* Send an event to log, pub/sub, user notification script.647 *648 * 'level' is the log level for logging. Only LL_WARNING events will trigger649 * the execution of the user notification script.650 *651 * 'type' is the message type, also used as a pub/sub channel name.652 *653 * 'ri', is the redis instance target of this event if applicable, and is654 * used to obtain the path of the notification script to execute.655 *656 * The remaining arguments are printf-alike.657 * If the format specifier starts with the two characters "%@" then ri is658 * not NULL, and the message is prefixed with an instance identifier in the659 * following format:660 *661 *  <instance type> <instance name> <ip> <port>662 *663 *  If the instance type is not master, than the additional string is664 *  added to specify the originating master:665 *666 *  @ <master name> <master ip> <master port>667 *668 *  Any other specifier after "%@" is processed by printf itself.669 */670void sentinelEvent(int level, char *type, sentinelRedisInstance *ri,671                   const char *fmt, ...) {672    va_list ap;673    char msg[LOG_MAX_LEN];674    robj *channel, *payload;675676    /* Handle %@ */677    if (fmt[0] == '%' && fmt[1] == '@') {678        sentinelRedisInstance *master = (ri->flags & SRI_MASTER) ?679                                         NULL : ri->master;680681        if (master) {682            snprintf(msg, sizeof(msg), "%s %s %s %d @ %s %s %d",683                sentinelRedisInstanceTypeStr(ri),684                ri->name, announceSentinelAddr(ri->addr), ri->addr->port,685                master->name, announceSentinelAddr(master->addr), master->addr->port);686        } else {687            snprintf(msg, sizeof(msg), "%s %s %s %d",688                sentinelRedisInstanceTypeStr(ri),689                ri->name, announceSentinelAddr(ri->addr), ri->addr->port);690        }691        fmt += 2;692    } else {693        msg[0] = '\0';694    }695696    /* Use vsprintf for the rest of the formatting if any. */697    if (fmt[0] != '\0') {698        va_start(ap, fmt);699        vsnprintf(msg+strlen(msg), sizeof(msg)-strlen(msg), fmt, ap);700        va_end(ap);701    }702703    /* Log the message if the log level allows it to be logged. */704    if (level >= server.verbosity)705        serverLog(level,"%s %s",type,msg);706707    /* Publish the message via Pub/Sub if it's not a debugging one. */708    if (level != LL_DEBUG) {709        channel = createStringObject(type,strlen(type));710        payload = createStringObject(msg,strlen(msg));711        pubsubPublishMessage(channel,payload,0);712        decrRefCount(channel);713        decrRefCount(payload);714    }715716    /* Call the notification script if applicable. */717    if (level == LL_WARNING && ri != NULL) {718        sentinelRedisInstance *master = (ri->flags & SRI_MASTER) ?719                                         ri : ri->master;720        if (master && master->notification_script) {721            sentinelScheduleScriptExecution(master->notification_script,722                type,msg,NULL);723        }724    }725}726727/* This function is called only at startup and is used to generate a728 * +monitor event for every configured master. The same events are also729 * generated when a master to monitor is added at runtime via the730 * SENTINEL MONITOR command. */731void sentinelGenerateInitialMonitorEvents(void) {732    dictIterator di;733    dictEntry *de;734735    dictInitIterator(&di, sentinel.masters);736    while((de = dictNext(&di)) != NULL) {737        sentinelRedisInstance *ri = dictGetVal(de);738        sentinelEvent(LL_WARNING,"+monitor",ri,"%@ quorum %d",ri->quorum);739    }740    dictResetIterator(&di);741}742743/* ============================ script execution ============================ */744745/* Release a script job structure and all the associated data. */746void sentinelReleaseScriptJob(sentinelScriptJob *sj) {747    int j = 0;748749    while(sj->argv[j]) sdsfree(sj->argv[j++]);750    zfree(sj->argv);751    zfree(sj);752}753754#define SENTINEL_SCRIPT_MAX_ARGS 16755void sentinelScheduleScriptExecution(char *path, ...) {756    va_list ap;757    char *argv[SENTINEL_SCRIPT_MAX_ARGS+1];758    int argc = 1;759    sentinelScriptJob *sj;760761    va_start(ap, path);762    while(argc < SENTINEL_SCRIPT_MAX_ARGS) {763        argv[argc] = va_arg(ap,char*);764        if (!argv[argc]) break;765        argv[argc] = sdsnew(argv[argc]); /* Copy the string. */766        argc++;767    }768    va_end(ap);769    argv[0] = sdsnew(path);770771    sj = zmalloc(sizeof(*sj));772    sj->flags = SENTINEL_SCRIPT_NONE;773    sj->retry_num = 0;774    sj->argv = zmalloc(sizeof(char*)*(argc+1));775    sj->start_time = 0;776    sj->pid = 0;777    memcpy(sj->argv,argv,sizeof(char*)*(argc+1));778779    listAddNodeTail(sentinel.scripts_queue,sj);780781    /* Remove the oldest non running script if we already hit the limit. */782    if (listLength(sentinel.scripts_queue) > SENTINEL_SCRIPT_MAX_QUEUE) {783        listNode *ln;784        listIter li;785786        listRewind(sentinel.scripts_queue,&li);787        while ((ln = listNext(&li)) != NULL) {788            sj = ln->value;789790            if (sj->flags & SENTINEL_SCRIPT_RUNNING) continue;791            /* The first node is the oldest as we add on tail. */792            listDelNode(sentinel.scripts_queue,ln);793            sentinelReleaseScriptJob(sj);794            break;795        }796        serverAssert(listLength(sentinel.scripts_queue) <=797                    SENTINEL_SCRIPT_MAX_QUEUE);798    }799}800801/* Lookup a script in the scripts queue via pid, and returns the list node802 * (so that we can easily remove it from the queue if needed). */803listNode *sentinelGetScriptListNodeByPid(pid_t pid) {804    listNode *ln;805    listIter li;806807    listRewind(sentinel.scripts_queue,&li);808    while ((ln = listNext(&li)) != NULL) {809        sentinelScriptJob *sj = ln->value;810811        if ((sj->flags & SENTINEL_SCRIPT_RUNNING) && sj->pid == pid)812            return ln;813    }814    return NULL;815}816817/* Run pending scripts if we are not already at max number of running818 * scripts. */819void sentinelRunPendingScripts(void) {820    listNode *ln;821    listIter li;822    mstime_t now = mstime();823824    /* Find jobs that are not running and run them, from the top to the825     * tail of the queue, so we run older jobs first. */826    listRewind(sentinel.scripts_queue,&li);827    while (sentinel.running_scripts < SENTINEL_SCRIPT_MAX_RUNNING &&828           (ln = listNext(&li)) != NULL)829    {830        sentinelScriptJob *sj = ln->value;831        pid_t pid;832833        /* Skip if already running. */834        if (sj->flags & SENTINEL_SCRIPT_RUNNING) continue;835836        /* Skip if it's a retry, but not enough time has elapsed. */837        if (sj->start_time && sj->start_time > now) continue;838839        sj->flags |= SENTINEL_SCRIPT_RUNNING;840        sj->start_time = mstime();841        sj->retry_num++;842        pid = fork();843844        if (pid == -1) {845            /* Parent (fork error).846             * We report fork errors as signal 99, in order to unify the847             * reporting with other kind of errors. */848            sentinelEvent(LL_WARNING,"-script-error",NULL,849                          "%s %d %d", sj->argv[0], 99, 0);850            sj->flags &= ~SENTINEL_SCRIPT_RUNNING;851            sj->pid = 0;852        } else if (pid == 0) {853            /* Child */854            connTypeCleanupAll();855            execve(sj->argv[0],sj->argv,environ);856            /* If we are here an error occurred. */857            _exit(2); /* Don't retry execution. */858        } else {859            sentinel.running_scripts++;860            sj->pid = pid;861            sentinelEvent(LL_DEBUG,"+script-child",NULL,"%ld",(long)pid);862        }863    }864}865866/* How much to delay the execution of a script that we need to retry after867 * an error?868 *869 * We double the retry delay for every further retry we do. So for instance870 * if RETRY_DELAY is set to 30 seconds and the max number of retries is 10871 * starting from the second attempt to execute the script the delays are:872 * 30 sec, 60 sec, 2 min, 4 min, 8 min, 16 min, 32 min, 64 min, 128 min. */873mstime_t sentinelScriptRetryDelay(int retry_num) {874    mstime_t delay = sentinel_script_retry_delay;875876    while (retry_num-- > 1) delay *= 2;877    return delay;878}879880/* Check for scripts that terminated, and remove them from the queue if the881 * script terminated successfully. If instead the script was terminated by882 * a signal, or returned exit code "1", it is scheduled to run again if883 * the max number of retries did not already elapsed. */884void sentinelCollectTerminatedScripts(void) {885    int statloc;886    pid_t pid;887888    while ((pid = waitpid(-1, &statloc, WNOHANG)) > 0) {889        int exitcode = WEXITSTATUS(statloc);890        int bysignal = 0;891        listNode *ln;892        sentinelScriptJob *sj;893894        if (WIFSIGNALED(statloc)) bysignal = WTERMSIG(statloc);895        sentinelEvent(LL_DEBUG,"-script-child",NULL,"%ld %d %d",896            (long)pid, exitcode, bysignal);897898        ln = sentinelGetScriptListNodeByPid(pid);899        if (ln == NULL) {900            serverLog(LL_WARNING,"waitpid() returned a pid (%ld) we can't find in our scripts execution queue!", (long)pid);901            continue;902        }903        sj = ln->value;904905        /* If the script was terminated by a signal or returns an906         * exit code of "1" (that means: please retry), we reschedule it907         * if the max number of retries is not already reached. */908        if ((bysignal || exitcode == 1) &&909            sj->retry_num != SENTINEL_SCRIPT_MAX_RETRY)910        {911            sj->flags &= ~SENTINEL_SCRIPT_RUNNING;912            sj->pid = 0;913            sj->start_time = mstime() +914                             sentinelScriptRetryDelay(sj->retry_num);915        } else {916            /* Otherwise let's remove the script, but log the event if the917             * execution did not terminated in the best of the ways. */918            if (bysignal || exitcode != 0) {919                sentinelEvent(LL_WARNING,"-script-error",NULL,920                              "%s %d %d", sj->argv[0], bysignal, exitcode);921            }922            listDelNode(sentinel.scripts_queue,ln);923            sentinelReleaseScriptJob(sj);924        }925        sentinel.running_scripts--;926    }927}928929/* Kill scripts in timeout, they'll be collected by the930 * sentinelCollectTerminatedScripts() function. */931void sentinelKillTimedoutScripts(void) {932    listNode *ln;933    listIter li;934    mstime_t now = mstime();935936    listRewind(sentinel.scripts_queue,&li);937    while ((ln = listNext(&li)) != NULL) {938        sentinelScriptJob *sj = ln->value;939940        if (sj->flags & SENTINEL_SCRIPT_RUNNING &&941            (now - sj->start_time) > sentinel_script_max_runtime)942        {943            sentinelEvent(LL_WARNING,"-script-timeout",NULL,"%s %ld",944                sj->argv[0], (long)sj->pid);945            kill(sj->pid,SIGKILL);946        }947    }948}949950/* Implements SENTINEL PENDING-SCRIPTS command. */951void sentinelPendingScriptsCommand(client *c) {952    listNode *ln;953    listIter li;954955    addReplyArrayLen(c,listLength(sentinel.scripts_queue));956    listRewind(sentinel.scripts_queue,&li);957    while ((ln = listNext(&li)) != NULL) {958        sentinelScriptJob *sj = ln->value;959        int j = 0;960961        addReplyMapLen(c,5);962963        addReplyBulkCString(c,"argv");964        while (sj->argv[j]) j++;965        addReplyArrayLen(c,j);966        j = 0;967        while (sj->argv[j]) addReplyBulkCString(c,sj->argv[j++]);968969        addReplyBulkCString(c,"flags");970        addReplyBulkCString(c,971            (sj->flags & SENTINEL_SCRIPT_RUNNING) ? "running" : "scheduled");972973        addReplyBulkCString(c,"pid");974        addReplyBulkLongLong(c,sj->pid);975976        if (sj->flags & SENTINEL_SCRIPT_RUNNING) {977            addReplyBulkCString(c,"run-time");978            addReplyBulkLongLong(c,mstime() - sj->start_time);979        } else {980            mstime_t delay = sj->start_time ? (sj->start_time-mstime()) : 0;981            if (delay < 0) delay = 0;982            addReplyBulkCString(c,"run-delay");983            addReplyBulkLongLong(c,delay);984        }985986        addReplyBulkCString(c,"retry-num");987        addReplyBulkLongLong(c,sj->retry_num);988    }989}990991/* This function calls, if any, the client reconfiguration script with the992 * following parameters:993 *994 * <master-name> <role> <state> <from-ip> <from-port> <to-ip> <to-port>995 *996 * It is called every time a failover is performed.997 *998 * <state> is currently always "start".999 * <role> is either "leader" or "observer".1000 *1001 * from/to fields are respectively master -> promoted slave addresses for1002 * "start" and "end". */1003void sentinelCallClientReconfScript(sentinelRedisInstance *master, int role, char *state, sentinelAddr *from, sentinelAddr *to) {1004    char fromport[32], toport[32];10051006    if (master->client_reconfig_script == NULL) return;1007    ll2string(fromport,sizeof(fromport),from->port);1008    ll2string(toport,sizeof(toport),to->port);1009    sentinelScheduleScriptExecution(master->client_reconfig_script,1010        master->name,1011        (role == SENTINEL_LEADER) ? "leader" : "observer",1012        state, announceSentinelAddr(from), fromport,1013        announceSentinelAddr(to), toport, NULL);1014}10151016/* =============================== instanceLink ============================= */10171018/* Create a not yet connected link object. */1019instanceLink *createInstanceLink(void) {1020    instanceLink *link = zmalloc(sizeof(*link));10211022    link->refcount = 1;1023    link->disconnected = 1;1024    link->pending_commands = 0;1025    link->cc = NULL;1026    link->pc = NULL;1027    link->cc_conn_time = 0;1028    link->pc_conn_time = 0;1029    link->last_reconn_time = 0;1030    link->pc_last_activity = 0;1031    /* We set the act_ping_time to "now" even if we actually don't have yet1032     * a connection with the node, nor we sent a ping.1033     * This is useful to detect a timeout in case we'll not be able to connect1034     * with the node at all. */1035    link->act_ping_time = mstime();1036    link->last_ping_time = 0;1037    link->last_avail_time = mstime();1038    link->last_pong_time = mstime();1039    return link;1040}10411042/* Disconnect a hiredis connection in the context of an instance link. */1043void instanceLinkCloseConnection(instanceLink *link, redisAsyncContext *c) {1044    if (c == NULL) return;10451046    if (link->cc == c) {1047        link->cc = NULL;1048        link->pending_commands = 0;1049    }1050    if (link->pc == c) link->pc = NULL;1051    c->data = NULL;1052    link->disconnected = 1;1053    redisAsyncFree(c);1054}10551056/* Decrement the refcount of a link object, if it drops to zero, actually1057 * free it and return NULL. Otherwise don't do anything and return the pointer1058 * to the object.1059 *1060 * If we are not going to free the link and ri is not NULL, we rebind all the1061 * pending requests in link->cc (hiredis connection for commands) to a1062 * callback that will just ignore them. This is useful to avoid processing1063 * replies for an instance that no longer exists. */1064instanceLink *releaseInstanceLink(instanceLink *link, sentinelRedisInstance *ri)1065{1066    serverAssert(link->refcount > 0);1067    link->refcount--;1068    if (link->refcount != 0) {1069        if (ri && ri->link->cc) {1070            /* This instance may have pending callbacks in the hiredis async1071             * context, having as 'privdata' the instance that we are going to1072             * free. Let's rewrite the callback list, directly exploiting1073             * hiredis internal data structures, in order to bind them with1074             * a callback that will ignore the reply at all. */1075            redisCallback *cb;1076            redisCallbackList *callbacks = &link->cc->replies;10771078            cb = callbacks->head;1079            while(cb) {1080                if (cb->privdata == ri) {1081                    cb->fn = sentinelDiscardReplyCallback;1082                    cb->privdata = NULL; /* Not strictly needed. */1083                }1084                cb = cb->next;1085            }1086        }1087        return link; /* Other active users. */1088    }10891090    instanceLinkCloseConnection(link,link->cc);1091    instanceLinkCloseConnection(link,link->pc);1092    zfree(link);1093    return NULL;1094}10951096/* This function will attempt to share the instance link we already have1097 * for the same Sentinel in the context of a different master, with the1098 * instance we are passing as argument.1099 *1100 * This way multiple Sentinel objects that refer all to the same physical1101 * Sentinel instance but in the context of different masters will use1102 * a single connection, will send a single PING per second for failure1103 * detection and so forth.1104 *1105 * Return C_OK if a matching Sentinel was found in the context of a1106 * different master and sharing was performed. Otherwise C_ERR1107 * is returned. */1108int sentinelTryConnectionSharing(sentinelRedisInstance *ri) {1109    serverAssert(ri->flags & SRI_SENTINEL);1110    dictIterator di;1111    dictEntry *de;11121113    if (ri->runid == NULL) return C_ERR; /* No way to identify it. */1114    if (ri->link->refcount > 1) return C_ERR; /* Already shared. */11151116    dictInitIterator(&di, sentinel.masters);1117    while((de = dictNext(&di)) != NULL) {1118        sentinelRedisInstance *master = dictGetVal(de), *match;1119        /* We want to share with the same physical Sentinel referenced1120         * in other masters, so skip our master. */1121        if (master == ri->master) continue;1122        match = getSentinelRedisInstanceByAddrAndRunID(master->sentinels,1123                                                       NULL,0,ri->runid);1124        if (match == NULL) continue; /* No match. */1125        if (match == ri) continue; /* Should never happen but... safer. */11261127        /* We identified a matching Sentinel, great! Let's free our link1128         * and use the one of the matching Sentinel. */1129        releaseInstanceLink(ri->link,NULL);1130        ri->link = match->link;1131        match->link->refcount++;1132        dictResetIterator(&di);1133        return C_OK;1134    }1135    dictResetIterator(&di);1136    return C_ERR;1137}11381139/* Disconnect the relevant master and its replicas. */1140void dropInstanceConnections(sentinelRedisInstance *ri) {1141    serverAssert(ri->flags & SRI_MASTER);11421143    /* Disconnect with the master. */1144    instanceLinkCloseConnection(ri->link, ri->link->cc);1145    instanceLinkCloseConnection(ri->link, ri->link->pc);1146    1147    /* Disconnect with all replicas. */1148    dictIterator di;1149    dictEntry *de;1150    sentinelRedisInstance *repl_ri;11511152    dictInitIterator(&di, ri->slaves);1153    while ((de = dictNext(&di)) != NULL) {1154        repl_ri = dictGetVal(de);1155        instanceLinkCloseConnection(repl_ri->link, repl_ri->link->cc);1156        instanceLinkCloseConnection(repl_ri->link, repl_ri->link->pc);1157    }1158    dictResetIterator(&di);1159}11601161/* Drop all connections to other sentinels. Returns the number of connections1162 * dropped.*/1163int sentinelDropConnections(void) {1164    dictIterator di;1165    dictEntry *de;1166    int dropped = 0;11671168    dictInitIterator(&di, sentinel.masters);1169    while ((de = dictNext(&di)) != NULL) {1170        dictIterator sdi;1171        dictEntry *sde;11721173        sentinelRedisInstance *ri = dictGetVal(de);1174        dictInitIterator(&sdi, ri->sentinels);1175        while ((sde = dictNext(&sdi)) != NULL) {1176            sentinelRedisInstance *si = dictGetVal(sde);1177            if (!si->link->disconnected) {1178                instanceLinkCloseConnection(si->link, si->link->pc);1179                instanceLinkCloseConnection(si->link, si->link->cc);1180                dropped++;1181            }1182        }1183        dictResetIterator(&sdi);1184    }1185    dictResetIterator(&di);11861187    return dropped;1188}11891190/* When we detect a Sentinel to switch address (reporting a different IP/port1191 * pair in Hello messages), let's update all the matching Sentinels in the1192 * context of other masters as well and disconnect the links, so that everybody1193 * will be updated.1194 *1195 * Return the number of updated Sentinel addresses. */1196int sentinelUpdateSentinelAddressInAllMasters(sentinelRedisInstance *ri) {1197    serverAssert(ri->flags & SRI_SENTINEL);1198    dictIterator di;1199    dictEntry *de;1200    int reconfigured = 0;12011202    dictInitIterator(&di, sentinel.masters);1203    while((de = dictNext(&di)) != NULL) {1204        sentinelRedisInstance *master = dictGetVal(de), *match;1205        match = getSentinelRedisInstanceByAddrAndRunID(master->sentinels,1206                                                       NULL,0,ri->runid);1207        /* If there is no match, this master does not know about this1208         * Sentinel, try with the next one. */1209        if (match == NULL) continue;12101211        /* Disconnect the old links if connected. */1212        if (match->link->cc != NULL)1213            instanceLinkCloseConnection(match->link,match->link->cc);1214        if (match->link->pc != NULL)1215            instanceLinkCloseConnection(match->link,match->link->pc);12161217        if (match == ri) continue; /* Address already updated for it. */12181219        /* Update the address of the matching Sentinel by copying the address1220         * of the Sentinel object that received the address update. */1221        releaseSentinelAddr(match->addr);1222        match->addr = dupSentinelAddr(ri->addr);1223        reconfigured++;1224    }1225    dictResetIterator(&di);1226    if (reconfigured)1227        sentinelEvent(LL_NOTICE,"+sentinel-address-update", ri,1228                    "%@ %d additional matching instances", reconfigured);1229    return reconfigured;1230}12311232/* This function is called when a hiredis connection reported an error.1233 * We set it to NULL and mark the link as disconnected so that it will be1234 * reconnected again.1235 *1236 * Note: we don't free the hiredis context as hiredis will do it for us1237 * for async connections. */1238void instanceLinkConnectionError(const redisAsyncContext *c) {1239    instanceLink *link = c->data;1240    int pubsub;12411242    if (!link) return;12431244    pubsub = (link->pc == c);1245    if (pubsub)1246        link->pc = NULL;1247    else1248        link->cc = NULL;1249    link->disconnected = 1;1250}12511252/* Hiredis connection established / disconnected callbacks. We need them1253 * just to cleanup our link state. */1254void sentinelLinkEstablishedCallback(const redisAsyncContext *c, int status) {1255    if (status != C_OK) instanceLinkConnectionError(c);1256}12571258void sentinelDisconnectCallback(const redisAsyncContext *c, int status) {1259    UNUSED(status);1260    instanceLinkConnectionError(c);1261}12621263/* ========================== sentinelRedisInstance ========================= */12641265/* Create a redis instance, the following fields must be populated by the1266 * caller if needed:1267 * runid: set to NULL but will be populated once INFO output is received.1268 * info_refresh: is set to 0 to mean that we never received INFO so far.1269 *1270 * If SRI_MASTER is set into initial flags the instance is added to1271 * sentinel.masters table.1272 *1273 * if SRI_SLAVE or SRI_SENTINEL is set then 'master' must be not NULL and the1274 * instance is added into master->slaves or master->sentinels table.1275 *1276 * If the instance is a slave, the name parameter is ignored and is created1277 * automatically as ip/hostname:port.1278 *1279 * The function fails if hostname can't be resolved or port is out of range.1280 * When this happens NULL is returned and errno is set accordingly to the1281 * createSentinelAddr() function.1282 *1283 * The function may also fail and return NULL with errno set to EBUSY if1284 * a master with the same name, a slave with the same address, or a sentinel1285 * with the same ID already exists. */12861287sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char *hostname, int port, int quorum, sentinelRedisInstance *master) {1288    sentinelRedisInstance *ri;1289    sentinelAddr *addr;1290    dict *table = NULL;1291    sds sdsname;12921293    serverAssert(flags & (SRI_MASTER|SRI_SLAVE|SRI_SENTINEL));1294    serverAssert((flags & SRI_MASTER) || master != NULL);12951296    /* Check address validity. */1297    addr = createSentinelAddr(hostname,port,1);1298    if (addr == NULL) return NULL;12991300    /* For slaves use ip/host:port as name. */1301    if (flags & SRI_SLAVE)1302        sdsname = announceSentinelAddrAndPort(addr);1303    else1304        sdsname = sdsnew(name);13051306    /* Make sure the entry is not duplicated. This may happen when the same1307     * name for a master is used multiple times inside the configuration or1308     * if we try to add multiple times a slave or sentinel with same ip/port1309     * to a master. */1310    if (flags & SRI_MASTER) table = sentinel.masters;1311    else if (flags & SRI_SLAVE) table = master->slaves;1312    else if (flags & SRI_SENTINEL) table = master->sentinels;1313    if (dictFind(table,sdsname)) {1314        releaseSentinelAddr(addr);1315        sdsfree(sdsname);1316        errno = EBUSY;1317        return NULL;1318    }13191320    /* Create the instance object. */1321    ri = zmalloc(sizeof(*ri));1322    /* Note that all the instances are started in the disconnected state,1323     * the event loop will take care of connecting them. */1324    ri->flags = flags;1325    ri->name = sdsname;1326    ri->runid = NULL;1327    ri->config_epoch = 0;1328    ri->addr = addr;1329    ri->link = createInstanceLink();1330    ri->last_pub_time = mstime();1331    ri->last_hello_time = mstime();1332    ri->last_master_down_reply_time = mstime();1333    ri->s_down_since_time = 0;1334    ri->o_down_since_time = 0;1335    ri->down_after_period = master ? master->down_after_period : sentinel_default_down_after;1336    ri->master_reboot_down_after_period = 0;1337    ri->master_link_down_time = 0;1338    ri->auth_pass = NULL;1339    ri->auth_user = NULL;1340    ri->slave_priority = SENTINEL_DEFAULT_SLAVE_PRIORITY;1341    ri->replica_announced = 1;1342    ri->slave_reconf_sent_time = 0;1343    ri->slave_master_host = NULL;1344    ri->slave_master_port = 0;1345    ri->slave_master_link_status = SENTINEL_MASTER_LINK_STATUS_DOWN;1346    ri->slave_repl_offset = 0;1347    ri->sentinels = dictCreate(&instancesDictType);1348    ri->quorum = quorum;1349    ri->parallel_syncs = SENTINEL_DEFAULT_PARALLEL_SYNCS;1350    ri->master = master;1351    ri->slaves = dictCreate(&instancesDictType);1352    ri->info_refresh = 0;1353    ri->renamed_commands = dictCreate(&renamedCommandsDictType);13541355    /* Failover state. */1356    ri->leader = NULL;1357    ri->leader_epoch = 0;1358    ri->failover_epoch = 0;1359    ri->failover_state = SENTINEL_FAILOVER_STATE_NONE;1360    ri->failover_state_change_time = 0;1361    ri->failover_start_time = 0;1362    ri->failover_timeout = sentinel_default_failover_timeout;1363    ri->failover_delay_logged = 0;1364    ri->promoted_slave = NULL;1365    ri->notification_script = NULL;1366    ri->client_reconfig_script = NULL;1367    ri->info = NULL;13681369    /* Role */1370    ri->role_reported = ri->flags & (SRI_MASTER|SRI_SLAVE);1371    ri->role_reported_time = mstime();1372    ri->slave_conf_change_time = mstime();13731374    /* Add into the right table. */1375    dictAdd(table, ri->name, ri);1376    return ri;1377}13781379/* Release this instance and all its slaves, sentinels, hiredis connections.1380 * This function does not take care of unlinking the instance from the main1381 * masters table (if it is a master) or from its master sentinels/slaves table1382 * if it is a slave or sentinel. */1383void releaseSentinelRedisInstance(sentinelRedisInstance *ri) {1384    /* Release all its slaves or sentinels if any. */1385    dictRelease(ri->sentinels);1386    dictRelease(ri->slaves);13871388    /* Disconnect the instance. */1389    releaseInstanceLink(ri->link,ri);13901391    /* Free other resources. */1392    sdsfree(ri->name);1393    sdsfree(ri->runid);1394    sdsfree(ri->notification_script);1395    sdsfree(ri->client_reconfig_script);1396    sdsfree(ri->slave_master_host);1397    sdsfree(ri->leader);1398    sdsfree(ri->auth_pass);1399    sdsfree(ri->auth_user);1400    sdsfree(ri->info);1401    releaseSentinelAddr(ri->addr);1402    dictRelease(ri->renamed_commands);14031404    /* Clear state into the master if needed. */1405    if ((ri->flags & SRI_SLAVE) && (ri->flags & SRI_PROMOTED) && ri->master)1406        ri->master->promoted_slave = NULL;14071408    zfree(ri);1409}14101411/* Lookup a slave in a master Redis instance, by ip and port. */1412sentinelRedisInstance *sentinelRedisInstanceLookupSlave(1413                sentinelRedisInstance *ri, char *slave_addr, int port)1414{1415    sds key;1416    sentinelRedisInstance *slave;1417    sentinelAddr *addr;14181419    serverAssert(ri->flags & SRI_MASTER);14201421    /* We need to handle a slave_addr that is potentially a hostname.1422     * If that is the case, depending on configuration we either resolve1423     * it and use the IP address or fail.1424     */1425    addr = createSentinelAddr(slave_addr, port, 0);1426    if (!addr) return NULL;1427    key = announceSentinelAddrAndPort(addr);1428    releaseSentinelAddr(addr);14291430    slave = dictFetchValue(ri->slaves,key);1431    sdsfree(key);1432    return slave;1433}14341435/* Return the name of the type of the instance as a string. */1436const char *sentinelRedisInstanceTypeStr(sentinelRedisInstance *ri) {1437    if (ri->flags & SRI_MASTER) return "master";1438    else if (ri->flags & SRI_SLAVE) return "slave";1439    else if (ri->flags & SRI_SENTINEL) return "sentinel";1440    else return "unknown";1441}14421443/* This function remove the Sentinel with the specified ID from the1444 * specified master.1445 *1446 * If "runid" is NULL the function returns ASAP.1447 *1448 * This function is useful because on Sentinels address switch, we want to1449 * remove our old entry and add a new one for the same ID but with the new1450 * address.1451 *1452 * The function returns 1 if the matching Sentinel was removed, otherwise1453 * 0 if there was no Sentinel with this ID. */1454int removeMatchingSentinelFromMaster(sentinelRedisInstance *master, char *runid) {1455    dictIterator di;1456    dictEntry *de;1457    int removed = 0;14581459    if (runid == NULL) return 0;14601461    dictInitSafeIterator(&di, master->sentinels);1462    while((de = dictNext(&di)) != NULL) {1463        sentinelRedisInstance *ri = dictGetVal(de);14641465        if (ri->runid && strcmp(ri->runid,runid) == 0) {1466            dictDelete(master->sentinels,ri->name);1467            removed++;1468        }1469    }1470    dictResetIterator(&di);1471    return removed;1472}14731474/* Search an instance with the same runid, ip and port into a dictionary1475 * of instances. Return NULL if not found, otherwise return the instance1476 * pointer.1477 *1478 * runid or addr can be NULL. In such a case the search is performed only1479 * by the non-NULL field. */1480sentinelRedisInstance *getSentinelRedisInstanceByAddrAndRunID(dict *instances, char *addr, int port, char *runid) {1481    dictIterator di;1482    dictEntry *de;1483    sentinelRedisInstance *instance = NULL;1484    sentinelAddr *ri_addr = NULL;14851486    serverAssert(addr || runid);   /* User must pass at least one search param. */1487    if (addr != NULL) {1488        /* Try to resolve addr. If hostnames are used, we're accepting an ri_addr1489         * that contains an hostname only and can still be matched based on that.1490         */1491        ri_addr = createSentinelAddr(addr,port,1);1492        if (!ri_addr) return NULL;1493    }1494    dictInitIterator(&di, instances);1495    while((de = dictNext(&di)) != NULL) {1496        sentinelRedisInstance *ri = dictGetVal(de);14971498        if (runid && !ri->runid) continue;1499        if ((runid == NULL || strcmp(ri->runid, runid) == 0) &&1500            (addr == NULL || sentinelAddrOrHostnameEqual(ri->addr, ri_addr)))1501        {1502            instance = ri;1503            break;1504        }1505    }1506    dictResetIterator(&di);1507    if (ri_addr != NULL)1508        releaseSentinelAddr(ri_addr);15091510    return instance;1511}15121513/* Master lookup by name */1514sentinelRedisInstance *sentinelGetMasterByName(char *name) {1515    sentinelRedisInstance *ri;1516    sds sdsname = sdsnew(name);15171518    ri = dictFetchValue(sentinel.masters,sdsname);1519    sdsfree(sdsname);1520    return ri;1521}15221523/* Reset the state of a monitored master:1524 * 1) Remove all slaves.1525 * 2) Remove all sentinels.1526 * 3) Remove most of the flags resulting from runtime operations.1527 * 4) Reset timers to their default value. For example after a reset it will be1528 *    possible to failover again the same master ASAP, without waiting the1529 *    failover timeout delay.1530 * 5) In the process of doing this undo the failover if in progress.1531 * 6) Disconnect the connections with the master (will reconnect automatically).1532 */15331534#define SENTINEL_RESET_NO_SENTINELS (1<<0)1535void sentinelResetMaster(sentinelRedisInstance *ri, int flags) {1536    serverAssert(ri->flags & SRI_MASTER);1537    dictRelease(ri->slaves);1538    ri->slaves = dictCreate(&instancesDictType);1539    if (!(flags & SENTINEL_RESET_NO_SENTINELS)) {1540        dictRelease(ri->sentinels);1541        ri->sentinels = dictCreate(&instancesDictType);1542    }1543    instanceLinkCloseConnection(ri->link,ri->link->cc);1544    instanceLinkCloseConnection(ri->link,ri->link->pc);1545    ri->flags &= SRI_MASTER;1546    if (ri->leader) {1547        sdsfree(ri->leader);1548        ri->leader = NULL;1549    }1550    ri->failover_state = SENTINEL_FAILOVER_STATE_NONE;1551    ri->failover_state_change_time = 0;1552    ri->failover_start_time = 0; /* We can failover again ASAP. */1553    ri->promoted_slave = NULL;1554    sdsfree(ri->runid);1555    sdsfree(ri->slave_master_host);1556    ri->runid = NULL;1557    ri->slave_master_host = NULL;1558    ri->link->act_ping_time = mstime();1559    ri->link->last_ping_time = 0;1560    ri->link->last_avail_time = mstime();1561    ri->link->last_pong_time = mstime();1562    ri->role_reported_time = mstime();1563    ri->role_reported = SRI_MASTER;1564    if (flags & SENTINEL_GENERATE_EVENT)1565        sentinelEvent(LL_WARNING,"+reset-master",ri,"%@");1566}15671568/* Call sentinelResetMaster() on every master with a name matching the specified1569 * pattern. */1570int sentinelResetMastersByPattern(char *pattern, int flags) {1571    dictIterator di;1572    dictEntry *de;1573    int reset = 0;15741575    dictInitIterator(&di, sentinel.masters);1576    while((de = dictNext(&di)) != NULL) {1577        sentinelRedisInstance *ri = dictGetVal(de);15781579        if (ri->name) {1580            if (stringmatch(pattern,ri->name,0)) {1581                sentinelResetMaster(ri,flags);1582                reset++;1583            }1584        }1585    }1586    dictResetIterator(&di);1587    return reset;1588}15891590/* Reset the specified master with sentinelResetMaster(), and also change1591 * the ip:port address, but take the name of the instance unmodified.1592 *1593 * This is used to handle the +switch-master event.1594 *1595 * The function returns C_ERR if the address can't be resolved for some1596 * reason. Otherwise C_OK is returned.  */1597int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *hostname, int port) {1598    sentinelAddr *oldaddr, *newaddr;1599    sentinelAddr **slaves = NULL;1600    int numslaves = 0, j;1601    dictIterator di;1602    dictEntry *de;16031604    newaddr = createSentinelAddr(hostname,port,0);1605    if (newaddr == NULL) return C_ERR;16061607    /* There can be only 0 or 1 slave that has the newaddr.1608     * and It can add old master 1 more slave. 1609     * so It allocates dictSize(master->slaves) + 1          */1610    slaves = zmalloc(sizeof(sentinelAddr*)*(dictSize(master->slaves) + 1));1611    1612    /* Don't include the one having the address we are switching to. */1613    dictInitIterator(&di, master->slaves);1614    while((de = dictNext(&di)) != NULL) {1615        sentinelRedisInstance *slave = dictGetVal(de);16161617        if (sentinelAddrOrHostnameEqual(slave->addr,newaddr)) continue;1618        slaves[numslaves++] = dupSentinelAddr(slave->addr);1619    }1620    dictResetIterator(&di);16211622    /* If we are switching to a different address, include the old address1623     * as a slave as well, so that we'll be able to sense / reconfigure1624     * the old master. */1625    if (!sentinelAddrOrHostnameEqual(newaddr,master->addr)) {1626        slaves[numslaves++] = dupSentinelAddr(master->addr);1627    }16281629    /* Reset and switch address. */1630    sentinelResetMaster(master,SENTINEL_RESET_NO_SENTINELS);1631    oldaddr = master->addr;1632    master->addr = newaddr;1633    master->o_down_since_time = 0;1634    master->s_down_since_time = 0;16351636    /* Add slaves back. */1637    for (j = 0; j < numslaves; j++) {1638        sentinelRedisInstance *slave;16391640        slave = createSentinelRedisInstance(NULL,SRI_SLAVE,slaves[j]->hostname,1641                    slaves[j]->port, master->quorum, master);1642        releaseSentinelAddr(slaves[j]);1643        if (slave) sentinelEvent(LL_NOTICE,"+slave",slave,"%@");1644    }1645    zfree(slaves);16461647    /* Release the old address at the end so we are safe even if the function1648     * gets the master->addr->ip and master->addr->port as arguments. */1649    releaseSentinelAddr(oldaddr);1650    sentinelFlushConfig();1651    return C_OK;1652}16531654/* Return non-zero if there was no SDOWN or ODOWN error associated to this1655 * instance in the latest 'ms' milliseconds. */1656int sentinelRedisInstanceNoDownFor(sentinelRedisInstance *ri, mstime_t ms) {1657    mstime_t most_recent;16581659    most_recent = ri->s_down_since_time;1660    if (ri->o_down_since_time > most_recent)1661        most_recent = ri->o_down_since_time;1662    return most_recent == 0 || (mstime() - most_recent) > ms;1663}16641665/* Return the current master address, that is, its address or the address1666 * of the promoted slave if already operational. */1667sentinelAddr *sentinelGetCurrentMasterAddress(sentinelRedisInstance *master) {1668    /* If we are failing over the master, and the state is already1669     * SENTINEL_FAILOVER_STATE_RECONF_SLAVES or greater, it means that we1670     * already have the new configuration epoch in the master, and the1671     * slave acknowledged the configuration switch. Advertise the new1672     * address. */1673    if ((master->flags & SRI_FAILOVER_IN_PROGRESS) &&1674        master->promoted_slave &&1675        master->failover_state >= SENTINEL_FAILOVER_STATE_RECONF_SLAVES)1676    {1677        return master->promoted_slave->addr;1678    } else {1679        return master->addr;1680    }1681}16821683/* This function sets the down_after_period field value in 'master' to all1684 * the slaves and sentinel instances connected to this master. */1685void sentinelPropagateDownAfterPeriod(sentinelRedisInstance *master) {1686    dictIterator di;1687    dictEntry *de;1688    int j;1689    dict *d[] = {master->slaves, master->sentinels, NULL};16901691    for (j = 0; d[j]; j++) {1692        dictInitIterator(&di, d[j]);1693        while((de = dictNext(&di)) != NULL) {1694            sentinelRedisInstance *ri = dictGetVal(de);1695            ri->down_after_period = master->down_after_period;1696        }1697        dictResetIterator(&di);1698    }1699}17001701/* This function is used in order to send commands to Redis instances: the1702 * commands we send from Sentinel may be renamed, a common case is a master1703 * with CONFIG and SLAVEOF commands renamed for security concerns. In that1704 * case we check the ri->renamed_command table (or if the instance is a slave,1705 * we check the one of the master), and map the command that we should send1706 * to the set of renamed commands. However, if the command was not renamed,1707 * we just return "command" itself. */1708char *sentinelInstanceMapCommand(sentinelRedisInstance *ri, char *command) {1709    sds sc = sdsnew(command);1710    if (ri->master) ri = ri->master;1711    char *retval = dictFetchValue(ri->renamed_commands, sc);1712    sdsfree(sc);1713    return retval ? retval : command;1714}17151716/* ============================ Config handling ============================= */17171718/* Generalise handling create instance error. Use SRI_MASTER, SRI_SLAVE or1719 * SRI_SENTINEL as a role value. */1720const char *sentinelCheckCreateInstanceErrors(int role) {1721    switch(errno) {1722    case EBUSY:1723        switch (role) {1724        case SRI_MASTER:1725            return "Duplicate master name.";1726        case SRI_SLAVE:1727            return "Duplicate hostname and port for replica.";1728        case SRI_SENTINEL:1729            return "Duplicate runid for sentinel.";1730        default:1731            serverAssert(0);1732            break;1733        }1734        break;1735    case ENOENT:1736        return "Can't resolve instance hostname.";1737    case EINVAL:1738        return "Invalid port number.";1739    default:1740        return "Unknown Error for creating instances.";1741    }1742}17431744/* init function for server.sentinel_config */1745void initializeSentinelConfig(void) {1746    server.sentinel_config = zmalloc(sizeof(struct sentinelConfig));1747    server.sentinel_config->monitor_cfg = listCreate();1748    server.sentinel_config->pre_monitor_cfg = listCreate();1749    server.sentinel_config->post_monitor_cfg = listCreate();1750    listSetFreeMethod(server.sentinel_config->monitor_cfg,freeSentinelLoadQueueEntry);1751    listSetFreeMethod(server.sentinel_config->pre_monitor_cfg,freeSentinelLoadQueueEntry);1752    listSetFreeMethod(server.sentinel_config->post_monitor_cfg,freeSentinelLoadQueueEntry);1753}17541755/* destroy function for server.sentinel_config */1756void freeSentinelConfig(void) {1757    /* release these three config queues since we will not use it anymore */1758    listRelease(server.sentinel_config->pre_monitor_cfg);1759    listRelease(server.sentinel_config->monitor_cfg);1760    listRelease(server.sentinel_config->post_monitor_cfg);1761    zfree(server.sentinel_config);1762    server.sentinel_config = NULL;1763}17641765/* Search config name in pre monitor config name array, return 1 if found,1766 * 0 if not found. */1767int searchPreMonitorCfgName(const char *name) {1768    for (unsigned int i = 0; i < sizeof(preMonitorCfgName)/sizeof(preMonitorCfgName[0]); i++) {1769        if (!strcasecmp(preMonitorCfgName[i],name)) return 1;1770    }1771    return 0;1772}17731774/* free method for sentinelLoadQueueEntry when release the list */1775void freeSentinelLoadQueueEntry(void *item) {1776    struct sentinelLoadQueueEntry *entry = item;1777    sdsfreesplitres(entry->argv,entry->argc);1778    sdsfree(entry->line);1779    zfree(entry);1780}17811782/* This function is used for queuing sentinel configuration, the main1783 * purpose of this function is to delay parsing the sentinel config option1784 * in order to avoid the order dependent issue from the config. */1785void queueSentinelConfig(sds *argv, int argc, int linenum, sds line) {1786    int i;1787    struct sentinelLoadQueueEntry *entry;17881789    /* initialize sentinel_config for the first call */1790    if (server.sentinel_config == NULL) initializeSentinelConfig();17911792    entry = zmalloc(sizeof(struct sentinelLoadQueueEntry));1793    entry->argv = zmalloc(sizeof(char*)*argc);1794    entry->argc = argc;1795    entry->linenum = linenum;1796    entry->line = sdsdup(line);1797    for (i = 0; i < argc; i++) {1798        entry->argv[i] = sdsdup(argv[i]);1799    }1800    /*  Separate config lines with pre monitor config, monitor config and1801     *  post monitor config, in order to parsing config dependencies1802     *  correctly. */1803    if (!strcasecmp(argv[0],"monitor")) {1804        listAddNodeTail(server.sentinel_config->monitor_cfg,entry);1805    } else if (searchPreMonitorCfgName(argv[0])) {1806        listAddNodeTail(server.sentinel_config->pre_monitor_cfg,entry);1807    } else{1808        listAddNodeTail(server.sentinel_config->post_monitor_cfg,entry);1809    }1810}18111812/* This function is used for loading the sentinel configuration from1813 * pre_monitor_cfg, monitor_cfg and post_monitor_cfg list */1814void loadSentinelConfigFromQueue(void) {1815    const char *err = NULL;1816    listIter li;1817    listNode *ln;1818    int linenum = 0;1819    sds line = NULL;1820    unsigned int j;18211822    /* if there is no sentinel_config entry, we can return immediately */1823    if (server.sentinel_config == NULL) return;18241825    list *sentinel_configs[3] = {1826        server.sentinel_config->pre_monitor_cfg,1827        server.sentinel_config->monitor_cfg,1828        server.sentinel_config->post_monitor_cfg1829    };1830    /* loading from pre monitor config queue first to avoid dependency issues1831     * loading from monitor config queue1832     * loading from the post monitor config queue */1833    for (j = 0; j < sizeof(sentinel_configs) / sizeof(sentinel_configs[0]); j++) {1834        listRewind(sentinel_configs[j],&li);1835        while((ln = listNext(&li))) {1836            struct sentinelLoadQueueEntry *entry = ln->value;1837            err = sentinelHandleConfiguration(entry->argv,entry->argc);1838            if (err) {1839                linenum = entry->linenum;1840                line = entry->line;1841                goto loaderr;1842            }1843        }1844    }18451846    /* free sentinel_config when config loading is finished */1847    freeSentinelConfig();1848    return;18491850loaderr:1851    fprintf(stderr, "\n*** FATAL CONFIG FILE ERROR (Redis %s) ***\n",1852        REDIS_VERSION);1853    fprintf(stderr, "Reading the configuration file, at line %d\n", linenum);1854    fprintf(stderr, ">>> '%s'\n", line);1855    fprintf(stderr, "%s\n", err);1856    exit(1);1857}18581859const char *sentinelHandleConfiguration(char **argv, int argc) {18601861    sentinelRedisInstance *ri;18621863    if (!strcasecmp(argv[0],"monitor") && argc == 5) {1864        /* monitor <name> <host> <port> <quorum> */1865        int quorum = atoi(argv[4]);18661867        if (quorum <= 0) return "Quorum must be 1 or greater.";1868        if (createSentinelRedisInstance(argv[1],SRI_MASTER,argv[2],1869                                        atoi(argv[3]),quorum,NULL) == NULL)1870        {1871            return sentinelCheckCreateInstanceErrors(SRI_MASTER);1872        }1873    } else if (!strcasecmp(argv[0],"down-after-milliseconds") && argc == 3) {1874        /* down-after-milliseconds <name> <milliseconds> */1875        ri = sentinelGetMasterByName(argv[1]);1876        if (!ri) return "No such master with specified name.";1877        ri->down_after_period = atoi(argv[2]);1878        if (ri->down_after_period <= 0)1879            return "negative or zero time parameter.";1880        sentinelPropagateDownAfterPeriod(ri);1881    } else if (!strcasecmp(argv[0],"failover-timeout") && argc == 3) {1882        /* failover-timeout <name> <milliseconds> */1883        ri = sentinelGetMasterByName(argv[1]);1884        if (!ri) return "No such master with specified name.";1885        ri->failover_timeout = atoi(argv[2]);1886        if (ri->failover_timeout <= 0)1887            return "negative or zero time parameter.";1888    } else if (!strcasecmp(argv[0],"parallel-syncs") && argc == 3) {1889        /* parallel-syncs <name> <milliseconds> */1890        ri = sentinelGetMasterByName(argv[1]);1891        if (!ri) return "No such master with specified name.";1892        ri->parallel_syncs = atoi(argv[2]);1893    } else if (!strcasecmp(argv[0],"notification-script") && argc == 3) {1894        /* notification-script <name> <path> */1895        ri = sentinelGetMasterByName(argv[1]);1896        if (!ri) return "No such master with specified name.";1897        if (access(argv[2],X_OK) == -1)1898            return "Notification script seems non existing or non executable.";1899        ri->notification_script = sdsnew(argv[2]);1900    } else if (!strcasecmp(argv[0],"client-reconfig-script") && argc == 3) {1901        /* client-reconfig-script <name> <path> */1902        ri = sentinelGetMasterByName(argv[1]);1903        if (!ri) return "No such master with specified name.";1904        if (access(argv[2],X_OK) == -1)1905            return "Client reconfiguration script seems non existing or "1906                   "non executable.";1907        ri->client_reconfig_script = sdsnew(argv[2]);1908    } else if (!strcasecmp(argv[0],"auth-pass") && argc == 3) {1909        /* auth-pass <name> <password> */1910        ri = sentinelGetMasterByName(argv[1]);1911        if (!ri) return "No such master with specified name.";1912        ri->auth_pass = sdsnew(argv[2]);1913    } else if (!strcasecmp(argv[0],"auth-user") && argc == 3) {1914        /* auth-user <name> <username> */1915        ri = sentinelGetMasterByName(argv[1]);1916        if (!ri) return "No such master with specified name.";1917        ri->auth_user = sdsnew(argv[2]);1918    } else if (!strcasecmp(argv[0],"current-epoch") && argc == 2) {1919        /* current-epoch <epoch> */1920        unsigned long long current_epoch = strtoull(argv[1],NULL,10);1921        if (current_epoch > sentinel.current_epoch)1922            sentinel.current_epoch = current_epoch;1923    } else if (!strcasecmp(argv[0],"myid") && argc == 2) {1924        if (strlen(argv[1]) != CONFIG_RUN_ID_SIZE)1925            return "Malformed Sentinel id in myid option.";1926        memcpy(sentinel.myid,argv[1],CONFIG_RUN_ID_SIZE);1927    } else if (!strcasecmp(argv[0],"config-epoch") && argc == 3) {1928        /* config-epoch <name> <epoch> */1929        ri = sentinelGetMasterByName(argv[1]);1930        if (!ri) return "No such master with specified name.";1931        ri->config_epoch = strtoull(argv[2],NULL,10);1932        /* The following update of current_epoch is not really useful as1933         * now the current epoch is persisted on the config file, but1934         * we leave this check here for redundancy. */1935        if (ri->config_epoch > sentinel.current_epoch)1936            sentinel.current_epoch = ri->config_epoch;1937    } else if (!strcasecmp(argv[0],"leader-epoch") && argc == 3) {1938        /* leader-epoch <name> <epoch> */1939        ri = sentinelGetMasterByName(argv[1]);1940        if (!ri) return "No such master with specified name.";1941        ri->leader_epoch = strtoull(argv[2],NULL,10);1942    } else if ((!strcasecmp(argv[0],"known-slave") ||1943                !strcasecmp(argv[0],"known-replica")) && argc == 4)1944    {1945        sentinelRedisInstance *slave;19461947        /* known-replica <name> <ip> <port> */1948        ri = sentinelGetMasterByName(argv[1]);1949        if (!ri) return "No such master with specified name.";1950        if ((slave = createSentinelRedisInstance(NULL,SRI_SLAVE,argv[2],1951                    atoi(argv[3]), ri->quorum, ri)) == NULL)1952        {1953            return sentinelCheckCreateInstanceErrors(SRI_SLAVE);1954        }1955    } else if (!strcasecmp(argv[0],"known-sentinel") &&1956               (argc == 4 || argc == 5)) {1957        sentinelRedisInstance *si;19581959        if (argc == 5) { /* Ignore the old form without runid. */1960            /* known-sentinel <name> <ip> <port> [runid] */1961            ri = sentinelGetMasterByName(argv[1]);1962            if (!ri) return "No such master with specified name.";1963            if ((si = createSentinelRedisInstance(argv[4],SRI_SENTINEL,argv[2],1964                        atoi(argv[3]), ri->quorum, ri)) == NULL)1965            {1966                return sentinelCheckCreateInstanceErrors(SRI_SENTINEL);1967            }1968            si->runid = sdsnew(argv[4]);1969            sentinelTryConnectionSharing(si);1970        }1971    } else if (!strcasecmp(argv[0],"rename-command") && argc == 4) {1972        /* rename-command <name> <command> <renamed-command> */1973        ri = sentinelGetMasterByName(argv[1]);1974        if (!ri) return "No such master with specified name.";1975        sds oldcmd = sdsnew(argv[2]);1976        sds newcmd = sdsnew(argv[3]);1977        if (dictAdd(ri->renamed_commands,oldcmd,newcmd) != DICT_OK) {1978            sdsfree(oldcmd);1979            sdsfree(newcmd);1980            return "Same command renamed multiple times with rename-command.";1981        }1982    } else if (!strcasecmp(argv[0],"announce-ip") && argc == 2) {1983        /* announce-ip <ip-address> */1984        if (strlen(argv[1]))1985            sentinel.announce_ip = sdsnew(argv[1]);1986    } else if (!strcasecmp(argv[0],"announce-port") && argc == 2) {1987        /* announce-port <port> */1988        sentinel.announce_port = atoi(argv[1]);1989    } else if (!strcasecmp(argv[0],"deny-scripts-reconfig") && argc == 2) {1990        /* deny-scripts-reconfig <yes|no> */1991        if ((sentinel.deny_scripts_reconfig = yesnotoi(argv[1])) == -1) {1992            return "Please specify yes or no for the "1993                   "deny-scripts-reconfig options.";1994        }1995    } else if (!strcasecmp(argv[0],"sentinel-user") && argc == 2) {1996        /* sentinel-user <user-name> */1997        if (strlen(argv[1]))1998            sentinel.sentinel_auth_user = sdsnew(argv[1]);1999    } else if (!strcasecmp(argv[0],"sentinel-pass") && argc == 2) {2000        /* sentinel-pass <password> */