1/*2 * Copyright (c) 2009-Present, Redis Ltd.3 * All rights reserved.4 *5 * Copyright (c) 2024-present, Valkey contributors.6 * All rights reserved.7 *8 * Licensed under your choice of (a) the Redis Source Available License 2.09 * (RSALv2); or (b) the Server Side Public License v1 (SSPLv1); or (c) the10 * GNU Affero General Public License v3 (AGPLv3).11 *12 * Portions of this file are available under BSD3 terms; see REDISCONTRIBUTIONS for more information.13 */1415#include "server.h"16#include "atomicvar.h"17#include "cluster.h"18#include "cluster_slot_stats.h"19#include "script.h"20#include "fpconv_dtoa.h"21#include "fmtargs.h"22#include "cluster_asm.h"23#include "memory_prefetch.h"24#include "connection.h"25#include <sys/socket.h>26#include <sys/uio.h>27#include <math.h>28#include <ctype.h>2930static void setProtocolError(const char *errstr, client *c);31static void pauseClientsByClient(mstime_t end, int isPauseClientAll);32char *getClientSockname(client *c);33static inline int clientTypeIsSlave(client *c);34static inline int _clientHasPendingRepliesSlave(client *c);35static inline int _clientHasPendingRepliesNonSlave(client *c);36static inline int _writeToClientNonSlave(client *c, ssize_t *nwritten);37static inline int _writeToClientSlave(client *c, ssize_t *nwritten);38static pendingCommand *acquirePendingCommand(void);39static inline void reclaimPendingCommand(client *c, pendingCommand *pcmd);40static size_t getClientOutputBufferLogicalSize(client *c);4142int ProcessingEventsWhileBlocked = 0; /* See processEventsWhileBlocked(). */43__thread sds thread_reusable_qb = NULL;44__thread int thread_reusable_qb_used = 0; /* Avoid multiple clients using reusable query45 * buffer due to nested command execution. */4647/* Return the size consumed from the allocator, for the specified SDS string,48 * including internal fragmentation. This function is used in order to compute49 * the client output buffer size. */50size_t sdsZmallocSize(sds s) {51 void *sh = sdsAllocPtr(s);52 return zmalloc_size(sh);53}5455/* Return the amount of memory used by the sds string at object->ptr56 * for a string object. This includes internal fragmentation. */57size_t getStringObjectSdsUsedMemory(robj *o) {58 serverAssertWithInfo(NULL,o,o->type == OBJ_STRING);59 switch(o->encoding) {60 case OBJ_ENCODING_RAW: return sdsZmallocSize(o->ptr);61 case OBJ_ENCODING_EMBSTR: return zmalloc_size(o)-sizeof(robj);62 default: return 0; /* Just integer encoding for now. */63 }64}6566/* Return the length of a string object.67 * This does NOT includes internal fragmentation or sds unused space. */68size_t getStringObjectLen(robj *o) {69 serverAssertWithInfo(NULL,o,o->type == OBJ_STRING);70 switch(o->encoding) {71 case OBJ_ENCODING_RAW: return sdslen(o->ptr);72 case OBJ_ENCODING_EMBSTR: return sdslen(o->ptr);73 default: return 0; /* Just integer encoding for now. */74 }75}7677/* Client.reply list dup and free methods. */78void *dupClientReplyValue(void *o) {79 clientReplyBlock *old = o;80 clientReplyBlock *buf = zmalloc(sizeof(clientReplyBlock) + old->size);81 memcpy(buf, o, sizeof(clientReplyBlock) + old->size);82 return buf;83}8485void freeClientReplyValue(void *o) {86 zfree(o);87}8889/* This function links the client to the global linked list of clients.90 * unlinkClient() does the opposite, among other things. */91void linkClient(client *c) {92 listAddNodeTail(server.clients,c);93 /* Note that we remember the linked list node where the client is stored,94 * this way removing the client in unlinkClient() will not require95 * a linear scan, but just a constant time operation. */96 c->client_list_node = listLast(server.clients);97 uint64_t id = htonu64(c->id);98 raxInsert(server.clients_index,(unsigned char*)&id,sizeof(id),c,NULL);99}100101/* Initialize client authentication state.102 */103static void clientSetDefaultAuth(client *c) {104 /* If the default user does not require authentication, the user is105 * directly authenticated. */106 clientSetUser(c, DefaultUser, 0);107 c->authenticated = (c->user->flags & USER_FLAG_NOPASS) &&108 !(c->user->flags & USER_FLAG_DISABLED);109}110111int authRequired(client *c) {112 /* Check if the user is authenticated. This check is skipped in case113 * the default user is flagged as "nopass" and is active. */114 uint32_t default_flags;115 atomicGet(DefaultUser->flags, default_flags);116 int auth_required = (!(default_flags & USER_FLAG_NOPASS) ||117 (default_flags & USER_FLAG_DISABLED)) &&118 !c->authenticated;119 return auth_required;120}121122client *createClient(connection *conn) {123 client *c = zmalloc(sizeof(client));124125 /* passing NULL as conn it is possible to create a non connected client.126 * This is useful since all the commands needs to be executed127 * in the context of a client. When commands are executed in other128 * contexts (for instance a Lua script) we need a non connected client. */129 if (conn) {130 connEnableTcpNoDelay(conn);131 if (server.tcpkeepalive)132 connKeepAlive(conn,server.tcpkeepalive);133 connSetReadHandler(conn, readQueryFromClient);134 connSetPrivateData(conn, c);135 }136 c->buf = zmalloc_usable(PROTO_REPLY_CHUNK_BYTES, &c->buf_usable_size);137 selectDb(c,0);138 uint64_t client_id;139 atomicGetIncr(server.next_client_id, client_id, 1);140 c->id = client_id;141 c->tid = IOTHREAD_MAIN_THREAD_ID;142 c->running_tid = IOTHREAD_MAIN_THREAD_ID;143 if (conn) server.io_threads_clients_num[c->tid]++;144#ifdef LOG_REQ_RES145 reqresReset(c, 0);146 c->resp = server.client_default_resp;147#else148 c->resp = 2;149#endif150 c->conn = conn;151 c->name = NULL;152 c->lib_name = NULL;153 c->lib_ver = NULL;154 c->bufpos = 0;155 c->buf_peak = c->buf_usable_size;156 c->buf_peak_last_reset_time = server.unixtime;157 c->buf_encoded = 0;158 c->last_header = NULL;159 c->ref_repl_buf_node = NULL;160 c->ref_block_pos = 0;161 c->io_curr_repl_node = NULL;162 c->io_curr_block_pos = 0;163 c->io_bound_repl_node = NULL;164 c->io_bound_block_pos = 0;165 c->qb_pos = 0;166 c->querybuf = NULL;167 c->querybuf_peak = 0;168 c->reqtype = 0;169 c->argc = 0;170 c->argv = NULL;171 c->argv_len = 0;172 c->all_argv_len_sum = 0;173 c->pending_cmds.head = c->pending_cmds.tail = NULL;174 c->pending_cmds.len = c->pending_cmds.ready_len = 0;175 c->current_pending_cmd = NULL;176 c->original_argc = 0;177 c->original_argv = NULL;178 c->deferred_objects = NULL;179 c->deferred_objects_num = 0;180 c->io_deferred_objects = NULL;181 c->io_deferred_objects_num = 0;182 c->io_deferred_objects_size = 0;183 c->cmd = c->lastcmd = c->realcmd = c->lookedcmd = NULL;184 c->cur_script = NULL;185 c->multibulklen = 0;186 c->bulklen = -1;187 c->sentlen = 0;188 c->flags = 0;189 c->io_flags = CLIENT_IO_READ_ENABLED | CLIENT_IO_WRITE_ENABLED;190 c->read_error = 0;191 c->slot = -1;192 c->cluster_compatibility_check_slot = -2;193 c->ctime = c->lastinteraction = server.unixtime;194 c->io_lastinteraction = 0;195 c->duration = 0;196 c->user = DefaultUser; /* Set a safe default value: clientSetDefaultAuth reads c->user. */197 clientSetDefaultAuth(c);198 c->replstate = REPL_STATE_NONE;199 c->repl_start_cmd_stream_on_ack = 0;200 c->reploff = 0;201 c->reploff_next = 0;202 c->read_reploff = 0;203 c->io_read_reploff = 0;204 c->repl_applied = 0;205 c->repl_ack_off = 0;206 c->repl_ack_time = 0;207 c->io_repl_ack_time = 0;208 c->repl_aof_off = 0;209 c->repl_last_partial_write = 0;210 c->slave_listening_port = 0;211 c->slave_addr = NULL;212 c->slave_capa = SLAVE_CAPA_NONE;213 c->slave_req = SLAVE_REQ_NONE;214 c->main_ch_client_id = 0;215 c->reply = listCreate();216 c->deferred_reply_errors = NULL;217 c->reply_bytes = c->reply_bytes_shared = c->reply_bytes_unshared = 0;218 c->obuf_soft_limit_reached_time = 0;219 listSetFreeMethod(c->reply,freeClientReplyValue);220 listSetDupMethod(c->reply,dupClientReplyValue);221 initClientBlockingState(c);222 c->woff = 0;223 c->watched_keys = listCreate();224 c->pubsub_channels = dictCreate(&objectKeyPointerValueDictType);225 c->pubsub_patterns = dictCreate(&objectKeyPointerValueDictType);226 c->pubsubshard_channels = dictCreate(&objectKeyPointerValueDictType);227 c->peerid = NULL;228 c->sockname = NULL;229 c->client_list_node = NULL;230 c->io_thread_client_list_node = NULL;231 c->io_thread_compression_clients_node = NULL;232 c->postponed_list_node = NULL;233 c->client_tracking_redirection = 0;234 c->client_tracking_prefixes = NULL;235 c->io_last_client_cron = 0;236 c->io_last_repl_cron = 0;237 c->last_memory_usage = 0;238 c->last_memory_type = CLIENT_TYPE_NORMAL;239 c->module_blocked_client = NULL;240 c->module_auth_ctx = NULL;241 c->auth_callback = NULL;242 c->auth_callback_privdata = NULL;243 c->auth_module = NULL;244 listInitNode(&c->clients_pending_write_node, c);245 listInitNode(&c->pending_ref_reply_node, c);246 c->mem_usage_bucket = NULL;247 c->mem_usage_bucket_node = NULL;248 c->net_input_bytes_curr_cmd = 0;249 c->net_output_bytes_curr_cmd = 0;250 if (conn) linkClient(c);251 initClientMultiState(c);252 c->net_input_bytes = 0;253 c->net_output_bytes = 0;254 c->commands_processed = 0;255 c->last_ts_when_counted_as_active = 0;256 c->stat_total_read_events = 0;257 c->stat_avg_pipeline_length_sum = 0;258 c->stat_avg_pipeline_length_cnt = 0;259 c->task = NULL;260 c->node_id = NULL;261 c->himport_fieldsets = NULL;262 c->compression_level = 0;263 c->compression_state = NULL;264 atomicSet(c->pending_read, 0);265 return c;266}267268void installClientWriteHandler(client *c) {269 int ae_barrier = 0;270 /* For the fsync=always policy, we want that a given FD is never271 * served for reading and writing in the same event loop iteration,272 * so that in the middle of receiving the query, and serving it273 * to the client, we'll call beforeSleep() that will do the274 * actual fsync of AOF to disk. the write barrier ensures that. */275 if (server.aof_state == AOF_ON &&276 server.aof_fsync == AOF_FSYNC_ALWAYS)277 {278 ae_barrier = 1;279 }280 if (connSetWriteHandlerWithBarrier(c->conn, sendReplyToClient, ae_barrier) == C_ERR) {281 freeClientAsync(c);282 }283}284285/* This function puts the client in the queue of clients that should write286 * their output buffers to the socket. Note that it does not *yet* install287 * the write handler, to start clients are put in a queue of clients that need288 * to write, so we try to do that before returning in the event loop (see the289 * handleClientsWithPendingWrites() function).290 * If we fail and there is more data to write, compared to what the socket291 * buffers can hold, then we'll really install the handler. */292void putClientInPendingWriteQueue(client *c) {293 /* Schedule the client to write the output buffers to the socket only294 * if not already done and, for slaves, if the slave can actually receive295 * writes at this stage. */296 if (!(c->flags & CLIENT_PENDING_WRITE) &&297 (c->replstate == REPL_STATE_NONE ||298 c->replstate == SLAVE_STATE_SEND_BULK_AND_STREAM ||299 (c->replstate == SLAVE_STATE_ONLINE && !c->repl_start_cmd_stream_on_ack)))300 {301 /* Here instead of installing the write handler, we just flag the302 * client and put it into a list of clients that have something303 * to write to the socket. This way before re-entering the event304 * loop, we can try to directly write to the client sockets avoiding305 * a system call. We'll only really install the write handler if306 * we'll not be able to write the whole reply at once. */307 c->flags |= CLIENT_PENDING_WRITE;308 listLinkNodeHead(server.clients_pending_write, &c->clients_pending_write_node);309 }310}311312static inline int _prepareClientToWrite(client *c) {313 const uint64_t _flags = c->flags;314 /* If it's the Lua client we always return ok without installing any315 * handler since there is no socket at all. */316 if (unlikely(_flags & (CLIENT_SCRIPT|CLIENT_MODULE))) return C_OK;317318 /* If CLIENT_CLOSE_ASAP flag is set, we need not write anything. */319 if (unlikely(_flags & CLIENT_CLOSE_ASAP)) return C_ERR;320321 /* CLIENT REPLY OFF / SKIP handling: don't send replies.322 * CLIENT_PUSHING handling: disables the reply silencing flags. */323 if (unlikely((_flags & (CLIENT_REPLY_OFF|CLIENT_REPLY_SKIP)) &&324 !(_flags & CLIENT_PUSHING))) return C_ERR;325326 /* Masters don't receive replies, unless CLIENT_MASTER_FORCE_REPLY flag327 * is set. */328 if (unlikely((_flags & CLIENT_MASTER) &&329 !(_flags & CLIENT_MASTER_FORCE_REPLY))) return C_ERR;330331 if (unlikely(!c->conn)) return C_ERR; /* Fake client for AOF loading. */332333 /* Schedule the client to write the output buffers to the socket, unless334 * it should already be setup to do so (it has already pending data).335 *336 * If the client runs in an IO thread, we should not put the client in the337 * pending write queue. Instead, we will install the write handler to the338 * corresponding IO thread’s event loop and let it handle the reply. */339 if (likely(c->running_tid == IOTHREAD_MAIN_THREAD_ID) && !clientHasPendingReplies(c))340 putClientInPendingWriteQueue(c);341342 /* Authorize the caller to queue in the output buffer of this client. */343 return C_OK;344}345346/* This function is called every time we are going to transmit new data347 * to the client. The behavior is the following:348 *349 * If the client should receive new data (normal clients will) the function350 * returns C_OK, and make sure to install the write handler in our event351 * loop so that when the socket is writable new data gets written.352 *353 * If the client should not receive new data, because it is a fake client354 * (used to load AOF in memory), a master or because the setup of the write355 * handler failed, the function returns C_ERR.356 *357 * The function may return C_OK without actually installing the write358 * event handler in the following cases:359 *360 * 1) The event handler should already be installed since the output buffer361 * already contains something.362 * 2) The client is a slave but not yet online, so we want to just accumulate363 * writes in the buffer but not actually sending them yet.364 *365 * Typically gets called every time a reply is built, before adding more366 * data to the clients output buffers. If the function returns C_ERR no367 * data should be appended to the output buffers. */368int prepareClientToWrite(client *c) {369 return _prepareClientToWrite(c);370}371372/* -----------------------------------------------------------------------------373 * Low level functions to add more data to output buffers.374 * -------------------------------------------------------------------------- */375376static int tryAddPayload(client *c, char *buf, size_t *used, size_t size, uint8_t type, const void *payload, size_t len) {377 if (*used + sizeof(payloadHeader) + len > size) return 0;378379 /* Start a new payload chunk */380 payloadHeader *header = (payloadHeader *)(buf + *used);381 header->payload_type = type;382 header->payload_len = len;383 memcpy((char *)header + sizeof(payloadHeader), payload, len);384 *used += sizeof(payloadHeader) + len;385386 /* Track referenced reply bytes for copy avoidance. */387 if (type == BULK_STR_REF) {388 const bulkStrRef *str_ref = (const bulkStrRef *)payload;389 c->reply_bytes_shared += sdslen(str_ref->obj->ptr);390 }391392 return 1;393}394395/* Adds the payload to the reply linked list.396 * Note: some edits to this function need to be relayed to AddReplyFromClient. */397static void _addReplyPayloadToList(client *c, list *reply_list, const char *payload, size_t len, uint8_t payload_type) {398 listNode *ln = listLast(reply_list);399 clientReplyBlock *tail = ln ? listNodeValue(ln) : NULL;400 /* Determine if encoded buffer is required */401 int encoded = payload_type == BULK_STR_REF;402403 /* Note that 'tail' may be NULL even if we have a tail node, because when404 * addReplyDeferredLen() is used, it sets a dummy node to NULL just405 * to fill it later, when the size of the bulk length is set. */406407 /* Append to tail node when possible. */408 if (tail) {409 if (unlikely(tail->buf_encoded)) {410 /* Try to add to encoded buffer */411 if (tryAddPayload(c, tail->buf, &tail->used, tail->size, payload_type, (void *)payload, len)) {412 /* For BULK_STR_REF payloads, tryAddPayload updates shared reply bytes413 * which accounts for referenced strings. */414 if (encoded) closeClientOnOutputBufferLimitReached(c, 1);415 return;416 }417 } else if (!encoded) {418 /* Both tail and new payload are non-encoded, can append directly */419 size_t avail = tail->size - tail->used;420 size_t copy = avail >= len ? len : avail;421 if (copy > 0) {422 memcpy(tail->buf + tail->used, payload, copy);423 tail->used += copy;424 payload += copy;425 len -= copy;426 }427 }428 /* else: tail is non-encoded but new payload needs encoding, can't append */429 }430431 if (len) {432 /* Create a new node, make sure it is allocated to at433 * least PROTO_REPLY_CHUNK_BYTES */434 size_t usable_size;435 size_t required_size = encoded ? len + sizeof(payloadHeader) : len;436 size_t size = required_size < PROTO_REPLY_CHUNK_BYTES ? PROTO_REPLY_CHUNK_BYTES : required_size;437 tail = zmalloc_usable(size + sizeof(clientReplyBlock), &usable_size);438 /* take over the allocation's internal fragmentation */439 tail->size = usable_size - sizeof(clientReplyBlock);440 tail->used = 0;441 tail->buf_encoded = encoded;442 if (tail->buf_encoded) {443 serverAssert(tryAddPayload(c, tail->buf, &tail->used, tail->size, payload_type, (void *)payload, len));444 } else {445 tail->used = len;446 memcpy(tail->buf, payload, len);447 }448 listAddNodeTail(reply_list, tail);449 c->reply_bytes += tail->size;450451 closeClientOnOutputBufferLimitReached(c, 1);452 }453}454455/* The subscribe / unsubscribe command family has a push as a reply,456 * or in other words, it responds with a push (or several of them457 * depending on how many arguments it got), and has no reply. */458int cmdHasPushAsReply(struct redisCommand *cmd) {459 if (!cmd) return 0;460 return cmd->proc == subscribeCommand || cmd->proc == unsubscribeCommand ||461 cmd->proc == psubscribeCommand || cmd->proc == punsubscribeCommand ||462 cmd->proc == ssubscribeCommand || cmd->proc == sunsubscribeCommand;463}464465/* Attempts to add the reply to the static buffer in the client struct.466 * Returns the length of data that is added to the reply buffer. */467static size_t _addReplyPayloadToBuffer(client *c, const void *payload, size_t len, uint8_t payload_type) {468 /* If there already are entries in the reply list, we cannot469 * add anything more to the static buffer. */470 if (listLength(c->reply) > 0) return 0;471472 size_t available = c->buf_usable_size - c->bufpos;473 size_t reply_len = min(available, len);474 if (c->buf_encoded) {475 if (!tryAddPayload(c, c->buf, &c->bufpos, c->buf_usable_size, payload_type, payload, len))476 return 0;477 reply_len = len;478 } else {479 memcpy(c->buf + c->bufpos, payload, reply_len);480 c->bufpos+=reply_len;481 }482483 /* We update the buffer peak after appending the reply to the buffer */484 if (c->buf_peak < (size_t)c->bufpos) c->buf_peak = (size_t)c->bufpos;485 return reply_len;486}487488/* Adds bulk string reference (i.e. pointer to object and pointer to string itself) to static buffer489 * Returns non-zero value if succeeded to add */490static size_t _addBulkStrRefToBuffer(client *c, const void *payload, size_t len) {491 size_t result;492 if (!c->buf_encoded) {493 /* If buffer is plain and not empty then can't add bulk string reference to it */494 if (c->bufpos) return 0;495 c->buf_encoded = 1; /* Set c->buf to encoded mode to allow bulk string reference to be stored in it */496 result = _addReplyPayloadToBuffer(c, payload, len, BULK_STR_REF);497 if (!result) {498 /* Failed to add bulk string reference to buffer, need to revert to plain mode. */499 c->buf_encoded = 0;500 return 0;501 }502 } else {503 result = _addReplyPayloadToBuffer(c, payload, len, BULK_STR_REF);504 if (!result) return 0;505 }506507 /* Even though the bulk string is stored by reference and the underlying508 * memory is shared, we still account this shared memory towards this509 * client's output buffer usage, so we need to check the output buffer limits. */510 closeClientOnOutputBufferLimitReached(c, 1);511 return result;512}513514/* One segment of a multi-part reply. */515typedef struct replySegment {516 const char *ptr;517 size_t len;518} replySegment;519520/* Append 'nseg' ordered segments as a single logical reply. The replica-reject,521 * push-postpone and buffer-then-list spillover chain runs once for the whole batch522 * instead of once per segment, and each segment's bytes are copied at most once523 * (into the static buffer, or spilled to the reply list once the buffer fills) so524 * callers can hand a payload over by reference instead of pre-assembling it.525 *526 * Spillover is implicit: once any segment overflows into the reply list,527 * _addReplyPayloadToBuffer short-circuits to 0 (list non-empty) for the remaining528 * segments, routing the whole tail to the list in order.529 *530 * always_inline so the single-segment wrapper below scalarizes its on-stack segment531 * and stays branch-for-branch identical to a direct append on the hot reply path. */532static inline __attribute__((always_inline))533void _addReplySegmentsToBufferOrList(client *c, const replySegment *seg, int nseg) {534 if (c->flags & CLIENT_CLOSE_AFTER_REPLY) return;535536 /* Replicas should normally not cause any writes to the reply buffer. In case a rogue replica sent a command on the537 * replication link that caused a reply to be generated we'll simply disconnect it.538 * Note this is the simplest way to check a command added a response. Replication links are used to write data but539 * not for responses, so we should normally never get here on a replica client. */540 if (unlikely(clientTypeIsSlave(c))) {541 sds cmdname = c->lastcmd ? c->lastcmd->fullname : NULL;542 logInvalidUseAndFreeClientAsync(c, "Replica generated a reply to command '%s'",543 cmdname ? cmdname : "<unknown>");544 return;545 }546547 for (int i = 0; i < nseg; i++) c->net_output_bytes_curr_cmd += seg[i].len;548 /* We call it here because this may affect the reply buffer offset (see the549 * reqres function comment); it is idempotent per request. */550 reqresSaveClientReplyOffset(c);551552 /* If we're processing a push message into the current client (i.e. executing PUBLISH553 * to a channel which we are subscribed to, then we wanna postpone that message to be added554 * after the command's reply (specifically important during multi-exec). the exception is555 * the SUBSCRIBE command family, which (currently) have a push message instead of a proper reply.556 * The check for executing_client also avoids affecting push messages that are part of eviction.557 * Check CLIENT_PUSHING first to avoid race conditions, as it's absent in module's fake client. */558 if ((c->flags & CLIENT_PUSHING) && c == server.current_client &&559 server.executing_client && !cmdHasPushAsReply(server.executing_client->cmd))560 {561 for (int i = 0; i < nseg; i++)562 if (seg[i].len)563 _addReplyPayloadToList(c, server.pending_push_messages,564 seg[i].ptr, seg[i].len, PLAIN_REPLY);565 return;566 }567568 for (int i = 0; i < nseg; i++) {569 const char *s = seg[i].ptr;570 size_t len = seg[i].len;571 if (!len) continue;572 size_t reply_len = _addReplyPayloadToBuffer(c, s, len, PLAIN_REPLY);573 if (len > reply_len)574 _addReplyPayloadToList(c, c->reply, s + reply_len, len - reply_len, PLAIN_REPLY);575 }576}577578/* Append a single contiguous reply segment (thin wrapper over the batched form). */579void _addReplyToBufferOrList(client *c, const char *s, size_t len) {580 replySegment seg = { s, len };581 _addReplySegmentsToBufferOrList(c, &seg, 1);582}583584/* Check if the client's pending_ref_reply_node is currently linked in the list.585 * A node is considered linked if it has neighbors (prev/next), or if it's the586 * only node in the list (head points to it). */587static inline int clientIsInPendingRefReplyList(client *c) {588 return listNextNode(&c->pending_ref_reply_node) != NULL ||589 listPrevNode(&c->pending_ref_reply_node) != NULL ||590 listFirst(server.clients_with_pending_ref_reply) == &c->pending_ref_reply_node;591}592593/* Format "$<len>\r\n" into str_ref->prefix and set prefix_cnt. Called at write time. */594unsigned int formatBulkStrRefPrefix(bulkStrRef *str_ref) {595 if (str_ref->prefix_cnt != 0) return str_ref->prefix_cnt;596 str_ref->prefix[0] = '$';597 size_t num_len = ll2string(str_ref->prefix + 1, sizeof(str_ref->prefix) - 3, sdslen(str_ref->obj->ptr));598 str_ref->prefix[num_len + 1] = '\r';599 str_ref->prefix[num_len + 2] = '\n';600 str_ref->prefix_cnt = (unsigned int)(num_len + 3);601 return str_ref->prefix_cnt;602}603604/* Increment reference to object and add pointer to object and605 * pointer to string itself to current reply buffer */606static void _addBulkStrRefToBufferOrList(client *c, robj *obj, size_t len) {607 if (c->flags & CLIENT_CLOSE_AFTER_REPLY) return;608609 bulkStrRef str_ref;610 str_ref.obj = obj;611 incrRefCount(obj); /* Refcount will be decremented in write handler */612 str_ref.prefix_cnt = 0; /* filled at write time */613 str_ref.crlf[0] = '\r';614 str_ref.crlf[1] = '\n';615616 /* Track output bytes: bulk string prefix + content + trailing CRLF */617 c->net_output_bytes_curr_cmd += digits10(len) + 3 + len + 2;618619 /* We call it here because this function may affect the reply620 * buffer offset (see function comment) */621 reqresSaveClientReplyOffset(c);622623 if (!_addBulkStrRefToBuffer(c, (void *)&str_ref, sizeof(str_ref))) {624 _addReplyPayloadToList(c, c->reply, (void *)&str_ref, sizeof(str_ref), BULK_STR_REF);625 }626627 /* Track clients with pending referenced reply objects for async flushdb protection. */628 if (!clientIsInPendingRefReplyList(c)) {629 listLinkNodeTail(server.clients_with_pending_ref_reply, &c->pending_ref_reply_node);630 }631}632633/* -----------------------------------------------------------------------------634 * Higher level functions to queue data on the client output buffer.635 * The following functions are the ones that commands implementations will call.636 * -------------------------------------------------------------------------- */637638/* Add the object 'obj' string representation to the client output buffer. */639void addReply(client *c, robj *obj) {640 if (_prepareClientToWrite(c) != C_OK) return;641642 if (sdsEncodedObject(obj)) {643 _addReplyToBufferOrList(c,obj->ptr,sdslen(obj->ptr));644 } else if (obj->encoding == OBJ_ENCODING_INT) {645 /* For integer encoded strings we just convert it into a string646 * using our optimized function, and attach the resulting string647 * to the output buffer. */648 char buf[32];649 size_t len = ll2string(buf,sizeof(buf),(long)obj->ptr);650 _addReplyToBufferOrList(c,buf,len);651 } else {652 serverPanic("Wrong obj->encoding in addReply()");653 }654}655656/* Add the SDS 's' string to the client output buffer, as a side effect657 * the SDS string is freed. */658void addReplySds(client *c, sds s) {659 if (_prepareClientToWrite(c) != C_OK) {660 /* The caller expects the sds to be free'd. */661 sdsfree(s);662 return;663 }664 _addReplyToBufferOrList(c,s,sdslen(s));665 sdsfree(s);666}667668/* This low level function just adds whatever protocol you send it to the669 * client buffer, trying the static buffer initially, and using the string670 * of objects if not possible.671 *672 * It is efficient because does not create an SDS object nor an Redis object673 * if not needed. The object will only be created by calling674 * _addReplyProtoToList() if we fail to extend the existing tail object675 * in the list of objects. */676void addReplyProto(client *c, const char *s, size_t len) {677 if (_prepareClientToWrite(c) != C_OK) return;678 _addReplyToBufferOrList(c,s,len);679}680681/* Low level function called by the addReplyError...() functions.682 * It emits the protocol for a Redis error, in the form:683 *684 * -ERRORCODE Error Message<CR><LF>685 *686 * If the error code is already passed in the string 's', the error687 * code provided is used, otherwise the string "-ERR " for the generic688 * error code is automatically added.689 * Note that 's' must NOT end with \r\n. */690void addReplyErrorLength(client *c, const char *s, size_t len) {691 /* If the string already starts with "-..." then the error code692 * is provided by the caller. Otherwise we use "-ERR". */693 if (!len || s[0] != '-') addReplyProto(c,"-ERR ",5);694 addReplyProto(c,s,len);695 addReplyProto(c,"\r\n",2);696}697698/* Do some actions after an error reply was sent (Log if needed, updates stats, etc.)699 * Possible flags:700 * * ERR_REPLY_FLAG_NO_STATS_UPDATE - indicate not to update any error stats. */701void afterErrorReply(client *c, const char *s, size_t len, int flags) {702 /* Module clients fall into two categories:703 * Calls to RM_Call, in which case the error isn't being returned to a client, so should not be counted.704 * Module thread safe context calls to RM_ReplyWithError, which will be added to a real client by the main thread later. */705 if (c->flags & CLIENT_MODULE) {706 if (!c->deferred_reply_errors) {707 c->deferred_reply_errors = listCreate();708 listSetFreeMethod(c->deferred_reply_errors, sdsfreegeneric);709 }710 listAddNodeTail(c->deferred_reply_errors, sdsnewlen(s, len));711 return;712 }713714 if (!(flags & ERR_REPLY_FLAG_NO_STATS_UPDATE)) {715 /* Increment the global error counter */716 server.stat_total_error_replies++;717 /* Increment the error stats718 * If the string already starts with "-..." then the error prefix719 * is provided by the caller ( we limit the search to 32 chars). Otherwise we use "-ERR". */720 if (s[0] != '-') {721 incrementErrorCount("ERR", 3);722 } else {723 const char *spaceloc = memchr(s, ' ', len < 32 ? len : 32);724 if (spaceloc) {725 const size_t errEndPos = (size_t)(spaceloc - s);726 incrementErrorCount(s+1, errEndPos-1);727 } else {728 /* Fallback to ERR if we can't retrieve the error prefix */729 incrementErrorCount("ERR", 3);730 }731 }732 } else {733 /* stat_total_error_replies will not be updated, which means that734 * the cmd stats will not be updated as well, we still want this command735 * to be counted as failed so we update it here. We update c->realcmd in736 * case c->cmd was changed (like in GEOADD). */737 c->realcmd->failed_calls++;738 }739740 /* Sometimes it could be normal that a slave replies to a master with741 * an error and this function gets called. Actually the error will never742 * be sent because addReply*() against master clients has no effect...743 * A notable example is:744 *745 * EVAL 'redis.call("incr",KEYS[1]); redis.call("nonexisting")' 1 x746 *747 * Where the master must propagate the first change even if the second748 * will produce an error. However it is useful to log such events since749 * they are rare and may hint at errors in a script or a bug in Redis. */750 int ctype = getClientType(c);751 if (ctype == CLIENT_TYPE_MASTER || ctype == CLIENT_TYPE_SLAVE || c->id == CLIENT_ID_AOF) {752 char *to, *from;753754 if (c->id == CLIENT_ID_AOF) {755 to = "AOF-loading-client";756 from = "server";757 } else if (ctype == CLIENT_TYPE_MASTER) {758 if (c->flags & CLIENT_ASM_IMPORTING) {759 to = "source";760 from = "destination";761 } else {762 to = "master";763 from = "replica";764 }765 } else {766 to = "replica";767 from = "master";768 }769770 if (len > 4096) len = 4096;771 sds cmdname = c->lastcmd ? c->lastcmd->fullname : NULL;772 serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error "773 "to its %s: '%.*s' after processing the command "774 "'%s'", from, to, (int)len, s, cmdname ? cmdname : "<unknown>");775 if (ctype == CLIENT_TYPE_MASTER && server.repl_backlog &&776 !(c->flags & CLIENT_ASM_IMPORTING) && server.repl_backlog->histlen > 0)777 {778 showLatestBacklog();779 }780 server.stat_unexpected_error_replies++;781782 /* Based off the propagation error behavior, check if we need to panic here. There783 * are currently two checked cases:784 * * If this command was from our master and we are not a writable replica.785 * * We are reading from an AOF file. */786 int panic_in_replicas = (ctype == CLIENT_TYPE_MASTER && server.repl_slave_ro)787 && (server.propagation_error_behavior == PROPAGATION_ERR_BEHAVIOR_PANIC ||788 server.propagation_error_behavior == PROPAGATION_ERR_BEHAVIOR_PANIC_ON_REPLICAS);789 int panic_in_aof = c->id == CLIENT_ID_AOF 790 && server.propagation_error_behavior == PROPAGATION_ERR_BEHAVIOR_PANIC;791 if (panic_in_replicas || panic_in_aof) {792 serverPanic("This %s panicked sending an error to its %s"793 " after processing the command '%s'",794 from, to, cmdname ? cmdname : "<unknown>");795 }796 }797}798799/* The 'err' object is expected to start with -ERRORCODE and end with \r\n.800 * Unlike addReplyErrorSds and others alike which rely on addReplyErrorLength. */801void addReplyErrorObject(client *c, robj *err) {802 addReply(c, err);803 afterErrorReply(c, err->ptr, sdslen(err->ptr)-2, 0); /* Ignore trailing \r\n */804}805806/* Sends either a reply or an error reply by checking the first char.807 * If the first char is '-' the reply is considered an error.808 * In any case the given reply is sent, if the reply is also recognize809 * as an error we also perform some post reply operations such as810 * logging and stats update. */811void addReplyOrErrorObject(client *c, robj *reply) {812 serverAssert(sdsEncodedObject(reply));813 sds rep = reply->ptr;814 if (sdslen(rep) > 1 && rep[0] == '-') {815 addReplyErrorObject(c, reply);816 } else {817 addReply(c, reply);818 }819}820821/* See addReplyErrorLength for expectations from the input string. */822void addReplyError(client *c, const char *err) {823 addReplyErrorLength(c,err,strlen(err));824 afterErrorReply(c,err,strlen(err),0);825}826827/* Add error reply to the given client.828 * Supported flags:829 * * ERR_REPLY_FLAG_NO_STATS_UPDATE - indicate not to perform any error stats updates */830void addReplyErrorSdsEx(client *c, sds err, int flags) {831 addReplyErrorLength(c,err,sdslen(err));832 afterErrorReply(c,err,sdslen(err),flags);833 sdsfree(err);834}835836/* See addReplyErrorLength for expectations from the input string.837 * The string is safe to contain \r and \n anywhere. */838void addReplyErrorSdsExSafe(client *c, sds err, int flags) {839 err = sdsmapchars(err, "\r\n", " ", 2);840 addReplyErrorSdsEx(c, err, flags);841}842843/* See addReplyErrorLength for expectations from the input string. */844/* As a side effect the SDS string is freed. */845void addReplyErrorSds(client *c, sds err) {846 addReplyErrorSdsEx(c, err, 0);847}848849/* See addReplyErrorLength for expectations from the input string. */850/* As a side effect the SDS string is freed. */851void addReplyErrorSdsSafe(client *c, sds err) {852 err = sdsmapchars(err, "\r\n", " ", 2);853 addReplyErrorSdsEx(c, err, 0);854}855856/* Internal function used by addReplyErrorFormat, addReplyErrorFormatEx and RM_ReplyWithErrorFormat.857 * Refer to afterErrorReply for more information about the flags. */858void addReplyErrorFormatInternal(client *c, int flags, const char *fmt, va_list ap) {859 va_list cpy;860 va_copy(cpy,ap);861 sds s = sdscatvprintf(sdsempty(),fmt,cpy);862 va_end(cpy);863 /* Trim any newlines at the end (ones will be added by addReplyErrorLength) */864 s = sdstrim(s, "\r\n");865 /* Make sure there are no newlines in the middle of the string, otherwise866 * invalid protocol is emitted. */867 s = sdsmapchars(s, "\r\n", " ", 2);868 addReplyErrorLength(c,s,sdslen(s));869 afterErrorReply(c,s,sdslen(s),flags);870 sdsfree(s);871}872873void addReplyErrorFormatEx(client *c, int flags, const char *fmt, ...) {874 va_list ap;875 va_start(ap,fmt);876 addReplyErrorFormatInternal(c, flags, fmt, ap);877 va_end(ap);878}879880/* See addReplyErrorLength for expectations from the formatted string.881 * The formatted string is safe to contain \r and \n anywhere. */882void addReplyErrorFormat(client *c, const char *fmt, ...) {883 va_list ap;884 va_start(ap,fmt);885 addReplyErrorFormatInternal(c, 0, fmt, ap);886 va_end(ap);887}888889void addReplyErrorArity(client *c) {890 addReplyErrorFormat(c, "wrong number of arguments for '%s' command",891 c->cmd->fullname);892}893894void addReplyErrorExpireTime(client *c) {895 addReplyErrorFormat(c, "invalid expire time in '%s' command",896 c->cmd->fullname);897}898899void addReplyStatusLength(client *c, const char *s, size_t len) {900 addReplyProto(c,"+",1);901 addReplyProto(c,s,len);902 addReplyProto(c,"\r\n",2);903}904905void addReplyStatus(client *c, const char *status) {906 addReplyStatusLength(c,status,strlen(status));907}908909/* Like addReplyStatus() but the string is safe to contain \r and \n anywhere. */910void addReplyStatusSafe(client *c, const char *status) {911 sds s = sdsmapchars(sdsnew(status), "\r\n", " ", 2);912 addReplyStatusLength(c,s,sdslen(s));913 sdsfree(s);914}915916void addReplyStatusFormat(client *c, const char *fmt, ...) {917 va_list ap;918 va_start(ap,fmt);919 sds s = sdscatvprintf(sdsempty(),fmt,ap);920 va_end(ap);921 /* Trim any newlines at the end (ones will be added by addReplyStatusLength) */922 s = sdstrim(s, "\r\n");923 /* Make sure there are no newlines in the middle of the string, otherwise924 * invalid protocol is emitted. */925 s = sdsmapchars(s, "\r\n", " ", 2);926 addReplyStatusLength(c,s,sdslen(s));927 sdsfree(s);928}929930/* Sometimes we are forced to create a new reply node, and we can't append to931 * the previous one, when that happens, we wanna try to trim the unused space932 * at the end of the last reply node which we won't use anymore. */933void trimReplyUnusedTailSpace(client *c) {934 listNode *ln = listLast(c->reply);935 clientReplyBlock *tail = ln? listNodeValue(ln): NULL;936937 /* Note that 'tail' may be NULL even if we have a tail node, because when938 * addReplyDeferredLen() is used */939 if (!tail) return;940941 /* We only try to trim the space is relatively high (more than a 1/4 of the942 * allocation), otherwise there's a high chance realloc will NOP.943 * Also, to avoid large memmove which happens as part of realloc, we only do944 * that if the used part is small. */945 if (tail->size - tail->used > tail->size / 4 &&946 tail->used < PROTO_REPLY_CHUNK_BYTES && !tail->buf_encoded)947 {948 size_t usable_size;949 size_t old_size = tail->size;950 tail = zrealloc_usable(tail, tail->used + sizeof(clientReplyBlock), &usable_size, NULL);951 /* take over the allocation's internal fragmentation (at least for952 * memory usage tracking) */953 tail->size = usable_size - sizeof(clientReplyBlock);954 c->reply_bytes = c->reply_bytes + tail->size - old_size;955 listNodeValue(ln) = tail;956 }957}958959/* Adds an empty object to the reply list that will contain the multi bulk960 * length, which is not known when this function is called. */961void *addReplyDeferredLen(client *c) {962 /* Note that we install the write event here even if the object is not963 * ready to be sent, since we are sure that before returning to the964 * event loop setDeferredAggregateLen() will be called. */965 if (_prepareClientToWrite(c) != C_OK) return NULL;966967 /* Replicas should normally not cause any writes to the reply buffer. In case a rogue replica sent a command on the968 * replication link that caused a reply to be generated we'll simply disconnect it.969 * Note this is the simplest way to check a command added a response. Replication links are used to write data but970 * not for responses, so we should normally never get here on a replica client. */971 if (unlikely(clientTypeIsSlave(c))) {972 sds cmdname = c->lastcmd ? c->lastcmd->fullname : NULL;973 logInvalidUseAndFreeClientAsync(c, "Replica generated a reply to command '%s'",974 cmdname ? cmdname : "<unknown>");975 return NULL;976 }977978 /* We call it here because this function conceptually affects the reply979 * buffer offset (see function comment) */980 reqresSaveClientReplyOffset(c);981982 trimReplyUnusedTailSpace(c);983 listAddNodeTail(c->reply,NULL); /* NULL is our placeholder. */984 return listLast(c->reply);985}986987void setDeferredReply(client *c, void *node, const char *s, size_t length) {988 listNode *ln = (listNode*)node;989 clientReplyBlock *next, *prev;990991 /* Abort when *node is NULL: when the client should not accept writes992 * we return NULL in addReplyDeferredLen() */993 if (node == NULL) return;994 serverAssert(!listNodeValue(ln));995996 /* Normally we fill this dummy NULL node, added by addReplyDeferredLen(),997 * with a new buffer structure containing the protocol needed to specify998 * the length of the array following. However sometimes there might be room999 * in the previous/next node so we can instead remove this NULL node, and1000 * suffix/prefix our data in the node immediately before/after it, in order1001 * to save a write(2) syscall later. Conditions needed to do it:1002 *1003 * - The prev node is non-NULL and has space in it or1004 * - The next node is non-NULL,1005 * - It has enough room already allocated1006 * - And not too large (avoid large memmove) */1007 if (ln->prev != NULL && (prev = listNodeValue(ln->prev)) &&1008 prev->used < prev->size && !prev->buf_encoded)1009 {1010 size_t len_to_copy = prev->size - prev->used;1011 if (len_to_copy > length)1012 len_to_copy = length;1013 memcpy(prev->buf + prev->used, s, len_to_copy);1014 c->net_output_bytes_curr_cmd += len_to_copy;1015 prev->used += len_to_copy;1016 length -= len_to_copy;1017 if (length == 0) {1018 listDelNode(c->reply, ln);1019 return;1020 }1021 s += len_to_copy;1022 }10231024 if (ln->next != NULL && (next = listNodeValue(ln->next)) &&1025 next->size - next->used >= length &&1026 next->used < PROTO_REPLY_CHUNK_BYTES * 4 &&1027 !next->buf_encoded)1028 {1029 memmove(next->buf + length, next->buf, next->used);1030 memcpy(next->buf, s, length);1031 c->net_output_bytes_curr_cmd += length;1032 next->used += length;1033 listDelNode(c->reply,ln);1034 } else {1035 /* Create a new node */1036 size_t usable_size;1037 clientReplyBlock *buf = zmalloc_usable(length + sizeof(clientReplyBlock), &usable_size);1038 /* Take over the allocation's internal fragmentation */1039 buf->size = usable_size - sizeof(clientReplyBlock);1040 buf->used = length;1041 buf->buf_encoded = 0;1042 memcpy(buf->buf, s, length);1043 c->net_output_bytes_curr_cmd += length;1044 listNodeValue(ln) = buf;1045 c->reply_bytes += buf->size;10461047 closeClientOnOutputBufferLimitReached(c, 1);1048 }1049}10501051/* Populate the length object and try gluing it to the next chunk. */1052void setDeferredAggregateLen(client *c, void *node, long length, char prefix) {1053 serverAssert(length >= 0);10541055 /* Abort when *node is NULL: when the client should not accept writes1056 * we return NULL in addReplyDeferredLen() */1057 if (node == NULL) return;10581059 /* Things like *2\r\n, %3\r\n or ~4\r\n are emitted very often by the protocol1060 * so we have a few shared objects to use if the integer is small1061 * like it is most of the times. */1062 const size_t hdr_len = OBJ_SHARED_HDR_STRLEN(length);1063 const int opt_hdr = length < OBJ_SHARED_BULKHDR_LEN;1064 if (prefix == '*' && opt_hdr) {1065 setDeferredReply(c, node, shared.mbulkhdr[length]->ptr, hdr_len);1066 return;1067 }1068 if (prefix == '%' && opt_hdr) {1069 setDeferredReply(c, node, shared.maphdr[length]->ptr, hdr_len);1070 return;1071 }1072 if (prefix == '~' && opt_hdr) {1073 setDeferredReply(c, node, shared.sethdr[length]->ptr, hdr_len);1074 return;1075 }10761077 char lenstr[128];1078 size_t lenstr_len = snprintf(lenstr, sizeof(lenstr), "%c%ld\r\n", prefix, length);1079 setDeferredReply(c, node, lenstr, lenstr_len);1080}10811082void setDeferredArrayLen(client *c, void *node, long length) {1083 setDeferredAggregateLen(c,node,length,'*');1084}10851086void setDeferredMapLen(client *c, void *node, long length) {1087 int prefix = c->resp == 2 ? '*' : '%';1088 if (c->resp == 2) length *= 2;1089 setDeferredAggregateLen(c,node,length,prefix);1090}10911092void setDeferredSetLen(client *c, void *node, long length) {1093 int prefix = c->resp == 2 ? '*' : '~';1094 setDeferredAggregateLen(c,node,length,prefix);1095}10961097void setDeferredAttributeLen(client *c, void *node, long length) {1098 serverAssert(c->resp >= 3);1099 setDeferredAggregateLen(c,node,length,'|');1100}11011102void setDeferredPushLen(client *c, void *node, long length) {1103 serverAssert(c->resp >= 3);1104 setDeferredAggregateLen(c,node,length,'>');1105}11061107/* Add a double as a bulk reply */1108void addReplyDouble(client *c, double d) {1109 if (c->resp == 3) {1110 char dbuf[MAX_D2STRING_CHARS+3];1111 dbuf[0] = ',';1112 const int dlen = d2string(dbuf+1,sizeof(dbuf)-1,d);1113 dbuf[dlen+1] = '\r';1114 dbuf[dlen+2] = '\n';1115 dbuf[dlen+3] = '\0';1116 addReplyProto(c,dbuf,dlen+3);1117 } else {1118 char dbuf[MAX_LONG_DOUBLE_CHARS+32];1119 /* In order to prepend the string length before the formatted number,1120 * but still avoid an extra memcpy of the whole number, we reserve space1121 * for maximum header `$0000\r\n`, print double, add the resp header in1122 * front of it, and then send the buffer with the right `start` offset. */1123 const int dlen = d2string(dbuf+7,sizeof(dbuf)-7,d);1124 int digits = digits10(dlen);1125 int start = 4 - digits;1126 serverAssert(start >= 0);1127 dbuf[start] = '$';11281129 /* Convert `dlen` to string, putting it's digits after '$' and before the1130 * formatted double string. */1131 for(int i = digits, val = dlen; val && i > 0 ; --i, val /= 10) {1132 dbuf[start + i] = "0123456789"[val % 10];1133 }1134 dbuf[5] = '\r';1135 dbuf[6] = '\n';1136 dbuf[dlen+7] = '\r';1137 dbuf[dlen+8] = '\n';1138 dbuf[dlen+9] = '\0';1139 addReplyProto(c,dbuf+start,dlen+9-start);1140 }1141}11421143void addReplyBigNum(client *c, const char* num, size_t len) {1144 if (c->resp == 2) {1145 addReplyBulkCBuffer(c, num, len);1146 } else {1147 addReplyProto(c,"(",1);1148 addReplyProto(c,num,len);1149 addReplyProto(c,"\r\n",2);1150 }1151}11521153/* Add a long double as a bulk reply, but uses a human readable formatting1154 * of the double instead of exposing the crude behavior of doubles to the1155 * dear user. */1156void addReplyHumanLongDouble(client *c, long double d) {1157 if (c->resp == 2) {1158 robj *o = createStringObjectFromLongDouble(d,1);1159 addReplyBulk(c,o);1160 decrRefCount(o);1161 } else {1162 char buf[MAX_LONG_DOUBLE_CHARS];1163 int len = ld2string(buf,sizeof(buf),d,LD_STR_HUMAN);1164 addReplyProto(c,",",1);1165 addReplyProto(c,buf,len);1166 addReplyProto(c,"\r\n",2);1167 }1168}11691170static inline void _addReplyLongLongSharedHdr(client *c, long long ll, char prefix,1171 robj *shared_hdr[OBJ_SHARED_BULKHDR_LEN])1172{1173 char buf[128];1174 int len;1175 const int opt_hdr = ll < OBJ_SHARED_BULKHDR_LEN && ll >= 0;11761177 if (opt_hdr) {1178 _addReplyToBufferOrList(c, shared_hdr[ll]->ptr, OBJ_SHARED_HDR_STRLEN(ll));1179 return;1180 }11811182 buf[0] = prefix;1183 len = ll2string(buf + 1, sizeof(buf) - 1, ll);1184 buf[len + 1] = '\r';1185 buf[len + 2] = '\n';1186 _addReplyToBufferOrList(c, buf, len + 3);1187}11881189static inline void _addReplyLongLongBulk(client *c, long long ll) {1190 _addReplyLongLongSharedHdr(c, ll, '$', shared.bulkhdr);1191}11921193static inline void _addReplyLongLongMBulk(client *c, long long ll) {1194 _addReplyLongLongSharedHdr(c, ll, '*', shared.mbulkhdr);1195}11961197/* Add a long long as integer reply or bulk len / multi bulk count.1198 * Basically this is used to output <prefix><long long><crlf>. */1199static void _addReplyLongLongWithPrefix(client *c, long long ll, char prefix) {1200 char buf[128];1201 int len;12021203 /* Things like $3\r\n or *2\r\n are emitted very often by the protocol1204 * so we have a few shared objects to use if the integer is small1205 * like it is most of the times. */1206 const int opt_hdr = ll < OBJ_SHARED_BULKHDR_LEN && ll >= 0;1207 const size_t hdr_len = OBJ_SHARED_HDR_STRLEN(ll);1208 if (prefix == '*' && opt_hdr) {1209 _addReplyToBufferOrList(c, shared.mbulkhdr[ll]->ptr, hdr_len);1210 return;1211 } else if (prefix == '$' && opt_hdr) {1212 _addReplyToBufferOrList(c, shared.bulkhdr[ll]->ptr, hdr_len);1213 return;1214 } else if (prefix == '%' && opt_hdr) {1215 _addReplyToBufferOrList(c, shared.maphdr[ll]->ptr, hdr_len);1216 return;1217 } else if (prefix == '~' && opt_hdr) {1218 _addReplyToBufferOrList(c, shared.sethdr[ll]->ptr, hdr_len);1219 return;1220 }12211222 buf[0] = prefix;1223 len = ll2string(buf + 1, sizeof(buf) - 1, ll);1224 buf[len + 1] = '\r';1225 buf[len + 2] = '\n';1226 _addReplyToBufferOrList(c, buf, len + 3);1227}12281229void addReplyLongLong(client *c, long long ll) {1230 if (ll == 0)1231 addReply(c,shared.czero);1232 else if (ll == 1)1233 addReply(c, shared.cone);1234 else {1235 if (_prepareClientToWrite(c) != C_OK) return;1236 _addReplyLongLongWithPrefix(c, ll, ':');1237 }1238}12391240void addReplyLongLongFromStr(client *c, robj *str) {1241 addReplyProto(c,":",1);1242 addReply(c,str);1243 addReplyProto(c,"\r\n",2);1244}12451246/* Reply with unsigned 64-bit value. Uses integer reply when value fits in1247 * signed long long, otherwise big number (RESP3) or bulk string (RESP2). */1248void addReplyUnsignedLongLong(client *c, uint64_t v) {1249 if (v <= (uint64_t)LLONG_MAX) {1250 addReplyLongLong(c, (long long)v);1251 } else {1252 char buf[LONG_STR_SIZE];1253 int len = ull2string(buf, sizeof(buf), v);1254 addReplyBigNum(c, buf, len);1255 }1256}12571258void addReplyAggregateLen(client *c, long length, int prefix) {1259 serverAssert(length >= 0);1260 if (_prepareClientToWrite(c) != C_OK) return;1261 _addReplyLongLongWithPrefix(c, length, prefix);1262}12631264void addReplyArrayLen(client *c, long length) {1265 serverAssert(length >= 0);1266 if (_prepareClientToWrite(c) != C_OK) return;1267 _addReplyLongLongMBulk(c, length);1268}12691270void addReplyMapLen(client *c, long length) {1271 int prefix = c->resp == 2 ? '*' : '%';1272 if (c->resp == 2) length *= 2;1273 addReplyAggregateLen(c,length,prefix);1274}12751276void addReplySetLen(client *c, long length) {1277 int prefix = c->resp == 2 ? '*' : '~';1278 addReplyAggregateLen(c,length,prefix);1279}12801281void addReplyAttributeLen(client *c, long length) {1282 serverAssert(c->resp >= 3);1283 addReplyAggregateLen(c,length,'|');1284}12851286void addReplyPushLen(client *c, long length) {1287 serverAssert(c->resp >= 3);1288 serverAssertWithInfo(c, NULL, c->flags & CLIENT_PUSHING);1289 addReplyAggregateLen(c,length,'>');1290}12911292void addReplyNull(client *c) {1293 if (c->resp == 2) {1294 addReplyProto(c,"$-1\r\n",5);1295 } else {1296 addReplyProto(c,"_\r\n",3);1297 }1298}12991300void addReplyBool(client *c, int b) {1301 if (c->resp == 2) {1302 addReply(c, b ? shared.cone : shared.czero);1303 } else {1304 addReplyProto(c, b ? "#t\r\n" : "#f\r\n",4);1305 }1306}13071308/* A null array is a concept that no longer exists in RESP3. However1309 * RESP2 had it, so API-wise we have this call, that will emit the correct1310 * RESP2 protocol, however for RESP3 the reply will always be just the1311 * Null type "_\r\n". */1312void addReplyNullArray(client *c) {1313 if (c->resp == 2) {1314 addReplyProto(c,"*-1\r\n",5);1315 } else {1316 addReplyProto(c,"_\r\n",3);1317 }1318}13191320/* Create the length prefix of a bulk reply, example: $2234 */1321void addReplyBulkLen(client *c, robj *obj) {1322 size_t len = stringObjectLen(obj);1323 if (_prepareClientToWrite(c) != C_OK) return;1324 _addReplyLongLongBulk(c, len);1325}13261327/* Check if copy avoidance is preferred for this client and object.1328 * Copy avoidance allows I/O threads to directly reference obj->ptr1329 * instead of copying data to reply buffers. */1330static int isCopyAvoidPreferred(client *c, robj *obj, size_t len) {1331 /* Don't use copy avoidance for fake clients. */1332 if (!c->conn || !server.reply_copy_avoidance_enabled) return 0;13331334 int type = getClientType(c);1335 if (type != CLIENT_TYPE_NORMAL) return 0;13361337 /* Don't use copy avoidance for push messages. Push messages need to be deferred1338 * to server.pending_push_messages when CLIENT_PUSHING is set. */1339 if (c->flags & CLIENT_PUSHING) return 0;13401341 if (obj->encoding != OBJ_ENCODING_RAW || obj->refcount >= OBJ_FIRST_SPECIAL_REFCOUNT) return 0;13421343 /* Copy avoidance is preferred for any string size starting certain number of I/O threads */1344 if (server.io_threads_num >= COPY_AVOID_MIN_IO_THREADS) return 1;13451346 /* Main thread only. No I/O threads */1347 if (server.io_threads_num == 1) {1348 /* Copy avoidance is preferred starting certain string size */1349 return len >= COPY_AVOID_MIN_STRING_SIZE;1350 }13511352 /* Main thread + I/O threads */1353 return len >= COPY_AVOID_MIN_STRING_SIZE_THREADED;1354}13551356/* Try to avoid whole bulk string copy to a reply buffer1357 * If copy avoidance allowed then only pointer to object and string will be copied to the buffer */1358static int tryAvoidBulkStrCopyToReply(client *c, robj *obj, size_t len) {1359 if (!isCopyAvoidPreferred(c, obj, len)) return C_ERR;1360 _addBulkStrRefToBufferOrList(c, obj, len);1361 return C_OK;1362}13631364/* Add a Redis Object as a bulk reply.1365 * If avoid_copy is non-zero, attempt to use copy avoidance optimization. */1366void addReplyBulkWithFlag(client *c, robj *obj, int avoid_copy) {1367 if (_prepareClientToWrite(c) != C_OK) return;13681369 if (sdsEncodedObject(obj)) {1370 const size_t len = sdslen(obj->ptr);1371 if (avoid_copy && tryAvoidBulkStrCopyToReply(c, obj, len) == C_OK)1372 return;1373 _addReplyLongLongBulk(c, len);1374 _addReplyToBufferOrList(c,obj->ptr,len);1375 _addReplyToBufferOrList(c,"\r\n",2);1376 } else if (obj->encoding == OBJ_ENCODING_INT) {1377 /* For integer encoded strings we just convert it into a string1378 * using our optimized function, and attach the resulting string1379 * to the output buffer. */1380 char buf[34];1381 size_t len = ll2string(buf,sizeof(buf),(long)obj->ptr);1382 buf[len] = '\r';1383 buf[len+1] = '\n';1384 _addReplyLongLongBulk(c, len);1385 _addReplyToBufferOrList(c,buf,len+2);1386 } else {1387 serverPanic("Wrong obj->encoding in addReply()");1388 }1389}13901391/* Add a Redis Object as a bulk reply */1392void addReplyBulk(client *c, robj *obj) {1393 addReplyBulkWithFlag(c, obj, 1);1394}13951396/* Add a C buffer as bulk reply.1397 *1398 * Assemble the "$<len>\r\n" header on the stack and emit the header, payload and1399 * trailing CRLF as a single batched append. The payload is handed over by1400 * reference rather than copied into a scratch buffer, so this collapses the1401 * three per-call passes into one for payloads of any size. */1402void addReplyBulkCBuffer(client *c, const void *p, size_t len) {1403 if (_prepareClientToWrite(c) != C_OK) return;1404 const char *hdr;1405 size_t hdr_len;1406 /* '$' + up to 20 digits (64-bit) + "\r\n"; LONG_STR_SIZE already budgets the NUL. */1407 char hdrbuf[LONG_STR_SIZE + 3];1408 if (likely(len < OBJ_SHARED_BULKHDR_LEN)) {1409 /* Point straight at the shared "$<len>\r\n" object — no copy needed. */1410 hdr = shared.bulkhdr[len]->ptr;1411 hdr_len = OBJ_SHARED_HDR_STRLEN(len);1412 } else {1413 char *h = hdrbuf;1414 *h++ = '$';1415 /* Room left after '$', reserving the 2 trailing bytes for "\r\n". */1416 h += ll2string(h, sizeof(hdrbuf) - (size_t)(h - hdrbuf) - 2, (long long)len);1417 *h++ = '\r';1418 *h++ = '\n';1419 hdr = hdrbuf;1420 hdr_len = (size_t)(h - hdrbuf);1421 }1422 replySegment seg[3] = {1423 { hdr, hdr_len },1424 { p, len },1425 { "\r\n", 2 },1426 };1427 _addReplySegmentsToBufferOrList(c, seg, sizeof(seg) / sizeof(seg[0]));1428}14291430/* Add sds to reply (takes ownership of sds and frees it) */1431void addReplyBulkSds(client *c, sds s) {1432 if (_prepareClientToWrite(c) != C_OK) {1433 sdsfree(s);1434 return;1435 }1436 _addReplyLongLongWithPrefix(c, sdslen(s), '$');1437 _addReplyToBufferOrList(c, s, sdslen(s));1438 sdsfree(s);1439 _addReplyToBufferOrList(c, "\r\n", 2);1440}14411442/* Set sds to a deferred reply (for symmetry with addReplyBulkSds it also frees the sds) */1443void setDeferredReplyBulkSds(client *c, void *node, sds s) {1444 sds reply = sdscatprintf(sdsempty(), "$%d\r\n%s\r\n", (unsigned)sdslen(s), s);1445 setDeferredReply(c, node, reply, sdslen(reply));1446 sdsfree(reply);1447 sdsfree(s);1448}14491450/* Add a C null term string as bulk reply */1451void addReplyBulkCString(client *c, const char *s) {1452 if (s == NULL) {1453 addReplyNull(c);1454 } else {1455 addReplyBulkCBuffer(c,s,strlen(s));1456 }1457}14581459/* Add a long long as a bulk reply */1460void addReplyBulkLongLong(client *c, long long ll) {1461 char buf[64];1462 int len;14631464 len = ll2string(buf,64,ll);1465 addReplyBulkCBuffer(c,buf,len);1466}14671468/* Reply with a verbatim type having the specified extension.1469 *1470 * The 'ext' is the "extension" of the file, actually just a three1471 * character type that describes the format of the verbatim string.1472 * For instance "txt" means it should be interpreted as a text only1473 * file by the receiver, "md " as markdown, and so forth. Only the1474 * three first characters of the extension are used, and if the1475 * provided one is shorter than that, the remaining is filled with1476 * spaces. */1477void addReplyVerbatim(client *c, const char *s, size_t len, const char *ext) {1478 if (c->resp == 2) {1479 addReplyBulkCBuffer(c,s,len);1480 } else {1481 char buf[32];1482 size_t preflen = snprintf(buf,sizeof(buf),"=%zu\r\nxxx:",len+4);1483 char *p = buf+preflen-4;1484 for (int i = 0; i < 3; i++) {1485 if (*ext == '\0') {1486 p[i] = ' ';1487 } else {1488 p[i] = *ext++;1489 }1490 }1491 addReplyProto(c,buf,preflen);1492 addReplyProto(c,s,len);1493 addReplyProto(c,"\r\n",2);1494 }1495}14961497/* This function is similar to the addReplyHelp function but adds the1498 * ability to pass in two arrays of strings. Some commands have1499 * some additional subcommands based on the specific feature implementation1500 * Redis is compiled with (currently just clustering). This function allows1501 * to pass is the common subcommands in `help` and any implementation1502 * specific subcommands in `extended_help`.1503 */1504void addExtendedReplyHelp(client *c, const char **help, const char **extended_help) {1505 sds cmd = sdsnew((char*) c->argv[0]->ptr);1506 void *blenp = addReplyDeferredLen(c);1507 int blen = 0;1508 int idx = 0;15091510 sdstoupper(cmd);1511 addReplyStatusFormat(c,1512 "%s <subcommand> [<arg> [value] [opt] ...]. Subcommands are:",cmd);1513 sdsfree(cmd);15141515 while (help[blen]) addReplyStatus(c,help[blen++]);1516 if (extended_help) {1517 while (extended_help[idx]) addReplyStatus(c,extended_help[idx++]);1518 }1519 blen += idx;15201521 addReplyStatus(c,"HELP");1522 addReplyStatus(c," Print this help.");15231524 blen += 1; /* Account for the header. */1525 blen += 2; /* Account for the footer. */1526 setDeferredArrayLen(c,blenp,blen);1527}15281529/* Add an array of C strings as status replies with a heading.1530 * This function is typically invoked by commands that support1531 * subcommands in response to the 'help' subcommand. The help array1532 * is terminated by NULL sentinel. */1533void addReplyHelp(client *c, const char **help) {1534 addExtendedReplyHelp(c, help, NULL);1535}15361537/* Add a suggestive error reply.1538 * This function is typically invoked by from commands that support1539 * subcommands in response to an unknown subcommand or argument error. */1540void addReplySubcommandSyntaxError(client *c) {1541 sds cmd = sdsnew((char*) c->argv[0]->ptr);1542 sdstoupper(cmd);1543 addReplyErrorFormat(c,1544 "unknown subcommand or wrong number of arguments for '%.128s'. Try %s HELP.",1545 (char*)c->argv[1]->ptr,cmd);1546 sdsfree(cmd);1547}15481549/* Move the reply blocks and deferred errors after the static buffer has been1550 * appended and the destination has been checked by the caller. Module-only1551 * callers may run without the GIL: keep their errors client-local and deferred. */1552static void appendReplyBuffersFromClient(client *dst, client *src) {1553 /* Concatenate the reply list into the dest */1554 if (listLength(src->reply))1555 listJoin(dst->reply,src->reply);1556 serverAssert(src->reply_bytes_shared == 0); /* It is non-normal client, never has references. */1557 dst->reply_bytes += src->reply_bytes;1558 src->reply_bytes = 0;1559 src->bufpos = 0;15601561 if (src->deferred_reply_errors) {1562 deferredAfterErrorReply(dst, src->deferred_reply_errors);1563 listRelease(src->deferred_reply_errors);1564 src->deferred_reply_errors = NULL;1565 }1566}15671568/* Append 'src' client output buffers into 'dst' client output buffers.1569 * This function clears the output buffers of 'src'. */1570void AddReplyFromClient(client *dst, client *src) {1571 /* If the source client contains a partial response due to client output1572 * buffer limits, propagate that to the dest rather than copy a partial1573 * reply. We don't wanna run the risk of copying partial response in case1574 * for some reason the output limits don't reach the same decision (maybe1575 * they changed) */1576 if (src->flags & CLIENT_CLOSE_ASAP) {1577 sds client = catClientInfoString(sdsempty(),dst);1578 freeClientAsync(dst);1579 serverLog(LL_WARNING,"Client %s scheduled to be closed ASAP for overcoming of output buffer limits.", client);1580 sdsfree(client);1581 return;1582 }15831584 /* First add the static buffer (either into the static buffer or reply list) */1585 addReplyProto(dst,src->buf, src->bufpos);15861587 /* We need to check with _prepareClientToWrite again (after addReplyProto)1588 * since addReplyProto may have changed something (like CLIENT_CLOSE_ASAP) */1589 if (_prepareClientToWrite(dst) != C_OK)1590 return;15911592 /* We're bypassing _addReplyProtoToList, so we need to add the pre/post1593 * checks in it. */1594 if (dst->flags & CLIENT_CLOSE_AFTER_REPLY) return;15951596 appendReplyBuffersFromClient(dst, src);15971598 /* Check output buffer limits */1599 closeClientOnOutputBufferLimitReached(dst, 1);1600}16011602/* Move replies between connectionless module clients with exclusive access to1603 * both. This path may run without the GIL: addReplyProto must bypass write1604 * scheduling and output-limit handling for these clients, and the shared1605 * transfer helper must keep errors deferred. Real clients use AddReplyFromClient1606 * with their normal synchronization instead. */1607void AddReplyFromModuleClient(client *dst, client *src) {1608 serverAssert(src != dst);1609 serverAssert((src->flags & CLIENT_MODULE) && (dst->flags & CLIENT_MODULE));1610 serverAssert(!src->conn && !dst->conn);1611 serverAssert(!((src->flags | dst->flags) &1612 (CLIENT_CLOSE_ASAP | CLIENT_CLOSE_AFTER_REPLY | CLIENT_SLAVE | CLIENT_PUSHING)));16131614 addReplyProto(dst, src->buf, src->bufpos);1615 appendReplyBuffersFromClient(dst, src);1616}16171618/* Append the listed errors to the server error statistics. the input1619 * list is not modified and remains the responsibility of the caller. */1620void deferredAfterErrorReply(client *c, list *errors) {1621 listIter li;1622 listNode *ln;1623 listRewind(errors,&li);1624 while((ln = listNext(&li))) {1625 sds err = ln->value;1626 afterErrorReply(c, err, sdslen(err), 0);1627 }1628}16291630/* Logically copy 'src' replica client buffers info to 'dst' replica.1631 * Basically increase referenced buffer block node reference count. */1632void copyReplicaOutputBuffer(client *dst, client *src) {1633 serverAssert(src->bufpos == 0 && listLength(src->reply) == 0); 1634 serverAssert(src->running_tid == IOTHREAD_MAIN_THREAD_ID &&1635 dst->running_tid == IOTHREAD_MAIN_THREAD_ID);1636 if (src->ref_repl_buf_node == NULL) return;1637 dst->ref_repl_buf_node = src->ref_repl_buf_node;1638 dst->ref_block_pos = src->ref_block_pos;1639 ((replBufBlock *)listNodeValue(dst->ref_repl_buf_node))->refcount++;1640}16411642static inline int _clientHasPendingRepliesNonSlave(client *c) {1643 return c->bufpos || listLength(c->reply);1644}16451646static inline int _clientHasPendingRepliesSlave(client *c) {1647 /* Replicas use global shared replication buffer instead of1648 * private output buffer. */1649 serverAssert(c->bufpos == 0 && listLength(c->reply) == 0);1650 if (c->ref_repl_buf_node == NULL) return 0;16511652 /* If the last replication buffer block content is totally sent,1653 * we have nothing to send. */1654 if (c->running_tid == IOTHREAD_MAIN_THREAD_ID) {1655 listNode *ln = listLast(server.repl_buffer_blocks);1656 replBufBlock *tail = listNodeValue(ln);1657 if (ln == c->ref_repl_buf_node &&1658 c->ref_block_pos == tail->used) return 0;1659 } else {1660 if (c->io_bound_repl_node == c->io_curr_repl_node &&1661 c->io_bound_block_pos == c->io_curr_block_pos) return 0;1662 }1663 return 1;1664}16651666/* Return true if the specified client has pending reply buffers to write to1667 * the socket. */1668int clientHasPendingReplies(client *c) {1669 if (unlikely(clientTypeIsSlave(c))) {1670 return _clientHasPendingRepliesSlave(c);1671 }1672 return _clientHasPendingRepliesNonSlave(c);1673}16741675void clientAcceptHandler(connection *conn) {1676 client *c = connGetPrivateData(conn);16771678 if (connGetState(conn) != CONN_STATE_CONNECTED) {1679 serverLog(LL_WARNING,1680 "Error accepting a client connection: %s (addr=%s laddr=%s)",1681 connGetLastError(conn), getClientPeerId(c), getClientSockname(c));1682 freeClientAsync(c);1683 return;1684 }16851686 /* If the server is running in protected mode (the default) and there1687 * is no password set, nor a specific interface is bound, we don't accept1688 * requests from non loopback interfaces. Instead we try to explain the1689 * user what to do to fix it if needed. */1690 if (server.protected_mode &&1691 DefaultUser->flags & USER_FLAG_NOPASS)1692 {1693 if (connIsLocal(conn) != 1) {1694 char *err =1695 "-DENIED Redis is running in protected mode because protected "1696 "mode is enabled and no password is set for the default user. "1697 "In this mode connections are only accepted from the loopback interface. "1698 "If you want to connect from external computers to Redis you "1699 "may adopt one of the following solutions: "1700 "1) Just disable protected mode sending the command "1701 "'CONFIG SET protected-mode no' from the loopback interface "1702 "by connecting to Redis from the same host the server is "1703 "running, however MAKE SURE Redis is not publicly accessible "1704 "from internet if you do so. Use CONFIG REWRITE to make this "1705 "change permanent. "1706 "2) Alternatively you can just disable the protected mode by "1707 "editing the Redis configuration file, and setting the protected "1708 "mode option to 'no', and then restarting the server. "1709 "3) If you started the server manually just for testing, restart "1710 "it with the '--protected-mode no' option. "1711 "4) Set up an authentication password for the default user. "1712 "NOTE: You only need to do one of the above things in order for "1713 "the server to start accepting connections from the outside.\r\n";1714 if (connWrite(c->conn,err,strlen(err)) == -1) {1715 /* Nothing to do, Just to avoid the warning... */1716 }1717 server.stat_rejected_conn++;1718 freeClientAsync(c);1719 return;1720 }1721 }17221723 /* Auto-authenticate from cert_user field if set */1724 sds username = connGetPeerUsername(conn);1725 if (username != NULL) {1726 user *u = ACLGetUserByName(username, sdslen(username));1727 if (u && !(u->flags & USER_FLAG_DISABLED)) {1728 c->authenticated = 1;1729 clientSetUser(c, u, 1);1730 moduleNotifyUserChanged(c);1731 serverLog(LL_VERBOSE, "TLS: Auto-authenticated client as %s",1732 server.hide_user_data_from_log ? "*redacted*" : u->name);1733 } else {1734 addACLLogEntry(c, ACL_INVALID_TLS_CERT_AUTH, ACL_LOG_CTX_TOPLEVEL, 0, username, NULL);1735 }1736 sdsfree(username);1737 }17381739 server.stat_numconnections++;1740 moduleFireServerEvent(REDISMODULE_EVENT_CLIENT_CHANGE,1741 REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED,1742 c);17431744 /* Assign the client to an IO thread */1745 if (server.io_threads_num > 1) assignClientToIOThread(c);1746}17471748void acceptCommonHandler(connection *conn, int flags, char *ip) {1749 client *c;1750 UNUSED(ip);17511752 if (connGetState(conn) != CONN_STATE_ACCEPTING) {1753 char addr[NET_ADDR_STR_LEN] = {0};1754 char laddr[NET_ADDR_STR_LEN] = {0};1755 connFormatAddr(conn, addr, sizeof(addr), 1);1756 connFormatAddr(conn, laddr, sizeof(laddr), 0);1757 serverLog(LL_VERBOSE,1758 "Accepted client connection in error state: %s (addr=%s laddr=%s)",1759 connGetLastError(conn), addr, laddr);1760 connClose(conn);1761 return;1762 }17631764 /* Limit the number of connections we take at the same time.1765 *1766 * Admission control will happen before a client is created and connAccept()1767 * called, because we don't want to even start transport-level negotiation1768 * if rejected. */1769 if (listLength(server.clients) + getClusterConnectionsCount()1770 >= server.maxclients)1771 {1772 char *err;1773 if (server.cluster_enabled)1774 err = "-ERR max number of clients + cluster "1775 "connections reached\r\n";1776 else1777 err = "-ERR max number of clients reached\r\n";17781779 /* That's a best effort error message, don't check write errors.1780 * Note that for TLS connections, no handshake was done yet so nothing1781 * is written and the connection will just drop. */1782 if (connWrite(conn,err,strlen(err)) == -1) {1783 /* Nothing to do, Just to avoid the warning... */1784 }1785 server.stat_rejected_conn++;1786 connClose(conn);1787 return;1788 }17891790 /* Create connection and client */1791 if ((c = createClient(conn)) == NULL) {1792 char addr[NET_ADDR_STR_LEN] = {0};1793 char laddr[NET_ADDR_STR_LEN] = {0};1794 connFormatAddr(conn, addr, sizeof(addr), 1);1795 connFormatAddr(conn, laddr, sizeof(laddr), 0);1796 serverLog(LL_WARNING,1797 "Error registering fd event for the new client connection: %s (addr=%s laddr=%s)",1798 connGetLastError(conn), addr, laddr);1799 connClose(conn); /* May be already closed, just ignore errors */1800 return;1801 }18021803 /* Last chance to keep flags */1804 c->flags |= flags;18051806 /* Initiate accept.1807 *1808 * Note that connAccept() is free to do two things here:1809 * 1. Call clientAcceptHandler() immediately;1810 * 2. Schedule a future call to clientAcceptHandler().1811 *1812 * Because of that, we must do nothing else afterwards.1813 */1814 if (connAccept(conn, clientAcceptHandler) == C_ERR) {1815 if (connGetState(conn) == CONN_STATE_ERROR)1816 serverLog(LL_WARNING,1817 "Error accepting a client connection: %s (addr=%s laddr=%s)",1818 connGetLastError(conn), getClientPeerId(c), getClientSockname(c));1819 freeClient(connGetPrivateData(conn));1820 return;1821 }1822}18231824static void freeDeferredObject(client *c, int type, void *ptr) {1825 if (type == DEFERRED_OBJECT_TYPE_PENDING_COMMAND) {1826 freePendingCommand(c, ptr);1827 } else if (type == DEFERRED_OBJECT_TYPE_ROBJ) {1828 decrRefCount(ptr);1829 } else {1830 serverPanic("Unknown deferred object type: %d", type);1831 }1832}18331834/* Attempt to defer freeing the object to the IO thread. We usually call this since1835 * we know the object is allocated in the IO thread, to avoid memory arena contention,1836 * and also reducing the load of the main thread. */1837void tryDeferFreeClientObject(client *c, int type, void *ptr) {1838 if (!c || c->tid == IOTHREAD_MAIN_THREAD_ID) {1839 freeDeferredObject(c, type, ptr);1840 return;1841 }18421843 /* Put the object in the deferred objects array. */1844 if (c->deferred_objects && c->deferred_objects_num < CLIENT_MAX_DEFERRED_OBJECTS) {1845 c->deferred_objects[c->deferred_objects_num].type = type;1846 c->deferred_objects[c->deferred_objects_num].ptr = ptr;1847 c->deferred_objects_num++;1848 } else {1849 freeDeferredObject(c, type, ptr);1850 }1851}18521853/* Free the objects in the deferred_pending_cmds array. If free_array is true1854 * then free the array itself as well. */1855void freeClientDeferredObjects(client *c, int free_array) {1856 for (int j = 0; j < c->deferred_objects_num; j++) {1857 deferredObject *obj = &c->deferred_objects[j];1858 freeDeferredObject(c, obj->type, obj->ptr);1859 }1860 c->deferred_objects_num = 0;18611862 if (free_array) {1863 zfree(c->deferred_objects);1864 c->deferred_objects = NULL;1865 }1866}18671868/* Queue an robj to be freed by the main thread when client returns from IO thread.1869 * This is used in IO thread write path to avoid refcount race conditions. */1870#define IO_DEFERRED_OBJECTS_INIT_SIZE 81871void ioDeferFreeRobj(client *c, robj *obj) {1872 if (c->io_deferred_objects_num >= c->io_deferred_objects_size) {1873 int new_size = !c->io_deferred_objects_size ?1874 IO_DEFERRED_OBJECTS_INIT_SIZE : c->io_deferred_objects_size * 2;1875 c->io_deferred_objects = zrealloc(c->io_deferred_objects, new_size * sizeof(robj *));1876 c->io_deferred_objects_size = new_size;1877 }1878 c->io_deferred_objects[c->io_deferred_objects_num++] = obj;1879}18801881/* Free all objects queued by IO thread for deferred freeing.1882 * Called by main thread when client returns from IO thread.1883 * If free_array is true then free the array itself as well. */1884void freeClientIODeferredObjects(client *c, int free_array) {1885 if (!c->conn) return;18861887 for (int i = 0; i < c->io_deferred_objects_num; i++) {1888 robj *obj = c->io_deferred_objects[i];1889 decrRefCount(obj);1890 }18911892 if (!free_array) {1893 /* If the utilization rate is less than 1/4, reduce the size to 1/2 to avoid thrashing */1894 if (c->io_deferred_objects_size > IO_DEFERRED_OBJECTS_INIT_SIZE &&1895 c->io_deferred_objects_num * 4 < c->io_deferred_objects_size)1896 {1897 int new_size = c->io_deferred_objects_size / 2;1898 c->io_deferred_objects = zrealloc(c->io_deferred_objects, new_size * sizeof(robj *));1899 c->io_deferred_objects_size = new_size;1900 }1901 c->io_deferred_objects_num = 0;1902 } else {1903 zfree(c->io_deferred_objects);1904 c->io_deferred_objects = NULL;1905 c->io_deferred_objects_num = 0;1906 c->io_deferred_objects_size = 0;1907 }1908}19091910void freeClientOriginalArgv(client *c) {1911 /* We didn't rewrite this client */1912 if (!c->original_argv) return;19131914 for (int j = 0; j < c->original_argc; j++)1915 decrRefCount(c->original_argv[j]);1916 zfree(c->original_argv);1917 c->original_argv = NULL;1918 c->original_argc = 0;1919}19201921static inline void freeClientArgvInternal(client *c, int free_argv) {1922 int j;1923 for (j = 0; j < c->argc; j++)1924 decrRefCount(c->argv[j]);1925 c->argc = 0;1926 c->cmd = NULL;1927 c->lookedcmd = NULL;1928 if (free_argv) {1929 c->argv_len = 0;1930 zfree(c->argv);1931 c->argv = NULL;1932 }1933}19341935void freeClientArgv(client *c) {1936 freeClientArgvInternal(c, 1);1937}19381939static inline void freeClientPendingCommands(client *c, int num_pcmds_to_free) {1940 /* (-1) means free all pending commands */1941 if (num_pcmds_to_free == -1)1942 num_pcmds_to_free = c->pending_cmds.len;19431944 while (num_pcmds_to_free--) {1945 pendingCommand *pcmd = popPendingCommandFromHead(&c->pending_cmds);1946 serverAssert(pcmd);1947 reclaimPendingCommand(c, pcmd);1948 }1949}19501951/* Close all the slaves connections. This is useful in chained replication1952 * when we resync with our own master and want to force all our slaves to1953 * resync with us as well. */1954void disconnectSlaves(void) {1955 listIter li;1956 listNode *ln;1957 listRewind(server.slaves,&li);1958 while((ln = listNext(&li))) {1959 freeClient((client*)ln->value);1960 }1961}19621963/* Check if there is any other slave waiting dumping RDB finished expect me.1964 * This function is useful to judge current dumping RDB can be used for full1965 * synchronization or not. */1966int anyOtherSlaveWaitRdb(client *except_me) {1967 listIter li;1968 listNode *ln;19691970 listRewind(server.slaves, &li);1971 while((ln = listNext(&li))) {1972 client *slave = ln->value;1973 if (slave != except_me &&1974 slave->replstate == SLAVE_STATE_WAIT_BGSAVE_END)1975 {1976 return 1;1977 }1978 }1979 return 0;1980}19811982/* Remove the specified client from global lists where the client could1983 * be referenced, not including the Pub/Sub channels.1984 * This is used by freeClient() and replicationCacheMaster(). */1985void unlinkClient(client *c) {1986 listNode *ln;19871988 /* If this is marked as current client unset it. */1989 if (c->conn && server.current_client == c) server.current_client = NULL;19901991 /* Certain operations must be done only if the client has an active connection.1992 * If the client was already unlinked or if it's a "fake client" the1993 * conn is already set to NULL. */1994 if (c->conn) {1995 /* Remove from the list of active clients. */1996 if (c->client_list_node) {1997 uint64_t id = htonu64(c->id);1998 raxRemove(server.clients_index,(unsigned char*)&id,sizeof(id),NULL);1999 listDelNode(server.clients,c->client_list_node);2000 c->client_list_node = NULL;
Findings
✓ No findings reported for this file.