1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */23/** \file4 * The main memcached header holding commonly used data5 * structures and function prototypes.6 */78#ifdef HAVE_CONFIG_H9#include "config.h"10#endif1112#include <sys/types.h>13#include <sys/socket.h>14#include <sys/time.h>15#include <netinet/in.h>16#include <event.h>17#include <netdb.h>18#include <pthread.h>19#include <unistd.h>20#include <assert.h>21#include <grp.h>22#include <signal.h>23#ifdef NEED_ALIGN24#include <string.h>25#endif26/* need this to get IOV_MAX on some platforms. */27#ifndef __need_IOV_MAX28#define __need_IOV_MAX29#endif30#include <limits.h>31/* FreeBSD 4.x doesn't have IOV_MAX exposed. */32#ifndef IOV_MAX33#if defined(__FreeBSD__) || defined(__APPLE__) || defined(__GNU__)34# define IOV_MAX 102435/* GNU/Hurd don't set MAXPATHLEN36 * http://www.gnu.org/software/hurd/hurd/porting/guidelines.html#PATH_MAX_tt_MAX_PATH_tt_MAXPATHL */37#ifndef MAXPATHLEN38#define MAXPATHLEN 409639#endif40#endif41#endif4243#if defined(__linux__)44# define SOCK_COOKIE_ID SO_MARK45#elif defined(__FreeBSD__)46# define SOCK_COOKIE_ID SO_USER_COOKIE47#elif defined(__OpenBSD__)48# define SOCK_COOKIE_ID SO_RTABLE49#endif5051#include "itoa_ljust.h"52#include "slabs_mover.h"53#include "protocol_binary.h"54#include "cache.h"55#include "logger.h"56#include "queue.h"57#include "util.h"5859#ifdef EXTSTORE60#include "crc32c.h"61#endif6263#include "sasl_defs.h"6465/* for NAPI pinning feature */66#ifndef SO_INCOMING_NAPI_ID67#define SO_INCOMING_NAPI_ID 5668#endif6970/** Maximum length of a key. */71#define KEY_MAX_LENGTH 2507273/** Maximum length of a uri encoded key. */74#define KEY_MAX_URI_ENCODED_LENGTH (KEY_MAX_LENGTH  * 3 + 1)7576/** Size of an incr buf. */77#define INCR_MAX_STORAGE_LEN 247879#define WRITE_BUFFER_SIZE 102480#define READ_BUFFER_SIZE 1638481#define READ_BUFFER_CACHED 082#define UDP_READ_BUFFER_SIZE 6553683#define UDP_MAX_PAYLOAD_SIZE 140084#define UDP_HEADER_SIZE 885#define UDP_DATA_SIZE 1392 // UDP_MAX_PAYLOAD_SIZE - UDP_HEADER_SIZE86#define MAX_SENDBUF_SIZE (256 * 1024 * 1024)8788/* Binary protocol stuff */89#define BIN_MAX_EXTLEN 20 // length of the _incr command is currently the longest.9091/* Initial power multiplier for the hash table */92#define HASHPOWER_DEFAULT 1693#define HASHPOWER_MAX 329495/* Abstract the size of an item's client flag suffix */96#ifdef LARGE_CLIENT_FLAGS97typedef uint64_t client_flags_t;98#define safe_strtoflags safe_strtoull99#else100typedef uint32_t client_flags_t;101#define safe_strtoflags safe_strtoul102#endif103104/*105 * We only reposition items in the LRU queue if they haven't been repositioned106 * in this many seconds. That saves us from churning on frequently-accessed107 * items.108 */109#define ITEM_UPDATE_INTERVAL 60110111/*112 * Valid range of the maximum size of an item, in bytes.113 */114#define ITEM_SIZE_MAX_LOWER_LIMIT 1024115#define ITEM_SIZE_MAX_UPPER_LIMIT 1024 * 1024 * 1024116117/* Slab sizing definitions. */118#define POWER_SMALLEST 1119#define POWER_LARGEST  256 /* actual cap is 255 */120#define SLAB_GLOBAL_PAGE_POOL 0 /* magic slab class for storing pages for reassignment */121#define CHUNK_ALIGN_BYTES 8122/* slab class max is a 6-bit number, -1. */123#define MAX_NUMBER_OF_SLAB_CLASSES (63 + 1)124125/** How long an object can reasonably be assumed to be locked before126    harvesting it on a low memory condition. Default: disabled. */127#define TAIL_REPAIR_TIME_DEFAULT 0128129/* warning: don't use these macros with a function, as it evals its arg twice */130#define ITEM_get_cas(i) (((i)->it_flags & ITEM_CAS) ? \131        (i)->data->cas : (uint64_t)0)132133#define ITEM_set_cas(i,v) { \134    if ((i)->it_flags & ITEM_CAS) { \135        (i)->data->cas = v; \136    } \137}138139#define ITEM_key(item) (((char*)&((item)->data)) \140         + (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0))141142#define ITEM_suffix(item) ((char*) &((item)->data) + (item)->nkey + 1 \143         + (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0))144145#define ITEM_data(item) ((char*) &((item)->data) + (item)->nkey + 1 \146         + (((item)->it_flags & ITEM_CFLAGS) ? sizeof(client_flags_t) : 0) \147         + (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0))148149#define ITEM_ntotal(item) (sizeof(struct _stritem) + (item)->nkey + 1 \150         + (item)->nbytes \151         + (((item)->it_flags & ITEM_CFLAGS) ? sizeof(client_flags_t) : 0) \152         + (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0))153154#define ITEM_clsid(item) ((item)->slabs_clsid & ~(3<<6))155#define ITEM_lruid(item) ((item)->slabs_clsid & (3<<6))156157#define STAT_KEY_LEN 128158#define STAT_VAL_LEN 128159160/** Append a simple stat with a stat name, value format and value */161#define APPEND_STAT(name, fmt, val) \162    append_stat(name, add_stats, c, fmt, val);163164/** Append an indexed stat with a stat name (with format), value format165    and value */166#define APPEND_NUM_FMT_STAT(name_fmt, num, name, fmt, val)          \167    klen = snprintf(key_str, STAT_KEY_LEN, name_fmt, num, name);    \168    vlen = snprintf(val_str, STAT_VAL_LEN, fmt, val);               \169    add_stats(key_str, klen, val_str, vlen, c);170171/** Common APPEND_NUM_FMT_STAT format. */172#define APPEND_NUM_STAT(num, name, fmt, val) \173    APPEND_NUM_FMT_STAT("%d:%s", num, name, fmt, val)174175#ifdef NEED_ALIGN176/** Item client flag conversion */177#define FLAGS_CONV(it, flag) { \178    if ((it)->it_flags & ITEM_CFLAGS) { \179        client_flags_t _flags_conv; \180        memcpy(&_flags_conv, ITEM_suffix((it)), sizeof(_flags_conv)); \181        flag = _flags_conv; \182    } else { \183        flag = 0; \184    } \185}186#else187/** Item client flag conversion */188#define FLAGS_CONV(it, flag) { \189    if ((it)->it_flags & ITEM_CFLAGS) { \190        flag = *((client_flags_t *)ITEM_suffix((it))); \191    } else { \192        flag = 0; \193    } \194}195#endif196197#define FLAGS_SIZE(item) (((item)->it_flags & ITEM_CFLAGS) ? sizeof(client_flags_t) : 0)198199/**200 * Callback for any function producing stats.201 *202 * @param key the stat's key203 * @param klen length of the key204 * @param val the stat's value in an ascii form (e.g. text form of a number)205 * @param vlen length of the value206 * @parm cookie magic callback cookie207 */208typedef void (*ADD_STAT)(const char *key, const uint16_t klen,209                         const char *val, const uint32_t vlen,210                         const void *cookie);211212/*213 * NOTE: If you modify this table you _MUST_ update the function state_text214 */215/**216 * Possible states of a connection.217 */218enum conn_states {219    conn_listening,  /**< the socket which listens for connections */220    conn_new_cmd,    /**< Prepare connection for next command */221    conn_waiting,    /**< waiting for a readable socket */222    conn_read,       /**< reading in a command line */223    conn_parse_cmd,  /**< try to parse a command from the input buffer */224    conn_write,      /**< writing out a simple response */225    conn_nread,      /**< reading in a fixed number of bytes */226    conn_swallow,    /**< swallowing unnecessary bytes w/o storing */227    conn_closing,    /**< closing this connection */228    conn_mwrite,     /**< writing out many items sequentially */229    conn_closed,     /**< connection is closed */230    conn_watch,      /**< held by the logger thread as a watcher */231    conn_io_queue,   /**< wait on async. process to get response object */232    conn_io_resume,  /**< ready to resume mwrite after async work */233    conn_io_pending, /**< got woken up while waiting for async work */234    conn_max_state   /**< Max state value (used for assertion) */235};236237enum bin_substates {238    bin_no_state,239    bin_reading_set_header,240    bin_reading_cas_header,241    bin_read_set_value,242    bin_reading_get_key,243    bin_reading_stat,244    bin_reading_del_header,245    bin_reading_incr_header,246    bin_read_flush_exptime,247    bin_reading_sasl_auth,248    bin_reading_sasl_auth_data,249    bin_reading_touch_key,250};251252enum protocol {253    ascii_prot = 3, /* arbitrary value. */254    binary_prot,255    negotiating_prot, /* Discovering the protocol */256#ifdef PROXY257    proxy_prot,258#endif259};260261enum network_transport {262    local_transport, /* Unix sockets*/263    tcp_transport,264    udp_transport265};266267enum pause_thread_types {268    PAUSE_WORKER_THREADS = 0,269    PAUSE_ALL_THREADS,270    RESUME_ALL_THREADS,271    RESUME_WORKER_THREADS272};273274enum stop_reasons {275    NOT_STOP,276    GRACE_STOP,277    EXIT_NORMALLY278};279280enum close_reasons {281    ERROR_CLOSE,282    NORMAL_CLOSE,283    IDLE_TIMEOUT_CLOSE,284    SHUTDOWN_CLOSE,285};286287#define IS_TCP(x) (x == tcp_transport)288#define IS_UDP(x) (x == udp_transport)289290#define NREAD_ADD 1291#define NREAD_SET 2292#define NREAD_REPLACE 3293#define NREAD_APPEND 4294#define NREAD_PREPEND 5295#define NREAD_CAS 6296#define NREAD_APPENDVIV 7 // specific to meta297#define NREAD_PREPENDVIV 8 // specific to meta298299#define CAS_ALLOW_STALE true300#define CAS_NO_STALE false301302#define LOG_TYPE_DELETE 1303#define LOG_TYPE_META_DELETE 2304305enum store_item_type {306    NOT_STORED=0, STORED, EXISTS, NOT_FOUND, TOO_LARGE, NO_MEMORY307};308309enum delta_result_type {310    OK, NON_NUMERIC, EOM, DELTA_ITEM_NOT_FOUND, DELTA_ITEM_CAS_MISMATCH311};312313/** Time relative to server start. Smaller than time_t on 64-bit systems. */314// TODO: Move to sub-header. needed in logger.h315//typedef unsigned int rel_time_t;316317/** Use X macros to avoid iterating over the stats fields during reset and318 * aggregation. No longer have to add new stats in 3+ places.319 */320321#define SLAB_STATS_FIELDS \322    X(set_cmds) \323    X(get_hits) \324    X(touch_hits) \325    X(delete_hits) \326    X(cas_hits) \327    X(cas_badval) \328    X(incr_hits) \329    X(decr_hits)330331/** Stats stored per slab (and per thread). */332struct slab_stats {333#define X(name) uint64_t    name;334    SLAB_STATS_FIELDS335#undef X336};337338#define THREAD_STATS_FIELDS \339    X(get_cmds) \340    X(get_misses) \341    X(get_expired) \342    X(get_flushed) \343    X(touch_cmds) \344    X(touch_misses) \345    X(delete_misses) \346    X(incr_misses) \347    X(decr_misses) \348    X(cas_misses) \349    X(meta_cmds) \350    X(bytes_read) \351    X(bytes_written) \352    X(flush_cmds) \353    X(conn_yields) /* # of yields for connections (-R option)*/ \354    X(auth_cmds) \355    X(auth_errors) \356    X(idle_kicks) /* idle connections killed */ \357    X(response_obj_oom) \358    X(response_obj_count) \359    X(response_obj_bytes) \360    X(read_buf_oom) \361    X(store_too_large) \362    X(store_no_memory)363364#ifdef EXTSTORE365#define EXTSTORE_THREAD_STATS_FIELDS \366    X(get_extstore) \367    X(get_aborted_extstore) \368    X(get_oom_extstore) \369    X(recache_from_extstore) \370    X(miss_from_extstore) \371    X(badcrc_from_extstore)372#endif373374#ifdef PROXY375#define PROXY_THREAD_STATS_FIELDS \376    X(proxy_conn_requests) \377    X(proxy_conn_errors) \378    X(proxy_conn_oom)379#endif380381/**382 * Stats stored per-thread.383 */384struct thread_stats {385    pthread_mutex_t   mutex;386#define X(name) uint64_t    name;387    THREAD_STATS_FIELDS388#ifdef EXTSTORE389    EXTSTORE_THREAD_STATS_FIELDS390#endif391#ifdef PROXY392    PROXY_THREAD_STATS_FIELDS393    int64_t proxy_req_active;394#endif395#undef X396    struct slab_stats slab_stats[MAX_NUMBER_OF_SLAB_CLASSES];397    uint64_t lru_hits[POWER_LARGEST];398    uint64_t read_buf_count;399    uint64_t read_buf_bytes;400    uint64_t read_buf_bytes_free;401};402403/**404 * Global stats. Only resettable stats should go into this structure.405 */406struct stats {407    uint64_t      total_items;408    uint64_t      total_conns;409    uint64_t      rejected_conns;410    uint64_t      malloc_fails;411    uint64_t      listen_disabled_num;412    uint64_t      slabs_moved;       /* times slabs were moved around */413    uint64_t      slab_reassign_rescues; /* items rescued during slab move */414    uint64_t      slab_reassign_inline_reclaim; /* valid items lost during slab move */415    uint64_t      slab_reassign_chunk_rescues; /* chunked-item chunks recovered */416    uint64_t      slab_reassign_busy_items; /* valid temporarily unmovable */417    uint64_t      slab_reassign_busy_deletes; /* refcounted items killed */418    uint64_t      slab_reassign_busy_nomem; /* valid items lost during slab move */419    const char *  slab_reassign_last_busy_status; /* text of last busy item status */420    uint64_t      lru_crawler_starts; /* Number of item crawlers kicked off */421    uint64_t      lru_maintainer_juggles; /* number of LRU bg pokes */422    uint64_t      time_in_listen_disabled_us;  /* elapsed time in microseconds while server unable to process new connections */423    uint64_t      log_worker_dropped; /* logs dropped by worker threads */424    uint64_t      log_worker_written; /* logs written by worker threads */425    uint64_t      log_watcher_skipped; /* logs watchers missed */426    uint64_t      log_watcher_sent; /* logs sent to watcher buffers */427#ifdef EXTSTORE428    uint64_t      extstore_compact_lost; /* items lost because they were locked */429    uint64_t      extstore_compact_rescues; /* items re-written during compaction */430    uint64_t      extstore_compact_skipped; /* unhit items skipped during compaction */431    uint64_t      extstore_compact_resc_cold; /* items re-written during compaction */432    uint64_t      extstore_compact_resc_old; /* items re-written during compaction */433#endif434#ifdef TLS435    uint64_t      ssl_proto_errors; /* TLS failures during SSL_read() and SSL_write() calls */436    uint64_t      ssl_handshake_errors; /* TLS failures at accept/handshake time */437    uint64_t      ssl_new_sessions; /* successfully negotiated new (non-reused) TLS sessions */438#endif439    struct timeval maxconns_entered;  /* last time maxconns entered */440    uint64_t      unexpected_napi_ids;  /* see doc/napi_ids.txt */441    uint64_t      round_robin_fallback; /* see doc/napi_ids.txt */442};443444/**445 * Global "state" stats. Reflects state that shouldn't be wiped ever.446 * Ordered for some cache line locality for commonly updated counters.447 */448struct stats_state {449    uint64_t      curr_items;450    uint64_t      curr_bytes;451    uint64_t      curr_conns;452    uint64_t      hash_bytes;       /* size used for hash tables */453    float         extstore_memory_pressure; /* when extstore might memory evict */454    unsigned int  conn_structs;455    unsigned int  reserved_fds;456    unsigned int  hash_power_level; /* Better hope it's not over 9000 */457    unsigned int  log_watchers; /* number of currently active watchers */458    bool          hash_is_expanding; /* If the hash table is being expanded */459    bool          accepting_conns;  /* whether we are currently accepting */460    bool          slab_reassign_running; /* slab reassign in progress */461    bool          lru_crawler_running; /* crawl in progress */462};463464#define MAX_VERBOSITY_LEVEL 2465466/* When adding a setting, be sure to update process_stat_settings */467/**468 * Globally accessible settings as derived from the commandline.469 */470struct settings {471    size_t maxbytes;472    int maxconns;473    int port;474    int udpport;475    char *inter;476    int verbose;477    rel_time_t oldest_live; /* ignore existing items older than this */478    int evict_to_free;479    char *socketpath;   /* path to unix socket if using local socket */480    char *auth_file;    /* path to user authentication file */481    int access;  /* access mask (a la chmod) for unix domain socket */482    double factor;          /* chunk size growth factor */483    int chunk_size;484    int num_threads;        /* number of worker (without dispatcher) libevent threads to run */485    int num_threads_per_udp; /* number of worker threads serving each udp socket */486    char prefix_delimiter;  /* character that marks a key prefix (for stats) */487    int detail_enabled;     /* nonzero if we're collecting detailed stats */488    int reqs_per_event;     /* Maximum number of io to process on each489                               io-event. */490    bool use_cas;491    enum protocol binding_protocol;492    int backlog;493    int item_size_max;        /* Maximum item size */494    int slab_chunk_size_max;  /* Upper end for chunks within slab pages. */495    int slab_page_size;     /* Slab's page units. */496    volatile sig_atomic_t sig_hup;  /* a HUP signal was received but not yet handled */497    bool sasl;              /* SASL on/off */498    bool maxconns_fast;     /* Whether or not to early close connections */499    bool lru_crawler;        /* Whether or not to enable the autocrawler thread */500    bool lru_maintainer_thread; /* LRU maintainer background thread */501    bool lru_segmented;     /* Use split or flat LRU's */502    bool slab_reassign;     /* Whether or not slab reassignment is allowed */503    bool ssl_enabled; /* indicates whether SSL is enabled */504    int slab_automove;     /* Whether or not to automatically move slabs */505    unsigned int slab_automove_version; /* bump if AM config args change */506    double slab_automove_ratio; /* youngest must be within pct of oldest */507    double slab_automove_freeratio; /* % of memory to hold free as buffer */508    unsigned int slab_automove_window; /* window mover for algorithm */509    int hashpower_init;     /* Starting hash power level */510    bool shutdown_command; /* allow shutdown command */511    int tail_repair_time;   /* LRU tail refcount leak repair time */512    bool flush_enabled;     /* flush_all enabled */513    bool dump_enabled;      /* whether cachedump/metadump commands work */514    char *hash_algorithm;     /* Hash algorithm in use */515    int lru_crawler_sleep;  /* Microsecond sleep between items */516    uint32_t lru_crawler_tocrawl; /* Number of items to crawl per run */517    int hot_lru_pct; /* percentage of slab space for HOT_LRU */518    int warm_lru_pct; /* percentage of slab space for WARM_LRU */519    double hot_max_factor; /* HOT tail age relative to COLD tail */520    double warm_max_factor; /* WARM tail age relative to COLD tail */521    int crawls_persleep; /* Number of LRU crawls to run before sleeping */522    bool temp_lru; /* TTL < temporary_ttl uses TEMP_LRU */523    uint32_t temporary_ttl; /* temporary LRU threshold */524    int idle_timeout;       /* Number of seconds to let connections idle */525    unsigned int logger_watcher_buf_size; /* size of logger's per-watcher buffer */526    unsigned int logger_buf_size; /* size of per-thread logger buffer */527    unsigned int read_buf_mem_limit; /* total megabytes allowable for net buffers */528    bool drop_privileges;   /* Whether or not to drop unnecessary process privileges */529    bool watch_enabled; /* allows watch commands to be dropped */530    bool relaxed_privileges;   /* Relax process restrictions when running testapp */531    struct slab_rebal_thread *slab_rebal; /* struct for page mover thread */532#ifdef EXTSTORE533    unsigned int ext_io_threadcount; /* number of IO threads to run. */534    unsigned int ext_page_size; /* size in megabytes of storage pages. */535    unsigned int ext_item_size; /* minimum size of items to store externally */536    unsigned int ext_item_age; /* max age of tail item before storing ext. */537    unsigned int ext_low_ttl; /* remaining TTL below this uses own pages */538    unsigned int ext_recache_rate; /* counter++ % recache_rate == 0 > recache */539    unsigned int ext_wbuf_size; /* read only note for the engine */540    unsigned int ext_compact_under; /* when fewer than this many pages, compact */541    unsigned int ext_drop_under; /* when fewer than this many pages, drop COLD items */542    unsigned int ext_max_sleep; /* maximum sleep time for extstore bg threads, in us */543    double ext_max_frag; /* ideal maximum page fragmentation */544    bool ext_drop_unread; /* skip unread items during compaction */545    /* start flushing to extstore after memory below this */546    unsigned int ext_global_pool_min;547#endif548#ifdef TLS549    void *ssl_ctx; /* holds the SSL server context which has the server certificate */550    char *ssl_chain_cert; /* path to the server SSL chain certificate */551    char *ssl_key; /* path to the server key */552    int ssl_verify_mode; /* client certificate verify mode */553    int ssl_keyformat; /* key format , default is PEM */554    char *ssl_ciphers; /* list of SSL ciphers */555    char *ssl_ca_cert; /* certificate with CAs. */556    rel_time_t ssl_last_cert_refresh_time; /* time of the last server certificate refresh */557    unsigned int ssl_wbuf_size; /* size of the write buffer used by ssl_sendmsg method */558    bool ssl_session_cache; /* enable SSL server session caching */559    bool ssl_kernel_tls; /* enable server kTLS */560    int ssl_min_version; /* minimum SSL protocol version to accept */561#endif562    int num_napi_ids;   /* maximum number of NAPI IDs */563    char *memory_file;  /* warm restart memory file path */564#ifdef PROXY565    bool proxy_enabled;566    bool proxy_uring; /* if the proxy should use io_uring */567    bool proxy_memprofile; /* output detail of lua allocations */568    char *proxy_startfile; /* lua file to run when workers start */569    char *proxy_startarg; /* string argument to pass to proxy */570    void *proxy_ctx; /* proxy's state context */571#endif572#ifdef SOCK_COOKIE_ID573    uint32_t sock_cookie_id;574#endif575};576577extern struct stats stats;578extern struct stats_state stats_state;579extern time_t process_started;580extern struct settings settings;581582#define ITEM_LINKED 1583#define ITEM_CAS 2584585/* temp */586#define ITEM_SLABBED 4587588/* Item was fetched at least once in its lifetime */589#define ITEM_FETCHED 8590/* Appended on fetch, removed on LRU shuffling */591#define ITEM_ACTIVE 16592/* If an item's storage are chained chunks. */593#define ITEM_CHUNKED 32594#define ITEM_CHUNK 64595/* ITEM_data bulk is external to item */596#define ITEM_HDR 128597/* additional 4 bytes for item client flags */598#define ITEM_CFLAGS 256599/* item has sent out a token already */600#define ITEM_TOKEN_SENT 512601/* reserved, in case tokens should be a 2-bit count in future */602#define ITEM_TOKEN_RESERVED 1024603/* if item has been marked as a stale value */604#define ITEM_STALE 2048605/* if item key was sent in binary */606#define ITEM_KEY_BINARY 4096607608#define ITEM_PRESERVE_FLAGS (ITEM_FETCHED|ITEM_TOKEN_SENT|ITEM_STALE|ITEM_KEY_BINARY)609610/**611 * Structure for storing items within memcached.612 */613typedef struct _stritem {614    /* Protected by LRU locks */615    struct _stritem *next;616    struct _stritem *prev;617    /* Rest are protected by an item lock */618    struct _stritem *h_next;    /* hash chain next */619    rel_time_t      time;       /* least recent access */620    rel_time_t      exptime;    /* expire time */621    int             nbytes;     /* size of data */622    unsigned short  refcount;623    uint16_t        it_flags;   /* ITEM_* above */624    uint8_t         slabs_clsid;/* which slab class we're in */625    uint8_t         nkey;       /* key length, w/terminating null and padding */626    /* this odd type prevents type-punning issues when we do627     * the little shuffle to save space when not using CAS. */628    union {629        uint64_t cas;630        char end;631    } data[];632    /* if it_flags & ITEM_CAS we have 8 bytes CAS */633    /* then null-terminated key */634    /* then " flags length\r\n" (no terminating null) */635    /* then data with terminating \r\n (no terminating null; it's binary!) */636} item;637638// TODO: If we eventually want user loaded modules, we can't use an enum :(639enum crawler_run_type {640    CRAWLER_AUTOEXPIRE=0, CRAWLER_EXPIRED, CRAWLER_METADUMP, CRAWLER_MGDUMP641};642643typedef struct {644    struct _stritem *next;645    struct _stritem *prev;646    struct _stritem *h_next;    /* hash chain next */647    rel_time_t      time;       /* least recent access */648    rel_time_t      exptime;    /* expire time */649    int             nbytes;     /* size of data */650    unsigned short  refcount;651    uint16_t        it_flags;   /* ITEM_* above */652    uint8_t         slabs_clsid;/* which slab class we're in */653    uint8_t         nkey;       /* key length, w/terminating null and padding */654    uint32_t        remaining;  /* Max keys to crawl per slab per invocation */655    uint64_t        reclaimed;  /* items reclaimed during this crawl. */656    uint64_t        unfetched;  /* items reclaimed unfetched during this crawl. */657    uint64_t        checked;    /* items examined during this crawl. */658} crawler;659660/* Header when an item is actually a chunk of another item. */661typedef struct _strchunk {662    struct _strchunk *next;     /* points within its own chain. */663    struct _strchunk *prev;     /* can potentially point to the head. */664    struct _stritem  *head;     /* always points to the owner chunk */665    int              size;      /* available chunk space in bytes */666    int              used;      /* chunk space used */667    int              nbytes;    /* used. */668    unsigned short   refcount;  /* used? */669    uint16_t         it_flags;  /* ITEM_* above. */670    uint8_t          slabs_clsid; /* Same as above. */671    uint8_t          orig_clsid; /* For obj hdr chunks slabs_clsid is fake. */672    char data[];673} item_chunk;674675#ifdef NEED_ALIGN676static inline char *ITEM_schunk(item *it) {677    int offset = it->nkey + 1678        + ((it->it_flags & ITEM_CFLAGS) ? sizeof(client_flags_t) : 0)679        + ((it->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0);680    int remain = offset % 8;681    if (remain != 0) {682        offset += 8 - remain;683    }684    return ((char *) &(it->data)) + offset;685}686#else687#define ITEM_schunk(item) ((char*) &((item)->data) + (item)->nkey + 1 \688         + (((item)->it_flags & ITEM_CFLAGS) ? sizeof(client_flags_t) : 0) \689         + (((item)->it_flags & ITEM_CAS) ? sizeof(uint64_t) : 0))690#endif691692#ifdef EXTSTORE693typedef struct {694    unsigned int page_version; /* from IO header */695    unsigned int offset; /* from IO header */696    unsigned short page_id; /* from IO header */697} item_hdr;698#endif699700#define IO_QUEUE_COUNT 3701702#define IO_QUEUE_NONE 0703#define IO_QUEUE_EXTSTORE 1704#define IO_QUEUE_PROXY 2705706#define IO_PENDING_TYPE_NONE 0707#define IO_PENDING_TYPE_BASIC 1708#define IO_PENDING_TYPE_EXTSTORE 2709#define IO_PENDING_TYPE_PROXY 3710711typedef STAILQ_HEAD(iop_head_s, _io_pending_t) iop_head_t;712typedef struct _io_pending_t io_pending_t;713typedef struct io_queue_s io_queue_t;714typedef void (*io_queue_stack_cb)(io_queue_t *q);715typedef void (*io_queue_cb)(io_pending_t *pending);716// IO pending objects are created and stacked into this structure. They are717// then sent off to remote threads.718// The objects are returned one at a time to the worker threads.719typedef struct io_queue_s {720    void *ctx; // untouched ptr for specific context721    iop_head_t stack;722    io_queue_stack_cb submit_cb; // callback given a full stack of pending IO's at once.723    int type;724} io_queue_t;725726struct thread_notify {727    struct event notify_event;  /* listen event for notify pipe or eventfd */728#ifdef HAVE_EVENTFD729    int notify_event_fd;        /* notify counter */730#else731    int notify_receive_fd;      /* receiving end of notify pipe */732    int notify_send_fd;         /* sending end of notify pipe */733#endif734};735736typedef struct _mc_resp_bundle mc_resp_bundle;737typedef struct {738    pthread_t thread_id;        /* unique ID of this thread */739    struct event_base *base;    /* libevent handle this thread uses */740    struct thread_notify n;     /* for thread notification */741    struct thread_notify ion;   /* for thread IO object notification */742    pthread_mutex_t ion_lock;   /* mutex for ion_head */743    iop_head_t ion_head;        /* queue for IO object return */744    int cur_sfd;                /* client fd for logging commands */745    int thread_baseid;          /* which "number" thread this is for data offsets */746    int conns_tosubmit;         /* number of conns which have put data in io_queue */747    struct thread_stats stats;  /* Stats generated by this thread */748    io_queue_t io_queues[IO_QUEUE_COUNT];749    struct conn_queue *ev_queue; /* Worker/conn event queue */750    cache_t *rbuf_cache;        /* static-sized read buffers */751    mc_resp_bundle *open_bundle;752    cache_t *io_cache;          /* IO objects */753#ifdef EXTSTORE754    void *storage;              /* data object for storage system */755#endif756    logger *l;                  /* logger buffer */757    void *lru_bump_buf;         /* async LRU bump buffer */758#ifdef TLS759    char   *ssl_wbuf;760#endif761    int napi_id;                /* napi id associated with this thread */762#ifdef PROXY763    void *proxy_ctx; // proxy global context764    void *L; // lua VM765    void *proxy_hooks;766    void *proxy_user_stats;767    void *proxy_int_stats;768    void *proxy_event_thread; // worker threads can also be proxy IO threads769    struct event *proxy_gc_timer; // periodic GC pushing.770    int proxy_vm_extra_kb;771    int proxy_vm_last_kb;772    unsigned int proxy_vm_negative_delta;773    int proxy_vm_gcrunning;774    int proxy_vm_gcpokemem;775    int64_t proxy_active_req_limit;776    uint64_t proxy_buffer_memory_limit;777    uint64_t proxy_buffer_memory_used;778    uint32_t proxy_rng[4]; // fast per-thread rng for lua.779    // TODO: add ctx object so we can attach to queue.780#endif781} LIBEVENT_THREAD;782783/**784 * Response objects785 */786#define MC_RESP_IOVCOUNT 4787typedef struct _mc_resp {788    mc_resp_bundle *bundle; // ptr back to bundle789    struct _mc_resp *next; // choo choo.790    int wbytes; // bytes to write out of wbuf: might be able to nuke this.791    int tosend; // total bytes to send for this response792    void *write_and_free; /** free this memory after finishing writing */793    io_pending_t *io_pending; /* pending IO descriptor for this response */794795    item *item; /* item associated with this response object, with reference held */796    struct iovec iov[MC_RESP_IOVCOUNT]; /* built-in iovecs to simplify network code */797    int chunked_total; /* total amount of chunked item data to send. */798    uint8_t iovcnt;799    uint8_t chunked_data_iov; /* this iov is a pointer to chunked data header */800801    /* instruct transmit to skip this response object. used by storage engines802     * to asynchronously kill an object that was queued to write803     */804    bool skip;805    bool suspended; // waiting for response from subsystem806    bool free; // double free detection.807    bool noreply; // TODO: remove if we can rely on skip-only808    bool binary_prot; // let main ascii prot know if we're binary or not809    bool mset_res; // whether we're completing an mset across a netread810    bool set_stale; // mset flag811#ifdef PROXY812    bool proxy_res; // we're handling a proxied response buffer.813#endif814    // UDP bits. Copied in from the client.815    uint16_t    request_id; /* Incoming UDP request ID, if this is a UDP "connection" */816    uint16_t    udp_sequence; /* packet counter when transmitting result */817    uint16_t    udp_total; /* total number of packets in sequence */818    struct sockaddr_in6 request_addr; /* udp: Who sent this request */819    socklen_t request_addr_size;820821    char wbuf[WRITE_BUFFER_SIZE];822} mc_resp;823824#define MAX_RESP_PER_BUNDLE ((READ_BUFFER_SIZE - sizeof(mc_resp_bundle)) / sizeof(mc_resp))825struct _mc_resp_bundle {826    uint8_t refcount;827    uint8_t next_check; // next object to check on assignment.828    LIBEVENT_THREAD *thread;829    struct _mc_resp_bundle *next;830    struct _mc_resp_bundle *prev;831    mc_resp r[];832};833834typedef struct conn conn;835836struct _io_pending_t {837    uint8_t io_queue_type; // which queue this entered on838    uint8_t io_sub_type; // sub-type. FIXME: can optimize out later.839    uint8_t payload;840    LIBEVENT_THREAD *thread;841    conn *c;842    mc_resp *resp; // associated response object843    io_queue_cb return_cb; // called on worker thread.844    io_queue_cb finalize_cb; // called back on the worker thread.845    STAILQ_ENTRY(_io_pending_t) iop_next; // queue chain.846    char data[120];847};848849/**850 * The structure representing a connection into memcached.851 */852struct conn {853    sasl_conn_t *sasl_conn;854    int    sfd;855    bool sasl_started;856    bool authenticated;857    bool close_after_write; /** flush write then move to close connection */858    bool rbuf_malloced; /** read buffer was malloc'ed for ascii mget, needs free() */859    bool item_malloced; /** item for conn_nread state is a temporary malloc */860    uint8_t ssl_enabled;861    void    *ssl;862#ifdef TLS863    char   *ssl_wbuf;864#endif865    enum conn_states  state;866    enum bin_substates substate;867    rel_time_t last_cmd_time;868    struct event event;869    short  ev_flags;870    short  which;   /** which events were just triggered */871872    char   *rbuf;   /** buffer to read commands into */873    char   *rcurr;  /** but if we parsed some already, this is where we stopped */874    int    rsize;   /** total allocated size of rbuf */875    int    rbytes;  /** how much data, starting from rcur, do we have unparsed */876877    mc_resp *resp; // tail response.878    mc_resp *resp_head; // first response in current stack.879    char   *ritem;  /** when we read in an item's value, it goes here */880    int    rlbytes;881882    /**883     * item is used to hold an item structure created after reading the command884     * line of set/add/replace commands, but before we finished reading the actual885     * data. The data is read into ITEM_data(item) to avoid extra copying.886     */887888    void   *item;     /* for commands set/add/replace  */889890    /* data for the swallow state */891    int    sbytes;    /* how many bytes to swallow */892893    int resps_suspended; /* see notes on io_queue_cb_t */894#ifdef PROXY895    void *proxy_rctx; /* pointer to active request context */896#endif897#ifdef EXTSTORE898    unsigned int recache_counter;899#endif900    enum protocol protocol;   /* which protocol this connection speaks */901    enum network_transport transport; /* what transport is used by this connection */902    enum close_reasons close_reason; /* reason for transition into conn_closing */903904    /* data for UDP clients */905    int    request_id; /* Incoming UDP request ID, if this is a UDP "connection" */906    struct sockaddr_in6 request_addr; /* udp: Who sent the most recent request */907    socklen_t request_addr_size;908909    /* current stats command */910    struct {911        char *buffer;912        size_t size;913        size_t offset;914    } stats;915916    /* Binary protocol stuff */917    /* This is where the binary header goes */918    protocol_binary_request_header binary_header;919    uint64_t cas; /* the cas to return */920    uint64_t tag; /* listener stocket tag */921    short cmd; /* current command being processed */922    int opaque;923    int keylen;924    conn   *next;     /* Used for generating a list of conn structures */925    LIBEVENT_THREAD *thread; /* Pointer to the thread object serving this connection */926    int (*try_read_command)(conn *c); /* pointer for top level input parser */927    ssize_t (*read)(conn  *c, void *buf, size_t count);928    ssize_t (*sendmsg)(conn *c, struct msghdr *msg, int flags);929    ssize_t (*write)(conn *c, void *buf, size_t count);930#ifdef MEMCACHED_DEBUG931    void *debug_item_reffed; // used for refcount leak testing932#endif933};934935/* array of conn structures, indexed by file descriptor */936extern conn **conns;937938/* current time of day (updated periodically) */939extern volatile rel_time_t current_time;940941#ifdef MEMCACHED_DEBUG942extern volatile bool is_paused;943extern volatile int64_t delta;944#endif945946#ifdef EXTSTORE947extern void *ext_storage;948#endif949/*950 * Functions951 */952void verify_default(const char* param, bool condition);953void do_accept_new_conns(const bool do_accept);954enum delta_result_type do_add_delta(LIBEVENT_THREAD *t, const char *key,955                                    const size_t nkey, const bool incr,956                                    const int64_t delta, char *buf,957                                    uint64_t *cas, const uint32_t hv,958                                    item **it_ret);959enum store_item_type do_store_item(item *item, int comm, LIBEVENT_THREAD *t, const uint32_t hv, int *nbytes, uint64_t *cas, const uint64_t cas_in, bool cas_stale);960void thread_io_queue_add(LIBEVENT_THREAD *t, int type, void *ctx, io_queue_stack_cb cb);961io_queue_t *thread_io_queue_get(LIBEVENT_THREAD *t, int type);962void thread_io_queue_submit(LIBEVENT_THREAD *t);963void conn_io_queue_return(io_pending_t *io);964#define conn_resp_suspend(c, resp) \965    do { \966        resp->suspended = true; \967        c->resps_suspended++; \968    } while (0)969#define conn_resp_unsuspend(c, resp) \970    do { \971        resp->suspended = false; \972        c->resps_suspended--; \973        assert(c->resps_suspended >= 0); \974        if (c->resps_suspended == 0) { \975            conn_worker_readd(c); \976        } \977    } while (0)978979conn *conn_new(const int sfd, const enum conn_states init_state, const int event_flags, const int read_buffer_size,980    enum network_transport transport, struct event_base *base, void *ssl, uint64_t conntag, enum protocol bproto);981982void conn_worker_readd(conn *c);983extern int daemonize(int nochdir, int noclose);984985#define mutex_lock(x) pthread_mutex_lock(x)986#define mutex_trylock(x) pthread_mutex_trylock(x)987#define mutex_unlock(x) pthread_mutex_unlock(x)988989#include "stats_prefix.h"990#include "slabs.h"991#include "assoc.h"992#include "items.h"993#include "crawler.h"994#include "trace.h"995#include "hash.h"996997/*998 * Functions such as the libevent-related calls that need to do cross-thread999 * communication in multithreaded mode (rather than actually doing the work1000 * in the current thread) are called via "dispatch_" frontends, which are1001 * also #define-d to directly call the underlying code in singlethreaded mode.1002 */1003void memcached_thread_init(int nthreads, void *arg);1004void redispatch_conn(conn *c);1005void timeout_conn(conn *c);1006#ifdef PROXY1007void proxy_reload_notify(LIBEVENT_THREAD *t);1008#endif1009void return_io_pending(io_pending_t *io);1010void dispatch_conn_new(int sfd, enum conn_states init_state, int event_flags, int read_buffer_size,1011    enum network_transport transport, void *ssl, uint64_t conntag, enum protocol bproto);1012void sidethread_conn_close(conn *c);10131014/* Lock wrappers for cache functions that are called from main loop. */1015enum delta_result_type add_delta(LIBEVENT_THREAD *t, const char *key,1016                                 const size_t nkey, bool incr,1017                                 const int64_t delta, char *buf,1018                                 uint64_t *cas);1019void accept_new_conns(const bool do_accept);1020void  conn_close_idle(conn *c);1021void  conn_close_all(void);1022item *item_alloc(const char *key, size_t nkey, client_flags_t flags, rel_time_t exptime, int nbytes);1023#define DO_UPDATE true1024#define DONT_UPDATE false1025item *item_get(const char *key, const size_t nkey, LIBEVENT_THREAD *t, const bool do_update);1026item *item_get_locked(const char *key, const size_t nkey, LIBEVENT_THREAD *t, const bool do_update, uint32_t *hv);1027item *item_touch(const char *key, const size_t nkey, uint32_t exptime, LIBEVENT_THREAD *t);1028int   item_link(item *it);1029void  item_remove(item *it);1030int   item_replace(item *it, item *new_it, const uint32_t hv, const uint64_t cas_in);1031void  item_unlink(item *it);10321033void item_lock(uint32_t hv);1034void *item_trylock(uint32_t hv);1035void item_trylock_unlock(void *arg);1036void item_unlock(uint32_t hv);1037void pause_threads(enum pause_thread_types type);1038void stop_threads(void);1039int stop_conn_timeout_thread(void);1040#define refcount_incr(it) ++(it->refcount)1041#define refcount_decr(it) --(it->refcount)1042void STATS_LOCK(void);1043void STATS_UNLOCK(void);1044#define THR_STATS_LOCK(t) pthread_mutex_lock(&t->stats.mutex)1045#define THR_STATS_UNLOCK(t) pthread_mutex_unlock(&t->stats.mutex)1046void threadlocal_stats_reset(void);1047void threadlocal_stats_aggregate(struct thread_stats *stats);1048void slab_stats_aggregate(struct thread_stats *stats, struct slab_stats *out);1049void thread_setname(pthread_t thread, const char *name);1050LIBEVENT_THREAD *get_worker_thread(int id);10511052/* Stat processing functions */1053void append_stat(const char *name, ADD_STAT add_stats, conn *c,1054                 const char *fmt, ...);10551056enum store_item_type store_item(item *item, int comm, LIBEVENT_THREAD *t, int *nbytes, uint64_t *cas, const uint64_t cas_in, bool cas_stale);10571058/* Protocol related code */1059void out_string(conn *c, const char *str);1060#define REALTIME_MAXDELTA 60*60*24*301061/* Negative exptimes can underflow and end up immortal. realtime() will1062   immediately expire values that are greater than REALTIME_MAXDELTA, but less1063   than process_started, so lets aim for that. */1064#define EXPTIME_TO_POSITIVE_TIME(exptime) (exptime < 0) ? \1065        REALTIME_MAXDELTA + 1 : exptime1066rel_time_t realtime(const time_t exptime);1067item* limited_get(const char *key, size_t nkey, LIBEVENT_THREAD *t, uint32_t exptime, bool should_touch, bool do_update, bool *overflow);1068item* limited_get_locked(const char *key, size_t nkey, LIBEVENT_THREAD *t, bool do_update, uint32_t *hv, bool *overflow);1069// Read/Response object handlers.1070void resp_reset(mc_resp *resp);1071void resp_add_iov(mc_resp *resp, const void *buf, int len);1072void resp_add_chunked_iov(mc_resp *resp, const void *buf, int len);1073bool resp_start(conn *c);1074mc_resp *resp_start_unlinked(conn *c);1075mc_resp* resp_finish(conn *c, mc_resp *resp);1076void resp_free(LIBEVENT_THREAD *th, mc_resp *resp);1077bool resp_has_stack(conn *c);1078bool rbuf_switch_to_malloc(conn *c);1079void conn_release_items(conn *c);1080void conn_set_state(conn *c, enum conn_states state);1081void out_of_memory(conn *c, char *ascii_error);1082void out_errstring(conn *c, const char *str);1083void write_and_free(conn *c, char *buf, int bytes);1084void server_stats(ADD_STAT add_stats, void *c);1085void append_stats(const char *key, const uint16_t klen,1086                  const char *val, const uint32_t vlen,1087                  const void *cookie);1088/** Return a datum for stats in binary protocol */1089bool get_stats(const char *stat_type, int nkey, ADD_STAT add_stats, void *c);1090void stats_reset(void);1091void process_stat_settings(ADD_STAT add_stats, void *c);1092void process_stats_conns(ADD_STAT add_stats, void *c);10931094#if HAVE_DROP_PRIVILEGES1095extern void setup_privilege_violations_handler(void);1096extern void drop_privileges(void);1097#else1098#define setup_privilege_violations_handler()1099#define drop_privileges()1100#endif11011102#if HAVE_DROP_WORKER_PRIVILEGES1103extern void drop_worker_privileges(void);1104#else1105#define drop_worker_privileges()1106#endif11071108/* If supported, give compiler hints for branch prediction. */1109#if !defined(__GNUC__) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)1110#define __builtin_expect(x, expected_value) (x)1111#endif11121113#define likely(x)       __builtin_expect((x),1)1114#define unlikely(x)     __builtin_expect((x),0)