1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */2/*3 * Thread management for memcached.4 */5#include "memcached.h"6#ifdef EXTSTORE7#include "storage.h"8#endif9#ifdef HAVE_EVENTFD10#include <sys/eventfd.h>11#endif12#ifdef PROXY13#include "proto_proxy.h"14#endif15#include <assert.h>16#include <stdio.h>17#include <stdlib.h>18#include <string.h>19#include <pthread.h>2021#include "queue.h"22#include "tls.h"2324#ifdef __sun25#include <atomic.h>26#endif2728#define ITEMS_PER_ALLOC 642930/* An item in the connection queue. */31enum conn_queue_item_modes {32    queue_new_conn,   /* brand new connection. */33    queue_pause,      /* pause thread */34    queue_timeout,    /* socket sfd timed out */35    queue_redispatch, /* return conn from side thread */36    queue_stop,       /* exit thread */37#ifdef PROXY38    queue_proxy_reload, /* signal proxy to reload worker VM */39#endif40};41typedef struct conn_queue_item CQ_ITEM;42struct conn_queue_item {43    int               sfd;44    enum conn_states  init_state;45    int               event_flags;46    int               read_buffer_size;47    enum network_transport     transport;48    enum conn_queue_item_modes mode;49    conn *c;50    void    *ssl;51    uint64_t conntag;52    enum protocol bproto;53    io_pending_t *io; // IO when used for deferred IO handling.54    STAILQ_ENTRY(conn_queue_item) i_next;55};5657/* A connection queue. */58typedef struct conn_queue CQ;59struct conn_queue {60    STAILQ_HEAD(conn_ev_head, conn_queue_item) head;61    pthread_mutex_t lock;62    cache_t *cache; /* freelisted objects */63};6465/* Locks for cache LRU operations */66pthread_mutex_t lru_locks[POWER_LARGEST];6768/* Connection lock around accepting new connections */69pthread_mutex_t conn_lock = PTHREAD_MUTEX_INITIALIZER;7071#if !defined(HAVE_GCC_ATOMICS) && !defined(__sun)72pthread_mutex_t atomics_mutex = PTHREAD_MUTEX_INITIALIZER;73#endif7475/* Lock for global stats */76static pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER;7778/* Lock to cause worker threads to hang up after being woken */79static pthread_mutex_t worker_hang_lock;8081static pthread_mutex_t *item_locks;82/* size of the item lock hash table */83static uint32_t item_lock_count;84static unsigned int item_lock_hashpower;85#define hashsize(n) ((unsigned long int)1<<(n))86#define hashmask(n) (hashsize(n)-1)8788/*89 * Each libevent instance has a wakeup pipe, which other threads90 * can use to signal that they've put a new connection on its queue.91 */92static LIBEVENT_THREAD *threads;9394/*95 * Number of worker threads that have finished setting themselves up.96 */97static int init_count = 0;98static pthread_mutex_t init_lock;99static pthread_cond_t init_cond;100101static void notify_worker(LIBEVENT_THREAD *t, CQ_ITEM *item);102static void notify_worker_fd(LIBEVENT_THREAD *t, int sfd, enum conn_queue_item_modes mode);103static CQ_ITEM *cqi_new(CQ *cq);104static void cq_push(CQ *cq, CQ_ITEM *item);105106static void thread_libevent_process(evutil_socket_t fd, short which, void *arg);107static void thread_libevent_ionotify(evutil_socket_t fd, short which, void *arg);108109/* item_lock() must be held for an item before any modifications to either its110 * associated hash bucket, or the structure itself.111 * LRU modifications must hold the item lock, and the LRU lock.112 * LRU's accessing items must item_trylock() before modifying an item.113 * Items accessible from an LRU must not be freed or modified114 * without first locking and removing from the LRU.115 */116117void item_lock(uint32_t hv) {118    mutex_lock(&item_locks[hv & hashmask(item_lock_hashpower)]);119}120121void *item_trylock(uint32_t hv) {122    pthread_mutex_t *lock = &item_locks[hv & hashmask(item_lock_hashpower)];123    if (pthread_mutex_trylock(lock) == 0) {124        return lock;125    }126    return NULL;127}128129void item_trylock_unlock(void *lock) {130    mutex_unlock((pthread_mutex_t *) lock);131}132133void item_unlock(uint32_t hv) {134    mutex_unlock(&item_locks[hv & hashmask(item_lock_hashpower)]);135}136137static void wait_for_thread_registration(int nthreads) {138    while (init_count < nthreads) {139        pthread_cond_wait(&init_cond, &init_lock);140    }141}142143static void register_thread_initialized(void) {144    pthread_mutex_lock(&init_lock);145    init_count++;146    pthread_cond_signal(&init_cond);147    pthread_mutex_unlock(&init_lock);148    /* Force worker threads to pile up if someone wants us to */149    pthread_mutex_lock(&worker_hang_lock);150    pthread_mutex_unlock(&worker_hang_lock);151}152153/* Must not be called with any deeper locks held */154void pause_threads(enum pause_thread_types type) {155    int i;156    bool pause_workers = false;157158    switch (type) {159        case PAUSE_ALL_THREADS:160            slab_maintenance_pause(settings.slab_rebal);161            lru_maintainer_pause();162            lru_crawler_pause();163#ifdef EXTSTORE164            storage_compact_pause();165            storage_write_pause();166#endif167        case PAUSE_WORKER_THREADS:168            pause_workers = true;169            pthread_mutex_lock(&worker_hang_lock);170            break;171        case RESUME_ALL_THREADS:172            slab_maintenance_resume(settings.slab_rebal);173            lru_maintainer_resume();174            lru_crawler_resume();175#ifdef EXTSTORE176            storage_compact_resume();177            storage_write_resume();178#endif179        case RESUME_WORKER_THREADS:180            pthread_mutex_unlock(&worker_hang_lock);181            break;182        default:183            fprintf(stderr, "Unknown lock type: %d\n", type);184            assert(1 == 0);185            break;186    }187188    /* Only send a message if we have one. */189    if (!pause_workers) {190        return;191    }192193    pthread_mutex_lock(&init_lock);194    init_count = 0;195    for (i = 0; i < settings.num_threads; i++) {196        notify_worker_fd(&threads[i], 0, queue_pause);197    }198    wait_for_thread_registration(settings.num_threads);199    pthread_mutex_unlock(&init_lock);200}201202// MUST not be called with any deeper locks held203// MUST be called only by parent thread204// Note: listener thread is the "main" event base, which has exited its205// loop in order to call this function.206void stop_threads(void) {207    int i;208209    // assoc can call pause_threads(), so we have to stop it first.210    stop_assoc_maintenance_thread();211    if (settings.verbose > 0)212        fprintf(stderr, "stopped assoc\n");213214    if (settings.verbose > 0)215        fprintf(stderr, "asking workers to stop\n");216217    pthread_mutex_lock(&worker_hang_lock);218    pthread_mutex_lock(&init_lock);219    init_count = 0;220    for (i = 0; i < settings.num_threads; i++) {221        notify_worker_fd(&threads[i], 0, queue_stop);222    }223    wait_for_thread_registration(settings.num_threads);224    pthread_mutex_unlock(&init_lock);225226    // All of the workers are hung but haven't done cleanup yet.227228    if (settings.verbose > 0)229        fprintf(stderr, "asking background threads to stop\n");230231    // stop each side thread.232    // TODO: Verify these all work if the threads are already stopped233    stop_item_crawler_thread(CRAWLER_WAIT);234    if (settings.verbose > 0)235        fprintf(stderr, "stopped lru crawler\n");236    if (settings.lru_maintainer_thread) {237        stop_lru_maintainer_thread();238        if (settings.verbose > 0)239            fprintf(stderr, "stopped maintainer\n");240    }241    if (settings.slab_reassign) {242        stop_slab_maintenance_thread(settings.slab_rebal);243        if (settings.verbose > 0)244            fprintf(stderr, "stopped slab mover\n");245    }246    logger_stop();247    if (settings.verbose > 0)248        fprintf(stderr, "stopped logger thread\n");249    stop_conn_timeout_thread();250    if (settings.verbose > 0)251        fprintf(stderr, "stopped idle timeout thread\n");252253    // Close all connections then let the workers finally exit.254    if (settings.verbose > 0)255        fprintf(stderr, "closing connections\n");256    conn_close_all();257    pthread_mutex_unlock(&worker_hang_lock);258    if (settings.verbose > 0)259        fprintf(stderr, "reaping worker threads\n");260    for (i = 0; i < settings.num_threads; i++) {261        pthread_join(threads[i].thread_id, NULL);262    }263264    if (settings.verbose > 0)265        fprintf(stderr, "all background threads stopped\n");266267    // At this point, every background thread must be stopped.268}269270/*271 * Initializes a connection queue.272 */273static void cq_init(CQ *cq) {274    pthread_mutex_init(&cq->lock, NULL);275    STAILQ_INIT(&cq->head);276    cq->cache = cache_create("cq", sizeof(CQ_ITEM), sizeof(char *));277    if (cq->cache == NULL) {278        fprintf(stderr, "Failed to create connection queue cache\n");279        exit(EXIT_FAILURE);280    }281}282283/*284 * Looks for an item on a connection queue, but doesn't block if there isn't285 * one.286 * Returns the item, or NULL if no item is available287 */288static CQ_ITEM *cq_pop(CQ *cq) {289    CQ_ITEM *item;290291    pthread_mutex_lock(&cq->lock);292    item = STAILQ_FIRST(&cq->head);293    if (item != NULL) {294        STAILQ_REMOVE_HEAD(&cq->head, i_next);295    }296    pthread_mutex_unlock(&cq->lock);297298    return item;299}300301/*302 * Adds an item to a connection queue.303 */304static void cq_push(CQ *cq, CQ_ITEM *item) {305    pthread_mutex_lock(&cq->lock);306    STAILQ_INSERT_TAIL(&cq->head, item, i_next);307    pthread_mutex_unlock(&cq->lock);308}309310/*311 * Returns a fresh connection queue item.312 */313static CQ_ITEM *cqi_new(CQ *cq) {314    CQ_ITEM *item = cache_alloc(cq->cache);315    if (item == NULL) {316        STATS_LOCK();317        stats.malloc_fails++;318        STATS_UNLOCK();319    }320    return item;321}322323/*324 * Frees a connection queue item (adds it to the freelist.)325 */326static void cqi_free(CQ *cq, CQ_ITEM *item) {327    cache_free(cq->cache, item);328}329330// TODO: Skip notify if queue wasn't empty?331// - Requires cq_push() returning a "was empty" flag332// - Requires event handling loop to pop the entire queue and work from that333// instead of the ev_count work there now.334// In testing this does result in a large performance uptick, but unclear how335// much that will transfer from a synthetic benchmark.336static void notify_worker(LIBEVENT_THREAD *t, CQ_ITEM *item) {337    cq_push(t->ev_queue, item);338#ifdef HAVE_EVENTFD339    uint64_t u = 1;340    if (write(t->n.notify_event_fd, &u, sizeof(uint64_t)) != sizeof(uint64_t)) {341        perror("failed writing to worker eventfd");342        /* TODO: This is a fatal problem. Can it ever happen temporarily? */343    }344#else345    char buf[1] = "c";346    if (write(t->n.notify_send_fd, buf, 1) != 1) {347        perror("Failed writing to notify pipe");348        /* TODO: This is a fatal problem. Can it ever happen temporarily? */349    }350#endif351}352353// NOTE: An external func that takes a conn *c might be cleaner overall.354static void notify_worker_fd(LIBEVENT_THREAD *t, int sfd, enum conn_queue_item_modes mode) {355    CQ_ITEM *item;356    while ( (item = cqi_new(t->ev_queue)) == NULL ) {357        // NOTE: most callers of this function cannot fail, but mallocs in358        // theory can fail. Small mallocs essentially never do without also359        // killing the process. Syscalls can also fail but the original code360        // never handled this either.361        // As a compromise, I'm leaving this note and this loop: This alloc362        // cannot fail, but pre-allocating the data is too much code in an363        // area I want to keep more lean. If this CQ business becomes a more364        // generic queue I'll reconsider.365    }366367    item->mode = mode;368    item->sfd = sfd;369    notify_worker(t, item);370}371372/*373 * Creates a worker thread.374 */375static void create_worker(void *(*func)(void *), void *arg) {376    pthread_attr_t  attr;377    int             ret;378379    pthread_attr_init(&attr);380381    if ((ret = pthread_create(&((LIBEVENT_THREAD*)arg)->thread_id, &attr, func, arg)) != 0) {382        fprintf(stderr, "Can't create thread: %s\n",383                strerror(ret));384        exit(1);385    }386387    thread_setname(((LIBEVENT_THREAD*)arg)->thread_id, "mc-worker");388}389390/*391 * Sets whether or not we accept new connections.392 */393void accept_new_conns(const bool do_accept) {394    pthread_mutex_lock(&conn_lock);395    do_accept_new_conns(do_accept);396    pthread_mutex_unlock(&conn_lock);397}398/****************************** LIBEVENT THREADS *****************************/399400static void setup_thread_notify(LIBEVENT_THREAD *me, struct thread_notify *tn,401        void(*cb)(int, short, void *)) {402#ifdef HAVE_EVENTFD403    event_set(&tn->notify_event, tn->notify_event_fd,404              EV_READ | EV_PERSIST, cb, me);405#else406    event_set(&tn->notify_event, tn->notify_receive_fd,407              EV_READ | EV_PERSIST, cb, me);408#endif409    event_base_set(me->base, &tn->notify_event);410411    if (event_add(&tn->notify_event, 0) == -1) {412        fprintf(stderr, "Can't monitor libevent notify pipe\n");413        exit(1);414    }415}416417/*418 * Set up a thread's information.419 */420static void setup_thread(LIBEVENT_THREAD *me) {421#if defined(LIBEVENT_VERSION_NUMBER) && LIBEVENT_VERSION_NUMBER >= 0x02000101422    struct event_config *ev_config;423    ev_config = event_config_new();424    event_config_set_flag(ev_config, EVENT_BASE_FLAG_NOLOCK);425    me->base = event_base_new_with_config(ev_config);426    event_config_free(ev_config);427#else428    me->base = event_init();429#endif430431    if (! me->base) {432        fprintf(stderr, "Can't allocate event base\n");433        exit(1);434    }435436    /* Listen for notifications from other threads */437    setup_thread_notify(me, &me->n, thread_libevent_process);438    setup_thread_notify(me, &me->ion, thread_libevent_ionotify);439    pthread_mutex_init(&me->ion_lock, NULL);440    STAILQ_INIT(&me->ion_head);441442    me->ev_queue = malloc(sizeof(struct conn_queue));443    if (me->ev_queue == NULL) {444        perror("Failed to allocate memory for connection queue");445        exit(EXIT_FAILURE);446    }447    cq_init(me->ev_queue);448449    if (pthread_mutex_init(&me->stats.mutex, NULL) != 0) {450        perror("Failed to initialize mutex");451        exit(EXIT_FAILURE);452    }453454    me->rbuf_cache = cache_create("rbuf", READ_BUFFER_SIZE, sizeof(char *));455    if (me->rbuf_cache == NULL) {456        fprintf(stderr, "Failed to create read buffer cache\n");457        exit(EXIT_FAILURE);458    }459    // Note: we were cleanly passing in num_threads before, but this now460    // relies on settings globals too much.461    if (settings.read_buf_mem_limit) {462        int limit = settings.read_buf_mem_limit / settings.num_threads;463        if (limit < READ_BUFFER_SIZE) {464            limit = 1;465        } else {466            limit = limit / READ_BUFFER_SIZE;467        }468        cache_set_limit(me->rbuf_cache, limit);469    }470471    me->io_cache = cache_create("io", sizeof(io_pending_t), sizeof(char*));472    if (me->io_cache == NULL) {473        fprintf(stderr, "Failed to create IO object cache\n");474        exit(EXIT_FAILURE);475    }476#ifdef TLS477    if (settings.ssl_enabled) {478        me->ssl_wbuf = (char *)malloc((size_t)settings.ssl_wbuf_size);479        if (me->ssl_wbuf == NULL) {480            fprintf(stderr, "Failed to allocate the SSL write buffer\n");481            exit(EXIT_FAILURE);482        }483    }484#endif485#ifdef EXTSTORE486    // me->storage is set just before this function is called.487    if (me->storage) {488        thread_io_queue_add(me, IO_QUEUE_EXTSTORE, me->storage,489            storage_submit_cb);490    }491#endif492#ifdef PROXY493    thread_io_queue_add(me, IO_QUEUE_PROXY, settings.proxy_ctx, proxy_submit_cb);494495    // TODO: maybe register hooks to be called here from sub-packages? ie;496    // extstore, TLS, proxy.497    if (settings.proxy_enabled) {498        proxy_thread_init(settings.proxy_ctx, me);499    }500#endif501    thread_io_queue_add(me, IO_QUEUE_NONE, NULL, NULL);502}503504/*505 * Worker thread: main event loop506 */507static void *worker_libevent(void *arg) {508    LIBEVENT_THREAD *me = arg;509510    /* Any per-thread setup can happen here; memcached_thread_init() will block until511     * all threads have finished initializing.512     */513    me->l = logger_create();514    me->lru_bump_buf = item_lru_bump_buf_create();515    if (me->l == NULL || me->lru_bump_buf == NULL) {516        abort();517    }518519    if (settings.drop_privileges) {520        drop_worker_privileges();521    }522523    register_thread_initialized();524    while (!event_base_got_exit(me->base)) {525        event_base_loop(me->base, EVLOOP_ONCE);526        // Run IO queues after the event loop to catch things like527        // re-submissions from proxy callbacks.528        thread_io_queue_submit(me);529#ifdef PROXY530        if (me->proxy_ctx) {531            proxy_gc_poke(me);532        }533#endif534    }535    // same mechanism used to watch for all threads exiting.536    register_thread_initialized();537538    event_base_free(me->base);539    return NULL;540}541542// Syscalls can be expensive enough that handling a few of them once here can543// save both throughput and overall latency.544#define MAX_PIPE_EVENTS 32545546// dedicated worker thread notify system for IO objects.547static void thread_libevent_ionotify(evutil_socket_t fd, short which, void *arg) {548    LIBEVENT_THREAD *me = arg;549    uint64_t ev_count = 0;550    iop_head_t head;551552    STAILQ_INIT(&head);553#ifdef HAVE_EVENTFD554    if (read(fd, &ev_count, sizeof(uint64_t)) != sizeof(uint64_t)) {555        if (settings.verbose > 0)556            fprintf(stderr, "Can't read from libevent pipe\n");557        return;558    }559#else560    char buf[MAX_PIPE_EVENTS];561562    ev_count = read(fd, buf, MAX_PIPE_EVENTS);563    if (ev_count == 0) {564        if (settings.verbose > 0)565            fprintf(stderr, "Can't read from libevent pipe\n");566        return;567    }568#endif569570    // pull entire queue and zero the thread head.571    // need to do this after reading a syscall as we are only guaranteed to572    // get syscalls if the queue is empty.573    pthread_mutex_lock(&me->ion_lock);574    STAILQ_CONCAT(&head, &me->ion_head);575    pthread_mutex_unlock(&me->ion_lock);576577    while (!STAILQ_EMPTY(&head)) {578        io_pending_t *io = STAILQ_FIRST(&head);579        STAILQ_REMOVE_HEAD(&head, iop_next);580        conn_io_queue_return(io);581    }582}583584/*585 * Processes an incoming "connection event" item. This is called when586 * input arrives on the libevent wakeup pipe.587 */588static void thread_libevent_process(evutil_socket_t fd, short which, void *arg) {589    LIBEVENT_THREAD *me = arg;590    CQ_ITEM *item;591    conn *c;592    uint64_t ev_count = 0; // max number of events to loop through this run.593#ifdef HAVE_EVENTFD594    // NOTE: unlike pipe we aren't limiting the number of events per read.595    // However we do limit the number of queue pulls to what the count was at596    // the time of this function firing.597    if (read(fd, &ev_count, sizeof(uint64_t)) != sizeof(uint64_t)) {598        if (settings.verbose > 0)599            fprintf(stderr, "Can't read from libevent pipe\n");600        return;601    }602#else603    char buf[MAX_PIPE_EVENTS];604605    ev_count = read(fd, buf, MAX_PIPE_EVENTS);606    if (ev_count == 0) {607        if (settings.verbose > 0)608            fprintf(stderr, "Can't read from libevent pipe\n");609        return;610    }611#endif612613    for (int x = 0; x < ev_count; x++) {614        item = cq_pop(me->ev_queue);615        if (item == NULL) {616            return;617        }618619        switch (item->mode) {620            case queue_new_conn:621                c = conn_new(item->sfd, item->init_state, item->event_flags,622                                   item->read_buffer_size, item->transport,623                                   me->base, item->ssl, item->conntag, item->bproto);624                if (c == NULL) {625                    if (IS_UDP(item->transport)) {626                        fprintf(stderr, "Can't listen for events on UDP socket\n");627                        exit(1);628                    } else {629                        if (settings.verbose > 0) {630                            fprintf(stderr, "Can't listen for events on fd %d\n",631                                item->sfd);632                        }633                        if (item->ssl) {634                            ssl_conn_close(item->ssl);635                            item->ssl = NULL;636                        }637                        close(item->sfd);638                    }639                } else {640                    c->thread = me;641#ifdef TLS642                    if (settings.ssl_enabled && c->ssl != NULL) {643                        assert(c->thread && c->thread->ssl_wbuf);644                        c->ssl_wbuf = c->thread->ssl_wbuf;645                    }646#endif647                }648                break;649            case queue_pause:650                /* we were told to pause and report in */651                register_thread_initialized();652                break;653            case queue_timeout:654                /* a client socket timed out */655                conn_close_idle(conns[item->sfd]);656                break;657            case queue_redispatch:658                /* a side thread redispatched a client connection */659                conn_worker_readd(conns[item->sfd]);660                break;661            case queue_stop:662                /* asked to stop */663                event_base_loopexit(me->base, NULL);664                break;665#ifdef PROXY666            case queue_proxy_reload:667                proxy_worker_reload(settings.proxy_ctx, me);668                break;669#endif670        }671672        cqi_free(me->ev_queue, item);673    }674}675676// Interface is slightly different on various platforms.677// On linux, at least, the len limit is 16 bytes.678#define THR_NAME_MAXLEN 16679void thread_setname(pthread_t thread, const char *name) {680assert(strlen(name) < THR_NAME_MAXLEN);681#if defined(__linux__) && defined(HAVE_PTHREAD_SETNAME_NP)682pthread_setname_np(thread, name);683#endif684}685#undef THR_NAME_MAXLEN686687// NOTE: need better encapsulation.688// used by the proxy module to iterate the worker threads.689LIBEVENT_THREAD *get_worker_thread(int id) {690    return &threads[id];691}692693/* Which thread we assigned a connection to most recently. */694static int last_thread = -1;695696/* Last thread we assigned to a connection based on napi_id */697static int last_thread_by_napi_id = -1;698699static LIBEVENT_THREAD *select_thread_round_robin(void)700{701    int tid = (last_thread + 1) % settings.num_threads;702703    last_thread = tid;704705    return threads + tid;706}707708static void reset_threads_napi_id(void)709{710    LIBEVENT_THREAD *thread;711    int i;712713    for (i = 0; i < settings.num_threads; i++) {714         thread = threads + i;715         thread->napi_id = 0;716    }717718    last_thread_by_napi_id = -1;719}720721/* Select a worker thread based on the NAPI ID of an incoming connection722 * request. NAPI ID is a globally unique ID that identifies a NIC RX queue723 * on which a flow is received.724 */725static LIBEVENT_THREAD *select_thread_by_napi_id(int sfd)726{727    LIBEVENT_THREAD *thread;728    int napi_id, err, i;729    socklen_t len;730    int tid = -1;731732    len = sizeof(socklen_t);733    err = getsockopt(sfd, SOL_SOCKET, SO_INCOMING_NAPI_ID, &napi_id, &len);734    if ((err == -1) || (napi_id == 0)) {735        STATS_LOCK();736        stats.round_robin_fallback++;737        STATS_UNLOCK();738        return select_thread_round_robin();739    }740741select:742    for (i = 0; i < settings.num_threads; i++) {743         thread = threads + i;744         if (last_thread_by_napi_id < i) {745             thread->napi_id = napi_id;746             last_thread_by_napi_id = i;747             tid = i;748             break;749         }750         if (thread->napi_id == napi_id) {751             tid = i;752             break;753         }754    }755756    if (tid == -1) {757        STATS_LOCK();758        stats.unexpected_napi_ids++;759        STATS_UNLOCK();760        reset_threads_napi_id();761        goto select;762    }763764    return threads + tid;765}766767/*768 * Dispatches a new connection to another thread. This is only ever called769 * from the main thread, either during initialization (for UDP) or because770 * of an incoming connection.771 */772void dispatch_conn_new(int sfd, enum conn_states init_state, int event_flags,773                       int read_buffer_size, enum network_transport transport, void *ssl,774                       uint64_t conntag, enum protocol bproto) {775    CQ_ITEM *item = NULL;776    LIBEVENT_THREAD *thread;777778    if (!settings.num_napi_ids)779        thread = select_thread_round_robin();780    else781        thread = select_thread_by_napi_id(sfd);782783    item = cqi_new(thread->ev_queue);784    if (item == NULL) {785        close(sfd);786        /* given that malloc failed this may also fail, but let's try */787        fprintf(stderr, "Failed to allocate memory for connection object\n");788        return;789    }790791    item->sfd = sfd;792    item->init_state = init_state;793    item->event_flags = event_flags;794    item->read_buffer_size = read_buffer_size;795    item->transport = transport;796    item->mode = queue_new_conn;797    item->ssl = ssl;798    item->conntag = conntag;799    item->bproto = bproto;800801    MEMCACHED_CONN_DISPATCH(sfd, (int64_t)thread->thread_id);802    notify_worker(thread, item);803}804805/*806 * Re-dispatches a connection back to the original thread. Can be called from807 * any side thread borrowing a connection.808 */809void redispatch_conn(conn *c) {810    notify_worker_fd(c->thread, c->sfd, queue_redispatch);811}812813void timeout_conn(conn *c) {814    notify_worker_fd(c->thread, c->sfd, queue_timeout);815}816#ifdef PROXY817void proxy_reload_notify(LIBEVENT_THREAD *t) {818    notify_worker_fd(t, 0, queue_proxy_reload);819}820#endif821822void return_io_pending(io_pending_t *io) {823    bool do_notify = false;824    LIBEVENT_THREAD *t = io->thread;825    pthread_mutex_lock(&t->ion_lock);826    if (STAILQ_EMPTY(&t->ion_head)) {827        do_notify = true;828    }829    STAILQ_INSERT_TAIL(&t->ion_head, io, iop_next);830    pthread_mutex_unlock(&t->ion_lock);831832    // skip the syscall if there was already data in the queue, as it's833    // already been notified.834    if (do_notify) {835#ifdef HAVE_EVENTFD836        uint64_t u = 1;837        if (write(t->ion.notify_event_fd, &u, sizeof(uint64_t)) != sizeof(uint64_t)) {838            perror("failed writing to worker eventfd");839            /* TODO: This is a fatal problem. Can it ever happen temporarily? */840        }841#else842        char buf[1] = "c";843        if (write(t->ion.notify_send_fd, buf, 1) != 1) {844            perror("Failed writing to notify pipe");845            /* TODO: This is a fatal problem. Can it ever happen temporarily? */846        }847#endif848    }849}850851/* This misses the allow_new_conns flag :( */852void sidethread_conn_close(conn *c) {853    if (settings.verbose > 1)854        fprintf(stderr, "<%d connection closing from side thread.\n", c->sfd);855856    c->state = conn_closing;857    // redispatch will see closing flag and properly close connection.858    redispatch_conn(c);859    return;860}861862/********************************* ITEM ACCESS *******************************/863864/*865 * Allocates a new item.866 */867item *item_alloc(const char *key, size_t nkey, client_flags_t flags, rel_time_t exptime, int nbytes) {868    item *it;869    /* do_item_alloc handles its own locks */870    it = do_item_alloc(key, nkey, flags, exptime, nbytes);871    return it;872}873874/*875 * Returns an item if it hasn't been marked as expired,876 * lazy-expiring as needed.877 */878item *item_get(const char *key, const size_t nkey, LIBEVENT_THREAD *t, const bool do_update) {879    item *it;880    uint32_t hv;881    hv = hash(key, nkey);882    item_lock(hv);883    it = do_item_get(key, nkey, hv, t, do_update);884    item_unlock(hv);885    return it;886}887888// returns an item with the item lock held.889// lock will still be held even if return is NULL, allowing caller to replace890// an item atomically if desired.891item *item_get_locked(const char *key, const size_t nkey, LIBEVENT_THREAD *t, const bool do_update, uint32_t *hv) {892    item *it;893    *hv = hash(key, nkey);894    item_lock(*hv);895    it = do_item_get(key, nkey, *hv, t, do_update);896    return it;897}898899item *item_touch(const char *key, size_t nkey, uint32_t exptime, LIBEVENT_THREAD *t) {900    item *it;901    uint32_t hv;902    hv = hash(key, nkey);903    item_lock(hv);904    it = do_item_touch(key, nkey, exptime, hv, t);905    item_unlock(hv);906    return it;907}908909/*910 * Decrements the reference count on an item and adds it to the freelist if911 * needed.912 */913void item_remove(item *item) {914    uint32_t hv;915    hv = hash(ITEM_key(item), item->nkey);916917    item_lock(hv);918    do_item_remove(item);919    item_unlock(hv);920}921922/*923 * Replaces one item with another in the hashtable.924 * Unprotected by a mutex lock since the core server does not require925 * it to be thread-safe.926 */927int item_replace(item *old_it, item *new_it, const uint32_t hv, const uint64_t cas_in) {928    return do_item_replace(old_it, new_it, hv, cas_in);929}930931/*932 * Unlinks an item from the LRU and hashtable.933 */934void item_unlink(item *item) {935    uint32_t hv;936    hv = hash(ITEM_key(item), item->nkey);937    item_lock(hv);938    do_item_unlink(item, hv);939    item_unlock(hv);940}941942/*943 * Does arithmetic on a numeric item value.944 */945enum delta_result_type add_delta(LIBEVENT_THREAD *t, const char *key,946                                 const size_t nkey, bool incr,947                                 const int64_t delta, char *buf,948                                 uint64_t *cas) {949    enum delta_result_type ret;950    uint32_t hv;951952    hv = hash(key, nkey);953    item_lock(hv);954    ret = do_add_delta(t, key, nkey, incr, delta, buf, cas, hv, NULL);955    item_unlock(hv);956    return ret;957}958959/*960 * Stores an item in the cache (high level, obeys set/add/replace semantics)961 */962enum store_item_type store_item(item *item, int comm, LIBEVENT_THREAD *t, int *nbytes, uint64_t *cas, const uint64_t cas_in, bool cas_stale) {963    enum store_item_type ret;964    uint32_t hv;965966    hv = hash(ITEM_key(item), item->nkey);967    item_lock(hv);968    ret = do_store_item(item, comm, t, hv, nbytes, cas, cas_in, cas_stale);969    item_unlock(hv);970    return ret;971}972973/******************************* GLOBAL STATS ******************************/974975void STATS_LOCK(void) {976    pthread_mutex_lock(&stats_lock);977}978979void STATS_UNLOCK(void) {980    pthread_mutex_unlock(&stats_lock);981}982983void threadlocal_stats_reset(void) {984    int ii;985    for (ii = 0; ii < settings.num_threads; ++ii) {986        pthread_mutex_lock(&threads[ii].stats.mutex);987#define X(name) threads[ii].stats.name = 0;988        THREAD_STATS_FIELDS989#ifdef EXTSTORE990        EXTSTORE_THREAD_STATS_FIELDS991#endif992#ifdef PROXY993        PROXY_THREAD_STATS_FIELDS994#endif995#undef X996997        memset(&threads[ii].stats.slab_stats, 0,998                sizeof(threads[ii].stats.slab_stats));999        memset(&threads[ii].stats.lru_hits, 0,1000                sizeof(uint64_t) * POWER_LARGEST);10011002        pthread_mutex_unlock(&threads[ii].stats.mutex);1003    }1004}10051006void threadlocal_stats_aggregate(struct thread_stats *stats) {1007    int ii, sid;10081009    /* The struct has a mutex, but we can safely set the whole thing1010     * to zero since it is unused when aggregating. */1011    memset(stats, 0, sizeof(*stats));10121013    for (ii = 0; ii < settings.num_threads; ++ii) {1014        pthread_mutex_lock(&threads[ii].stats.mutex);1015#define X(name) stats->name += threads[ii].stats.name;1016        THREAD_STATS_FIELDS1017#ifdef EXTSTORE1018        EXTSTORE_THREAD_STATS_FIELDS1019#endif1020#ifdef PROXY1021        PROXY_THREAD_STATS_FIELDS1022        stats->proxy_req_active += threads[ii].stats.proxy_req_active;1023#endif1024#undef X10251026        for (sid = 0; sid < MAX_NUMBER_OF_SLAB_CLASSES; sid++) {1027#define X(name) stats->slab_stats[sid].name += \1028            threads[ii].stats.slab_stats[sid].name;1029            SLAB_STATS_FIELDS1030#undef X1031        }10321033        for (sid = 0; sid < POWER_LARGEST; sid++) {1034            stats->lru_hits[sid] +=1035                threads[ii].stats.lru_hits[sid];1036            stats->slab_stats[CLEAR_LRU(sid)].get_hits +=1037                threads[ii].stats.lru_hits[sid];1038        }10391040        stats->read_buf_count += threads[ii].rbuf_cache->total;1041        stats->read_buf_bytes += threads[ii].rbuf_cache->total * READ_BUFFER_SIZE;1042        stats->read_buf_bytes_free += threads[ii].rbuf_cache->freecurr * READ_BUFFER_SIZE;1043        pthread_mutex_unlock(&threads[ii].stats.mutex);1044    }1045}10461047void slab_stats_aggregate(struct thread_stats *stats, struct slab_stats *out) {1048    int sid;10491050    memset(out, 0, sizeof(*out));10511052    for (sid = 0; sid < MAX_NUMBER_OF_SLAB_CLASSES; sid++) {1053#define X(name) out->name += stats->slab_stats[sid].name;1054        SLAB_STATS_FIELDS1055#undef X1056    }1057}10581059static void memcached_thread_notify_init(struct thread_notify *tn) {1060#ifdef HAVE_EVENTFD1061        tn->notify_event_fd = eventfd(0, EFD_NONBLOCK);1062        if (tn->notify_event_fd == -1) {1063            perror("failed creating eventfd for worker thread");1064            exit(1);1065        }1066#else1067        int fds[2];1068        if (pipe(fds)) {1069            perror("Can't create notify pipe");1070            exit(1);1071        }10721073        tn->notify_receive_fd = fds[0];1074        tn->notify_send_fd = fds[1];1075#endif1076}10771078/*1079 * Initializes the thread subsystem, creating various worker threads.1080 *1081 * nthreads  Number of worker event handler threads to spawn1082 */1083void memcached_thread_init(int nthreads, void *arg) {1084    int         i;1085    int         power;10861087    for (i = 0; i < POWER_LARGEST; i++) {1088        pthread_mutex_init(&lru_locks[i], NULL);1089    }1090    pthread_mutex_init(&worker_hang_lock, NULL);10911092    pthread_mutex_init(&init_lock, NULL);1093    pthread_cond_init(&init_cond, NULL);10941095    /* Want a wide lock table, but don't waste memory */1096    if (nthreads < 3) {1097        power = 10;1098    } else if (nthreads < 4) {1099        power = 11;1100    } else if (nthreads < 5) {1101        power = 12;1102    } else if (nthreads <= 10) {1103        power = 13;1104    } else if (nthreads <= 20) {1105        power = 14;1106    } else {1107        /* 32k buckets. just under the hashpower default. */1108        power = 15;1109    }11101111    if (power >= hashpower) {1112        fprintf(stderr, "Hash table power size (%d) cannot be equal to or less than item lock table (%d)\n", hashpower, power);1113        fprintf(stderr, "Item lock table grows with `-t N` (worker threadcount)\n");1114        fprintf(stderr, "Hash table grows with `-o hashpower=N` \n");1115        exit(1);1116    }11171118    item_lock_count = hashsize(power);1119    item_lock_hashpower = power;11201121    item_locks = calloc(item_lock_count, sizeof(pthread_mutex_t));1122    if (! item_locks) {1123        perror("Can't allocate item locks");1124        exit(1);1125    }1126    for (i = 0; i < item_lock_count; i++) {1127        pthread_mutex_init(&item_locks[i], NULL);1128    }11291130    threads = calloc(nthreads, sizeof(LIBEVENT_THREAD));1131    if (! threads) {1132        perror("Can't allocate thread descriptors");1133        exit(1);1134    }11351136    for (i = 0; i < nthreads; i++) {1137        memcached_thread_notify_init(&threads[i].n);1138        memcached_thread_notify_init(&threads[i].ion);1139#ifdef EXTSTORE1140        threads[i].storage = arg;1141#endif1142        threads[i].thread_baseid = i;1143        setup_thread(&threads[i]);1144        /* Reserve three fds for the libevent base, and two for the pipe */1145        stats_state.reserved_fds += 5;1146    }11471148    /* Create threads after we've done all the libevent setup. */1149    for (i = 0; i < nthreads; i++) {1150        create_worker(worker_libevent, &threads[i]);1151    }11521153    /* Wait for all the threads to set themselves up before returning. */1154    pthread_mutex_lock(&init_lock);1155    wait_for_thread_registration(nthreads);1156    pthread_mutex_unlock(&init_lock);1157}