1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */2#include "memcached.h"3#include "bipbuffer.h"4#include "storage.h"5#include "slabs_mover.h"6#include <sys/stat.h>7#include <sys/socket.h>8#include <sys/resource.h>9#include <fcntl.h>10#include <netinet/in.h>11#include <stdlib.h>12#include <stdio.h>13#include <string.h>14#include <time.h>15#include <assert.h>16#include <unistd.h>17#include <poll.h>1819/* Forward Declarations */20static void item_link_q(item *it);21static void item_unlink_q(item *it);2223static unsigned int lru_type_map[4] = {HOT_LRU, WARM_LRU, COLD_LRU, TEMP_LRU};2425#define LARGEST_ID POWER_LARGEST26typedef struct {27    uint64_t evicted;28    uint64_t evicted_nonzero;29    uint64_t reclaimed;30    uint64_t outofmemory;31    uint64_t tailrepairs;32    uint64_t expired_unfetched; /* items reclaimed but never touched */33    uint64_t evicted_unfetched; /* items evicted but never touched */34    uint64_t evicted_active; /* items evicted that should have been shuffled */35    uint64_t crawler_reclaimed;36    uint64_t crawler_items_checked;37    uint64_t lrutail_reflocked;38    uint64_t moves_to_cold;39    uint64_t moves_to_warm;40    uint64_t moves_within_lru;41    uint64_t direct_reclaims;42    uint64_t hits_to_hot;43    uint64_t hits_to_warm;44    uint64_t hits_to_cold;45    uint64_t hits_to_temp;46    uint64_t mem_requested;47    rel_time_t evicted_time;48} itemstats_t;4950static item *heads[LARGEST_ID];51static item *tails[LARGEST_ID];52static itemstats_t itemstats[LARGEST_ID];53static unsigned int sizes[LARGEST_ID];54static uint64_t sizes_bytes[LARGEST_ID];55static unsigned int *stats_sizes_hist = NULL;56static int stats_sizes_buckets = 0;57static uint64_t cas_id = 1;5859static volatile int do_run_lru_maintainer_thread = 0;60static pthread_mutex_t lru_maintainer_lock = PTHREAD_MUTEX_INITIALIZER;61static pthread_mutex_t cas_id_lock = PTHREAD_MUTEX_INITIALIZER;6263void item_stats_reset(void) {64    int i;65    for (i = 0; i < LARGEST_ID; i++) {66        pthread_mutex_lock(&lru_locks[i]);67        memset(&itemstats[i], 0, sizeof(itemstats_t));68        pthread_mutex_unlock(&lru_locks[i]);69    }70}7172/* called with class lru lock held */73void do_item_stats_add_crawl(const int i, const uint64_t reclaimed,74        const uint64_t unfetched, const uint64_t checked) {75    itemstats[i].crawler_reclaimed += reclaimed;76    itemstats[i].expired_unfetched += unfetched;77    itemstats[i].crawler_items_checked += checked;78}7980typedef struct _lru_bump_buf {81    struct _lru_bump_buf *prev;82    struct _lru_bump_buf *next;83    pthread_mutex_t mutex;84    bipbuf_t *buf;85    uint64_t dropped;86} lru_bump_buf;8788typedef struct {89    item *it;90    uint32_t hv;91} lru_bump_entry;9293static lru_bump_buf *bump_buf_head = NULL;94static lru_bump_buf *bump_buf_tail = NULL;95static pthread_mutex_t bump_buf_lock = PTHREAD_MUTEX_INITIALIZER;96/* TODO: tunable? Need bench results */97#define LRU_BUMP_BUF_SIZE 81929899static bool lru_bump_async(lru_bump_buf *b, item *it, uint32_t hv);100static uint64_t lru_total_bumps_dropped(void);101102/* Get the next CAS id for a new item. */103/* TODO: refactor some atomics for this. */104uint64_t get_cas_id(void) {105    pthread_mutex_lock(&cas_id_lock);106    uint64_t next_id = ++cas_id;107    pthread_mutex_unlock(&cas_id_lock);108    return next_id;109}110111void set_cas_id(uint64_t new_cas) {112    pthread_mutex_lock(&cas_id_lock);113    cas_id = new_cas;114    pthread_mutex_unlock(&cas_id_lock);115}116117int item_is_flushed(item *it) {118    rel_time_t oldest_live = settings.oldest_live;119    if (it->time <= oldest_live && oldest_live <= current_time)120        return 1;121122    return 0;123}124125/* must be locked before call */126unsigned int do_get_lru_size(uint32_t id) {127    return sizes[id];128}129130/* Enable this for reference-count debugging. */131#if 0132# define DEBUG_REFCNT(it,op) \133                fprintf(stderr, "item %x refcnt(%c) %d %c%c%c\n", \134                        it, op, it->refcount, \135                        (it->it_flags & ITEM_LINKED) ? 'L' : ' ', \136                        (it->it_flags & ITEM_SLABBED) ? 'S' : ' ')137#else138# define DEBUG_REFCNT(it,op) while(0)139#endif140141/**142 * Generates the variable-sized part of the header for an object.143 *144 * nkey    - The length of the key145 * flags   - key flags146 * nbytes  - Number of bytes to hold value and addition CRLF terminator147 * suffix  - Buffer for the "VALUE" line suffix (flags, size).148 * nsuffix - The length of the suffix is stored here.149 *150 * Returns the total size of the header.151 */152static size_t item_make_header(const uint8_t nkey, const client_flags_t flags, const int nbytes,153                     char *suffix, uint8_t *nsuffix) {154    if (flags == 0) {155        *nsuffix = 0;156    } else {157        *nsuffix = sizeof(flags);158    }159    return sizeof(item) + nkey + *nsuffix + nbytes;160}161162item *do_item_alloc_pull(const size_t ntotal, const unsigned int id) {163    item *it = NULL;164    int i;165    /* If no memory is available, attempt a direct LRU juggle/eviction */166    /* This is a race in order to simplify lru_pull_tail; in cases where167     * locked items are on the tail, you want them to fall out and cause168     * occasional OOM's, rather than internally work around them.169     * This also gives one fewer code path for slab alloc/free170     */171    for (i = 0; i < 10; i++) {172        /* Try to reclaim memory first */173        if (!settings.lru_segmented) {174            lru_pull_tail(id, COLD_LRU, 0, 0, 0, NULL);175        }176        it = slabs_alloc(id, 0);177178        if (it == NULL) {179            // We send '0' in for "total_bytes" as this routine is always180            // pulling to evict, or forcing HOT -> COLD migration.181            // As of this writing, total_bytes isn't at all used with COLD_LRU.182            if (lru_pull_tail(id, COLD_LRU, 0, LRU_PULL_EVICT, 0, NULL) <= 0) {183                if (settings.lru_segmented) {184                    lru_pull_tail(id, HOT_LRU, 0, 0, 0, NULL);185                } else {186                    break;187                }188            }189        } else {190            break;191        }192    }193194    if (i > 0) {195        pthread_mutex_lock(&lru_locks[id]);196        itemstats[id].direct_reclaims += i;197        pthread_mutex_unlock(&lru_locks[id]);198    }199200    return it;201}202203/* Chain another chunk onto this chunk. */204/* slab mover: if it finds a chunk without ITEM_CHUNK flag, and no ITEM_LINKED205 * flag, it counts as busy and skips.206 * I think it might still not be safe to do linking outside of the slab lock207 */208item_chunk *do_item_alloc_chunk(item_chunk *ch, const size_t bytes_remain) {209    // TODO: Should be a cleaner way of finding real size with slabber calls210    size_t size = bytes_remain + sizeof(item_chunk);211    if (size > settings.slab_chunk_size_max)212        size = settings.slab_chunk_size_max;213    unsigned int id = slabs_clsid(size);214215    item_chunk *nch = (item_chunk *) do_item_alloc_pull(size, id);216    if (nch == NULL) {217        // The final chunk in a large item will attempt to be a more218        // appropriately sized chunk to minimize memory overhead. However, if219        // there's no memory available in the lower slab classes we fail the220        // SET. In these cases as a fallback we ensure we attempt to evict a221        // max-size item and reuse a large chunk.222        if (size == settings.slab_chunk_size_max) {223            return NULL;224        } else {225            size = settings.slab_chunk_size_max;226            id = slabs_clsid(size);227            nch = (item_chunk *) do_item_alloc_pull(size, id);228229            if (nch == NULL)230                return NULL;231        }232    }233234    // link in.235    // ITEM_CHUNK[ED] bits need to be protected by the slabs lock.236    slabs_mlock();237    nch->head = ch->head;238    ch->next = nch;239    nch->prev = ch;240    nch->next = 0;241    nch->used = 0;242    nch->slabs_clsid = id;243    nch->size = size - sizeof(item_chunk);244    nch->it_flags |= ITEM_CHUNK;245    slabs_munlock();246    return nch;247}248249item *do_item_alloc(const char *key, const size_t nkey, const client_flags_t flags,250                    const rel_time_t exptime, const int nbytes) {251    uint8_t nsuffix;252    item *it = NULL;253    char suffix[40];254    // Avoid potential underflows.255    if (nbytes < 2)256        return 0;257258    size_t ntotal = item_make_header(nkey + 1, flags, nbytes, suffix, &nsuffix);259    if (settings.use_cas) {260        ntotal += sizeof(uint64_t);261    }262263    unsigned int id = slabs_clsid(ntotal);264    unsigned int hdr_id = 0;265    if (id == 0)266        return 0;267268    /* This is a large item. Allocate a header object now, lazily allocate269     *  chunks while reading the upload.270     */271    if (ntotal > settings.slab_chunk_size_max) {272        /* We still link this item into the LRU for the larger slab class, but273         * we're pulling a header from an entirely different slab class. The274         * free routines handle large items specifically.275         */276        int htotal = nkey + 1 + nsuffix + sizeof(item) + sizeof(item_chunk);277        if (settings.use_cas) {278            htotal += sizeof(uint64_t);279        }280#ifdef NEED_ALIGN281        // header chunk needs to be padded on some systems282        int remain = htotal % 8;283        if (remain != 0) {284            htotal += 8 - remain;285        }286#endif287        hdr_id = slabs_clsid(htotal);288        it = do_item_alloc_pull(htotal, hdr_id);289        /* setting ITEM_CHUNKED is fine here because we aren't LINKED yet. */290        if (it != NULL)291            it->it_flags |= ITEM_CHUNKED;292    } else {293        it = do_item_alloc_pull(ntotal, id);294    }295296    if (it == NULL) {297        pthread_mutex_lock(&lru_locks[id]);298        itemstats[id].outofmemory++;299        pthread_mutex_unlock(&lru_locks[id]);300        return NULL;301    }302303    assert(it->it_flags == 0 || it->it_flags == ITEM_CHUNKED);304    //assert(it != heads[id]);305306    /* Refcount is seeded to 1 by slabs_alloc() */307    it->next = it->prev = 0;308309    /* Items are initially loaded into the HOT_LRU. This is '0' but I want at310     * least a note here. Compiler (hopefully?) optimizes this out.311     */312    if (settings.temp_lru &&313            exptime - current_time <= settings.temporary_ttl) {314        id |= TEMP_LRU;315    } else if (settings.lru_segmented) {316        id |= HOT_LRU;317    } else {318        /* There is only COLD in compat-mode */319        id |= COLD_LRU;320    }321    it->slabs_clsid = id;322323    DEBUG_REFCNT(it, '*');324    it->it_flags |= settings.use_cas ? ITEM_CAS : 0;325    it->it_flags |= nsuffix != 0 ? ITEM_CFLAGS : 0;326    it->nkey = nkey;327    it->nbytes = nbytes;328    memcpy(ITEM_key(it), key, nkey);329    it->exptime = exptime;330    if (nsuffix > 0) {331        memcpy(ITEM_suffix(it), &flags, sizeof(flags));332    }333334    /* Initialize internal chunk. */335    if (it->it_flags & ITEM_CHUNKED) {336        item_chunk *chunk = (item_chunk *) ITEM_schunk(it);337338        chunk->next = 0;339        chunk->prev = 0;340        chunk->used = 0;341        chunk->size = 0;342        chunk->head = it;343        chunk->orig_clsid = hdr_id;344    }345    it->h_next = 0;346347    return it;348}349350void item_free(item *it) {351    unsigned int clsid;352    assert((it->it_flags & ITEM_LINKED) == 0);353    assert(it != heads[it->slabs_clsid]);354    assert(it != tails[it->slabs_clsid]);355    assert(it->refcount == 0);356357    /* so slab size changer can tell later if item is already free or not */358    clsid = ITEM_clsid(it);359    DEBUG_REFCNT(it, 'F');360    slabs_free(it, clsid);361}362363/**364 * Returns true if an item will fit in the cache (its size does not exceed365 * the maximum for a cache entry.)366 */367bool item_size_ok(const size_t nkey, const client_flags_t flags, const int nbytes) {368    char prefix[40];369    uint8_t nsuffix;370    if (nbytes < 2)371        return false;372373    size_t ntotal = item_make_header(nkey + 1, flags, nbytes,374                                     prefix, &nsuffix);375    if (settings.use_cas) {376        ntotal += sizeof(uint64_t);377    }378379    return slabs_clsid(ntotal) != 0;380}381382/* fixing stats/references during warm start */383void do_item_link_fixup(item *it) {384    item **head, **tail;385    int ntotal = ITEM_ntotal(it);386    uint32_t hv = hash(ITEM_key(it), it->nkey);387    assoc_insert(it, hv);388389    head = &heads[it->slabs_clsid];390    tail = &tails[it->slabs_clsid];391    if (it->prev == 0 && *head == 0) *head = it;392    if (it->next == 0 && *tail == 0) *tail = it;393    sizes[it->slabs_clsid]++;394    sizes_bytes[it->slabs_clsid] += ntotal;395396    STATS_LOCK();397    stats_state.curr_bytes += ntotal;398    stats_state.curr_items += 1;399    stats.total_items += 1;400    STATS_UNLOCK();401402    item_stats_sizes_add(it);403404    return;405}406407static void do_item_link_q(item *it) { /* item is the new head */408    item **head, **tail;409    assert((it->it_flags & ITEM_SLABBED) == 0);410411    head = &heads[it->slabs_clsid];412    tail = &tails[it->slabs_clsid];413    assert(it != *head);414    assert((*head && *tail) || (*head == 0 && *tail == 0));415    it->prev = 0;416    it->next = *head;417    if (it->next) it->next->prev = it;418    *head = it;419    if (*tail == 0) *tail = it;420    sizes[it->slabs_clsid]++;421#ifdef EXTSTORE422    if (it->it_flags & ITEM_HDR) {423        sizes_bytes[it->slabs_clsid] += (ITEM_ntotal(it) - it->nbytes) + sizeof(item_hdr);424    } else {425        sizes_bytes[it->slabs_clsid] += ITEM_ntotal(it);426    }427#else428    sizes_bytes[it->slabs_clsid] += ITEM_ntotal(it);429#endif430431    return;432}433434static void item_link_q(item *it) {435    pthread_mutex_lock(&lru_locks[it->slabs_clsid]);436    do_item_link_q(it);437    pthread_mutex_unlock(&lru_locks[it->slabs_clsid]);438}439440static void item_link_q_warm(item *it) {441    pthread_mutex_lock(&lru_locks[it->slabs_clsid]);442    do_item_link_q(it);443    itemstats[it->slabs_clsid].moves_to_warm++;444    pthread_mutex_unlock(&lru_locks[it->slabs_clsid]);445}446447static void do_item_unlink_q(item *it) {448    item **head, **tail;449    head = &heads[it->slabs_clsid];450    tail = &tails[it->slabs_clsid];451452    if (*head == it) {453        assert(it->prev == 0);454        *head = it->next;455    }456    if (*tail == it) {457        assert(it->next == 0);458        *tail = it->prev;459    }460    assert(it->next != it);461    assert(it->prev != it);462463    if (it->next) it->next->prev = it->prev;464    if (it->prev) it->prev->next = it->next;465    sizes[it->slabs_clsid]--;466#ifdef EXTSTORE467    if (it->it_flags & ITEM_HDR) {468        sizes_bytes[it->slabs_clsid] -= (ITEM_ntotal(it) - it->nbytes) + sizeof(item_hdr);469    } else {470        sizes_bytes[it->slabs_clsid] -= ITEM_ntotal(it);471    }472#else473    sizes_bytes[it->slabs_clsid] -= ITEM_ntotal(it);474#endif475476    return;477}478479static void item_unlink_q(item *it) {480    pthread_mutex_lock(&lru_locks[it->slabs_clsid]);481    do_item_unlink_q(it);482    pthread_mutex_unlock(&lru_locks[it->slabs_clsid]);483}484485int do_item_link(item *it, const uint32_t hv, const uint64_t cas) {486    MEMCACHED_ITEM_LINK(ITEM_key(it), it->nkey, it->nbytes);487    assert((it->it_flags & (ITEM_LINKED|ITEM_SLABBED)) == 0);488    it->it_flags |= ITEM_LINKED;489    it->time = current_time;490491    STATS_LOCK();492    stats_state.curr_bytes += ITEM_ntotal(it);493    stats_state.curr_items += 1;494    stats.total_items += 1;495    STATS_UNLOCK();496497    /* Allocate a new CAS ID on link. */498    ITEM_set_cas(it, cas);499    assoc_insert(it, hv);500    item_link_q(it);501    refcount_incr(it);502    item_stats_sizes_add(it);503504    return 1;505}506507void do_item_unlink(item *it, const uint32_t hv) {508    MEMCACHED_ITEM_UNLINK(ITEM_key(it), it->nkey, it->nbytes);509    if ((it->it_flags & ITEM_LINKED) != 0) {510        it->it_flags &= ~ITEM_LINKED;511        STATS_LOCK();512        stats_state.curr_bytes -= ITEM_ntotal(it);513        stats_state.curr_items -= 1;514        STATS_UNLOCK();515        item_stats_sizes_remove(it);516        assoc_delete(ITEM_key(it), it->nkey, hv);517        item_unlink_q(it);518        do_item_remove(it);519    }520}521522/* FIXME: Is it necessary to keep this copy/pasted code? */523void do_item_unlink_nolock(item *it, const uint32_t hv) {524    MEMCACHED_ITEM_UNLINK(ITEM_key(it), it->nkey, it->nbytes);525    if ((it->it_flags & ITEM_LINKED) != 0) {526        it->it_flags &= ~ITEM_LINKED;527        STATS_LOCK();528        stats_state.curr_bytes -= ITEM_ntotal(it);529        stats_state.curr_items -= 1;530        STATS_UNLOCK();531        item_stats_sizes_remove(it);532        assoc_delete(ITEM_key(it), it->nkey, hv);533        do_item_unlink_q(it);534        do_item_remove(it);535    }536}537538void do_item_remove(item *it) {539    MEMCACHED_ITEM_REMOVE(ITEM_key(it), it->nkey, it->nbytes);540    assert((it->it_flags & ITEM_SLABBED) == 0);541    assert(it->refcount > 0);542543    if (refcount_decr(it) == 0) {544        item_free(it);545    }546}547548/* Bump the last accessed time, or relink if we're in compat mode */549void do_item_update(item *it) {550    MEMCACHED_ITEM_UPDATE(ITEM_key(it), it->nkey, it->nbytes);551552    /* Hits to COLD_LRU immediately move to WARM. */553    if (settings.lru_segmented) {554        assert((it->it_flags & ITEM_SLABBED) == 0);555        if ((it->it_flags & ITEM_LINKED) != 0) {556            if (ITEM_lruid(it) == COLD_LRU && (it->it_flags & ITEM_ACTIVE)) {557                it->time = current_time;558                item_unlink_q(it);559                it->slabs_clsid = ITEM_clsid(it);560                it->slabs_clsid |= WARM_LRU;561                it->it_flags &= ~ITEM_ACTIVE;562                item_link_q_warm(it);563            } else {564                it->time = current_time;565            }566        }567    } else if (it->time < current_time - ITEM_UPDATE_INTERVAL) {568        assert((it->it_flags & ITEM_SLABBED) == 0);569570        if ((it->it_flags & ITEM_LINKED) != 0) {571            it->time = current_time;572            item_unlink_q(it);573            item_link_q(it);574        }575    }576}577578int do_item_replace(item *it, item *new_it, const uint32_t hv, const uint64_t cas) {579    MEMCACHED_ITEM_REPLACE(ITEM_key(it), it->nkey, it->nbytes,580                           ITEM_key(new_it), new_it->nkey, new_it->nbytes);581    assert((it->it_flags & ITEM_SLABBED) == 0);582583    do_item_unlink(it, hv);584    return do_item_link(new_it, hv, cas);585}586587void item_flush_expired(void) {588    int i;589    item *iter, *next;590    if (settings.oldest_live == 0)591        return;592    for (i = 0; i < LARGEST_ID; i++) {593        /* The LRU is sorted in decreasing time order, and an item's timestamp594         * is never newer than its last access time, so we only need to walk595         * back until we hit an item older than the oldest_live time.596         * The oldest_live checking will auto-expire the remaining items.597         */598        pthread_mutex_lock(&lru_locks[i]);599        for (iter = heads[i]; iter != NULL; iter = next) {600            void *hold_lock = NULL;601            next = iter->next;602            if (iter->time == 0 && iter->nkey == 0 && iter->it_flags == 1) {603                continue; // crawler item.604            }605            uint32_t hv = hash(ITEM_key(iter), iter->nkey);606            // if we can't lock the item, just give up.607            // we can't block here because the lock order is inverted.608            if ((hold_lock = item_trylock(hv)) == NULL) {609                continue;610            }611612            if (iter->time >= settings.oldest_live) {613                // note: not sure why SLABBED check is here. linked and slabbed614                // are mutually exclusive, but this can't hurt and I don't615                // want to validate it right now.616                if ((iter->it_flags & ITEM_SLABBED) == 0) {617                    STORAGE_delete(ext_storage, iter);618                    // nolock version because we hold the LRU lock already.619                    do_item_unlink_nolock(iter, hash(ITEM_key(iter), iter->nkey));620                }621                item_trylock_unlock(hold_lock);622            } else {623                /* We've hit the first old item. Continue to the next queue. */624                item_trylock_unlock(hold_lock);625                break;626            }627        }628        pthread_mutex_unlock(&lru_locks[i]);629    }630}631632/*@null@*/633/* This is walking the line of violating lock order, but I think it's safe.634 * If the LRU lock is held, an item in the LRU cannot be wiped and freed.635 * The data could possibly be overwritten, but this is only accessing the636 * headers.637 * It may not be the best idea to leave it like this, but for now it's safe.638 */639char *item_cachedump(const unsigned int slabs_clsid, const unsigned int limit, unsigned int *bytes) {640    unsigned int memlimit = 2 * 1024 * 1024;   /* 2MB max response size */641    char *buffer;642    unsigned int bufcurr;643    item *it;644    unsigned int len;645    unsigned int shown = 0;646    char key_temp[KEY_MAX_LENGTH + 1];647    char temp[512];648    unsigned int id = slabs_clsid;649    id |= COLD_LRU;650651    pthread_mutex_lock(&lru_locks[id]);652    it = heads[id];653654    buffer = malloc((size_t)memlimit);655    if (buffer == 0) {656        pthread_mutex_unlock(&lru_locks[id]);657        return NULL;658    }659    bufcurr = 0;660661    while (it != NULL && (limit == 0 || shown < limit)) {662        assert(it->nkey <= KEY_MAX_LENGTH);663        // protect from printing binary keys.664        if ((it->nbytes == 0 && it->nkey == 0) || (it->it_flags & ITEM_KEY_BINARY)) {665            it = it->next;666            continue;667        }668        /* Copy the key since it may not be null-terminated in the struct */669        strncpy(key_temp, ITEM_key(it), it->nkey);670        key_temp[it->nkey] = 0x00; /* terminate */671        len = snprintf(temp, sizeof(temp), "ITEM %s [%d b; %llu s]\r\n",672                       key_temp, it->nbytes - 2,673                       it->exptime == 0 ? 0 :674                       (unsigned long long)it->exptime + process_started);675        if (bufcurr + len + 6 > memlimit)  /* 6 is END\r\n\0 */676            break;677        memcpy(buffer + bufcurr, temp, len);678        bufcurr += len;679        shown++;680        it = it->next;681    }682683    memcpy(buffer + bufcurr, "END\r\n", 6);684    bufcurr += 5;685686    *bytes = bufcurr;687    pthread_mutex_unlock(&lru_locks[id]);688    return buffer;689}690691/* With refactoring of the various stats code the automover shouldn't need a692 * custom function here.693 */694void fill_item_stats_automove(item_stats_automove *am) {695    int n;696    for (n = 0; n < MAX_NUMBER_OF_SLAB_CLASSES; n++) {697        item_stats_automove *cur = &am[n];698699        // outofmemory records into HOT700        int i = n | HOT_LRU;701        pthread_mutex_lock(&lru_locks[i]);702        cur->outofmemory = itemstats[i].outofmemory;703        pthread_mutex_unlock(&lru_locks[i]);704705        // evictions and tail age are from COLD706        i = n | COLD_LRU;707        pthread_mutex_lock(&lru_locks[i]);708        cur->evicted = itemstats[i].evicted;709        if (!tails[i]) {710            cur->age = 0;711        } else if (tails[i]->nbytes == 0 && tails[i]->nkey == 0 && tails[i]->it_flags == 1) {712            /* it's a crawler, check previous entry */713            if (tails[i]->prev) {714               cur->age = current_time - tails[i]->prev->time;715            } else {716               cur->age = 0;717            }718        } else {719            cur->age = current_time - tails[i]->time;720        }721        pthread_mutex_unlock(&lru_locks[i]);722     }723}724725void item_stats_totals(ADD_STAT add_stats, void *c) {726    itemstats_t totals;727    memset(&totals, 0, sizeof(itemstats_t));728    int n;729    for (n = 0; n < MAX_NUMBER_OF_SLAB_CLASSES; n++) {730        int x;731        int i;732        for (x = 0; x < 4; x++) {733            i = n | lru_type_map[x];734            pthread_mutex_lock(&lru_locks[i]);735            totals.evicted += itemstats[i].evicted;736            totals.reclaimed += itemstats[i].reclaimed;737            totals.expired_unfetched += itemstats[i].expired_unfetched;738            totals.evicted_unfetched += itemstats[i].evicted_unfetched;739            totals.evicted_active += itemstats[i].evicted_active;740            totals.crawler_reclaimed += itemstats[i].crawler_reclaimed;741            totals.crawler_items_checked += itemstats[i].crawler_items_checked;742            totals.lrutail_reflocked += itemstats[i].lrutail_reflocked;743            totals.moves_to_cold += itemstats[i].moves_to_cold;744            totals.moves_to_warm += itemstats[i].moves_to_warm;745            totals.moves_within_lru += itemstats[i].moves_within_lru;746            totals.direct_reclaims += itemstats[i].direct_reclaims;747            pthread_mutex_unlock(&lru_locks[i]);748        }749    }750    APPEND_STAT("expired_unfetched", "%llu",751                (unsigned long long)totals.expired_unfetched);752    APPEND_STAT("evicted_unfetched", "%llu",753                (unsigned long long)totals.evicted_unfetched);754    if (settings.lru_maintainer_thread) {755        APPEND_STAT("evicted_active", "%llu",756                    (unsigned long long)totals.evicted_active);757    }758    APPEND_STAT("evictions", "%llu",759                (unsigned long long)totals.evicted);760    APPEND_STAT("reclaimed", "%llu",761                (unsigned long long)totals.reclaimed);762    APPEND_STAT("crawler_reclaimed", "%llu",763                (unsigned long long)totals.crawler_reclaimed);764    APPEND_STAT("crawler_items_checked", "%llu",765                (unsigned long long)totals.crawler_items_checked);766    APPEND_STAT("lrutail_reflocked", "%llu",767                (unsigned long long)totals.lrutail_reflocked);768    if (settings.lru_maintainer_thread) {769        APPEND_STAT("moves_to_cold", "%llu",770                    (unsigned long long)totals.moves_to_cold);771        APPEND_STAT("moves_to_warm", "%llu",772                    (unsigned long long)totals.moves_to_warm);773        APPEND_STAT("moves_within_lru", "%llu",774                    (unsigned long long)totals.moves_within_lru);775        APPEND_STAT("direct_reclaims", "%llu",776                    (unsigned long long)totals.direct_reclaims);777        APPEND_STAT("lru_bumps_dropped", "%llu",778                    (unsigned long long)lru_total_bumps_dropped());779    }780}781782void item_stats(ADD_STAT add_stats, void *c) {783    struct thread_stats thread_stats;784    threadlocal_stats_aggregate(&thread_stats);785    itemstats_t totals;786    int n;787    for (n = 0; n < MAX_NUMBER_OF_SLAB_CLASSES; n++) {788        memset(&totals, 0, sizeof(itemstats_t));789        int x;790        int i;791        unsigned int size = 0;792        unsigned int age  = 0;793        unsigned int age_hot = 0;794        unsigned int age_warm = 0;795        unsigned int lru_size_map[4];796        const char *fmt = "items:%d:%s";797        char key_str[STAT_KEY_LEN];798        char val_str[STAT_VAL_LEN];799        int klen = 0, vlen = 0;800        for (x = 0; x < 4; x++) {801            i = n | lru_type_map[x];802            pthread_mutex_lock(&lru_locks[i]);803            totals.evicted += itemstats[i].evicted;804            totals.evicted_nonzero += itemstats[i].evicted_nonzero;805            totals.reclaimed += itemstats[i].reclaimed;806            totals.outofmemory += itemstats[i].outofmemory;807            totals.tailrepairs += itemstats[i].tailrepairs;808            totals.expired_unfetched += itemstats[i].expired_unfetched;809            totals.evicted_unfetched += itemstats[i].evicted_unfetched;810            totals.evicted_active += itemstats[i].evicted_active;811            totals.crawler_reclaimed += itemstats[i].crawler_reclaimed;812            totals.crawler_items_checked += itemstats[i].crawler_items_checked;813            totals.lrutail_reflocked += itemstats[i].lrutail_reflocked;814            totals.moves_to_cold += itemstats[i].moves_to_cold;815            totals.moves_to_warm += itemstats[i].moves_to_warm;816            totals.moves_within_lru += itemstats[i].moves_within_lru;817            totals.direct_reclaims += itemstats[i].direct_reclaims;818            totals.mem_requested += sizes_bytes[i];819            size += sizes[i];820            lru_size_map[x] = sizes[i];821            if (lru_type_map[x] == COLD_LRU && tails[i] != NULL) {822                age = current_time - tails[i]->time;823            } else if (lru_type_map[x] == HOT_LRU && tails[i] != NULL) {824                age_hot = current_time - tails[i]->time;825            } else if (lru_type_map[x] == WARM_LRU && tails[i] != NULL) {826                age_warm = current_time - tails[i]->time;827            }828            if (lru_type_map[x] == COLD_LRU)829                totals.evicted_time = itemstats[i].evicted_time;830            switch (lru_type_map[x]) {831                case HOT_LRU:832                    totals.hits_to_hot = thread_stats.lru_hits[i];833                    break;834                case WARM_LRU:835                    totals.hits_to_warm = thread_stats.lru_hits[i];836                    break;837                case COLD_LRU:838                    totals.hits_to_cold = thread_stats.lru_hits[i];839                    break;840                case TEMP_LRU:841                    totals.hits_to_temp = thread_stats.lru_hits[i];842                    break;843            }844            pthread_mutex_unlock(&lru_locks[i]);845        }846        if (size == 0)847            continue;848        APPEND_NUM_FMT_STAT(fmt, n, "number", "%u", size);849        if (settings.lru_maintainer_thread) {850            APPEND_NUM_FMT_STAT(fmt, n, "number_hot", "%u", lru_size_map[0]);851            APPEND_NUM_FMT_STAT(fmt, n, "number_warm", "%u", lru_size_map[1]);852            APPEND_NUM_FMT_STAT(fmt, n, "number_cold", "%u", lru_size_map[2]);853            if (settings.temp_lru) {854                APPEND_NUM_FMT_STAT(fmt, n, "number_temp", "%u", lru_size_map[3]);855            }856            APPEND_NUM_FMT_STAT(fmt, n, "age_hot", "%u", age_hot);857            APPEND_NUM_FMT_STAT(fmt, n, "age_warm", "%u", age_warm);858        }859        APPEND_NUM_FMT_STAT(fmt, n, "age", "%u", age);860        APPEND_NUM_FMT_STAT(fmt, n, "mem_requested", "%llu", (unsigned long long)totals.mem_requested);861        APPEND_NUM_FMT_STAT(fmt, n, "evicted",862                            "%llu", (unsigned long long)totals.evicted);863        APPEND_NUM_FMT_STAT(fmt, n, "evicted_nonzero",864                            "%llu", (unsigned long long)totals.evicted_nonzero);865        APPEND_NUM_FMT_STAT(fmt, n, "evicted_time",866                            "%u", totals.evicted_time);867        APPEND_NUM_FMT_STAT(fmt, n, "outofmemory",868                            "%llu", (unsigned long long)totals.outofmemory);869        APPEND_NUM_FMT_STAT(fmt, n, "tailrepairs",870                            "%llu", (unsigned long long)totals.tailrepairs);871        APPEND_NUM_FMT_STAT(fmt, n, "reclaimed",872                            "%llu", (unsigned long long)totals.reclaimed);873        APPEND_NUM_FMT_STAT(fmt, n, "expired_unfetched",874                            "%llu", (unsigned long long)totals.expired_unfetched);875        APPEND_NUM_FMT_STAT(fmt, n, "evicted_unfetched",876                            "%llu", (unsigned long long)totals.evicted_unfetched);877        if (settings.lru_maintainer_thread) {878            APPEND_NUM_FMT_STAT(fmt, n, "evicted_active",879                                "%llu", (unsigned long long)totals.evicted_active);880        }881        APPEND_NUM_FMT_STAT(fmt, n, "crawler_reclaimed",882                            "%llu", (unsigned long long)totals.crawler_reclaimed);883        APPEND_NUM_FMT_STAT(fmt, n, "crawler_items_checked",884                            "%llu", (unsigned long long)totals.crawler_items_checked);885        APPEND_NUM_FMT_STAT(fmt, n, "lrutail_reflocked",886                            "%llu", (unsigned long long)totals.lrutail_reflocked);887        if (settings.lru_maintainer_thread) {888            APPEND_NUM_FMT_STAT(fmt, n, "moves_to_cold",889                                "%llu", (unsigned long long)totals.moves_to_cold);890            APPEND_NUM_FMT_STAT(fmt, n, "moves_to_warm",891                                "%llu", (unsigned long long)totals.moves_to_warm);892            APPEND_NUM_FMT_STAT(fmt, n, "moves_within_lru",893                                "%llu", (unsigned long long)totals.moves_within_lru);894            APPEND_NUM_FMT_STAT(fmt, n, "direct_reclaims",895                                "%llu", (unsigned long long)totals.direct_reclaims);896            APPEND_NUM_FMT_STAT(fmt, n, "hits_to_hot",897                                "%llu", (unsigned long long)totals.hits_to_hot);898899            APPEND_NUM_FMT_STAT(fmt, n, "hits_to_warm",900                                "%llu", (unsigned long long)totals.hits_to_warm);901902            APPEND_NUM_FMT_STAT(fmt, n, "hits_to_cold",903                                "%llu", (unsigned long long)totals.hits_to_cold);904905            APPEND_NUM_FMT_STAT(fmt, n, "hits_to_temp",906                                "%llu", (unsigned long long)totals.hits_to_temp);907908        }909    }910911    /* getting here means both ascii and binary terminators fit */912    add_stats(NULL, 0, NULL, 0, c);913}914915bool item_stats_sizes_status(void) {916    bool ret = false;917    if (stats_sizes_hist != NULL)918        ret = true;919    return ret;920}921922void item_stats_sizes_init(void) {923    if (stats_sizes_hist != NULL)924        return;925    stats_sizes_buckets = settings.item_size_max / 32 + 1;926    stats_sizes_hist = calloc(stats_sizes_buckets, sizeof(int));927}928929void item_stats_sizes_add(item *it) {930    if (stats_sizes_hist == NULL)931        return;932    int ntotal = ITEM_ntotal(it);933    int bucket = ntotal / 32;934    if ((ntotal % 32) != 0) bucket++;935    if (bucket < stats_sizes_buckets) stats_sizes_hist[bucket]++;936}937938/* I think there's no way for this to be accurate without using the CAS value.939 * Since items getting their time value bumped will pass this validation.940 */941void item_stats_sizes_remove(item *it) {942    if (stats_sizes_hist == NULL)943        return;944    int ntotal = ITEM_ntotal(it);945    int bucket = ntotal / 32;946    if ((ntotal % 32) != 0) bucket++;947    if (bucket < stats_sizes_buckets) stats_sizes_hist[bucket]--;948}949950/** dumps out a list of objects of each size, with granularity of 32 bytes */951/*@null@*/952/* Locks are correct based on a technicality. Holds LRU lock while doing the953 * work, so items can't go invalid, and it's only looking at header sizes954 * which don't change.955 */956void item_stats_sizes(ADD_STAT add_stats, void *c) {957    if (stats_sizes_hist != NULL) {958        int i;959        for (i = 0; i < stats_sizes_buckets; i++) {960            if (stats_sizes_hist[i] != 0) {961                char key[12];962                snprintf(key, sizeof(key), "%d", i * 32);963                APPEND_STAT(key, "%u", stats_sizes_hist[i]);964            }965        }966    } else {967        APPEND_STAT("sizes_status", "disabled", "");968    }969970    add_stats(NULL, 0, NULL, 0, c);971}972973/** wrapper around assoc_find which does the lazy expiration logic */974item *do_item_get(const char *key, const size_t nkey, const uint32_t hv, LIBEVENT_THREAD *t, const bool do_update) {975    item *it = assoc_find(key, nkey, hv);976    if (it != NULL) {977        refcount_incr(it);978    }979    int was_found = 0;980981    if (it != NULL) {982        was_found = 1;983        if (item_is_flushed(it)) {984            do_item_unlink(it, hv);985            STORAGE_delete(t->storage, it);986            do_item_remove(it);987            it = NULL;988            pthread_mutex_lock(&t->stats.mutex);989            t->stats.get_flushed++;990            pthread_mutex_unlock(&t->stats.mutex);991            was_found = 2;992        } else if (it->exptime != 0 && it->exptime <= current_time) {993            do_item_unlink(it, hv);994            STORAGE_delete(t->storage, it);995            do_item_remove(it);996            it = NULL;997            pthread_mutex_lock(&t->stats.mutex);998            t->stats.get_expired++;999            pthread_mutex_unlock(&t->stats.mutex);1000            was_found = 3;1001        } else {1002            if (do_update) {1003                do_item_bump(t, it, hv);1004            }1005            DEBUG_REFCNT(it, '+');1006        }1007    }10081009    if (settings.verbose > 2) {1010        fprintf(stderr, "> %s ", was_found ? "FOUND KEY" : "NOT FOUND");1011        for (int ii = 0; ii < nkey; ++ii) {1012            fprintf(stderr, "%c", key[ii]);1013        }1014        if (was_found == 2) {1015            fprintf(stderr, " -removed by flush");1016        } else if (was_found == 3) {1017            fprintf(stderr, " -removed by expire");1018        }10191020        fprintf(stderr, "\n");1021    }1022    /* For now this is in addition to the above verbose logging. */1023    LOGGER_LOG(t->l, LOG_FETCHERS, LOGGER_ITEM_GET, NULL, was_found, key,1024               nkey, (it) ? it->nbytes : 0, (it) ? ITEM_clsid(it) : 0, t->cur_sfd);10251026    return it;1027}10281029// Requires lock held for item.1030// Split out of do_item_get() to allow mget functions to look through header1031// data before losing state modified via the bump function.1032void do_item_bump(LIBEVENT_THREAD *t, item *it, const uint32_t hv) {1033    /* We update the hit markers only during fetches.1034     * An item needs to be hit twice overall to be considered1035     * ACTIVE, but only needs a single hit to maintain activity1036     * afterward.1037     * FETCHED tells if an item has ever been active.1038     */1039    if (settings.lru_segmented) {1040        if ((it->it_flags & ITEM_ACTIVE) == 0) {1041            if ((it->it_flags & ITEM_FETCHED) == 0) {1042                it->it_flags |= ITEM_FETCHED;1043            } else {1044                it->it_flags |= ITEM_ACTIVE;1045                if (ITEM_lruid(it) != COLD_LRU) {1046                    it->time = current_time; // only need to bump time.1047                } else if (!lru_bump_async(t->lru_bump_buf, it, hv)) {1048                    // add flag before async bump to avoid race.1049                    it->it_flags &= ~ITEM_ACTIVE;1050                }1051            }1052        }1053    } else {1054        it->it_flags |= ITEM_FETCHED;1055        do_item_update(it);1056    }1057}10581059item *do_item_touch(const char *key, size_t nkey, uint32_t exptime,1060                    const uint32_t hv, LIBEVENT_THREAD *t) {1061    item *it = do_item_get(key, nkey, hv, t, DO_UPDATE);1062    if (it != NULL) {1063        it->exptime = exptime;1064    }1065    return it;1066}10671068/*** LRU MAINTENANCE THREAD ***/10691070/* Returns number of items remove, expired, or evicted.1071 * Callable from worker threads or the LRU maintainer thread */1072int lru_pull_tail(const int orig_id, const int cur_lru,1073        const uint64_t total_bytes, const uint8_t flags, const rel_time_t max_age,1074        struct lru_pull_tail_return *ret_it) {1075    item *it = NULL;1076    int id = orig_id;1077    int removed = 0;1078    if (id == 0)1079        return 0;10801081    int tries = 5;1082    item *search;1083    item *next_it;1084    void *hold_lock = NULL;1085    unsigned int move_to_lru = 0;1086    uint64_t limit = 0;1087    bool do_slab_reassign = false;10881089    id |= cur_lru;1090    pthread_mutex_lock(&lru_locks[id]);1091    search = tails[id];1092    /* We walk up *only* for locked items, and if bottom is expired. */1093    for (; tries > 0 && search != NULL; tries--, search=next_it) {1094        /* we might relink search mid-loop, so search->prev isn't reliable */1095        next_it = search->prev;1096        if (search->nbytes == 0 && search->nkey == 0 && search->it_flags == 1) {1097            /* We are a crawler, ignore it. */1098            if (flags & LRU_PULL_CRAWL_BLOCKS) {1099                pthread_mutex_unlock(&lru_locks[id]);1100                return 0;1101            }1102            tries++;1103            continue;1104        }1105        uint32_t hv = hash(ITEM_key(search), search->nkey);1106        /* Attempt to hash item lock the "search" item. If locked, no1107         * other callers can incr the refcount. Also skip ourselves. */1108        if ((hold_lock = item_trylock(hv)) == NULL)1109            continue;1110        /* Now see if the item is refcount locked */1111        if (refcount_incr(search) != 2) {1112            /* Note pathological case with ref'ed items in tail.1113             * Can still unlink the item, but it won't be reusable yet */1114            itemstats[id].lrutail_reflocked++;1115            /* In case of refcount leaks, enable for quick workaround. */1116            /* WARNING: This can cause terrible corruption */1117            if (settings.tail_repair_time &&1118                    search->time + settings.tail_repair_time < current_time) {1119                itemstats[id].tailrepairs++;1120                search->refcount = 1;1121                /* This will call item_remove -> item_free since refcnt is 1 */1122                STORAGE_delete(ext_storage, search);1123                do_item_unlink_nolock(search, hv);1124                item_trylock_unlock(hold_lock);1125                continue;1126            }1127        }11281129        /* Expired or flushed */1130        if ((search->exptime != 0 && search->exptime < current_time)1131            || item_is_flushed(search)) {1132            itemstats[id].reclaimed++;1133            if ((search->it_flags & ITEM_FETCHED) == 0) {1134                itemstats[id].expired_unfetched++;1135            }1136            /* refcnt 2 -> 1 */1137            do_item_unlink_nolock(search, hv);1138            STORAGE_delete(ext_storage, search);1139            /* refcnt 1 -> 0 -> item_free */1140            do_item_remove(search);1141            item_trylock_unlock(hold_lock);1142            removed++;11431144            /* If all we're finding are expired, can keep going */1145            continue;1146        }11471148        /* If we're HOT_LRU or WARM_LRU and over size limit, send to COLD_LRU.1149         * If we're COLD_LRU, send to WARM_LRU unless we need to evict1150         */1151        switch (cur_lru) {1152            case HOT_LRU:1153                limit = total_bytes * settings.hot_lru_pct / 100;1154            case WARM_LRU:1155                if (limit == 0)1156                    limit = total_bytes * settings.warm_lru_pct / 100;1157                /* Rescue ACTIVE items aggressively */1158                if ((search->it_flags & ITEM_ACTIVE) != 0) {1159                    search->it_flags &= ~ITEM_ACTIVE;1160                    removed++;1161                    if (cur_lru == WARM_LRU) {1162                        itemstats[id].moves_within_lru++;1163                        do_item_unlink_q(search);1164                        do_item_link_q(search);1165                        do_item_remove(search);1166                        item_trylock_unlock(hold_lock);1167                    } else {1168                        /* Active HOT_LRU items flow to WARM */1169                        itemstats[id].moves_to_warm++;1170                        move_to_lru = WARM_LRU;1171                        do_item_unlink_q(search);1172                        it = search;1173                    }1174                } else if (sizes_bytes[id] > limit ||1175                           current_time - search->time > max_age) {1176                    itemstats[id].moves_to_cold++;1177                    move_to_lru = COLD_LRU;1178                    do_item_unlink_q(search);1179                    it = search;1180                    removed++;1181                    break;1182                } else {1183                    /* Don't want to move to COLD, not active, bail out */1184                    it = search;1185                }1186                break;1187            case COLD_LRU:1188                it = search; /* No matter what, we're stopping */1189                if (flags & LRU_PULL_EVICT) {1190                    if (settings.evict_to_free == 0) {1191                        /* Don't think we need a counter for this. It'll OOM.  */1192                        break;1193                    }1194                    itemstats[id].evicted++;1195                    itemstats[id].evicted_time = current_time - search->time;1196                    if (search->exptime != 0)1197                        itemstats[id].evicted_nonzero++;1198                    if ((search->it_flags & ITEM_FETCHED) == 0) {1199                        itemstats[id].evicted_unfetched++;1200                    }1201                    if ((search->it_flags & ITEM_ACTIVE)) {1202                        itemstats[id].evicted_active++;1203                    }1204                    LOGGER_LOG(NULL, LOG_EVICTIONS, LOGGER_EVICTION, search);1205                    STORAGE_delete(ext_storage, search);1206                    do_item_unlink_nolock(search, hv);1207                    removed++;1208                    if (settings.slab_automove == 2) {1209                        do_slab_reassign = true;1210                    }1211                } else if (flags & LRU_PULL_RETURN_ITEM) {1212                    /* Keep a reference to this item and return it. */1213                    ret_it->it = it;1214                    ret_it->hv = hv;1215                } else if ((search->it_flags & ITEM_ACTIVE) != 01216                        && settings.lru_segmented) {1217                    itemstats[id].moves_to_warm++;1218                    search->it_flags &= ~ITEM_ACTIVE;1219                    move_to_lru = WARM_LRU;1220                    do_item_unlink_q(search);1221                    removed++;1222                }1223                break;1224            case TEMP_LRU:1225                it = search; /* Kill the loop. Parent only interested in reclaims */1226                break;1227        }1228        if (it != NULL)1229            break;1230    }12311232    pthread_mutex_unlock(&lru_locks[id]);12331234    if (it != NULL) {1235        if (move_to_lru) {1236            it->slabs_clsid = ITEM_clsid(it);1237            it->slabs_clsid |= move_to_lru;1238            item_link_q(it);1239        }1240        if ((flags & LRU_PULL_RETURN_ITEM) == 0) {1241            do_item_remove(it);1242            item_trylock_unlock(hold_lock);1243        }1244    }12451246    if (do_slab_reassign) {1247        slabs_reassign(settings.slab_rebal, -1, orig_id, SLABS_REASSIGN_ALLOW_EVICTIONS);1248    }12491250    return removed;1251}125212531254/* TODO: Third place this code needs to be deduped */1255static void lru_bump_buf_link_q(lru_bump_buf *b) {1256    pthread_mutex_lock(&bump_buf_lock);1257    assert(b != bump_buf_head);12581259    b->prev = 0;1260    b->next = bump_buf_head;1261    if (b->next) b->next->prev = b;1262    bump_buf_head = b;1263    if (bump_buf_tail == 0) bump_buf_tail = b;1264    pthread_mutex_unlock(&bump_buf_lock);1265    return;1266}12671268void *item_lru_bump_buf_create(void) {1269    lru_bump_buf *b = calloc(1, sizeof(lru_bump_buf));1270    if (b == NULL) {1271        return NULL;1272    }12731274    b->buf = bipbuf_new(sizeof(lru_bump_entry) * LRU_BUMP_BUF_SIZE);1275    if (b->buf == NULL) {1276        free(b);1277        return NULL;1278    }12791280    pthread_mutex_init(&b->mutex, NULL);12811282    lru_bump_buf_link_q(b);1283    return b;1284}12851286static bool lru_bump_async(lru_bump_buf *b, item *it, uint32_t hv) {1287    bool ret = true;1288    refcount_incr(it);1289    pthread_mutex_lock(&b->mutex);1290    lru_bump_entry *be = (lru_bump_entry *) bipbuf_request(b->buf, sizeof(lru_bump_entry));1291    if (be != NULL) {1292        be->it = it;1293        be->hv = hv;1294        if (bipbuf_push(b->buf, sizeof(lru_bump_entry)) == 0) {1295            ret = false;1296            b->dropped++;1297        }1298    } else {1299        ret = false;1300        b->dropped++;1301    }1302    if (!ret) {1303        refcount_decr(it);1304    }1305    pthread_mutex_unlock(&b->mutex);1306    return ret;1307}13081309/* TODO: Might be worth a micro-optimization of having bump buffers link1310 * themselves back into the central queue when queue goes from zero to1311 * non-zero, then remove from list if zero more than N times.1312 * If very few hits on cold this would avoid extra memory barriers from LRU1313 * maintainer thread. If many hits, they'll just stay in the list.1314 */1315static bool lru_maintainer_bumps(void) {1316    lru_bump_buf *b;1317    lru_bump_entry *be;1318    unsigned int size;1319    unsigned int todo;1320    bool bumped = false;1321    pthread_mutex_lock(&bump_buf_lock);1322    for (b = bump_buf_head; b != NULL; b=b->next) {1323        pthread_mutex_lock(&b->mutex);1324        be = (lru_bump_entry *) bipbuf_peek_all(b->buf, &size);1325        pthread_mutex_unlock(&b->mutex);13261327        if (be == NULL) {1328            continue;1329        }1330        todo = size;1331        bumped = true;13321333        while (todo) {1334            item_lock(be->hv);1335            do_item_update(be->it);1336            do_item_remove(be->it);1337            item_unlock(be->hv);1338            be++;1339            todo -= sizeof(lru_bump_entry);1340        }13411342        pthread_mutex_lock(&b->mutex);1343        be = (lru_bump_entry *) bipbuf_poll(b->buf, size);1344        pthread_mutex_unlock(&b->mutex);1345    }1346    pthread_mutex_unlock(&bump_buf_lock);1347    return bumped;1348}13491350static uint64_t lru_total_bumps_dropped(void) {1351    uint64_t total = 0;1352    lru_bump_buf *b;1353    pthread_mutex_lock(&bump_buf_lock);1354    for (b = bump_buf_head; b != NULL; b=b->next) {1355        pthread_mutex_lock(&b->mutex);1356        total += b->dropped;1357        pthread_mutex_unlock(&b->mutex);1358    }1359    pthread_mutex_unlock(&bump_buf_lock);1360    return total;1361}13621363/* Loop up to N times:1364 * If too many items are in HOT_LRU, push to COLD_LRU1365 * If too many items are in WARM_LRU, push to COLD_LRU1366 * If too many items are in COLD_LRU, poke COLD_LRU tail1367 * 1000 loops with 1ms min sleep gives us under 1m items shifted/sec. The1368 * locks can't handle much more than that. Leaving a TODO for how to1369 * autoadjust in the future.1370 */1371static int lru_maintainer_juggle(const int slabs_clsid) {1372    int i;1373    int did_moves = 0;1374    uint64_t total_bytes = 0;1375    unsigned int chunks_perslab = 0;1376    //unsigned int chunks_free = 0;1377    /* TODO: if free_chunks below high watermark, increase aggressiveness */1378    slabs_available_chunks(slabs_clsid, NULL,1379            &chunks_perslab);1380    if (settings.temp_lru) {1381        /* Only looking for reclaims. Run before we size the LRU. */1382        for (i = 0; i < 500; i++) {1383            if (lru_pull_tail(slabs_clsid, TEMP_LRU, 0, 0, 0, NULL) <= 0) {1384                break;1385            } else {1386                did_moves++;1387            }1388        }1389    }13901391    rel_time_t cold_age = 0;1392    rel_time_t hot_age = 0;1393    rel_time_t warm_age = 0;1394    /* If LRU is in flat mode, force items to drain into COLD via max age of 0 */1395    if (settings.lru_segmented) {1396        pthread_mutex_lock(&lru_locks[slabs_clsid|COLD_LRU]);1397        if (tails[slabs_clsid|COLD_LRU]) {1398            cold_age = current_time - tails[slabs_clsid|COLD_LRU]->time;1399        }1400        // Also build up total_bytes for the classes.1401        total_bytes += sizes_bytes[slabs_clsid|COLD_LRU];1402        pthread_mutex_unlock(&lru_locks[slabs_clsid|COLD_LRU]);14031404        hot_age = cold_age * settings.hot_max_factor;1405        warm_age = cold_age * settings.warm_max_factor;14061407        // total_bytes doesn't have to be exact. cache it for the juggles.1408        pthread_mutex_lock(&lru_locks[slabs_clsid|HOT_LRU]);1409        total_bytes += sizes_bytes[slabs_clsid|HOT_LRU];1410        pthread_mutex_unlock(&lru_locks[slabs_clsid|HOT_LRU]);14111412        pthread_mutex_lock(&lru_locks[slabs_clsid|WARM_LRU]);1413        total_bytes += sizes_bytes[slabs_clsid|WARM_LRU];1414        pthread_mutex_unlock(&lru_locks[slabs_clsid|WARM_LRU]);1415    }14161417    /* Juggle HOT/WARM up to N times */1418    for (i = 0; i < 500; i++) {1419        int do_more = 0;1420        if (lru_pull_tail(slabs_clsid, HOT_LRU, total_bytes, LRU_PULL_CRAWL_BLOCKS, hot_age, NULL) ||1421            lru_pull_tail(slabs_clsid, WARM_LRU, total_bytes, LRU_PULL_CRAWL_BLOCKS, warm_age, NULL)) {1422            do_more++;1423        }1424        if (settings.lru_segmented) {1425            do_more += lru_pull_tail(slabs_clsid, COLD_LRU, total_bytes, LRU_PULL_CRAWL_BLOCKS, 0, NULL);1426        }1427        if (do_more == 0)1428            break;1429        did_moves++;1430    }1431    return did_moves;1432}14331434/* Will crawl all slab classes a minimum of once per hour */1435#define MAX_MAINTCRAWL_WAIT 60 * 6014361437/* Hoping user input will improve this function. This is all a wild guess.1438 * Operation: Kicks crawler for each slab id. Crawlers take some statistics as1439 * to items with nonzero expirations. It then buckets how many items will1440 * expire per minute for the next hour.1441 * This function checks the results of a run, and if it things more than 1% of1442 * expirable objects are ready to go, kick the crawler again to reap.1443 * It will also kick the crawler once per minute regardless, waiting a minute1444 * longer for each time it has no work to do, up to an hour wait time.1445 * The latter is to avoid newly started daemons from waiting too long before1446 * retrying a crawl.1447 */1448static void lru_maintainer_crawler_check(struct crawler_expired_data *cdata, logger *l) {1449    int i;1450    static rel_time_t next_crawls[POWER_LARGEST];1451    static rel_time_t next_crawl_wait[POWER_LARGEST];1452    uint8_t todo[POWER_LARGEST];1453    memset(todo, 0, sizeof(uint8_t) * POWER_LARGEST);1454    bool do_run = false;1455    unsigned int tocrawl_limit = 0;14561457    // TODO: If not segmented LRU, skip non-cold1458    for (i = POWER_SMALLEST; i < POWER_LARGEST; i++) {1459        crawlerstats_t *s = &cdata->crawlerstats[i];1460        /* We've not successfully kicked off a crawl yet. */1461        if (s->run_complete) {1462            char *lru_name = "na";1463            pthread_mutex_lock(&cdata->lock);1464            int x;1465            /* Should we crawl again? */1466            uint64_t possible_reclaims = s->seen - s->noexp;1467            uint64_t available_reclaims = 0;1468            /* Need to think we can free at least 1% of the items before1469             * crawling. */1470            /* FIXME: Configurable? */1471            uint64_t low_watermark = (possible_reclaims / 100) + 1;1472            rel_time_t since_run = current_time - s->end_time;1473            /* Don't bother if the payoff is too low. */1474            for (x = 0; x < 60; x++) {1475                available_reclaims += s->histo[x];1476                if (available_reclaims > low_watermark) {1477                    if (next_crawl_wait[i] < (x * 60)) {1478                        next_crawl_wait[i] += 60;1479                    } else if (next_crawl_wait[i] >= 60) {1480                        next_crawl_wait[i] -= 60;1481                    }1482                    break;1483                }1484            }14851486            if (available_reclaims == 0) {1487                next_crawl_wait[i] += 60;1488            }14891490            if (next_crawl_wait[i] > MAX_MAINTCRAWL_WAIT) {1491                next_crawl_wait[i] = MAX_MAINTCRAWL_WAIT;1492            }14931494            next_crawls[i] = current_time + next_crawl_wait[i] + 5;1495            switch (GET_LRU(i)) {1496                case HOT_LRU:1497                    lru_name = "hot";1498                    break;1499                case WARM_LRU:1500                    lru_name = "warm";1501                    break;1502                case COLD_LRU:1503                    lru_name = "cold";1504                    break;1505                case TEMP_LRU:1506                    lru_name = "temp";1507                    break;1508            }1509            LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_CRAWLER_STATUS, NULL,1510                    CLEAR_LRU(i),1511                    lru_name,1512                    (unsigned long long)low_watermark,1513                    (unsigned long long)available_reclaims,1514                    (unsigned int)since_run,1515                    next_crawls[i] - current_time,1516                    s->end_time - s->start_time,1517                    s->seen,1518                    s->reclaimed);1519            // Got our calculation, avoid running until next actual run.1520            s->run_complete = false;1521            pthread_mutex_unlock(&cdata->lock);1522        }1523        if (current_time > next_crawls[i]) {1524            pthread_mutex_lock(&lru_locks[i]);1525            if (sizes[i] > tocrawl_limit) {1526                tocrawl_limit = sizes[i];1527            }1528            pthread_mutex_unlock(&lru_locks[i]);1529            todo[i] = 1;1530            do_run = true;1531            next_crawls[i] = current_time + 5; // minimum retry wait.1532        }1533    }1534    if (do_run) {1535        if (settings.lru_crawler_tocrawl && settings.lru_crawler_tocrawl < tocrawl_limit) {1536            tocrawl_limit = settings.lru_crawler_tocrawl;1537        }1538        lru_crawler_start(todo, tocrawl_limit, CRAWLER_AUTOEXPIRE, cdata, NULL, 0);1539    }1540}15411542static pthread_t lru_maintainer_tid;15431544#define MAX_LRU_MAINTAINER_SLEEP (1000000-1)1545#define MIN_LRU_MAINTAINER_SLEEP 100015461547static void *lru_maintainer_thread(void *arg) {1548    int i;1549    useconds_t to_sleep = MIN_LRU_MAINTAINER_SLEEP;1550    useconds_t last_sleep = MIN_LRU_MAINTAINER_SLEEP;1551    rel_time_t last_crawler_check = 0;1552    useconds_t next_juggles[MAX_NUMBER_OF_SLAB_CLASSES] = {0};1553    useconds_t backoff_juggles[MAX_NUMBER_OF_SLAB_CLASSES] = {0};1554    struct crawler_expired_data *cdata =1555        calloc(1, sizeof(struct crawler_expired_data));1556    if (cdata == NULL) {1557        fprintf(stderr, "Failed to allocate crawler data for LRU maintainer thread\n");1558        abort();1559    }1560    pthread_mutex_init(&cdata->lock, NULL);1561    cdata->crawl_complete = true; // kick off the crawler.1562    logger *l = logger_create();1563    if (l == NULL) {1564        fprintf(stderr, "Failed to allocate logger for LRU maintainer thread\n");1565        abort();1566    }15671568    pthread_mutex_lock(&lru_maintainer_lock);1569    if (settings.verbose > 2)1570        fprintf(stderr, "Starting LRU maintainer background thread\n");1571    while (do_run_lru_maintainer_thread) {1572        pthread_mutex_unlock(&lru_maintainer_lock);1573        if (to_sleep)1574            usleep(to_sleep);1575        pthread_mutex_lock(&lru_maintainer_lock);1576        /* A sleep of zero counts as a minimum of a 1ms wait */1577        last_sleep = to_sleep > 1000 ? to_sleep : 1000;1578        to_sleep = MAX_LRU_MAINTAINER_SLEEP;15791580        STATS_LOCK();1581        stats.lru_maintainer_juggles++;1582        STATS_UNLOCK();15831584        /* Each slab class gets its own sleep to avoid hammering locks */1585        for (i = POWER_SMALLEST; i < MAX_NUMBER_OF_SLAB_CLASSES; i++) {1586            next_juggles[i] = next_juggles[i] > last_sleep ? next_juggles[i] - last_sleep : 0;15871588            if (next_juggles[i] > 0) {1589                // Sleep the thread just for the minimum amount (or not at all)1590                if (next_juggles[i] < to_sleep)1591                    to_sleep = next_juggles[i];1592                continue;1593            }15941595            int did_moves = lru_maintainer_juggle(i);1596            if (did_moves == 0) {1597                if (backoff_juggles[i] != 0) {1598                    backoff_juggles[i] += backoff_juggles[i] / 8;1599                } else {1600                    backoff_juggles[i] = MIN_LRU_MAINTAINER_SLEEP;1601                }1602                if (backoff_juggles[i] > MAX_LRU_MAINTAINER_SLEEP)1603                    backoff_juggles[i] = MAX_LRU_MAINTAINER_SLEEP;1604            } else if (backoff_juggles[i] > 0) {1605                backoff_juggles[i] /= 2;1606                if (backoff_juggles[i] < MIN_LRU_MAINTAINER_SLEEP) {1607                    backoff_juggles[i] = 0;1608                }1609            }1610            next_juggles[i] = backoff_juggles[i];1611            if (next_juggles[i] < to_sleep)1612                to_sleep = next_juggles[i];1613        }16141615        /* Minimize the sleep if we had async LRU bumps to process */1616        if (settings.lru_segmented && lru_maintainer_bumps() && to_sleep > 1000) {1617            to_sleep = 1000;1618        }16191620        /* Once per second at most */1621        if (settings.lru_crawler && last_crawler_check != current_time) {1622            lru_maintainer_crawler_check(cdata, l);1623            last_crawler_check = current_time;1624        }1625    }1626    pthread_mutex_unlock(&lru_maintainer_lock);1627    // LRU crawler *must* be stopped.1628    free(cdata);1629    if (settings.verbose > 2)1630        fprintf(stderr, "LRU maintainer thread stopping\n");16311632    return NULL;1633}16341635int stop_lru_maintainer_thread(void) {1636    int ret;1637    pthread_mutex_lock(&lru_maintainer_lock);1638    /* LRU thread is a sleep loop, will die on its own */1639    do_run_lru_maintainer_thread = 0;1640    pthread_mutex_unlock(&lru_maintainer_lock);1641    if ((ret = pthread_join(lru_maintainer_tid, NULL)) != 0) {1642        fprintf(stderr, "Failed to stop LRU maintainer thread: %s\n", strerror(ret));1643        return -1;1644    }1645    settings.lru_maintainer_thread = false;1646    return 0;1647}16481649int start_lru_maintainer_thread(void *arg) {1650    int ret;16511652    pthread_mutex_lock(&lru_maintainer_lock);1653    do_run_lru_maintainer_thread = 1;1654    settings.lru_maintainer_thread = true;1655    if ((ret = pthread_create(&lru_maintainer_tid, NULL,1656        lru_maintainer_thread, arg)) != 0) {1657        fprintf(stderr, "Can't create LRU maintainer thread: %s\n",1658            strerror(ret));1659        pthread_mutex_unlock(&lru_maintainer_lock);1660        return -1;1661    }1662    thread_setname(lru_maintainer_tid, "mc-lrumaint");1663    pthread_mutex_unlock(&lru_maintainer_lock);16641665    return 0;1666}16671668/* If we hold this lock, crawler can't wake up or move */1669void lru_maintainer_pause(void) {1670    pthread_mutex_lock(&lru_maintainer_lock);1671}16721673void lru_maintainer_resume(void) {1674    pthread_mutex_unlock(&lru_maintainer_lock);1675}16761677/* Tail linkers and crawler for the LRU crawler. */1678void do_item_linktail_q(item *it) { /* item is the new tail */1679    item **head, **tail;1680    assert(it->it_flags == 1);1681    assert(it->nbytes == 0);16821683    head = &heads[it->slabs_clsid];1684    tail = &tails[it->slabs_clsid];1685    //assert(*tail != 0);1686    assert(it != *tail);1687    assert((*head && *tail) || (*head == 0 && *tail == 0));1688    it->prev = *tail;1689    it->next = 0;1690    if (it->prev) {1691        assert(it->prev->next == 0);1692        it->prev->next = it;1693    }1694    *tail = it;1695    if (*head == 0) *head = it;1696    return;1697}16981699void do_item_unlinktail_q(item *it) {1700    item **head, **tail;1701    head = &heads[it->slabs_clsid];1702    tail = &tails[it->slabs_clsid];17031704    if (*head == it) {1705        assert(it->prev == 0);1706        *head = it->next;1707    }1708    if (*tail == it) {1709        assert(it->next == 0);1710        *tail = it->prev;1711    }1712    assert(it->next != it);1713    assert(it->prev != it);17141715    if (it->next) it->next->prev = it->prev;1716    if (it->prev) it->prev->next = it->next;1717    return;1718}17191720/* This is too convoluted, but it's a difficult shuffle. Try to rewrite it1721 * more clearly. */1722item *do_item_crawl_q(item *it) {1723    item **head, **tail;1724    assert(it->it_flags == 1);1725    assert(it->nbytes == 0);1726    head = &heads[it->slabs_clsid];1727    tail = &tails[it->slabs_clsid];17281729    /* We've hit the head, pop off */1730    if (it->prev == 0) {1731        assert(*head == it);1732        if (it->next) {1733            *head = it->next;1734            assert(it->next->prev == it);1735            it->next->prev = 0;1736        }1737        return NULL; /* Done */1738    }17391740    /* Swing ourselves in front of the next item */1741    /* NB: If there is a prev, we can't be the head */1742    assert(it->prev != it);1743    if (it->prev) {1744        if (*head == it->prev) {1745            /* Prev was the head, now we're the head */1746            *head = it;1747        }1748        if (*tail == it) {1749            /* We are the tail, now they are the tail */1750            *tail = it->prev;1751        }1752        assert(it->next != it);1753        if (it->next) {1754            assert(it->prev->next == it);1755            it->prev->next = it->next;1756            it->next->prev = it->prev;1757        } else {1758            /* Tail. Move this above? */1759            it->prev->next = 0;1760        }1761        /* prev->prev's next is it->prev */1762        it->next = it->prev;1763        it->prev = it->next->prev;1764        it->next->prev = it;1765        /* New it->prev now, if we're not at the head. */1766        if (it->prev) {1767            it->prev->next = it;1768        }1769    }1770    assert(it->next != it);1771    assert(it->prev != it);17721773    return it->next; /* success */1774}