1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */2/*3 * Hash table4 *5 * The hash function used here is by Bob Jenkins, 1996:6 *    <http://burtleburtle.net/bob/hash/doobs.html>7 *       "By Bob Jenkins, 1996.  bob_jenkins@burtleburtle.net.8 *       You may use this code any way you wish, private, educational,9 *       or commercial.  It's free."10 *11 * The rest of the file is licensed under the BSD license.  See LICENSE.12 */1314#include "memcached.h"15#include <sys/stat.h>16#include <sys/socket.h>17#include <sys/resource.h>18#include <signal.h>19#include <fcntl.h>20#include <netinet/in.h>21#include <errno.h>22#include <stdlib.h>23#include <stdio.h>24#include <string.h>25#include <assert.h>26#include <pthread.h>2728static pthread_cond_t maintenance_cond = PTHREAD_COND_INITIALIZER;29static pthread_mutex_t maintenance_lock = PTHREAD_MUTEX_INITIALIZER;3031/* how many powers of 2's worth of buckets we use */32unsigned int hashpower = HASHPOWER_DEFAULT;3334#define hashsize(n) ((uint64_t)1<<(n))35#define hashmask(n) (hashsize(n)-1)3637/* Main hash table. This is where we look except during expansion. */38static item** primary_hashtable = 0;3940/*41 * Previous hash table. During expansion, we look here for keys that haven't42 * been moved over to the primary yet.43 */44static item** old_hashtable = 0;4546/* Flag: Are we in the middle of expanding now? */47static bool expanding = false;4849/*50 * During expansion we migrate values with bucket granularity; this is how51 * far we've gotten so far. Ranges from 0 .. hashsize(hashpower - 1) - 1.52 */53static uint64_t expand_bucket = 0;5455void assoc_init(const int hashtable_init) {56    if (hashtable_init) {57        hashpower = hashtable_init;58    }59    primary_hashtable = calloc(hashsize(hashpower), sizeof(void *));60    if (! primary_hashtable) {61        fprintf(stderr, "Failed to init hashtable.\n");62        exit(EXIT_FAILURE);63    }64    STATS_LOCK();65    stats_state.hash_power_level = hashpower;66    stats_state.hash_bytes = hashsize(hashpower) * sizeof(void *);67    STATS_UNLOCK();68}6970item *assoc_find(const char *key, const size_t nkey, const uint32_t hv) {71    item *it;72    uint64_t oldbucket;7374    if (expanding &&75        (oldbucket = (hv & hashmask(hashpower - 1))) >= expand_bucket)76    {77        it = old_hashtable[oldbucket];78    } else {79        it = primary_hashtable[hv & hashmask(hashpower)];80    }8182    item *ret = NULL;83#ifdef ENABLE_DTRACE84    int depth = 0;85#endif86    while (it) {87        if ((nkey == it->nkey) && (memcmp(key, ITEM_key(it), nkey) == 0)) {88            ret = it;89            break;90        }91        it = it->h_next;92#ifdef ENABLE_DTRACE93        ++depth;94#endif95    }96    MEMCACHED_ASSOC_FIND(key, nkey, depth);97    return ret;98}99100/* returns the address of the item pointer before the key.  if *item == 0,101   the item wasn't found */102103static item** _hashitem_before (const char *key, const size_t nkey, const uint32_t hv) {104    item **pos;105    uint64_t oldbucket;106107    if (expanding &&108        (oldbucket = (hv & hashmask(hashpower - 1))) >= expand_bucket)109    {110        pos = &old_hashtable[oldbucket];111    } else {112        pos = &primary_hashtable[hv & hashmask(hashpower)];113    }114115    while (*pos && ((nkey != (*pos)->nkey) || memcmp(key, ITEM_key(*pos), nkey))) {116        pos = &(*pos)->h_next;117    }118    return pos;119}120121/* grows the hashtable to the next power of 2. */122static void assoc_expand(void) {123    old_hashtable = primary_hashtable;124125    primary_hashtable = calloc(hashsize(hashpower + 1), sizeof(void *));126    if (primary_hashtable) {127        if (settings.verbose > 1)128            fprintf(stderr, "Hash table expansion starting\n");129        hashpower++;130        expanding = true;131        expand_bucket = 0;132        STATS_LOCK();133        stats_state.hash_power_level = hashpower;134        stats_state.hash_bytes += hashsize(hashpower) * sizeof(void *);135        stats_state.hash_is_expanding = true;136        STATS_UNLOCK();137    } else {138        primary_hashtable = old_hashtable;139        /* Bad news, but we can keep running. */140    }141}142143void assoc_start_expand(uint64_t curr_items) {144    if (pthread_mutex_trylock(&maintenance_lock) == 0) {145        if (curr_items > (hashsize(hashpower) * 3) / 2 && hashpower < HASHPOWER_MAX) {146            pthread_cond_signal(&maintenance_cond);147        }148        pthread_mutex_unlock(&maintenance_lock);149    }150}151152/* Note: this isn't an assoc_update.  The key must not already exist to call this */153int assoc_insert(item *it, const uint32_t hv) {154    uint64_t oldbucket;155156//    assert(assoc_find(ITEM_key(it), it->nkey) == 0);  /* shouldn't have duplicately named things defined */157158    if (expanding &&159        (oldbucket = (hv & hashmask(hashpower - 1))) >= expand_bucket)160    {161        it->h_next = old_hashtable[oldbucket];162        old_hashtable[oldbucket] = it;163    } else {164        it->h_next = primary_hashtable[hv & hashmask(hashpower)];165        primary_hashtable[hv & hashmask(hashpower)] = it;166    }167168    MEMCACHED_ASSOC_INSERT(ITEM_key(it), it->nkey);169    return 1;170}171172void assoc_delete(const char *key, const size_t nkey, const uint32_t hv) {173    item **before = _hashitem_before(key, nkey, hv);174175    if (*before) {176        item *nxt;177        /* The DTrace probe cannot be triggered as the last instruction178         * due to possible tail-optimization by the compiler179         */180        MEMCACHED_ASSOC_DELETE(key, nkey);181        nxt = (*before)->h_next;182        (*before)->h_next = 0;   /* probably pointless, but whatever. */183        *before = nxt;184        return;185    }186    /* Note:  we never actually get here.  the callers don't delete things187       they can't find. */188    assert(*before != 0);189}190191192static volatile int do_run_maintenance_thread = 1;193194#define DEFAULT_HASH_BULK_MOVE 1195int hash_bulk_move = DEFAULT_HASH_BULK_MOVE;196197static void *assoc_maintenance_thread(void *arg) {198199    mutex_lock(&maintenance_lock);200    while (do_run_maintenance_thread) {201        int ii = 0;202203        /* There is only one expansion thread, so no need to global lock. */204        for (ii = 0; ii < hash_bulk_move && expanding; ++ii) {205            item *it, *next;206            uint64_t bucket;207            void *item_lock = NULL;208209            /* bucket = hv & hashmask(hashpower) =>the bucket of hash table210             * is the lowest N bits of the hv, and the bucket of item_locks is211             *  also the lowest M bits of hv, and N is greater than M.212             *  So we can process expanding with only one item_lock. cool! */213            if ((item_lock = item_trylock(expand_bucket))) {214                    for (it = old_hashtable[expand_bucket]; NULL != it; it = next) {215                        next = it->h_next;216                        bucket = hash(ITEM_key(it), it->nkey) & hashmask(hashpower);217                        it->h_next = primary_hashtable[bucket];218                        primary_hashtable[bucket] = it;219                    }220221                    old_hashtable[expand_bucket] = NULL;222223                    expand_bucket++;224                    if (expand_bucket == hashsize(hashpower - 1)) {225                        expanding = false;226                        free(old_hashtable);227                        STATS_LOCK();228                        stats_state.hash_bytes -= hashsize(hashpower - 1) * sizeof(void *);229                        stats_state.hash_is_expanding = false;230                        STATS_UNLOCK();231                        if (settings.verbose > 1)232                            fprintf(stderr, "Hash table expansion done\n");233                    }234235            } else {236                usleep(10*1000 - 1);237            }238239            if (item_lock) {240                item_trylock_unlock(item_lock);241                item_lock = NULL;242            }243        }244245        if (!expanding) {246            /* We are done expanding.. just wait for next invocation */247            pthread_cond_wait(&maintenance_cond, &maintenance_lock);248            /* assoc_expand() swaps out the hash table entirely, so we need249             * all threads to not hold any references related to the hash250             * table while this happens.251             * This is instead of a more complex, possibly slower algorithm to252             * allow dynamic hash table expansion without causing significant253             * wait times.254             */255            if (do_run_maintenance_thread) {256                pause_threads(PAUSE_ALL_THREADS);257                assoc_expand();258                pause_threads(RESUME_ALL_THREADS);259            }260        }261    }262    mutex_unlock(&maintenance_lock);263    return NULL;264}265266static pthread_t maintenance_tid;267268int start_assoc_maintenance_thread(void) {269    int ret;270    char *env = getenv("MEMCACHED_HASH_BULK_MOVE");271    if (env != NULL) {272        hash_bulk_move = atoi(env);273        if (hash_bulk_move == 0) {274            hash_bulk_move = DEFAULT_HASH_BULK_MOVE;275        }276    }277278    if ((ret = pthread_create(&maintenance_tid, NULL,279                              assoc_maintenance_thread, NULL)) != 0) {280        fprintf(stderr, "Can't create thread: %s\n", strerror(ret));281        return -1;282    }283    thread_setname(maintenance_tid, "mc-assocmaint");284    return 0;285}286287void stop_assoc_maintenance_thread(void) {288    mutex_lock(&maintenance_lock);289    do_run_maintenance_thread = 0;290    pthread_cond_signal(&maintenance_cond);291    mutex_unlock(&maintenance_lock);292293    /* Wait for the maintenance thread to stop */294    pthread_join(maintenance_tid, NULL);295}296297struct assoc_iterator {298    uint64_t bucket;299    item *it;300    item *next;301    bool bucket_locked;302};303304void *assoc_get_iterator(void) {305    struct assoc_iterator *iter = calloc(1, sizeof(struct assoc_iterator));306    if (iter == NULL) {307        return NULL;308    }309    // this will hang the caller while a hash table expansion is running.310    if (mutex_trylock(&maintenance_lock) == 0) {311        return iter;312    } else {313        free(iter);314        return NULL;315    }316}317318bool assoc_iterate(void *iterp, item **it) {319    struct assoc_iterator *iter = (struct assoc_iterator *) iterp;320    *it = NULL;321    // - if locked bucket and next, update next and return322    if (iter->bucket_locked) {323        if (iter->next != NULL) {324            iter->it = iter->next;325            iter->next = iter->it->h_next;326            *it = iter->it;327        } else {328            // unlock previous bucket, if any329            item_unlock(iter->bucket);330            // iterate the bucket post since it starts at 0.331            iter->bucket++;332            iter->bucket_locked = false;333            *it = NULL;334        }335        return true;336    }337338    // - loop until we hit the end or find something.339    if (iter->bucket != hashsize(hashpower)) {340        // - lock next bucket341        item_lock(iter->bucket);342        iter->bucket_locked = true;343        // - only check the primary hash table since expand is blocked.344        iter->it = primary_hashtable[iter->bucket];345        if (iter->it != NULL) {346            // - set it, next and return347            iter->next = iter->it->h_next;348            *it = iter->it;349        } else {350            // - nothing found in this bucket, try next.351            item_unlock(iter->bucket);352            iter->bucket_locked = false;353            iter->bucket++;354        }355    } else {356        return false;357    }358359    return true;360}361362void assoc_iterate_final(void *iterp) {363    struct assoc_iterator *iter = (struct assoc_iterator *) iterp;364    if (iter->bucket_locked) {365        item_unlock(iter->bucket);366    }367    mutex_unlock(&maintenance_lock);368    free(iter);369}