1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */2/*3 * Slabs memory allocation, based on powers-of-N. Slabs are up to 1MB in size4 * and are divided into chunks. The chunk sizes start off at the size of the5 * "item" structure plus space for a small key and value. They increase by6 * a multiplier factor from there, up to half the maximum slab size.7 */8#include "memcached.h"9#include <sys/mman.h>10#include <sys/stat.h>11#include <sys/socket.h>12#include <sys/resource.h>13#include <fcntl.h>14#include <netinet/in.h>15#include <stdlib.h>16#include <stdio.h>17#include <string.h>18#include <assert.h>19#include <pthread.h>2021//#define DEBUG_SLAB_MOVER22/* powers-of-N allocation structures */2324typedef struct {25    uint32_t size;      /* sizes of items */26    uint32_t perslab;   /* how many items per slab */2728    void *slots;           /* list of item ptrs */29    unsigned int sl_curr;   /* total free items in list */3031    unsigned int slabs;     /* how many slabs were allocated for this class */3233    void **slab_list;       /* array of slab pointers */34    unsigned int list_size; /* size of prev array */35} slabclass_t;3637static slabclass_t slabclass[MAX_NUMBER_OF_SLAB_CLASSES];38static size_t mem_limit = 0;39static size_t mem_malloced = 0;40/* If the memory limit has been hit once. Used as a hint to decide when to41 * early-wake the LRU maintenance thread */42static bool mem_limit_reached = false;43static int power_largest;4445static void *mem_base = NULL;46static void *mem_current = NULL;47static size_t mem_avail = 0;48/**49 * Access to the slab allocator is protected by this lock50 */51static pthread_mutex_t slabs_lock = PTHREAD_MUTEX_INITIALIZER;5253/*54 * Forward Declarations55 */56static int do_grow_slab_list(const unsigned int id);57static int do_slabs_newslab(const unsigned int id);58static void *memory_allocate(size_t size);59static void do_slabs_free(void *ptr, unsigned int id);6061/* Preallocate as many slab pages as possible (called from slabs_init)62   on start-up, so users don't get confused out-of-memory errors when63   they do have free (in-slab) space, but no space to make new slabs.64   if maxslabs is 18 (POWER_LARGEST - POWER_SMALLEST + 1), then all65   slab types can be made.  if max memory is less than 18 MB, only the66   smaller ones will be made.  */67static void slabs_preallocate (const unsigned int maxslabs);6869/*70 * Figures out which slab class (chunk size) is required to store an item of71 * a given size.72 *73 * Given object size, return id to use when allocating/freeing memory for object74 * 0 means error: can't store such a large object75 */7677unsigned int slabs_clsid(const size_t size) {78    int res = POWER_SMALLEST;7980    if (size == 0 || size > settings.item_size_max)81        return 0;82    while (size > slabclass[res].size)83        if (res++ == power_largest)     /* won't fit in the biggest slab */84            return power_largest;85    return res;86}8788bool slabs_class_check(const int id) {89    if (id < SLAB_GLOBAL_PAGE_POOL || id > power_largest) {90        return false;91    }92    return true;93}9495unsigned int slabs_size(const int clsid) {96    return slabclass[clsid].size;97}9899// TODO: could this work with the restartable memory?100// Docs say hugepages only work with private shm allocs.101/* Function split out for better error path handling */102static void * alloc_large_chunk(const size_t limit)103{104    void *ptr = NULL;105#if defined(__linux__) && defined(MADV_HUGEPAGE)106    size_t pagesize = 0;107    FILE *fp;108    int ret;109110    /* Get the size of huge pages */111    fp = fopen("/proc/meminfo", "r");112    if (fp != NULL) {113        char buf[64];114115        while ((fgets(buf, sizeof(buf), fp)))116            if (!strncmp(buf, "Hugepagesize:", 13)) {117                ret = sscanf(buf + 13, "%zu\n", &pagesize);118119                /* meminfo huge page size is in KiBs */120                pagesize <<= 10;121            }122        fclose(fp);123    }124125    if (!pagesize) {126        fprintf(stderr, "Failed to get supported huge page size\n");127        return NULL;128    }129130    if (settings.verbose > 1)131        fprintf(stderr, "huge page size: %zu\n", pagesize);132133    /* This works because glibc simply uses mmap when the alignment is134     * above a certain limit. */135    ret = posix_memalign(&ptr, pagesize, limit);136    if (ret != 0) {137        fprintf(stderr, "Failed to get aligned memory chunk: %d\n", ret);138        return NULL;139    }140141    ret = madvise(ptr, limit, MADV_HUGEPAGE);142    if (ret < 0) {143        fprintf(stderr, "Failed to set transparent hugepage hint: %d\n", ret);144        free(ptr);145        ptr = NULL;146    }147#elif defined(__FreeBSD__)148    size_t align = (sizeof(size_t) * 8 - (__builtin_clzl(4095)));149    ptr = mmap(NULL, limit, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON | MAP_ALIGNED(align) | MAP_ALIGNED_SUPER, -1, 0);150    if (ptr == MAP_FAILED) {151        fprintf(stderr, "Failed to set super pages\n");152        ptr = NULL;153    }154#else155    ptr = malloc(limit);156#endif157    return ptr;158}159160unsigned int slabs_fixup(char *chunk, const int border) {161    slabclass_t *p;162    item *it = (item *)chunk;163    int id = ITEM_clsid(it);164165    // memory isn't used yet. shunt to global pool.166    // (which must be 0)167    if (id == 0) {168        //assert(border == 0);169        p = &slabclass[0];170        do_grow_slab_list(0);171        p->slab_list[p->slabs++] = (char*)chunk;172        return -1;173    }174    p = &slabclass[id];175176    // if we're on a page border, add the slab to slab class177    if (border == 0) {178        do_grow_slab_list(id);179        p->slab_list[p->slabs++] = chunk;180    }181182    // increase free count if ITEM_SLABBED183    if (it->it_flags == ITEM_SLABBED) {184        // if ITEM_SLABBED re-stack on freelist.185        // don't have to run pointer fixups.186        it->prev = 0;187        it->next = p->slots;188        if (it->next) it->next->prev = it;189        p->slots = it;190191        p->sl_curr++;192        //fprintf(stderr, "replacing into freelist\n");193    }194195    return p->size;196}197198/**199 * Determines the chunk sizes and initializes the slab class descriptors200 * accordingly.201 */202void slabs_init(const size_t limit, const double factor, const bool prealloc, const uint32_t *slab_sizes, void *mem_base_external, bool reuse_mem) {203    int i = POWER_SMALLEST - 1;204    unsigned int size = sizeof(item) + settings.chunk_size;205206    /* Some platforms use runtime transparent hugepages. If for any reason207     * the initial allocation fails, the required settings do not persist208     * for remaining allocations. As such it makes little sense to do slab209     * preallocation. */210    bool __attribute__ ((unused)) do_slab_prealloc = false;211212    mem_limit = limit;213214    if (prealloc && mem_base_external == NULL) {215        mem_base = alloc_large_chunk(mem_limit);216        if (mem_base) {217            do_slab_prealloc = true;218            mem_current = mem_base;219            mem_avail = mem_limit;220        } else {221            fprintf(stderr, "Warning: Failed to allocate requested memory in"222                    " one large chunk.\nWill allocate in smaller chunks\n");223        }224    } else if (prealloc && mem_base_external != NULL) {225        // Can't (yet) mix hugepages with mmap allocations, so separate the226        // logic from above. Reusable memory also force-preallocates memory227        // pages into the global pool, which requires turning mem_* variables.228        do_slab_prealloc = true;229        mem_base = mem_base_external;230        // _current shouldn't be used in this case, but we set it to where it231        // should be anyway.232        if (reuse_mem) {233            mem_current = ((char*)mem_base) + mem_limit;234            mem_avail = 0;235        } else {236            mem_current = mem_base;237            mem_avail = mem_limit;238        }239    }240241    memset(slabclass, 0, sizeof(slabclass));242243    while (++i < MAX_NUMBER_OF_SLAB_CLASSES-1) {244        if (slab_sizes != NULL) {245            if (slab_sizes[i-1] == 0)246                break;247            size = slab_sizes[i-1];248        } else if (size >= settings.slab_chunk_size_max / factor) {249            break;250        }251        /* Make sure items are always n-byte aligned */252        if (size % CHUNK_ALIGN_BYTES)253            size += CHUNK_ALIGN_BYTES - (size % CHUNK_ALIGN_BYTES);254255        slabclass[i].size = size;256        slabclass[i].perslab = settings.slab_page_size / slabclass[i].size;257        if (slab_sizes == NULL)258            size *= factor;259        if (settings.verbose > 1) {260            fprintf(stderr, "slab class %3d: chunk size %9u perslab %7u\n",261                    i, slabclass[i].size, slabclass[i].perslab);262        }263    }264265    power_largest = i;266    slabclass[power_largest].size = settings.slab_chunk_size_max;267    slabclass[power_largest].perslab = settings.slab_page_size / settings.slab_chunk_size_max;268    if (settings.verbose > 1) {269        fprintf(stderr, "slab class %3d: chunk size %9u perslab %7u\n",270                i, slabclass[i].size, slabclass[i].perslab);271    }272273    /* for the test suite:  faking of how much we've already malloc'd */274    {275        char *t_initial_malloc = getenv("T_MEMD_INITIAL_MALLOC");276        if (t_initial_malloc) {277            int64_t env_malloced;278            if (safe_strtoll((const char *)t_initial_malloc, &env_malloced)) {279                mem_malloced = (size_t)env_malloced;280            }281        }282283    }284285    if (do_slab_prealloc) {286        if (!reuse_mem) {287            slabs_preallocate(power_largest);288        }289    }290}291292void slabs_prefill_global(void) {293    void *ptr;294    slabclass_t *p = &slabclass[0];295    int len = settings.slab_page_size;296297    while (mem_malloced < mem_limit298            && (ptr = memory_allocate(len)) != NULL) {299        do_grow_slab_list(0);300        // Ensure the front header is zero'd to avoid confusing restart code.301        // It's probably good enough to cast it and just zero slabs_clsid, but302        // this is extra paranoid.303        memset(ptr, 0, sizeof(item));304        p->slab_list[p->slabs++] = ptr;305    }306    mem_limit_reached = true;307}308309static void slabs_preallocate(const unsigned int maxslabs) {310    int i;311    unsigned int prealloc = 0;312313    /* pre-allocate a 1MB slab in every size class so people don't get314       confused by non-intuitive "SERVER_ERROR out of memory"315       messages.  this is the most common question on the mailing316       list.  if you really don't want this, you can rebuild without317       these three lines.  */318319    for (i = POWER_SMALLEST; i < MAX_NUMBER_OF_SLAB_CLASSES; i++) {320        if (++prealloc > maxslabs)321            break;322        if (do_slabs_newslab(i) == 0) {323            fprintf(stderr, "Error while preallocating slab memory!\n"324                "If using -L or other prealloc options, max memory must be "325                "at least %d megabytes.\n", power_largest);326            exit(1);327        }328    }329}330331static int do_grow_slab_list(const unsigned int id) {332    if (id > power_largest)333        return 0;334335    slabclass_t *p = &slabclass[id];336    if (p->slabs == p->list_size) {337        size_t new_size =  (p->list_size != 0) ? p->list_size * 2 : 16;338        void *new_list = realloc(p->slab_list, new_size * sizeof(void *));339        if (new_list == 0) return 0;340        p->list_size = new_size;341        p->slab_list = new_list;342    }343    return 1;344}345346int slabs_grow_slab_list(const unsigned int id) {347    int ret = 0;348    pthread_mutex_lock(&slabs_lock);349    ret = do_grow_slab_list(id);350    pthread_mutex_unlock(&slabs_lock);351    return ret;352}353354static void split_slab_page_into_freelist(char *ptr, const unsigned int id) {355    slabclass_t *p = &slabclass[id];356    int x;357    for (x = 0; x < p->perslab; x++) {358        do_slabs_free(ptr, id);359        ptr += p->size;360    }361}362363/* Fast FIFO queue */364static void *get_page_from_global_pool(void) {365    slabclass_t *p = &slabclass[SLAB_GLOBAL_PAGE_POOL];366    if (p->slabs < 1) {367        return NULL;368    }369    char *ret = p->slab_list[p->slabs - 1];370    p->slabs--;371    return ret;372}373374static int do_slabs_newslab(const unsigned int id) {375    slabclass_t *p = &slabclass[id];376    slabclass_t *g = &slabclass[SLAB_GLOBAL_PAGE_POOL];377    int len = (settings.slab_reassign || settings.slab_chunk_size_max != settings.slab_page_size)378        ? settings.slab_page_size379        : p->size * p->perslab;380    char *ptr;381382    if ((mem_limit && mem_malloced + len > mem_limit && p->slabs > 0383         && g->slabs == 0)) {384        mem_limit_reached = true;385        MEMCACHED_SLABS_SLABCLASS_ALLOCATE_FAILED(id);386        return 0;387    }388389    if ((do_grow_slab_list(id) == 0) ||390        (((ptr = get_page_from_global_pool()) == NULL) &&391        ((ptr = memory_allocate((size_t)len)) == 0))) {392393        MEMCACHED_SLABS_SLABCLASS_ALLOCATE_FAILED(id);394        return 0;395    }396397    // Always wipe the memory at this stage: in restart mode the mmap memory398    // could be unused, yet still full of data. Better for usability if we're399    // wiping memory as it's being pulled out of the global pool instead of400    // blocking startup all at once.401    memset(ptr, 0, (size_t)len);402    split_slab_page_into_freelist(ptr, id);403404    p->slab_list[p->slabs++] = ptr;405    MEMCACHED_SLABS_SLABCLASS_ALLOCATE(id);406407    return 1;408}409410/*@null@*/411static void *do_slabs_alloc(unsigned int id,412        unsigned int flags) {413    slabclass_t *p;414    void *ret = NULL;415    item *it = NULL;416417    if (id < POWER_SMALLEST || id > power_largest) {418        MEMCACHED_SLABS_ALLOCATE_FAILED(id);419        return NULL;420    }421    p = &slabclass[id];422    assert(p->sl_curr == 0 || (((item *)p->slots)->it_flags & ITEM_SLABBED));423424    /* fail unless we have space at the end of a recently allocated page,425       we have something on our freelist, or we could allocate a new page */426    if (p->sl_curr == 0 && flags != SLABS_ALLOC_NO_NEWPAGE) {427        do_slabs_newslab(id);428    }429430    if (p->sl_curr != 0) {431        /* return off our freelist */432        it = (item *)p->slots;433        p->slots = it->next;434        if (it->next) it->next->prev = 0;435        /* Kill flag and initialize refcount here for lock safety in slab436         * mover's freeness detection. */437        it->it_flags &= ~ITEM_SLABBED;438        it->refcount = 1;439        p->sl_curr--;440        ret = (void *)it;441    } else {442        ret = NULL;443    }444445    if (ret) {446        MEMCACHED_SLABS_ALLOCATE(id, p->size, ret);447    } else {448        MEMCACHED_SLABS_ALLOCATE_FAILED(id);449    }450451    return ret;452}453454static void do_slabs_free_chunked(item *it) {455    item_chunk *chunk = (item_chunk *) ITEM_schunk(it);456    slabclass_t *p;457458    it->it_flags = ITEM_SLABBED;459    // FIXME: refresh on how this works?460    //it->slabs_clsid = 0;461    it->prev = 0;462    // header object's original classid is stored in chunk.463    p = &slabclass[chunk->orig_clsid];464    // original class id needs to be set on free memory.465    it->slabs_clsid = chunk->orig_clsid;466    if (chunk->next) {467        chunk = chunk->next;468        chunk->prev = 0;469    } else {470        // header with no attached chunk471        chunk = NULL;472    }473474    // return the header object.475    // TODO: This is in three places, here and in do_slabs_free().476    it->prev = 0;477    it->next = p->slots;478    if (it->next) it->next->prev = it;479    p->slots = it;480    p->sl_curr++;481482    item_chunk *next_chunk;483    while (chunk) {484        assert(chunk->it_flags == ITEM_CHUNK);485        chunk->it_flags = ITEM_SLABBED;486        p = &slabclass[chunk->slabs_clsid];487        next_chunk = chunk->next;488489        chunk->prev = 0;490        chunk->next = p->slots;491        if (chunk->next) chunk->next->prev = chunk;492        p->slots = chunk;493        p->sl_curr++;494495        chunk = next_chunk;496    }497498    return;499}500501static void do_slabs_free(void *ptr, unsigned int id) {502    slabclass_t *p;503    item *it;504505    assert(id >= POWER_SMALLEST && id <= power_largest);506    if (id < POWER_SMALLEST || id > power_largest)507        return;508509    MEMCACHED_SLABS_FREE(id, ptr);510    p = &slabclass[id];511512    it = (item *)ptr;513    if ((it->it_flags & ITEM_CHUNKED) == 0) {514        it->it_flags = ITEM_SLABBED;515        it->slabs_clsid = id;516        it->prev = 0;517        it->next = p->slots;518        if (it->next) it->next->prev = it;519        p->slots = it;520521        p->sl_curr++;522    } else {523        do_slabs_free_chunked(it);524    }525    return;526}527528/* With refactoring of the various stats code the automover won't need a529 * custom function here.530 */531void fill_slab_stats_automove(slab_stats_automove *am) {532    int n;533    pthread_mutex_lock(&slabs_lock);534    for (n = 0; n < MAX_NUMBER_OF_SLAB_CLASSES; n++) {535        slabclass_t *p = &slabclass[n];536        slab_stats_automove *cur = &am[n];537        cur->chunks_per_page = p->perslab;538        cur->free_chunks = p->sl_curr;539        cur->total_pages = p->slabs;540        cur->chunk_size = p->size;541    }542    pthread_mutex_unlock(&slabs_lock);543}544545/* TODO: slabs_available_chunks should grow up to encompass this.546 * mem_flag is redundant with the other function.547 */548unsigned int global_page_pool_size(bool *mem_flag) {549    unsigned int ret = 0;550    pthread_mutex_lock(&slabs_lock);551    if (mem_flag != NULL)552        *mem_flag = mem_malloced >= mem_limit ? true : false;553    ret = slabclass[SLAB_GLOBAL_PAGE_POOL].slabs;554    pthread_mutex_unlock(&slabs_lock);555    return ret;556}557558/*@null@*/559static void do_slabs_stats(ADD_STAT add_stats, void *c) {560    int i, total;561    /* Get the per-thread stats which contain some interesting aggregates */562    struct thread_stats thread_stats;563    threadlocal_stats_aggregate(&thread_stats);564565    total = 0;566    for(i = POWER_SMALLEST; i <= power_largest; i++) {567        slabclass_t *p = &slabclass[i];568        if (p->slabs != 0) {569            uint32_t perslab, slabs;570            slabs = p->slabs;571            perslab = p->perslab;572573            char key_str[STAT_KEY_LEN];574            char val_str[STAT_VAL_LEN];575            int klen = 0, vlen = 0;576577            APPEND_NUM_STAT(i, "chunk_size", "%u", p->size);578            APPEND_NUM_STAT(i, "chunks_per_page", "%u", perslab);579            APPEND_NUM_STAT(i, "total_pages", "%u", slabs);580            APPEND_NUM_STAT(i, "total_chunks", "%u", slabs * perslab);581            APPEND_NUM_STAT(i, "used_chunks", "%u",582                            slabs*perslab - p->sl_curr);583            APPEND_NUM_STAT(i, "free_chunks", "%u", p->sl_curr);584            /* Stat is dead, but displaying zero instead of removing it. */585            APPEND_NUM_STAT(i, "free_chunks_end", "%u", 0);586            APPEND_NUM_STAT(i, "get_hits", "%llu",587                    (unsigned long long)thread_stats.slab_stats[i].get_hits);588            APPEND_NUM_STAT(i, "cmd_set", "%llu",589                    (unsigned long long)thread_stats.slab_stats[i].set_cmds);590            APPEND_NUM_STAT(i, "delete_hits", "%llu",591                    (unsigned long long)thread_stats.slab_stats[i].delete_hits);592            APPEND_NUM_STAT(i, "incr_hits", "%llu",593                    (unsigned long long)thread_stats.slab_stats[i].incr_hits);594            APPEND_NUM_STAT(i, "decr_hits", "%llu",595                    (unsigned long long)thread_stats.slab_stats[i].decr_hits);596            APPEND_NUM_STAT(i, "cas_hits", "%llu",597                    (unsigned long long)thread_stats.slab_stats[i].cas_hits);598            APPEND_NUM_STAT(i, "cas_badval", "%llu",599                    (unsigned long long)thread_stats.slab_stats[i].cas_badval);600            APPEND_NUM_STAT(i, "touch_hits", "%llu",601                    (unsigned long long)thread_stats.slab_stats[i].touch_hits);602            total++;603        }604    }605606    /* add overall slab stats and append terminator */607608    APPEND_STAT("active_slabs", "%d", total);609    APPEND_STAT("total_malloced", "%llu", (unsigned long long)mem_malloced);610    add_stats(NULL, 0, NULL, 0, c);611}612613static void *memory_allocate(size_t size) {614    void *ret;615616    if (mem_base == NULL) {617        /* We are not using a preallocated large memory chunk */618        ret = malloc(size);619    } else {620        ret = mem_current;621622        if (size > mem_avail) {623            return NULL;624        }625626        /* mem_current pointer _must_ be aligned!!! */627        if (size % CHUNK_ALIGN_BYTES) {628            size += CHUNK_ALIGN_BYTES - (size % CHUNK_ALIGN_BYTES);629        }630631        mem_current = ((char*)mem_current) + size;632        if (size < mem_avail) {633            mem_avail -= size;634        } else {635            mem_avail = 0;636        }637    }638    mem_malloced += size;639640    return ret;641}642643/* Must only be used if all pages are item_size_max */644static void memory_release(void) {645    void *p = NULL;646    if (mem_base != NULL)647        return;648649    if (!settings.slab_reassign)650        return;651652    while (mem_malloced > mem_limit &&653            (p = get_page_from_global_pool()) != NULL) {654        free(p);655        mem_malloced -= settings.slab_page_size;656    }657}658659void *slabs_alloc(unsigned int id, unsigned int flags) {660    void *ret;661662    pthread_mutex_lock(&slabs_lock);663    ret = do_slabs_alloc(id, flags);664    pthread_mutex_unlock(&slabs_lock);665    return ret;666}667668void slabs_free(void *ptr, unsigned int id) {669    pthread_mutex_lock(&slabs_lock);670    do_slabs_free(ptr, id);671    pthread_mutex_unlock(&slabs_lock);672}673674void slabs_stats(ADD_STAT add_stats, void *c) {675    pthread_mutex_lock(&slabs_lock);676    do_slabs_stats(add_stats, c);677    pthread_mutex_unlock(&slabs_lock);678}679680static bool do_slabs_adjust_mem_limit(size_t new_mem_limit) {681    /* Cannot adjust memory limit at runtime if prealloc'ed */682    if (mem_base != NULL)683        return false;684    settings.maxbytes = new_mem_limit;685    mem_limit = new_mem_limit;686    mem_limit_reached = false; /* Will reset on next alloc */687    memory_release(); /* free what might already be in the global pool */688    return true;689}690691bool slabs_adjust_mem_limit(size_t new_mem_limit) {692    bool ret;693    pthread_mutex_lock(&slabs_lock);694    ret = do_slabs_adjust_mem_limit(new_mem_limit);695    pthread_mutex_unlock(&slabs_lock);696    return ret;697}698699unsigned int slabs_available_chunks(const unsigned int id, bool *mem_flag,700        unsigned int *chunks_perslab) {701    unsigned int ret;702    slabclass_t *p;703704    pthread_mutex_lock(&slabs_lock);705    p = &slabclass[id];706    ret = p->sl_curr;707    if (mem_flag != NULL)708        *mem_flag = mem_malloced >= mem_limit ? true : false;709    if (chunks_perslab != NULL)710        *chunks_perslab = p->perslab;711    pthread_mutex_unlock(&slabs_lock);712    return ret;713}714715void *slabs_peek_page(const unsigned int id, uint32_t *size, uint32_t *perslab) {716    slabclass_t *s_cls;717    void *page = NULL;718    if (id > power_largest) {719        return NULL;720    }721    pthread_mutex_lock(&slabs_lock);722    s_cls = &slabclass[id];723    if (s_cls->slabs < 2) {724        pthread_mutex_unlock(&slabs_lock);725        return NULL;726    }727    *size = s_cls->size;728    *perslab = s_cls->perslab;729730    page = s_cls->slab_list[0];731732    pthread_mutex_unlock(&slabs_lock);733734    return page;735}736737/* detaches item/chunk from freelist.738 * for use with page mover.739 * lock _must_ be held.740 */741void do_slabs_unlink_free_chunk(const unsigned int id, item *it) {742    slabclass_t *s_cls = &slabclass[id];743    /* Ensure this was on the freelist and nothing else. */744    assert(it->it_flags == ITEM_SLABBED);745    if (s_cls->slots == it) {746        s_cls->slots = it->next;747    }748    if (it->next) it->next->prev = it->prev;749    if (it->prev) it->prev->next = it->next;750    s_cls->sl_curr--;751}752753void slabs_finalize_page_move(const unsigned int sid, const unsigned int did, void *page) {754    pthread_mutex_lock(&slabs_lock);755    slabclass_t *s_cls = &slabclass[sid];756    slabclass_t *d_cls = &slabclass[did];757758    s_cls->slabs--;759    for (int x = 0; x < s_cls->slabs; x++) {760        s_cls->slab_list[x] = s_cls->slab_list[x+1];761    }762763    // FIXME: it's nearly impossible for this to fail. If it does it can cause764    // writing beyond the border, lost pages, etc.765    // For now we won't handle the error, and a subsequent commit should766    // remove the need to resize the slab list.767    do_grow_slab_list(did);768    assert(d_cls->slab_list != NULL);769    d_cls->slab_list[d_cls->slabs++] = page;770    /* Don't need to split the page into chunks if we're just storing it */771    if (did > SLAB_GLOBAL_PAGE_POOL) {772        memset(page, 0, (size_t)settings.slab_page_size);773        split_slab_page_into_freelist(page, did);774    } else if (did == SLAB_GLOBAL_PAGE_POOL) {775        /* memset just enough to signal restart handler to skip */776        memset(page, 0, sizeof(item));777        /* mem_malloc'ed might be higher than mem_limit. */778        mem_limit_reached = false;779        memory_release();780    }781782    pthread_mutex_unlock(&slabs_lock);783}784/* Iterate at most once through the slab classes and pick a "random" source.785 * I like this better than calling rand() since rand() is slow enough that we786 * can just check all of the classes once instead.787 */788int slabs_pick_any_for_reassign(const unsigned int did) {789    pthread_mutex_lock(&slabs_lock);790    static int cur = POWER_SMALLEST - 1;791    int tries = MAX_NUMBER_OF_SLAB_CLASSES - POWER_SMALLEST + 1;792    for (; tries > 0; tries--) {793        cur++;794        if (cur >= MAX_NUMBER_OF_SLAB_CLASSES)795            cur = POWER_SMALLEST;796        if (cur == did)797            continue;798        if (slabclass[cur].slabs > 1) {799            pthread_mutex_unlock(&slabs_lock);800            return cur;801        }802    }803    pthread_mutex_unlock(&slabs_lock);804    return -1;805}806807int slabs_page_count(const unsigned int id) {808    int ret;809    pthread_mutex_lock(&slabs_lock);810    ret = slabclass[id].slabs;811    pthread_mutex_unlock(&slabs_lock);812    return ret;813}814815int slabs_locked_callback(slabs_cb cb, void *arg) {816    int ret = 0;817    pthread_mutex_lock(&slabs_lock);818    ret = cb(arg);819    pthread_mutex_unlock(&slabs_lock);820821    return ret;822}823824/* The slabber system could avoid needing to understand much, if anything,825 * about items if callbacks were strategically used. Due to how the slab mover826 * works, certain flag bits can only be adjusted while holding the slabs lock.827 * Using these functions, isolate sections of code needing this and turn them828 * into callbacks when an interface becomes more obvious.829 */830void slabs_mlock(void) {831    pthread_mutex_lock(&slabs_lock);832}833834void slabs_munlock(void) {835    pthread_mutex_unlock(&slabs_lock);836}