1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */2#include "memcached.h"3#include "slabs_mover.h"4#include "slab_automove.h"5#ifdef EXTSTORE6#include "slab_automove_extstore.h"7#endif8#include "storage.h"9#include <sys/mman.h>10#include <sys/stat.h>11#include <sys/socket.h>12#include <sys/resource.h>13#include <fcntl.h>14#include <netinet/in.h>15#include <stdlib.h>16#include <stdio.h>17#include <string.h>18#include <assert.h>19#include <pthread.h>2021struct slab_rebalance {22    void *slab_start;23    void *slab_end;24    void *slab_pos;25    unsigned int s_clsid;26    unsigned int d_clsid;27    uint32_t cls_size;28    uint32_t busy_items;29    uint32_t rescues;30    uint32_t inline_reclaim;31    uint32_t chunk_rescues;32    uint32_t busy_nomem;33    uint32_t busy_deletes;34    uint32_t busy_loops;35    int last_busy_status;36    uint8_t done;37    uint8_t *completed;38};3940struct slab_rebal_thread {41    bool run_thread;42    bool allow_evictions; // global pool empty or manual page move43    void *storage; // extstore instance.44    item *new_it; // memory for swapping out valid items.45    // TODO: logger instance.46    pthread_mutex_t lock;47    pthread_cond_t cond;48    pthread_t tid;49    logger *l;50    unsigned int am_version; // re-generate am object if version changes51    struct timespec am_last; // last time automover algo ran52    slab_automove_reg_t *sam; // active algorithm module53    void *active_am; // automover memory54    struct slab_rebalance rebal;55};5657enum move_status {58    MOVE_NONE=0, MOVE_PASS, MOVE_FROM_SLAB, MOVE_FROM_LRU, MOVE_BUSY,59    MOVE_BUSY_UPLOADING, MOVE_BUSY_ACTIVE, MOVE_BUSY_FLOATING, MOVE_LOCKED60};6162char *const move_status_text[] = {63    [MOVE_NONE] = "none",64    [MOVE_PASS] = "pass",65    [MOVE_FROM_SLAB] = "from_slab",66    [MOVE_FROM_LRU] = "from_lru",67    [MOVE_BUSY] = "busy",68    [MOVE_BUSY_UPLOADING] = "busy_uploading",69    [MOVE_BUSY_ACTIVE] = "busy_active",70    [MOVE_BUSY_FLOATING] = "busy_floating",71    [MOVE_LOCKED] = "locked",72};7374static slab_automove_reg_t slab_automove_default = {75    .init = slab_automove_init,76    .free = slab_automove_free,77    .run = slab_automove_run78};79#ifdef EXTSTORE80static slab_automove_reg_t slab_automove_extstore = {81    .init = slab_automove_extstore_init,82    .free = slab_automove_extstore_free,83    .run = slab_automove_extstore_run84};85#endif8687// the sleep inbetween loops is short, so 1000 loops is < 1s88#define SLAB_MOVE_MAX_LOOPS 500089static enum reassign_result_type do_slabs_reassign(struct slab_rebal_thread *t, int src, int dst, int flags);9091static int slab_rebalance_start(struct slab_rebal_thread *t) {92    uint32_t size;93    uint32_t perslab;94    bool mem_limit_reached;9596    if (t->rebal.s_clsid == t->rebal.d_clsid) {97        return -1;98    }99100    // check once at the start of a page move if the global pool is full.101    // since we're the only thing that can put memory back into the global102    // pool, this can't change until we complete.103    // unless the user changes the memory limit manually, which should be104    // rare.105    unsigned int global = global_page_pool_size(&mem_limit_reached);106    if (mem_limit_reached && global == 0) {107        t->allow_evictions = true;108    }109110    void *page = slabs_peek_page(t->rebal.s_clsid, &size, &perslab);111    // We _almost_ never get called in a condition where this can fail.112    if (page == NULL) {113        return -1;114    }115116    // Bit-vector to keep track of completed chunks117    t->rebal.completed = (uint8_t*)calloc(perslab,sizeof(uint8_t));118    if (!t->rebal.completed) {119        return -1;120    }121122    /* Always kill the first available slab page as it is most likely to123     * contain the oldest items124     */125    t->rebal.slab_start = t->rebal.slab_pos = page;126    t->rebal.slab_end   = (char *)page + (size * perslab);127    t->rebal.done       = 0;128    t->rebal.cls_size   = size;129    // Don't need to do chunk move work if page is in global pool.130    if (t->rebal.s_clsid == SLAB_GLOBAL_PAGE_POOL) {131        t->rebal.done = 1;132    }133134    // FIXME: remove this. query the structure from outside to see if we're135    // running.136    STATS_LOCK();137    stats_state.slab_reassign_running = true;138    STATS_UNLOCK();139140    return 0;141}142143#define MC_ALLOW_NEWPAGE 1144#define MC_NO_NEWPAGE 0145static void *slab_rebalance_alloc(struct slab_rebal_thread *t, unsigned int id, int flags) {146    item *new_it = NULL;147    // translating our flags to downstream flags...148    int sa_flags = flags == MC_ALLOW_NEWPAGE ? 0 : SLABS_ALLOC_NO_NEWPAGE;149150    // We will either wipe the whole page if unused, or run out of memory in151    // the page and return NULL.152    while (1) {153        new_it = slabs_alloc(id, sa_flags);154        if (new_it == NULL) {155            break;156        }157        /* check that memory isn't within the range to clear */158        if ((void *)new_it >= t->rebal.slab_start159            && (void *)new_it < t->rebal.slab_end) {160            /* Pulled something we intend to free. Mark it as freed since161             * we've already done the work of unlinking it from the freelist.162             */163            new_it->refcount = 0;164            new_it->it_flags = ITEM_SLABBED|ITEM_FETCHED;165#ifdef DEBUG_SLAB_MOVER166            memcpy(ITEM_key(new_it), "deadbeef", 8);167#endif168            new_it = NULL;169            t->rebal.inline_reclaim++;170        } else {171            break;172        }173    }174    return new_it;175}176177// To call move, we first need a free chunk of memory.178static void slab_rebalance_prep(struct slab_rebal_thread *t) {179    unsigned int s_clsid = t->rebal.s_clsid;180    if (t->new_it) {181        // move didn't use the memory from the last loop.182        return;183    }184185    t->new_it = slab_rebalance_alloc(t, s_clsid, MC_NO_NEWPAGE);186    // we could free the entire page in the above alloc call, but not get any187    // other memory to work with.188    // We try to busy-loop the page mover at least a few times in this case,189    // so it will pick up on all of the memory being freed already.190    if (t->new_it == NULL) {191        // If we've been stuck for a long time and don't have memory to192        // allocate with, flip allow_evictions. It starts on if we193        // start rebalance with completely full memory. However it's194        // possible to start rebalance before memory is full then it fills195        // while we rebalance and get stuck forever.196        if (!t->allow_evictions && t->rebal.busy_loops > SLAB_MOVE_MAX_LOOPS) {197            bool mem_limit_reached = false;198            global_page_pool_size(&mem_limit_reached);199200            // This is a stupid corner case that should self-resolve as memory201            // fills. If we haven't malloc'ed all possible memory but we're202            // trying to move memory for some reason, causing an eviction is203            // nonsense. So we allocate a page to push a page out.204            if (mem_limit_reached) {205                t->allow_evictions = true;206            } else {207                t->new_it = slab_rebalance_alloc(t, s_clsid, MC_ALLOW_NEWPAGE);208                if (t->new_it) {209                    return;210                }211            }212        }213214        if (t->allow_evictions) {215            // global is empty and memory limit is reached. we have to evict216            // memory to move forward.217            for (int x = 0; x < 10; x++) {218                if (lru_pull_tail(s_clsid, COLD_LRU, 0, LRU_PULL_EVICT, 0, NULL) <= 0) {219                    if (settings.lru_segmented) {220                        lru_pull_tail(s_clsid, HOT_LRU, 0, 0, 0, NULL);221                    }222                }223                t->new_it = slab_rebalance_alloc(t, s_clsid, MC_NO_NEWPAGE);224                if (t->new_it != NULL) {225                    break;226                }227            }228        }229    }230}231232/* refcount == 0 is safe since nobody can incr while item_lock is held.233 * refcount != 0 is impossible since flags/etc can be modified in other234 * threads. instead, note we found a busy one and bail.235 * NOTE: This is checking it_flags outside of an item lock. I believe this236 * works since it_flags is 8 bits, and we're only ever comparing a single bit237 * regardless. ITEM_SLABBED bit will always be correct since we're holding the238 * lock which modifies that bit. ITEM_LINKED won't exist if we're between an239 * item having ITEM_SLABBED removed, and the key hasn't been added to the item240 * yet. The memory barrier from the slabs lock should order the key write and the241 * flags to the item?242 * If ITEM_LINKED did exist and was just removed, but we still see it, that's243 * still safe since it will have a valid key, which we then lock, and then244 * recheck everything.245 * This may not be safe on all platforms; If not, slabs_alloc() will need to246 * seed the item key while holding slabs_lock.247 */248249struct _locked_st {250    item *it;251    item_chunk *ch;252    void *hold_lock; // held lock from trylock.253    uint32_t hv;254    unsigned int s_clsid;255    unsigned int d_clsid;256};257258// called while slabs lock is held so we can safely inspect a chunk of memory259// and do an inverted item lock.260static int _slabs_locked_cb(void *arg) {261    struct _locked_st *a = arg;262    int status = MOVE_PASS;263    item *it = a->it;264265    if (it->it_flags & ITEM_CHUNK) {266        /* This chunk is a chained part of a larger item. */267        a->ch = (item_chunk *) it;268        /* Instead, we use the head chunk to find the item and effectively269         * lock the entire structure. If a chunk has ITEM_CHUNK flag, its270         * head cannot be slabbed, so the normal routine is safe. */271        it = a->ch->head;272        assert(it->it_flags & ITEM_CHUNKED);273    }274275    /* ITEM_FETCHED when ITEM_SLABBED is overloaded to mean we've cleared276     * the chunk for move. Only these two flags should exist.277     */278    // TODO: bad failure mode if MOVE_PASS and we decide to skip later279    // but the item is actually alive for whatever reason.280    // default to MOVE_BUSY and set MOVE_PASS explicitly if the item is S|F?281    if (it->it_flags != (ITEM_SLABBED|ITEM_FETCHED)) {282        int refcount = 0;283284        /* ITEM_SLABBED can only be added/removed under the slabs_lock */285        if (it->it_flags & ITEM_SLABBED) {286            assert(a->ch == NULL);287            // must unlink our free item while slab lock held.288            // since we don't have an item lock to cover us.289            do_slabs_unlink_free_chunk(a->s_clsid, it);290            it->it_flags = 0; // fix flags outside of the slab lock.291            status = MOVE_FROM_SLAB;292        } else if ((it->it_flags & ITEM_LINKED) != 0) {293            /* If it doesn't have ITEM_SLABBED, the item could be in any294             * state on its way to being freed or written to. If no295             * ITEM_SLABBED, but it's had ITEM_LINKED, it must be active296             * and have the key written to it already.297             */298            a->hv = hash(ITEM_key(it), it->nkey);299            if ((a->hold_lock = item_trylock(a->hv)) == NULL) {300                status = MOVE_LOCKED;301            } else {302                bool is_linked = (it->it_flags & ITEM_LINKED);303                refcount = refcount_incr(it);304                if (refcount == 2) { /* item is linked but not busy */305                    /* Double check ITEM_LINKED flag here, since we're306                     * past a memory barrier from the mutex. */307                    if (is_linked) {308                        status = MOVE_FROM_LRU;309                    } else {310                        /* refcount == 1 + !ITEM_LINKED means the item is being311                         * uploaded to, or was just unlinked but hasn't been freed312                         * yet. Let it bleed off on its own and try again later */313                        status = MOVE_BUSY_UPLOADING;314                    }315                } else if (refcount > 2 && is_linked) {316                    status = MOVE_BUSY_ACTIVE;317                } else {318                    status = MOVE_BUSY;319                }320            }321        } else {322            /* See above comment. No ITEM_SLABBED or ITEM_LINKED. Mark323             * busy and wait for item to complete its upload. */324            status = MOVE_BUSY_FLOATING;325        }326    }327328    return status;329}330331static void slab_rebalance_rescue(struct slab_rebal_thread *t, struct _locked_st *a) {332    int cls_size = t->rebal.cls_size;333    item *it = a->it;334    item_chunk *ch = a->ch;335    item *new_it = t->new_it;336337    if (ch == NULL) {338        assert((new_it->it_flags & ITEM_CHUNKED) == 0);339        /* if free memory, memcpy. clear prev/next/h_bucket */340        memcpy(new_it, it, cls_size);341        new_it->prev = 0;342        new_it->next = 0;343        new_it->h_next = 0;344        /* These are definitely required. else fails assert */345        new_it->it_flags &= ~ITEM_LINKED;346        new_it->refcount = 0;347        do_item_replace(it, new_it, a->hv, ITEM_get_cas(it));348        /* Need to walk the chunks and repoint head  */349        if (new_it->it_flags & ITEM_CHUNKED) {350            item_chunk *fch = (item_chunk *) ITEM_schunk(new_it);351            fch->next->prev = fch;352            while (fch) {353                fch->head = new_it;354                fch = fch->next;355            }356        }357        it->refcount = 0;358        it->it_flags = ITEM_SLABBED|ITEM_FETCHED;359#ifdef DEBUG_SLAB_MOVER360        memcpy(ITEM_key(it), "deadbeef", 8);361#endif362        t->rebal.rescues++;363    } else {364        item_chunk *nch = (item_chunk *) new_it;365        /* Chunks always have head chunk (the main it) */366        ch->prev->next = nch;367        if (ch->next)368            ch->next->prev = nch;369        memcpy(nch, ch, ch->used + sizeof(item_chunk));370        ch->refcount = 0;371        ch->it_flags = ITEM_SLABBED|ITEM_FETCHED;372        t->rebal.chunk_rescues++;373#ifdef DEBUG_SLAB_MOVER374        memcpy(ITEM_key((item *)ch), "deadbeef", 8);375#endif376        refcount_decr(it);377    }378379    // we've used the temporary memory.380    t->new_it = NULL;381}382383// TODO: in order to rescue active chunked items we need to first do more work384// on chunked items:385// - individual chunks need to be refcounted, with refcounts protected by item386// lock. then they can be swapped out an released on refcount reduction387// - for chunked item headers I don't know how off-hand.388static int slab_rebalance_active_rescue(struct slab_rebal_thread *t, struct _locked_st *a) {389    int cls_size = t->rebal.cls_size;390    item *it = a->it;391    item_chunk *ch = a->ch;392    item *new_it = t->new_it;393394    // Can only rescue active non-chunked items right now.395    if (ch == NULL && (it->it_flags & ITEM_CHUNKED) == 0) {396        memcpy(new_it, it, cls_size);397        new_it->prev = 0;398        new_it->next = 0;399        new_it->h_next = 0;400401        new_it->it_flags &= ~ITEM_LINKED;402        new_it->refcount = 0;403        do_item_replace(it, new_it, a->hv, ITEM_get_cas(it));404        t->rebal.rescues++;405406        // old it is now unlinked. can't immediately rescue item.407        t->new_it = NULL;408        return 0;409    } else {410        // else if chunked, check if we've been busy-waiting too long and411        // delete the item.412        if (t->rebal.busy_loops > SLAB_MOVE_MAX_LOOPS) {413            // TODO: add indicator for source of eviction414            LOGGER_LOG(t->l, LOG_EVICTIONS, LOGGER_EVICTION, it);415            STORAGE_delete(t->storage, it);416            do_item_unlink(it, a->hv);417            t->rebal.busy_deletes++;418        }419    }420421    // failed to rescue busy item.422    return 1;423}424425// try to free up a chunk of memory, if not already free.426// if we have memory available outside of the source page, rescue any valid427// items.428// we still attempt to move data even if no memory is available for a rescue,429// in case the item is already free, expired, busy, etc.430static int slab_rebalance_move(struct slab_rebal_thread *t) {431    struct _locked_st cbarg;432    memset(&cbarg, 0, sizeof(cbarg));433434    // the offset to check if completed or not435    int offset = ((char*)t->rebal.slab_pos-(char*)t->rebal.slab_start)/(t->rebal.cls_size);436437    // skip acquiring the slabs lock for items we've already fully processed.438    if (t->rebal.completed[offset] == 0) {439        item *it;440        cbarg.it = it = t->rebal.slab_pos;441        cbarg.s_clsid = t->rebal.s_clsid;442        cbarg.d_clsid = t->rebal.d_clsid;443        // it is returned _locked_ if successful. _must_ unlock it!444        int status = slabs_locked_callback(_slabs_locked_cb, &cbarg);445446        item_chunk *ch = cbarg.ch;447        if (ch) {448            // swap item under examination to the chunk head if we're449            // attempting to move a chunk within a larger item.450            cbarg.it = it = ch->head;451        }452        switch (status) {453            case MOVE_BUSY_ACTIVE:454            case MOVE_FROM_LRU:455                /* Lock order is LRU locks -> slabs_lock. unlink uses LRU lock.456                 * We only need to hold the slabs_lock while initially looking457                 * at an item, and at this point we have an exclusive refcount458                 * (2) + the item is locked. Drop slabs lock, drop item to459                 * refcount 1 (just our own, then fall through and wipe it460                 */461                /* Check if expired or flushed */462                if ((it->exptime != 0 && it->exptime < current_time)463                    || item_is_flushed(it)) {464                    /* Expired, don't save. */465                    STORAGE_delete(t->storage, it);466                    if (!ch && (it->it_flags & ITEM_CHUNKED) == 0) {467                        do_item_unlink(it, cbarg.hv);468                        if (it->refcount == 1) {469                            it->it_flags = ITEM_SLABBED|ITEM_FETCHED;470                            it->refcount = 0;471#ifdef DEBUG_SLAB_MOVER472                            memcpy(ITEM_key(it), "deadbeef", 8);473#endif474                            t->rebal.completed[offset] = 1;475                        } else {476                            // expired, but busy.477                            do_item_remove(it);478                            t->rebal.busy_items++;479                        }480                    } else {481                        // chunked: unlink it and grab on next loop482                        // same code regardless of refcount483                        do_item_unlink(it, cbarg.hv);484                        do_item_remove(it);485                        t->rebal.busy_items++;486                    }487                } else {488                    // we should try to rescue the item.489                    if (t->new_it == NULL) {490                        // we don't actually have memory: need to mark as busy491                        // and try again in a future loop.492                        t->rebal.busy_items++;493                        t->rebal.busy_nomem++;494                        refcount_decr(it);495                    } else {496                        if (it->refcount == 2) {497                            slab_rebalance_rescue(t, &cbarg);498                            t->rebal.completed[offset] = 1;499                        } else {500                            assert(it->refcount > 2);501                            // need to wait for ref'ed owners to free *it502                            slab_rebalance_active_rescue(t, &cbarg);503                            t->rebal.busy_items++;504                            do_item_remove(it);505                        }506                    }507                }508509                item_trylock_unlock(cbarg.hold_lock);510                break;511            case MOVE_FROM_SLAB:512                it->refcount = 0;513                it->it_flags = ITEM_SLABBED|ITEM_FETCHED;514#ifdef DEBUG_SLAB_MOVER515                memcpy(ITEM_key(it), "deadbeef", 8);516#endif517                t->rebal.completed[offset] = 1;518                break;519            case MOVE_BUSY:520            case MOVE_BUSY_UPLOADING:521                assert(it->refcount != 0);522                t->rebal.busy_items++;523                refcount_decr(it);524                item_trylock_unlock(cbarg.hold_lock);525                break;526            case MOVE_LOCKED:527            case MOVE_BUSY_FLOATING:528                t->rebal.busy_items++;529                break;530            case MOVE_PASS:531                // already freed and unlinked, probably during an alloc532                t->rebal.completed[offset] = 1;533                break;534        }535536        t->rebal.last_busy_status = status;537    }538539    t->rebal.slab_pos = (char *)t->rebal.slab_pos + t->rebal.cls_size;540541    if (t->rebal.slab_pos >= t->rebal.slab_end) {542        /* Some items were busy, start again from the top */543        if (t->rebal.busy_items) {544            t->rebal.slab_pos = t->rebal.slab_start;545            STATS_LOCK();546            stats.slab_reassign_busy_items += t->rebal.busy_items;547            stats.slab_reassign_last_busy_status = move_status_text[t->rebal.last_busy_status];548            STATS_UNLOCK();549            t->rebal.busy_items = 0;550            t->rebal.busy_loops++;551            return 1;552        } else {553            t->rebal.done++;554            return 2;555        }556    }557558    return 0;559}560561static void slab_rebalance_finish(struct slab_rebal_thread *t) {562#ifdef DEBUG_SLAB_MOVER563    /* If the algorithm is broken, live items can sneak in. */564    slab_rebal.slab_pos = slab_rebal.slab_start;565    while (1) {566        item *it = slab_rebal.slab_pos;567        assert(it->it_flags == (ITEM_SLABBED|ITEM_FETCHED));568        assert(memcmp(ITEM_key(it), "deadbeef", 8) == 0);569        it->it_flags = ITEM_SLABBED|ITEM_FETCHED;570        slab_rebal.slab_pos = (char *)slab_rebal.slab_pos + slab_rebal.cls_size;571        if (slab_rebal.slab_pos >= slab_rebal.slab_end)572            break;573    }574#endif575576    // release any temporary memory we didn't end up using.577    if (t->new_it) {578        slabs_free(t->new_it, t->rebal.s_clsid);579        t->new_it = NULL;580    }581582    /* At this point the stolen slab is completely clear.583     * We always kill the "first"/"oldest" slab page in the slab_list, so584     * shuffle the page list backwards and decrement.585     */586    slabs_finalize_page_move(t->rebal.s_clsid, t->rebal.d_clsid,587            t->rebal.slab_start);588589    STATS_LOCK();590    stats.slabs_moved++;591    stats.slab_reassign_rescues += t->rebal.rescues;592    stats.slab_reassign_inline_reclaim += t->rebal.inline_reclaim;593    stats.slab_reassign_chunk_rescues += t->rebal.chunk_rescues;594    stats.slab_reassign_busy_deletes += t->rebal.busy_deletes;595    stats.slab_reassign_busy_nomem += t->rebal.busy_nomem;596    stats_state.slab_reassign_running = false;597    STATS_UNLOCK();598599    free(t->rebal.completed);600    memset(&t->rebal, 0, sizeof(t->rebal));601    t->allow_evictions = false;602}603604static int slab_rebalance_check_automove(struct slab_rebal_thread *t,605        struct timespec *now) {606    int src, dst;607    if (settings.slab_automove == 0) {608        // not enabled609        return 0;610    }611612    if (t->am_last.tv_sec == now->tv_sec) {613        // run once per second-ish.614        return 0;615    }616617    if (settings.slab_automove_version != t->am_version) {618        void *am_new = t->sam->init(&settings);619        // only replace if we successfully re-init'ed620        if (am_new) {621            t->sam->free(t->active_am);622            t->active_am = am_new;623        }624        t->am_version = settings.slab_automove_version;625    }626627    t->sam->run(t->active_am, &src, &dst);628    if (src != -1 && dst != -1) {629        // rebalancer lock already held, call directly.630        const char *msg = "ok";631        if (do_slabs_reassign(t, src, dst, 0) != REASSIGN_OK) {632            msg = "fail";633        }634        LOGGER_LOG(t->l, LOG_SYSEVENTS, LOGGER_SLAB_MOVE, NULL, src, dst, msg);635        if (dst != 0) {636            // if not reclaiming to global, rate limit to one per second.637            t->am_last.tv_sec = now->tv_sec;638        }639        // run the thread since we're moving a page.640        return 1;641    }642643    return 0;644}645646/* Slab mover thread.647 * Sits waiting for a condition to jump off and shovel some memory about648 */649// TODO: add back the "max busy loops" and bail the page move650static void *slab_rebalance_thread(void *arg) {651    struct slab_rebal_thread *t = arg;652    struct slab_rebalance *r = &t->rebal;653    int backoff_timer = 1;654    int backoff_max = 5000;655    int algo_backoff = 0;656    // create logger in thread for setspecific657    t->l = logger_create();658    /* Go into cond_wait with the mutex held */659    mutex_lock(&t->lock);660661    /* Must finish moving page before stopping */662    while (t->run_thread) {663        // are we running a rebalance?664        if (r->s_clsid != 0 || r->d_clsid != 0) {665            // do we need to kick it off?666            if (r->slab_start == NULL) {667                if (slab_rebalance_start(t) < 0) {668                    r->s_clsid = 0;669                    r->d_clsid = 0;670                    continue;671                }672            }673674            if (r->done) {675                slab_rebalance_finish(t);676            } else {677                // attempt to get some prepared memory678                slab_rebalance_prep(t);679                // attempt to free up memory in a page680                int res = slab_rebalance_move(t);681                if (res == 1) {682                    // ran through a page but had busy items; sleep.683                    usleep(backoff_timer);684                    backoff_timer = backoff_timer * 2;685                    if (backoff_timer > backoff_max)686                        backoff_timer = backoff_max;687                } else if (res == 2) {688                    // ran through a page successfully. reset any sleep.689                    backoff_timer = 1;690                }691                // else we're working through a page and shouldn't sleep.692            }693        } else {694            struct timespec now;695            clock_gettime(CLOCK_REALTIME, &now);696            if (slab_rebalance_check_automove(t, &now) == 0) {697                struct timespec next = { .tv_sec = 0, .tv_nsec = 100000000 };698                if (algo_backoff++ > 10) {699                    // start sleeping once per second since we've gone at700                    // least a second without doing anything.701                    now.tv_sec++;702                } else {703                    // else do a shorter sleep to be more responsive.704                    mc_timespec_add(&now, &next);705                }706                // standard delay707                // wait for signal to start another move.708                pthread_cond_timedwait(&t->cond, &t->lock, &now);709            } else { // else don't wait, run again immediately.710                algo_backoff = 0;711            }712        }713    }714715    // TODO: cancel in-flight slab page move716    mutex_unlock(&t->lock);717    return NULL;718}719720static enum reassign_result_type do_slabs_reassign(struct slab_rebal_thread *t, int src, int dst, int flags) {721    if (src == dst)722        return REASSIGN_SRC_DST_SAME;723724    /* Special indicator to choose ourselves. */725    if (src == -1) {726        src = slabs_pick_any_for_reassign(dst);727        /* TODO: If we end up back at -1, return a new error type */728    }729730    if (!slabs_class_check(src) || !slabs_class_check(dst)) {731        return REASSIGN_BADCLASS;732    }733734    if (slabs_page_count(src) < 2) {735        return REASSIGN_NOSPARE;736    }737738    t->rebal.s_clsid = src;739    t->rebal.d_clsid = dst;740    if (flags & SLABS_REASSIGN_ALLOW_EVICTIONS) {741        t->allow_evictions = true;742    }743744    pthread_cond_signal(&t->cond);745746    return REASSIGN_OK;747}748749enum reassign_result_type slabs_reassign(struct slab_rebal_thread *t, int src, int dst, int flags) {750    enum reassign_result_type ret;751    if (pthread_mutex_trylock(&t->lock) != 0) {752        return REASSIGN_RUNNING;753    }754    ret = do_slabs_reassign(t, src, dst, flags);755    pthread_mutex_unlock(&t->lock);756    return ret;757}758759/* If we hold this lock, rebalancer can't wake up or move */760void slab_maintenance_pause(struct slab_rebal_thread *t) {761    pthread_mutex_lock(&t->lock);762}763764void slab_maintenance_resume(struct slab_rebal_thread *t) {765    pthread_mutex_unlock(&t->lock);766}767768struct slab_rebal_thread *start_slab_maintenance_thread(void *storage) {769    int ret;770    struct slab_rebal_thread *t = calloc(1, sizeof(*t));771    if (t == NULL)772        return NULL;773774    pthread_mutex_init(&t->lock, NULL);775    pthread_cond_init(&t->cond, NULL);776    t->run_thread = true;777    if (storage) {778#ifdef EXTSTORE779        t->storage = storage;780        t->sam = &slab_automove_extstore;781#endif782    } else {783        t->sam = &slab_automove_default;784    }785    t->active_am = t->sam->init(&settings);786    if (t->active_am == NULL) {787        fprintf(stderr, "Can't create slab rebalancer thread: failed to allocate automover memory\n");788        return NULL;789    }790791    if ((ret = pthread_create(&t->tid, NULL,792                              slab_rebalance_thread, t)) != 0) {793        fprintf(stderr, "Can't create slab rebalancer thread: %s\n", strerror(ret));794        return NULL;795    }796    thread_setname(t->tid, "mc-slabmaint");797    return t;798}799800/* The maintenance thread is on a sleep/loop cycle, so it should join after a801 * short wait */802void stop_slab_maintenance_thread(struct slab_rebal_thread *t) {803    pthread_mutex_lock(&t->lock);804    t->run_thread = false;805    pthread_cond_signal(&t->cond);806    pthread_mutex_unlock(&t->lock);807808    /* Wait for the maintenance thread to stop */809    pthread_join(t->tid, NULL);810811    pthread_mutex_destroy(&t->lock);812    pthread_cond_destroy(&t->cond);813    if (t->rebal.completed) {814        free(t->rebal.completed);815    }816    t->sam->free(t->active_am);817    free(t);818819    // TODO: there is no logger_destroy() yet.820}