1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */2#include "memcached.h"3#ifdef EXTSTORE45#include "storage.h"6#include "extstore.h"7#include <stdlib.h>8#include <stdio.h>9#include <stddef.h>10#include <string.h>11#include <limits.h>12#include <ctype.h>1314#define PAGE_BUCKET_DEFAULT 015#define PAGE_BUCKET_COMPACT 116#define PAGE_BUCKET_CHUNKED 217#define PAGE_BUCKET_LOWTTL  318#define PAGE_BUCKET_COLDCOMPACT 419#define PAGE_BUCKET_OLD     520// Not another bucket; this is the total number of buckets.21#define PAGE_BUCKET_COUNT   62223/*24 * API functions25 */26static void storage_finalize_cb(io_pending_t *pending);27static void storage_return_cb(io_pending_t *pending);2829// re-cast an io_pending_t into this more descriptive structure.30// the first few items _must_ match the original struct.31typedef struct _io_pending_storage_t {32    uint8_t io_queue_type;33    uint8_t io_sub_type;34    uint8_t payload; // payload offset35    LIBEVENT_THREAD *thread;36    conn *c;37    mc_resp *resp;38    io_queue_cb return_cb;    // called on worker thread.39    io_queue_cb finalize_cb;  // called back on the worker thread.40    STAILQ_ENTRY(io_pending_t) iop_next; // queue chain.41                              /* original struct ends here */42    item *hdr_it;             /* original header item. */43    obj_io io_ctx;            /* embedded extstore IO header */44    unsigned int iovec_data;  /* specific index of data iovec */45    bool noreply;             /* whether the response had noreply set */46    bool miss;                /* signal a miss to unlink hdr_it */47    bool badcrc;              /* signal a crc failure */48    bool active;              /* tells if IO was dispatched or not */49} io_pending_storage_t;5051static pthread_t storage_compact_tid;52static pthread_mutex_t storage_compact_plock;53static pthread_cond_t storage_compact_cond;5455// Only call this if item has ITEM_HDR56bool storage_validate_item(void *e, item *it) {57    item_hdr *hdr = (item_hdr *)ITEM_data(it);58    if (extstore_check(e, hdr->page_id, hdr->page_version) != 0) {59        return false;60    } else {61        return true;62    }63}6465void storage_delete(void *e, item *it) {66    if (it->it_flags & ITEM_HDR) {67        item_hdr *hdr = (item_hdr *)ITEM_data(it);68        extstore_delete(e, hdr->page_id, hdr->page_version,69                1, ITEM_ntotal(it));70    }71}7273// Function for the extra stats called from a protocol.74// NOTE: This either needs a name change or a wrapper, perhaps?75// it's defined here to reduce exposure of extstore.h to the rest of memcached76// but feels a little off being defined here.77// At very least maybe "process_storage_stats" in line with making this more78// of a generic wrapper module.79void process_extstore_stats(ADD_STAT add_stats, void *c) {80    int i;81    char key_str[STAT_KEY_LEN];82    char val_str[STAT_VAL_LEN];83    int klen = 0, vlen = 0;84    struct extstore_stats st;8586    assert(add_stats);8788    void *storage = ext_storage;89    if (storage == NULL) {90        return;91    }92    extstore_get_stats(storage, &st);93    st.page_data = calloc(st.page_count, sizeof(struct extstore_page_data));94    extstore_get_page_data(storage, &st);9596    for (i = 0; i < st.page_count; i++) {97        APPEND_NUM_STAT(i, "version", "%llu",98                (unsigned long long) st.page_data[i].version);99        APPEND_NUM_STAT(i, "bytes", "%llu",100                (unsigned long long) st.page_data[i].bytes_used);101        APPEND_NUM_STAT(i, "bucket", "%u",102                st.page_data[i].bucket);103        APPEND_NUM_STAT(i, "free_bucket", "%u",104                st.page_data[i].free_bucket);105    }106107    free(st.page_data);108}109110// Additional storage stats for the main stats output.111void storage_stats(ADD_STAT add_stats, void *c) {112    struct extstore_stats st;113    if (ext_storage) {114        STATS_LOCK();115        APPEND_STAT("extstore_memory_pressure", "%.2f", stats_state.extstore_memory_pressure);116        APPEND_STAT("extstore_compact_lost", "%llu", (unsigned long long)stats.extstore_compact_lost);117        APPEND_STAT("extstore_compact_rescues", "%llu", (unsigned long long)stats.extstore_compact_rescues);118        APPEND_STAT("extstore_compact_resc_cold", "%llu", (unsigned long long)stats.extstore_compact_resc_cold);119        APPEND_STAT("extstore_compact_resc_old", "%llu", (unsigned long long)stats.extstore_compact_resc_old);120        APPEND_STAT("extstore_compact_skipped", "%llu", (unsigned long long)stats.extstore_compact_skipped);121        STATS_UNLOCK();122        extstore_get_stats(ext_storage, &st);123        APPEND_STAT("extstore_page_allocs", "%llu", (unsigned long long)st.page_allocs);124        APPEND_STAT("extstore_page_evictions", "%llu", (unsigned long long)st.page_evictions);125        APPEND_STAT("extstore_page_reclaims", "%llu", (unsigned long long)st.page_reclaims);126        APPEND_STAT("extstore_pages_free", "%llu", (unsigned long long)st.pages_free);127        APPEND_STAT("extstore_pages_used", "%llu", (unsigned long long)st.pages_used);128        APPEND_STAT("extstore_objects_evicted", "%llu", (unsigned long long)st.objects_evicted);129        APPEND_STAT("extstore_objects_read", "%llu", (unsigned long long)st.objects_read);130        APPEND_STAT("extstore_objects_written", "%llu", (unsigned long long)st.objects_written);131        APPEND_STAT("extstore_objects_used", "%llu", (unsigned long long)st.objects_used);132        APPEND_STAT("extstore_bytes_evicted", "%llu", (unsigned long long)st.bytes_evicted);133        APPEND_STAT("extstore_bytes_written", "%llu", (unsigned long long)st.bytes_written);134        APPEND_STAT("extstore_bytes_read", "%llu", (unsigned long long)st.bytes_read);135        APPEND_STAT("extstore_bytes_used", "%llu", (unsigned long long)st.bytes_used);136        APPEND_STAT("extstore_bytes_fragmented", "%llu", (unsigned long long)st.bytes_fragmented);137        APPEND_STAT("extstore_limit_maxbytes", "%llu", (unsigned long long)(st.page_count * st.page_size));138        APPEND_STAT("extstore_io_queue", "%llu", (unsigned long long)(st.io_queue));139    }140141}142143// This callback runs in the IO thread.144// TODO: Some or all of this should move to the145// io_pending's callback back in the worker thread.146// It might make sense to keep the crc32c check here though.147static void _storage_get_item_cb(void *e, obj_io *io, int ret) {148    // FIXME: assumes success149    io_pending_storage_t *p = (io_pending_storage_t *)io->data;150    mc_resp *resp = p->resp;151    assert(p->active == true);152    item *read_it = (item *)io->buf;153    bool miss = false;154155    // TODO: How to do counters for hit/misses?156    if (ret < 1) {157        miss = true;158    } else {159        uint32_t crc2;160        uint32_t crc = (uint32_t) read_it->exptime;161        int x;162        // item is chunked, crc the iov's163        if (io->iov != NULL) {164            // first iov is the header, which we don't use beyond crc165            crc2 = crc32c(0, (char *)io->iov[0].iov_base+STORE_OFFSET, io->iov[0].iov_len-STORE_OFFSET);166            // make sure it's not sent. hack :(167            io->iov[0].iov_len = 0;168            for (x = 1; x < io->iovcnt; x++) {169                crc2 = crc32c(crc2, (char *)io->iov[x].iov_base, io->iov[x].iov_len);170            }171        } else {172            crc2 = crc32c(0, (char *)read_it+STORE_OFFSET, io->len-STORE_OFFSET);173        }174175        if (crc != crc2) {176            miss = true;177            p->badcrc = true;178        }179    }180181    if (miss) {182        if (p->noreply) {183            // In all GET cases, noreply means we send nothing back.184            resp->skip = true;185        } else {186            // TODO: This should be movable to the worker thread.187            // Convert the binprot response into a miss response.188            // The header requires knowing a bunch of stateful crap, so rather189            // than simply writing out a "new" miss response we mangle what's190            // already there.191            if (resp->binary_prot) {192                protocol_binary_response_header *header =193                    (protocol_binary_response_header *)resp->wbuf;194195                // cut the extra nbytes off of the body_len196                uint32_t body_len = ntohl(header->response.bodylen);197                uint8_t hdr_len = header->response.extlen;198                body_len -= resp->iov[p->iovec_data].iov_len + hdr_len;199                resp->tosend -= resp->iov[p->iovec_data].iov_len + hdr_len;200                header->response.extlen = 0;201                header->response.status = (uint16_t)htons(PROTOCOL_BINARY_RESPONSE_KEY_ENOENT);202                header->response.bodylen = htonl(body_len);203204                // truncate the data response.205                resp->iov[p->iovec_data].iov_len = 0;206                // wipe the extlen iov... wish it was just a flat buffer.207                // this is unconditionally the second iov.208                resp->iov[1].iov_len = 0;209                resp->chunked_data_iov = 0;210            } else {211                int i;212                // Meta commands have EN status lines for miss, rather than213                // END as a trailer as per normal ascii.214                if (resp->iov[0].iov_len >= 3215                        && memcmp(resp->iov[0].iov_base, "VA ", 3) == 0) {216                    // TODO: These miss translators should use specific callback217                    // functions attached to the io wrap. This is weird :(218                    resp->iovcnt = 1;219                    resp->iov[0].iov_len = 4;220                    resp->iov[0].iov_base = "EN\r\n";221                    resp->tosend = 4;222                } else {223                    // Wipe the iovecs up through our data injection.224                    // Allows trailers to be returned (END)225                    for (i = 0; i <= p->iovec_data; i++) {226                        resp->tosend -= resp->iov[i].iov_len;227                        resp->iov[i].iov_len = 0;228                        resp->iov[i].iov_base = NULL;229                    }230                }231                resp->chunked_total = 0;232                resp->chunked_data_iov = 0;233            }234        }235        p->miss = true;236    } else {237        assert(read_it->slabs_clsid != 0);238        // TODO: should always use it instead of ITEM_data to kill more239        // chunked special casing.240        if ((read_it->it_flags & ITEM_CHUNKED) == 0) {241            resp->iov[p->iovec_data].iov_base = ITEM_data(read_it);242        }243        p->miss = false;244    }245246    p->active = false;247    //assert(c->io_wrapleft >= 0);248249    return_io_pending((io_pending_t *)p);250}251252int storage_get_item(LIBEVENT_THREAD *t, item *it, mc_resp *resp) {253#ifdef NEED_ALIGN254    item_hdr hdr;255    memcpy(&hdr, ITEM_data(it), sizeof(hdr));256#else257    item_hdr *hdr = (item_hdr *)ITEM_data(it);258#endif259    io_queue_t *q = thread_io_queue_get(t, IO_QUEUE_EXTSTORE);260    size_t ntotal = ITEM_ntotal(it);261    unsigned int clsid = slabs_clsid(ntotal);262    item *new_it;263    bool chunked = false;264    if (ntotal > settings.slab_chunk_size_max) {265        // Pull a chunked item header.266        client_flags_t flags;267        FLAGS_CONV(it, flags);268        new_it = item_alloc(ITEM_key(it), it->nkey, flags, it->exptime, it->nbytes);269        assert(new_it == NULL || (new_it->it_flags & ITEM_CHUNKED));270        chunked = true;271    } else {272        new_it = do_item_alloc_pull(ntotal, clsid);273    }274    if (new_it == NULL)275        return -1;276    // so we can free the chunk on a miss277    new_it->slabs_clsid = clsid;278279    io_pending_storage_t *p = do_cache_alloc(t->io_cache);280    // this is a re-cast structure, so assert that we never outsize it.281    assert(sizeof(io_pending_t) >= sizeof(io_pending_storage_t));282    memset(p, 0, sizeof(io_pending_storage_t));283    p->active = true;284    p->miss = false;285    p->badcrc = false;286    p->noreply = resp->noreply;287    p->thread = t;288    p->return_cb = storage_return_cb;289    p->finalize_cb = storage_finalize_cb;290    // io_pending owns the reference for this object now.291    p->hdr_it = it;292    p->resp = resp;293    p->io_queue_type = IO_QUEUE_EXTSTORE;294    p->payload = offsetof(io_pending_storage_t, io_ctx);295    obj_io *eio = &p->io_ctx;296297    // FIXME: error handling.298    if (chunked) {299        unsigned int ciovcnt = 0;300        size_t remain = new_it->nbytes;301        item_chunk *chunk = (item_chunk *) ITEM_schunk(new_it);302        // TODO: This might make sense as a _global_ cache vs a per-thread.303        // but we still can't load objects requiring > IOV_MAX iovs.304        // In the meantime, these objects are rare/slow enough that305        // malloc/freeing a statically sized object won't cause us much pain.306        eio->iov = malloc(sizeof(struct iovec) * IOV_MAX);307        if (eio->iov == NULL) {308            item_remove(new_it);309            do_cache_free(t->io_cache, p);310            return -1;311        }312313        // fill the header so we can get the full data + crc back.314        eio->iov[0].iov_base = new_it;315        eio->iov[0].iov_len = ITEM_ntotal(new_it) - new_it->nbytes;316        ciovcnt++;317318        while (remain > 0) {319            chunk = do_item_alloc_chunk(chunk, remain);320            // FIXME: _pure evil_, silently erroring if item is too large.321            if (chunk == NULL || ciovcnt > IOV_MAX-1) {322                item_remove(new_it);323                free(eio->iov);324                // TODO: wrapper function for freeing up an io wrap?325                eio->iov = NULL;326                do_cache_free(t->io_cache, p);327                return -1;328            }329            eio->iov[ciovcnt].iov_base = chunk->data;330            eio->iov[ciovcnt].iov_len = (remain < chunk->size) ? remain : chunk->size;331            chunk->used = (remain < chunk->size) ? remain : chunk->size;332            remain -= chunk->size;333            ciovcnt++;334        }335336        eio->iovcnt = ciovcnt;337    }338339    // Chunked or non chunked we reserve a response iov here.340    p->iovec_data = resp->iovcnt;341    int iovtotal = (resp->binary_prot) ? it->nbytes - 2 : it->nbytes;342    if (chunked) {343        resp_add_chunked_iov(resp, new_it, iovtotal);344    } else {345        resp_add_iov(resp, "", iovtotal);346    }347348    // We can't bail out anymore, so mc_resp owns the IO from here.349    resp->io_pending = (io_pending_t *)p;350351    eio->buf = (void *)new_it;352353    STAILQ_INSERT_TAIL(&q->stack, (io_pending_t *)p, iop_next);354355    // reference ourselves for the callback.356    eio->data = (void *)p;357358    // Now, fill in io->io based on what was in our header.359#ifdef NEED_ALIGN360    eio->page_version = hdr.page_version;361    eio->page_id = hdr.page_id;362    eio->offset = hdr.offset;363#else364    eio->page_version = hdr->page_version;365    eio->page_id = hdr->page_id;366    eio->offset = hdr->offset;367#endif368    eio->len = ntotal;369    eio->mode = OBJ_IO_READ;370    eio->cb = _storage_get_item_cb;371372    // FIXME: This stat needs to move to reflect # of flash hits vs misses373    // for now it's a good gauge on how often we request out to flash at374    // least.375    pthread_mutex_lock(&t->stats.mutex);376    t->stats.get_extstore++;377    pthread_mutex_unlock(&t->stats.mutex);378379    return 0;380}381382void storage_submit_cb(io_queue_t *q) {383    // TODO: until we decide to port extstore's internal code to use384    // io_pending objs we "port" the IOP's into an obj_io chain just before385    // submission here.386    void *eio_head = NULL;387    while(!STAILQ_EMPTY(&q->stack)) {388        io_pending_t *p = STAILQ_FIRST(&q->stack);389        STAILQ_REMOVE_HEAD(&q->stack, iop_next);390        // FIXME: re-evaluate this.391        obj_io *io_ctx = (obj_io *) ((char *)p + p->payload);392        io_ctx->next = eio_head;393        eio_head = io_ctx;394    }395    extstore_submit(q->ctx, eio_head);396}397398// Runs locally in worker thread.399static void recache_or_free(io_pending_t *pending) {400    // re-cast to our specific struct.401    io_pending_storage_t *p = (io_pending_storage_t *)pending;402403    conn *c = p->c;404    obj_io *io = &p->io_ctx;405    assert(io != NULL);406    item *it = (item *)io->buf;407    assert(c != NULL);408    bool do_free = true;409    if (p->active) {410        // If request never dispatched, free the read buffer but leave the411        // item header alone.412        do_free = false;413        size_t ntotal = ITEM_ntotal(p->hdr_it);414        slabs_free(it, slabs_clsid(ntotal));415416        p->resp->suspended = false;417        c->resps_suspended--;418        // this is an unsubmitted IO, so unlink ourselves.419        // (note: slow but should be very rare op)420        io_queue_t *q = thread_io_queue_get(p->thread, p->io_queue_type);421        STAILQ_REMOVE(&q->stack, pending, _io_pending_t, iop_next);422423        pthread_mutex_lock(&c->thread->stats.mutex);424        c->thread->stats.get_aborted_extstore++;425        pthread_mutex_unlock(&c->thread->stats.mutex);426    } else if (p->miss) {427        // If request was ultimately a miss, unlink the header.428        do_free = false;429        size_t ntotal = ITEM_ntotal(p->hdr_it);430        item_unlink(p->hdr_it);431        slabs_free(it, slabs_clsid(ntotal));432        pthread_mutex_lock(&c->thread->stats.mutex);433        c->thread->stats.miss_from_extstore++;434        if (p->badcrc)435            c->thread->stats.badcrc_from_extstore++;436        pthread_mutex_unlock(&c->thread->stats.mutex);437    } else if (settings.ext_recache_rate) {438        // hashvalue is cuddled during store439        uint32_t hv = (uint32_t)it->time;440        // opt to throw away rather than wait on a lock.441        void *hold_lock = item_trylock(hv);442        if (hold_lock != NULL) {443            item *h_it = p->hdr_it;444            uint8_t flags = ITEM_LINKED|ITEM_FETCHED|ITEM_ACTIVE;445            // Item must be recently hit at least twice to recache.446            if (((h_it->it_flags & flags) == flags) &&447                    h_it->time > current_time - ITEM_UPDATE_INTERVAL &&448                    c->recache_counter++ % settings.ext_recache_rate == 0) {449                do_free = false;450                // In case it's been updated.451                it->exptime = h_it->exptime;452                it->it_flags &= ~ITEM_LINKED;453                it->it_flags |= (h_it->it_flags & (ITEM_PRESERVE_FLAGS));454                it->refcount = 0;455                it->h_next = NULL; // might not be necessary.456                STORAGE_delete(c->thread->storage, h_it);457                item_replace(h_it, it, hv, ITEM_get_cas(h_it));458                pthread_mutex_lock(&c->thread->stats.mutex);459                c->thread->stats.recache_from_extstore++;460                pthread_mutex_unlock(&c->thread->stats.mutex);461            }462        }463        if (hold_lock)464            item_trylock_unlock(hold_lock);465    }466    if (do_free)467        slabs_free(it, ITEM_clsid(it));468469    p->io_ctx.buf = NULL;470    p->io_ctx.next = NULL;471    p->active = false;472473    // TODO: reuse lock and/or hv.474    item_remove(p->hdr_it);475}476477// Called after an IO has been returned to the worker thread.478static void storage_return_cb(io_pending_t *pending) {479    conn_resp_unsuspend(pending->c, pending->resp);480}481482// Called after responses have been transmitted. Need to free up related data.483static void storage_finalize_cb(io_pending_t *pending) {484    recache_or_free(pending);485    io_pending_storage_t *p = (io_pending_storage_t *)pending;486    obj_io *io = &p->io_ctx;487    // malloc'ed iovec list used for chunked extstore fetches.488    if (io->iov) {489        free(io->iov);490        io->iov = NULL;491    }492    // don't need to free the main context, since it's embedded.493}494495/*496 * WRITE FLUSH THREAD497 */498499static int storage_write(void *storage, const int clsid, const int item_age) {500    int did_moves = 0;501    struct lru_pull_tail_return it_info;502503    it_info.it = NULL;504    lru_pull_tail(clsid, COLD_LRU, 0, LRU_PULL_RETURN_ITEM, 0, &it_info);505    /* Item is locked, and we have a reference to it. */506    if (it_info.it == NULL) {507        return did_moves;508    }509510    obj_io io;511    item *it = it_info.it;512    /* First, storage for the header object */513    size_t orig_ntotal = ITEM_ntotal(it);514    client_flags_t flags;515    if ((it->it_flags & ITEM_HDR) == 0 &&516            (item_age == 0 || current_time - it->time > item_age)) {517        FLAGS_CONV(it, flags);518        item *hdr_it = do_item_alloc(ITEM_key(it), it->nkey, flags, it->exptime, sizeof(item_hdr));519        /* Run the storage write understanding the start of the item is dirty.520         * We will fill it (time/exptime/etc) from the header item on read.521         */522        if (hdr_it != NULL) {523            int bucket = (it->it_flags & ITEM_CHUNKED) ?524                PAGE_BUCKET_CHUNKED : PAGE_BUCKET_DEFAULT;525            // Compress soon to expire items into similar pages.526            if (it->exptime - current_time < settings.ext_low_ttl) {527                bucket = PAGE_BUCKET_LOWTTL;528            }529            hdr_it->it_flags |= ITEM_HDR;530            io.len = orig_ntotal;531            io.mode = OBJ_IO_WRITE;532            // NOTE: when the item is read back in, the slab mover533            // may see it. Important to have refcount>=2 or ~ITEM_LINKED534            assert(it->refcount >= 2);535            // NOTE: write bucket vs free page bucket will disambiguate once536            // lowttl feature is better understood.537            if (extstore_write_request(storage, bucket, bucket, &io) == 0) {538                // cuddle the hash value into the time field so we don't have539                // to recalculate it.540                item *buf_it = (item *) io.buf;541                buf_it->time = it_info.hv;542                // copy from past the headers + time headers.543                // TODO: should be in items.c544                if (it->it_flags & ITEM_CHUNKED) {545                    // Need to loop through the item and copy546                    item_chunk *sch = (item_chunk *) ITEM_schunk(it);547                    int remain = orig_ntotal;548                    int copied = 0;549                    // copy original header550                    int hdrtotal = ITEM_ntotal(it) - it->nbytes;551                    memcpy((char *)io.buf+STORE_OFFSET, (char *)it+STORE_OFFSET, hdrtotal - STORE_OFFSET);552                    copied = hdrtotal;553                    // copy data in like it were one large object.554                    while (sch && remain) {555                        assert(remain >= sch->used);556                        memcpy((char *)io.buf+copied, sch->data, sch->used);557                        // FIXME: use one variable?558                        remain -= sch->used;559                        copied += sch->used;560                        sch = sch->next;561                    }562                } else {563                    memcpy((char *)io.buf+STORE_OFFSET, (char *)it+STORE_OFFSET, io.len-STORE_OFFSET);564                }565                // crc what we copied so we can do it sequentially.566                buf_it->it_flags &= ~ITEM_LINKED;567                buf_it->exptime = crc32c(0, (char*)io.buf+STORE_OFFSET, orig_ntotal-STORE_OFFSET);568                extstore_write(storage, &io);569                item_hdr *hdr = (item_hdr *) ITEM_data(hdr_it);570                hdr->page_version = io.page_version;571                hdr->page_id = io.page_id;572                hdr->offset  = io.offset;573                // overload nbytes for the header it574                hdr_it->nbytes = it->nbytes;575                hdr_it->it_flags |= (it->it_flags & (ITEM_PRESERVE_FLAGS));576                /* success! Now we need to fill relevant data into the new577                 * header and replace. Most of this requires the item lock578                 */579                /* CAS gets set while linking. Copy post-replace */580                item_replace(it, hdr_it, it_info.hv, ITEM_get_cas(it));581                do_item_remove(hdr_it);582                did_moves = 1;583                LOGGER_LOG(NULL, LOG_EVICTIONS, LOGGER_EXTSTORE_WRITE, it, bucket);584            } else {585                /* Failed to write for some reason, can't continue. */586                slabs_free(hdr_it, ITEM_clsid(hdr_it));587            }588        }589    }590    do_item_remove(it);591    item_unlock(it_info.hv);592    return did_moves;593}594595static pthread_t storage_write_tid;596static pthread_mutex_t storage_write_plock;597#define WRITE_SLEEP_MIN 200598599static void *storage_write_thread(void *arg) {600    void *storage = arg;601    // NOTE: ignoring overflow since that would take years of uptime in a602    // specific load pattern of never going to sleep.603    unsigned int backoff[MAX_NUMBER_OF_SLAB_CLASSES] = {0};604    unsigned int counter = 0;605    useconds_t to_sleep = WRITE_SLEEP_MIN;606    logger *l = logger_create();607    if (l == NULL) {608        fprintf(stderr, "Failed to allocate logger for storage compaction thread\n");609        abort();610    }611612    pthread_mutex_lock(&storage_write_plock);613    // The compaction checker is CPU intensive, so we do a loose fudging to614    // only activate it once every "slab page size" worth of bytes written.615    // I was calling the compact checker once per run through this main loop,616    // but we can end up doing lots of short loops without sleeping and end up617    // calling the compact checker pretty frequently.618    int check_compact = settings.slab_page_size;619620    while (1) {621        // cache per-loop to avoid calls to the slabs_clsid() search loop622        int min_class = slabs_clsid(settings.ext_item_size);623        unsigned int global_pages = global_page_pool_size(NULL);624        bool do_sleep = true;625        int target_pages = 0;626        if (global_pages < settings.ext_global_pool_min) {627            target_pages = settings.ext_global_pool_min - global_pages;628        } else if (global_pages == settings.ext_global_pool_min) {629            // start flushing a little early to lessen pressure on page mover630            target_pages = 1;631        }632        counter++;633        if (to_sleep > settings.ext_max_sleep)634            to_sleep = settings.ext_max_sleep;635636        // the largest items have the least overhead from going to disk.637        for (int x = MAX_NUMBER_OF_SLAB_CLASSES-1; x > 0; x--) {638            bool did_move = false;639            bool mem_limit_reached = false;640            unsigned int chunks_free;641            int item_age;642643            if (min_class > x || (backoff[x] && (counter % backoff[x] != 0))) {644                continue;645            }646647            // Avoid extra slab lock calls during heavy writing.648            unsigned int chunks_perpage = 0;649            chunks_free = slabs_available_chunks(x, &mem_limit_reached,650                    &chunks_perpage);651652            if (chunks_perpage == 0) {653                // no slab class here, skip.654                continue;655            }656            // Loose estimate for cutting the calls to compacter657            unsigned int chunk_size = settings.slab_page_size / chunks_perpage;658            // NOTE: stupid heuristic: we need to avoid over-flushing small659            // slab classes because the relative size of the headers is close660            // enough to cause runaway problems.661            unsigned int max_pages = 0;662            unsigned int target = 0;663            if (chunk_size < 500) {664                max_pages = 3;665            } else if (chunk_size < 1000) {666                max_pages = 4;667            } else if (chunk_size < 2000) {668                max_pages = 5;669            } else {670                max_pages = target_pages;671            }672673            if (target_pages > max_pages) {674                target = chunks_perpage * max_pages;675            } else {676                target = chunks_perpage * target_pages;677            }678679            // storage_write() will fail and cut loop after filling write buffer.680            while (1) {681                // if we are low on chunks and no spare, push out early.682                if (chunks_free < target) {683                    item_age = 0;684                } else {685                    item_age = settings.ext_item_age;686                }687                if (storage_write(storage, x, item_age)) {688                    chunks_free++; // Allow stopping if we've done enough this loop689                    check_compact -= chunk_size;690                    // occasionally kick the compact checker.691                    if (check_compact < 0) {692                        pthread_cond_signal(&storage_compact_cond);693                        check_compact = settings.slab_page_size;694                    }695                    did_move = true;696                    do_sleep = false;697                    if (to_sleep > WRITE_SLEEP_MIN)698                        to_sleep /= 2;699                } else {700                    break;701                }702            }703704            if (!did_move) {705                backoff[x]++;706            } else {707                backoff[x] = 1;708            }709        }710711        // flip lock so we can be paused or stopped712        pthread_mutex_unlock(&storage_write_plock);713        if (do_sleep) {714            // Only do backoffs on other slab classes if we're actively715            // flushing at least one class.716            for (int x = 0; x < MAX_NUMBER_OF_SLAB_CLASSES; x++) {717                backoff[x] = 1;718            }719720            // call the compact checker occasionally even if we're just721            // sleeping.722            check_compact -= to_sleep * 10;723            if (check_compact < 0) {724                pthread_cond_signal(&storage_compact_cond);725                check_compact = settings.slab_page_size;726            }727728            usleep(to_sleep);729            to_sleep++;730        }731        pthread_mutex_lock(&storage_write_plock);732    }733    return NULL;734}735736// TODO737// logger needs logger_destroy() to exist/work before this is safe.738/*int stop_storage_write_thread(void) {739    int ret;740    pthread_mutex_lock(&lru_maintainer_lock);741    do_run_lru_maintainer_thread = 0;742    pthread_mutex_unlock(&lru_maintainer_lock);743    // WAKEUP SIGNAL744    if ((ret = pthread_join(lru_maintainer_tid, NULL)) != 0) {745        fprintf(stderr, "Failed to stop LRU maintainer thread: %s\n", strerror(ret));746        return -1;747    }748    settings.lru_maintainer_thread = false;749    return 0;750}*/751752void storage_write_pause(void) {753    pthread_mutex_lock(&storage_write_plock);754}755756void storage_write_resume(void) {757    pthread_mutex_unlock(&storage_write_plock);758}759760int start_storage_write_thread(void *arg) {761    int ret;762763    pthread_mutex_init(&storage_write_plock, NULL);764    if ((ret = pthread_create(&storage_write_tid, NULL,765        storage_write_thread, arg)) != 0) {766        fprintf(stderr, "Can't create storage_write thread: %s\n",767            strerror(ret));768        return -1;769    }770    thread_setname(storage_write_tid, "mc-ext-write");771772    return 0;773}774775/*** COMPACTOR ***/776typedef struct __storage_buk {777    unsigned int bucket;778    unsigned int low_page;779    unsigned int lowest_page;780    uint64_t low_version;781    uint64_t lowest_version;782    unsigned int pages_free;783    unsigned int pages_used;784    unsigned int pages_total;785    unsigned int bytes_fragmented; // fragmented bytes for low page786    bool do_compact; // indicate this bucket should do a compaction.787    bool do_compact_drop;788} _storage_buk;789790struct _compact_flags {791    unsigned int drop_unread : 1;792    unsigned int has_coldcompact : 1;793    unsigned int has_old : 1;794    unsigned int use_old : 1;795};796797/* Fetch stats from the external storage system and decide to compact.798 */799static int storage_compact_check(void *storage, logger *l,800        uint32_t *page_id, uint64_t *page_version,801        uint64_t *page_size, struct _compact_flags *flags) {802    struct extstore_stats st;803    _storage_buk buckets[PAGE_BUCKET_COUNT];804    _storage_buk *buk = NULL;805    uint64_t frag_limit;806    extstore_get_stats(storage, &st);807    if (st.pages_used == 0)808        return 0;809810    for (int x = 0; x < PAGE_BUCKET_COUNT; x++) {811        memset(&buckets[x], 0, sizeof(_storage_buk));812        buckets[x].low_version = ULLONG_MAX;813        buckets[x].lowest_version = ULLONG_MAX;814    }815    flags->drop_unread = 0;816817    frag_limit = st.page_size * settings.ext_max_frag;818    LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_FRAGINFO,819            NULL, settings.ext_max_frag, frag_limit);820    st.page_data = calloc(st.page_count, sizeof(struct extstore_page_data));821    extstore_get_page_data(storage, &st);822823    // find either the most fragmented page or the lowest version.824    for (int x = 0; x < st.page_count; x++) {825        buk = &buckets[st.page_data[x].free_bucket];826        buk->pages_total++;827        if (st.page_data[x].version == 0) {828            buk->pages_free++;829            // free pages don't contribute after this point.830            continue;831        } else {832            buk->pages_used++;833        }834835        // skip pages actively being used.836        if (st.page_data[x].active) {837            continue;838        }839840        if (st.page_data[x].version < buk->lowest_version) {841            buk->lowest_page = x;842            buk->lowest_version = st.page_data[x].version;843        }844        // track the most fragmented page.845        unsigned int frag = st.page_size - st.page_data[x].bytes_used;846        if (st.page_data[x].bytes_used < frag_limit && frag > buk->bytes_fragmented) {847            buk->low_page = x;848            buk->low_version = st.page_data[x].version;849            buk->bytes_fragmented = frag;850        }851    }852    *page_size = st.page_size;853    free(st.page_data);854855    buk = &buckets[PAGE_BUCKET_COLDCOMPACT];856    if (buk->pages_total != 0) {857        flags->has_coldcompact = 1;858        if (buk->pages_free == 0 && buk->lowest_version != ULLONG_MAX) {859            extstore_evict_page(storage, buk->lowest_page, buk->lowest_version);860            return 0;861        }862    }863864    buk = &buckets[PAGE_BUCKET_OLD];865    if (buk->pages_total != 0) {866        flags->has_old = 1;867        if (buk->pages_free == 0 && buk->lowest_version != ULLONG_MAX) {868            extstore_evict_page(storage, buk->lowest_page, buk->lowest_version);869            return 0;870        }871    }872873    for (int x = 0; x < PAGE_BUCKET_COUNT; x++) {874        buk = &buckets[x];875        assert(buk->pages_total == (buk->pages_used + buk->pages_free));876        unsigned int pages_total = buk->pages_total;877        // only process buckets which have dedicated pages assigned.878        // LOWTTL skips compaction.879        if (pages_total == 0 || x == PAGE_BUCKET_LOWTTL)880            continue;881882        if (buk->pages_free < settings.ext_compact_under) {883            if (buk->low_version != ULLONG_MAX) {884                // found a normally defraggable page.885                *page_id = buk->low_page;886                *page_version = buk->low_version;887                return 1;888            } else if (buk->pages_free < settings.ext_drop_under889                    && buk->lowest_version != ULLONG_MAX) {890891                if (x == PAGE_BUCKET_COLDCOMPACT || x == PAGE_BUCKET_OLD) {892                    // this freeing technique doesn't apply to these buckets.893                    // instead these buckets are eviction or normal894                    // defragmentation only.895                    continue;896                }897                // Nothing defraggable. Check for other usable conditions.898                if (settings.ext_drop_unread) {899                    flags->drop_unread = 1;900                }901902                // If OLD and/or COLDCOMPACT pages exist we should always have903                // one free page in those buckets, so we can always attempt to904                // defrag into them.905                // If only COLDCOMPACT exists this will attempt to segment off906                // parts of a page that haven't been used.907                // If OLD exists everything else in this "oldest page" goes908                // into the OLD stream.909                if (flags->drop_unread || flags->has_coldcompact || flags->has_old) {910                    // only actually use the old flag if we can't compact.911                    flags->use_old = flags->has_old;912                    *page_id = buk->lowest_page;913                    *page_version = buk->lowest_version;914                    return 1;915                }916            }917        }918    }919920    return 0;921}922923#define MIN_STORAGE_COMPACT_SLEEP 1000924925struct storage_compact_wrap {926    obj_io io;927    pthread_mutex_t lock; // gates the bools.928    bool done;929    bool submitted;930    bool miss; // version flipped out from under us931};932933static void storage_compact_readback(void *storage, logger *l,934        struct _compact_flags flags, char *readback_buf,935        uint32_t page_id, uint64_t page_version, uint32_t page_offset, uint64_t read_size) {936    uint64_t offset = 0;937    unsigned int rescues = 0;938    // rescues which required a memory realloc (informational)939    unsigned int rescues_realloc = 0;940    unsigned int lost = 0;941    // failed rescues because of OOM vs disk space942    unsigned int lost_oom = 0;943    unsigned int skipped = 0;944    unsigned int rescue_cold = 0;945    unsigned int rescue_old = 0;946    // Sleep this many times during the readback before becoming impatient.947    // Each sleep is 1ms.948    int max_waits = 5000;949950    while (offset < read_size) {951        item *hdr_it = NULL;952        item_hdr *hdr = NULL;953        item *it = (item *)(readback_buf+offset);954        unsigned int ntotal;955        // probably zeroed out junk at the end of the wbuf956        if (it->nkey == 0) {957            break;958        }959960        ntotal = ITEM_ntotal(it);961        uint32_t hv = (uint32_t)it->time;962        item_lock(hv);963        // We don't have a conn and don't need to do most of do_item_get964        hdr_it = assoc_find(ITEM_key(it), it->nkey, hv);965        if (hdr_it != NULL) {966            bool do_write = false;967            int bucket = flags.use_old ? PAGE_BUCKET_OLD : PAGE_BUCKET_COMPACT;968            refcount_incr(hdr_it);969970            // Check validity but don't bother removing it.971            if ((hdr_it->it_flags & ITEM_HDR) && !item_is_flushed(hdr_it) &&972                   (hdr_it->exptime == 0 || hdr_it->exptime > current_time)) {973                hdr = (item_hdr *)ITEM_data(hdr_it);974                if (hdr->page_id == page_id && hdr->page_version == page_version975                        && hdr->offset == (int)offset + page_offset) {976                    // Item header is still completely valid.977                    extstore_delete(storage, page_id, page_version, 1, ntotal);978                    // special case inactive items.979                    do_write = true;980                    if (GET_LRU(hdr_it->slabs_clsid) == COLD_LRU) {981                        if (flags.has_coldcompact) {982                            // Write the cold items to a different stream.983                            bucket = PAGE_BUCKET_COLDCOMPACT;984                        } else if (flags.drop_unread) {985                            do_write = false;986                            skipped++;987                        }988                    }989                }990            }991992            if (do_write) {993                bool do_update = false;994                int tries = max_waits;995                obj_io io;996                io.len = ntotal;997                io.mode = OBJ_IO_WRITE;998                // Wait for a write buffer or page to become available.999                if (tries < 10) {1000                    // Wait a minimum of 10ms1001                    tries = 10;1002                }1003                for (; tries > 0; tries--) {1004                    if (extstore_write_request(storage, bucket, bucket, &io) == 0) {1005                        memcpy(io.buf, it, io.len);1006                        extstore_write(storage, &io);1007                        do_update = true;1008                        break;1009                    } else {1010                        max_waits--;1011                        usleep(1000);1012                    }1013                }10141015                if (do_update) {1016                    bool rescued = false;1017                    if (hdr_it->refcount == 2) {1018                        hdr->page_version = io.page_version;1019                        hdr->page_id = io.page_id;1020                        hdr->offset = io.offset;1021                        rescued = true;1022                    } else {1023                        // re-alloc and replace header.1024                        client_flags_t flags;1025                        FLAGS_CONV(hdr_it, flags);1026                        item *new_it = do_item_alloc(ITEM_key(hdr_it), hdr_it->nkey, flags, hdr_it->exptime, sizeof(item_hdr));1027                        if (new_it) {1028                            // need to preserve the original item flags, but we1029                            // start unlinked, with linked being added during1030                            // item_replace below.1031                            new_it->it_flags = hdr_it->it_flags & (~ITEM_LINKED);1032                            new_it->time = hdr_it->time;1033                            new_it->nbytes = hdr_it->nbytes;10341035                            // copy the hdr data.1036                            item_hdr *new_hdr = (item_hdr *) ITEM_data(new_it);1037                            new_hdr->page_version = io.page_version;1038                            new_hdr->page_id = io.page_id;1039                            new_hdr->offset = io.offset;10401041                            // replace the item in the hash table.1042                            item_replace(hdr_it, new_it, hv, ITEM_get_cas(hdr_it));1043                            do_item_remove(new_it); // release our reference.1044                            rescued = true;1045                            rescues_realloc++;1046                        } else {1047                            lost++;1048                            lost_oom++;1049                        }1050                    }10511052                    if (rescued) {1053                        rescues++;1054                        if (bucket == PAGE_BUCKET_COLDCOMPACT) {1055                            rescue_cold++;1056                        } else if (bucket == PAGE_BUCKET_OLD) {1057                            rescue_old++;1058                        }1059                    }1060                } else {1061                    lost++;1062                }1063            }10641065            do_item_remove(hdr_it);1066        }10671068        item_unlock(hv);1069        offset += ntotal;1070        if (read_size - offset < sizeof(struct _stritem))1071            break;1072    }10731074    STATS_LOCK();1075    stats.extstore_compact_lost += lost;1076    stats.extstore_compact_rescues += rescues;1077    stats.extstore_compact_skipped += skipped;1078    stats.extstore_compact_resc_cold += rescue_cold;1079    stats.extstore_compact_resc_old += rescue_old;1080    STATS_UNLOCK();1081    LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_READ_END,1082            NULL, page_id, offset, rescues, rescues_realloc, lost, lost_oom, skipped);1083}10841085// wrap lock is held while waiting for this callback, preventing caller thread1086// from fast-looping.1087static void _storage_compact_cb(void *e, obj_io *io, int ret) {1088    struct storage_compact_wrap *wrap = (struct storage_compact_wrap *)io->data;1089    assert(wrap->submitted == true);10901091    if (ret < 1) {1092        wrap->miss = true;1093    }1094    wrap->done = true;10951096    pthread_mutex_unlock(&wrap->lock);1097}10981099// TODO: hoist the storage bits from lru_maintainer_thread in here.1100// would be nice if they could avoid hammering the same locks though?1101// I guess it's only COLD. that's probably fine.1102static void *storage_compact_thread(void *arg) {1103    void *storage = arg;1104    bool compacting = false;1105    uint64_t page_version = 0;1106    uint64_t page_size = 0;1107    uint32_t page_offset = 0;1108    uint32_t page_id = 0;1109    struct _compact_flags flags;1110    char *readback_buf = NULL;1111    struct storage_compact_wrap wrap;1112    memset(&flags, 0, sizeof(flags));11131114    logger *l = logger_create();1115    if (l == NULL) {1116        fprintf(stderr, "Failed to allocate logger for storage compaction thread\n");1117        abort();1118    }11191120    readback_buf = malloc(settings.ext_wbuf_size);1121    if (readback_buf == NULL) {1122        fprintf(stderr, "Failed to allocate readback buffer for storage compaction thread\n");1123        abort();1124    }11251126    pthread_mutex_init(&wrap.lock, NULL);1127    wrap.done = false;1128    wrap.submitted = false;1129    wrap.io.data = &wrap;1130    wrap.io.iov = NULL;1131    wrap.io.buf = (void *)readback_buf;11321133    wrap.io.len = settings.ext_wbuf_size;1134    wrap.io.mode = OBJ_IO_READ;1135    wrap.io.cb = _storage_compact_cb;1136    pthread_mutex_lock(&storage_compact_plock);11371138    while (1) {1139        if (!compacting && storage_compact_check(storage, l,1140                    &page_id, &page_version, &page_size, &flags)) {1141            page_offset = 0;1142            compacting = true;1143            LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_START,1144                    NULL, page_id, page_version);1145        } else {1146            pthread_cond_wait(&storage_compact_cond, &storage_compact_plock);1147        }11481149        while (compacting) {1150            pthread_mutex_lock(&wrap.lock);1151            if (page_offset < page_size && !wrap.done && !wrap.submitted) {1152                wrap.io.page_version = page_version;1153                wrap.io.page_id = page_id;1154                wrap.io.offset = page_offset;1155                // FIXME: should be smarter about io->next (unlink at use?)1156                wrap.io.next = NULL;1157                wrap.submitted = true;1158                wrap.miss = false;11591160                extstore_submit_bg(storage, &wrap.io);1161            } else if (wrap.miss) {1162                LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_ABORT,1163                        NULL, page_id);1164                wrap.done = false;1165                wrap.submitted = false;1166                compacting = false;1167            } else if (wrap.submitted && wrap.done) {1168                LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_READ_START,1169                        NULL, page_id, page_offset);1170                storage_compact_readback(storage, l, flags,1171                        readback_buf, page_id, page_version, page_offset,1172                        settings.ext_wbuf_size);1173                page_offset += settings.ext_wbuf_size;1174                wrap.done = false;1175                wrap.submitted = false;1176            } else if (page_offset >= page_size) {1177                compacting = false;1178                wrap.done = false;1179                wrap.submitted = false;1180                extstore_close_page(storage, page_id, page_version);1181                LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_END,1182                        NULL, page_id);1183                // short cooling period between defragmentation runs.1184                usleep(MIN_STORAGE_COMPACT_SLEEP);1185            }1186            pthread_mutex_unlock(&wrap.lock);1187        }1188    }1189    free(readback_buf);11901191    return NULL;1192}11931194// TODO1195// logger needs logger_destroy() to exist/work before this is safe.1196/*int stop_storage_compact_thread(void) {1197    int ret;1198    pthread_mutex_lock(&lru_maintainer_lock);1199    do_run_lru_maintainer_thread = 0;1200    pthread_mutex_unlock(&lru_maintainer_lock);1201    if ((ret = pthread_join(lru_maintainer_tid, NULL)) != 0) {1202        fprintf(stderr, "Failed to stop LRU maintainer thread: %s\n", strerror(ret));1203        return -1;1204    }1205    settings.lru_maintainer_thread = false;1206    return 0;1207}*/12081209void storage_compact_pause(void) {1210    pthread_mutex_lock(&storage_compact_plock);1211}12121213void storage_compact_resume(void) {1214    pthread_mutex_unlock(&storage_compact_plock);1215}12161217int start_storage_compact_thread(void *arg) {1218    int ret;12191220    pthread_mutex_init(&storage_compact_plock, NULL);1221    pthread_cond_init(&storage_compact_cond, NULL);1222    if ((ret = pthread_create(&storage_compact_tid, NULL,1223        storage_compact_thread, arg)) != 0) {1224        fprintf(stderr, "Can't create storage_compact thread: %s\n",1225            strerror(ret));1226        return -1;1227    }1228    thread_setname(storage_compact_tid, "mc-ext-compact");12291230    return 0;1231}12321233/*** UTILITY ***/1234// /path/to/file:100G:bucket11235// FIXME: Modifies argument. copy instead?1236struct extstore_conf_file *storage_conf_parse(char *arg) {1237    struct extstore_conf_file *cf = NULL;1238    char *b = NULL;1239    char *p = strtok_r(arg, ":", &b);1240    char unit = 0;1241    uint64_t multiplier = 0;1242    int base_size = 0;1243    if (p == NULL)1244        goto error;1245    // First arg is the filepath.1246    cf = calloc(1, sizeof(struct extstore_conf_file));1247    cf->file = strdup(p);12481249    p = strtok_r(NULL, ":", &b);1250    if (p == NULL) {1251        fprintf(stderr, "must supply size to ext_path, ie: ext_path=/f/e:64m (M|G|T|P supported)\n");1252        goto error;1253    }1254    unit = tolower(p[strlen(p)-1]);1255    p[strlen(p)-1] = '\0';1256    // sigh.1257    switch (unit) {1258        case 'm':1259            multiplier = 1024 * 1024;1260            break;1261        case 'g':1262            multiplier = 1024 * 1024 * 1024;1263            break;1264        case 't':1265            multiplier = 1024 * 1024;1266            multiplier *= 1024 * 1024;1267            break;1268        case 'p':1269            multiplier = 1024 * 1024;1270            multiplier *= 1024 * 1024 * 1024;1271            break;1272        default:1273            fprintf(stderr, "must supply size to ext_path, ie: ext_path=/f/e:64m (m|g|t|p supported)\n");1274            goto error;1275    }1276    base_size = atoi(p);1277    cf->total_size = multiplier * base_size;12781279    // final token would be a default free bucket1280    p = strtok_r(NULL, ":", &b);1281    // TODO: We reuse the original DEFINES for now,1282    // but if lowttl gets split up this needs to be its own set.1283    if (p != NULL) {1284        if (strcmp(p, "compact") == 0) {1285            cf->free_bucket = PAGE_BUCKET_COMPACT;1286        } else if (strcmp(p, "lowttl") == 0) {1287            cf->free_bucket = PAGE_BUCKET_LOWTTL;1288        } else if (strcmp(p, "chunked") == 0) {1289            cf->free_bucket = PAGE_BUCKET_CHUNKED;1290        } else if (strcmp(p, "default") == 0) {1291            cf->free_bucket = PAGE_BUCKET_DEFAULT;1292        } else if (strcmp(p, "coldcompact") == 0) {1293            cf->free_bucket = PAGE_BUCKET_COLDCOMPACT;1294        } else if (strcmp(p, "old") == 0) {1295            cf->free_bucket = PAGE_BUCKET_OLD;1296        } else {1297            fprintf(stderr, "Unknown extstore bucket: %s\n", p);1298            goto error;1299        }1300    } else {1301        // TODO: is this necessary?1302        cf->free_bucket = PAGE_BUCKET_DEFAULT;1303    }13041305    return cf;1306error:1307    if (cf) {1308        if (cf->file)1309            free(cf->file);1310        free(cf);1311    }1312    return NULL;1313}13141315struct storage_settings {1316    struct extstore_conf_file *storage_file;1317    struct extstore_conf ext_cf;1318};13191320void *storage_init_config(struct settings *s) {1321    struct storage_settings *cf = calloc(1, sizeof(struct storage_settings));13221323    s->ext_item_size = 512;1324    s->ext_item_age = UINT_MAX;1325    s->ext_low_ttl = 0;1326    s->ext_recache_rate = 2000;1327    s->ext_max_frag = 0.8;1328    s->ext_drop_unread = false;1329    s->ext_wbuf_size = 1024 * 1024 * 4;1330    s->ext_compact_under = 0;1331    s->ext_drop_under = 0;1332    s->ext_max_sleep = 1000000;1333    s->slab_automove_freeratio = 0.01;1334    s->ext_page_size = 1024 * 1024 * 64;1335    s->ext_io_threadcount = 1;1336    cf->ext_cf.page_size = settings.ext_page_size;1337    cf->ext_cf.wbuf_size = settings.ext_wbuf_size;1338    cf->ext_cf.io_threadcount = settings.ext_io_threadcount;1339    cf->ext_cf.io_depth = 1;1340    cf->ext_cf.page_buckets = PAGE_BUCKET_COUNT;1341    cf->ext_cf.wbuf_count = cf->ext_cf.page_buckets;13421343    return cf;1344}13451346// TODO: pass settings struct?1347int storage_read_config(void *conf, char **subopt) {1348    struct storage_settings *cf = conf;1349    struct extstore_conf *ext_cf = &cf->ext_cf;1350    char *subopts_value;13511352    enum {1353        EXT_PAGE_SIZE,1354        EXT_WBUF_SIZE,1355        EXT_THREADS,1356        EXT_IO_DEPTH,1357        EXT_PATH,1358        EXT_ITEM_SIZE,1359        EXT_ITEM_AGE,1360        EXT_LOW_TTL,1361        EXT_RECACHE_RATE,1362        EXT_COMPACT_UNDER,1363        EXT_DROP_UNDER,1364        EXT_MAX_SLEEP,1365        EXT_MAX_FRAG,1366        EXT_DROP_UNREAD,1367        SLAB_AUTOMOVE_FREERATIO, // FIXME: move this back?1368    };13691370    char *const subopts_tokens[] = {1371        [EXT_PAGE_SIZE] = "ext_page_size",1372        [EXT_WBUF_SIZE] = "ext_wbuf_size",1373        [EXT_THREADS] = "ext_threads",1374        [EXT_IO_DEPTH] = "ext_io_depth",1375        [EXT_PATH] = "ext_path",1376        [EXT_ITEM_SIZE] = "ext_item_size",1377        [EXT_ITEM_AGE] = "ext_item_age",1378        [EXT_LOW_TTL] = "ext_low_ttl",1379        [EXT_RECACHE_RATE] = "ext_recache_rate",1380        [EXT_COMPACT_UNDER] = "ext_compact_under",1381        [EXT_DROP_UNDER] = "ext_drop_under",1382        [EXT_MAX_SLEEP] = "ext_max_sleep",1383        [EXT_MAX_FRAG] = "ext_max_frag",1384        [EXT_DROP_UNREAD] = "ext_drop_unread",1385        [SLAB_AUTOMOVE_FREERATIO] = "slab_automove_freeratio",1386        NULL1387    };13881389    switch (getsubopt(subopt, subopts_tokens, &subopts_value)) {1390        case EXT_PAGE_SIZE:1391            if (subopts_value == NULL) {1392                fprintf(stderr, "Missing ext_page_size argument\n");1393                return 1;1394            }1395            if (!safe_strtoul(subopts_value, &ext_cf->page_size)) {1396                fprintf(stderr, "could not parse argument to ext_page_size\n");1397                return 1;1398            }1399            ext_cf->page_size *= 1024 * 1024; /* megabytes */1400            break;1401        case EXT_WBUF_SIZE:1402            if (subopts_value == NULL) {1403                fprintf(stderr, "Missing ext_wbuf_size argument\n");1404                return 1;1405            }1406            if (!safe_strtoul(subopts_value, &ext_cf->wbuf_size)) {1407                fprintf(stderr, "could not parse argument to ext_wbuf_size\n");1408                return 1;1409            }1410            ext_cf->wbuf_size *= 1024 * 1024; /* megabytes */1411            settings.ext_wbuf_size = ext_cf->wbuf_size;1412            break;1413        case EXT_THREADS:1414            if (subopts_value == NULL) {1415                fprintf(stderr, "Missing ext_threads argument\n");1416                return 1;1417            }1418            if (!safe_strtoul(subopts_value, &ext_cf->io_threadcount)) {1419                fprintf(stderr, "could not parse argument to ext_threads\n");1420                return 1;1421            }1422            break;1423        case EXT_IO_DEPTH:1424            if (subopts_value == NULL) {1425                fprintf(stderr, "Missing ext_io_depth argument\n");1426                return 1;1427            }1428            if (!safe_strtoul(subopts_value, &ext_cf->io_depth)) {1429                fprintf(stderr, "could not parse argument to ext_io_depth\n");1430                return 1;1431            }1432            break;1433        case EXT_ITEM_SIZE:1434            if (subopts_value == NULL) {1435                fprintf(stderr, "Missing ext_item_size argument\n");1436                return 1;1437            }1438            if (!safe_strtoul(subopts_value, &settings.ext_item_size)) {1439                fprintf(stderr, "could not parse argument to ext_item_size\n");1440                return 1;1441            }1442            break;1443        case EXT_ITEM_AGE:1444            if (subopts_value == NULL) {1445                fprintf(stderr, "Missing ext_item_age argument\n");1446                return 1;1447            }1448            if (!safe_strtoul(subopts_value, &settings.ext_item_age)) {1449                fprintf(stderr, "could not parse argument to ext_item_age\n");1450                return 1;1451            }1452            break;1453        case EXT_LOW_TTL:1454            if (subopts_value == NULL) {1455                fprintf(stderr, "Missing ext_low_ttl argument\n");1456                return 1;1457            }1458            if (!safe_strtoul(subopts_value, &settings.ext_low_ttl)) {1459                fprintf(stderr, "could not parse argument to ext_low_ttl\n");1460                return 1;1461            }1462            break;1463        case EXT_RECACHE_RATE:1464            if (subopts_value == NULL) {1465                fprintf(stderr, "Missing ext_recache_rate argument\n");1466                return 1;1467            }1468            if (!safe_strtoul(subopts_value, &settings.ext_recache_rate)) {1469                fprintf(stderr, "could not parse argument to ext_recache_rate\n");1470                return 1;1471            }1472            break;1473        case EXT_COMPACT_UNDER:1474            if (subopts_value == NULL) {1475                fprintf(stderr, "Missing ext_compact_under argument\n");1476                return 1;1477            }1478            if (!safe_strtoul(subopts_value, &settings.ext_compact_under)) {1479                fprintf(stderr, "could not parse argument to ext_compact_under\n");1480                return 1;1481            }1482            break;1483        case EXT_DROP_UNDER:1484            if (subopts_value == NULL) {1485                fprintf(stderr, "Missing ext_drop_under argument\n");1486                return 1;1487            }1488            if (!safe_strtoul(subopts_value, &settings.ext_drop_under)) {1489                fprintf(stderr, "could not parse argument to ext_drop_under\n");1490                return 1;1491            }1492            break;1493        case EXT_MAX_SLEEP:1494            if (subopts_value == NULL) {1495                fprintf(stderr, "Missing ext_max_sleep argument\n");1496                return 1;1497            }1498            if (!safe_strtoul(subopts_value, &settings.ext_max_sleep)) {1499                fprintf(stderr, "could not parse argument to ext_max_sleep\n");1500                return 1;1501            }1502            break;1503        case EXT_MAX_FRAG:1504            if (subopts_value == NULL) {1505                fprintf(stderr, "Missing ext_max_frag argument\n");1506                return 1;1507            }1508            if (!safe_strtod(subopts_value, &settings.ext_max_frag)) {1509                fprintf(stderr, "could not parse argument to ext_max_frag\n");1510                return 1;1511            }1512            break;1513        case SLAB_AUTOMOVE_FREERATIO:1514            if (subopts_value == NULL) {1515                fprintf(stderr, "Missing slab_automove_freeratio argument\n");1516                return 1;1517            }1518            if (!safe_strtod(subopts_value, &settings.slab_automove_freeratio)) {1519                fprintf(stderr, "could not parse argument to slab_automove_freeratio\n");1520                return 1;1521            }1522            break;1523        case EXT_DROP_UNREAD:1524            settings.ext_drop_unread = true;1525            break;1526        case EXT_PATH:1527            if (subopts_value) {1528                struct extstore_conf_file *tmp = storage_conf_parse(subopts_value);1529                if (tmp == NULL) {1530                    fprintf(stderr, "failed to parse ext_path argument\n");1531                    return 1;1532                }1533                if (cf->storage_file != NULL) {1534                    tmp->next = cf->storage_file;1535                }1536                cf->storage_file = tmp;1537            } else {1538                fprintf(stderr, "missing argument to ext_path, ie: ext_path=/d/file:5G\n");1539                return 1;1540            }1541            break;1542        default:1543            fprintf(stderr, "Illegal suboption \"%s\"\n", subopts_value);1544            return 1;1545    }15461547    return 0;1548}15491550int storage_check_config(void *conf) {1551    struct storage_settings *cf = conf;1552    struct extstore_conf *ext_cf = &cf->ext_cf;15531554    if (cf->storage_file) {1555        if (settings.item_size_max > ext_cf->wbuf_size) {1556            fprintf(stderr, "-I (item_size_max: %d) cannot be larger than ext_wbuf_size: %d\n",1557                settings.item_size_max, ext_cf->wbuf_size);1558            return 1;1559        }15601561        if (settings.udpport) {1562            fprintf(stderr, "Cannot use UDP with extstore enabled (-U 0 to disable)\n");1563            return 1;1564        }15651566        struct extstore_conf_file *fh = cf->storage_file;1567        while (fh != NULL) {1568            // page_count is nearest-but-not-larger-than pages * psize1569            fh->page_count = fh->total_size / cf->ext_cf.page_size;1570            assert(cf->ext_cf.page_size * fh->page_count <= fh->total_size);1571            if (fh->page_count == 0) {1572                fprintf(stderr, "supplied ext_path has zero size, cannot use\n");1573                return 1;1574            }1575            fh = fh->next;1576        }15771578        return 0;1579    }15801581    return 2;1582}15831584void *storage_init(void *conf) {1585    struct storage_settings *cf = conf;1586    struct extstore_conf *ext_cf = &cf->ext_cf;15871588    enum extstore_res eres;1589    void *storage = NULL;1590    if (settings.ext_compact_under == 0) {1591        // If changing the default fraction, change the help text as well.1592        settings.ext_compact_under = cf->storage_file->page_count * 0.01;1593        settings.ext_drop_under = cf->storage_file->page_count * 0.01;1594        if (settings.ext_compact_under < 1) {1595            settings.ext_compact_under = 1;1596        }1597        if (settings.ext_drop_under < 1) {1598            settings.ext_drop_under = 1;1599        }1600    }1601    crc32c_init();16021603    settings.ext_global_pool_min = 0;1604    storage = extstore_init(cf->storage_file, ext_cf, &eres);1605    if (storage == NULL) {1606        fprintf(stderr, "Failed to initialize external storage: %s\n",1607                extstore_err(eres));1608        if (eres == EXTSTORE_INIT_OPEN_FAIL) {1609            perror("extstore open");1610        }1611        return NULL;1612    }16131614    return storage;1615}16161617#endif