1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */2/*3 * Functions for handling the proxy layer. wraps text protocols4 *5 * NOTE: many lua functions generate pointers via "lua_newuserdatauv" or6 * similar. Normal memory checking isn't done as lua will throw a high level7 * error if malloc fails. Must keep this in mind while allocating data so any8 * manually malloc'ed information gets freed properly.9 */1011#include "proxy.h"1213#define PROCESS_MULTIGET true14#define PROCESS_NORMAL false15#define PROXY_GC_BACKGROUND_SECONDS 416#define PROXY_GC_DEFAULT_RATIO 2.017static void proxy_process_command(conn *c, char *command, size_t cmdlen, bool multiget);18static void *mcp_profile_alloc(void *ud, void *ptr, size_t osize, size_t nsize);1920/******** EXTERNAL FUNCTIONS ******/21// functions starting with _ are breakouts for the public functions.2223static inline void _proxy_advance_lastkb(lua_State *L, LIBEVENT_THREAD *t) {24    int new_kb = lua_gc(L, LUA_GCCOUNT);25    // We need to slew the increase in "gc pause" because the lua GC actually26    // needs to run twice to free a userdata: once to run the _gc's and again27    // to actually clean up the object.28    // Meaning we will continually increase in size.29    if (new_kb > t->proxy_vm_last_kb) {30        new_kb = t->proxy_vm_last_kb + (new_kb - t->proxy_vm_last_kb) * 0.50;31    }3233    // remove the memory freed during this cycle so we can kick off the GC34    // early if we're very aggressively making garbage.35    // carry our negative delta forward so a huge reclaim can push for a36    // couple cycles.37    if (t->proxy_vm_negative_delta >= new_kb) {38        t->proxy_vm_negative_delta -= new_kb;39        new_kb = 1;40    } else {41        new_kb -= t->proxy_vm_negative_delta;42        t->proxy_vm_negative_delta = 0;43    }4445    t->proxy_vm_last_kb = new_kb;46}4748// The lua GC is paused while running requests. Run it manually inbetween49// processing network events.50void proxy_gc_poke(LIBEVENT_THREAD *t) {51    lua_State *L = t->L;52    proxy_ctx_t *ctx = t->proxy_ctx;53    float ratio = ctx->tunables.gc_ratio;54    struct proxy_int_stats *is = t->proxy_int_stats;55    int vm_kb = lua_gc(L, LUA_GCCOUNT) + t->proxy_vm_extra_kb;56    WSTAT_L(t);57    is->vm_memory_kb = vm_kb;58    WSTAT_UL(t);5960    // equivalent of luagc "pause" value61    int last = t->proxy_vm_last_kb;62    if (t->proxy_vm_gcrunning <= 0 && vm_kb > last * ratio) {63        t->proxy_vm_gcrunning = 1;64        //fprintf(stderr, "PROXYGC: proxy_gc_poke START [cur: %d - last: %d - ratio: %f]\n", vm_kb, last, ratio);65    }6667    // We configure small GC "steps" then increase the number of times we run68    // a step based on current memory usage.69    if (t->proxy_vm_gcrunning > 0) {70        t->proxy_vm_gcpokemem = 0;71        int loops = t->proxy_vm_gcrunning;72        int done = 0;73        /*fprintf(stderr, "PROXYGC: proxy_gc_poke [cur: %d - last: %d - loops: %d]\n",74            vm_kb,75            t->proxy_vm_last_kb,76            loops);*/77        while (loops-- && !done) {78            // reset counters once full GC cycle has completed79            done = lua_gc(L, LUA_GCSTEP, 0);80        }8182        int vm_kb_after = lua_gc(L, LUA_GCCOUNT);83        int vm_kb_clean = vm_kb - t->proxy_vm_extra_kb;84        if (vm_kb_clean > vm_kb_after) {85            // track the amount of memory freed during the GC cycle.86            t->proxy_vm_negative_delta += vm_kb_clean - vm_kb_after;87        }8889        if (done) {90            _proxy_advance_lastkb(L, t);91            t->proxy_vm_extra_kb = 0;92            t->proxy_vm_gcrunning = 0;93            WSTAT_L(t);94            is->vm_gc_runs++;95            WSTAT_UL(t);96            //fprintf(stderr, "PROXYGC: proxy_gc_poke COMPLETE [cur: %d next: %d]\n", lua_gc(L, LUA_GCCOUNT), t->proxy_vm_last_kb);97        } else if ((last*ratio) * (1 + t->proxy_vm_gcrunning*0.20) < vm_kb) {98            // increase the aggressiveness by memory bloat level.99            t->proxy_vm_gcrunning++;100            //fprintf(stderr, "PROXYGC: proxy_gc_poke INCREASING AGGRESSIVENESS [cur: %d - aggro: %d]\n", t->proxy_vm_last_kb, t->proxy_vm_gcrunning);101        }102    }103}104105// every couple seconds we force-run one GC step.106// this is needed until after API1 is retired and pool objects are no longer107// managed by the GC.108// We use a negative value so a "timer poke" GC run doesn't cause requests to109// suddenly aggressively run the GC.110static void proxy_gc_timerpoke(evutil_socket_t fd, short event, void *arg) {111    LIBEVENT_THREAD *t = arg;112    struct timeval next = { PROXY_GC_BACKGROUND_SECONDS, 0 };113    evtimer_add(t->proxy_gc_timer, &next);114    // if GC ran recently, don't do anything.115    // also if memory changed recently, don't do anything.116    int curmem = lua_gc(t->L, LUA_GCCOUNT);117    if (t->proxy_vm_gcpokemem == 0 || curmem != t->proxy_vm_gcpokemem) {118        t->proxy_vm_gcpokemem = curmem;119        return;120    }121122    // if we weren't told to skip and there's otherwise no GC running, start a123    // GC run.124    if (t->proxy_vm_gcrunning == 0) {125        t->proxy_vm_gcrunning = -1;126    }127128    // only advance GC if we're doing our own timer run.129    if (t->proxy_vm_gcrunning == -1) {130        if (lua_gc(t->L, LUA_GCSTEP, 0)) {131            // don't advance last_kb: let the main algo decide when to run.132            t->proxy_vm_gcrunning = 0;133            t->proxy_vm_gcpokemem = 0;134        } else {135            // only continue running if memory stays where we expect it.136            t->proxy_vm_gcpokemem = lua_gc(t->L, LUA_GCCOUNT);137        }138    }139}140141bool proxy_bufmem_checkadd(LIBEVENT_THREAD *t, int len) {142    bool oom = false;143    if (t->proxy_buffer_memory_used > t->proxy_buffer_memory_limit) {144        oom = true;145    } else {146        t->proxy_buffer_memory_used += len;147    }148    return oom;149}150151// see also: process_extstore_stats()152void proxy_stats(void *arg, ADD_STAT add_stats, void *c) {153    if (arg == NULL) {154       return;155    }156    proxy_ctx_t *ctx = arg;157158    STAT_L(ctx);159    APPEND_STAT("proxy_config_reloads", "%llu", (unsigned long long)ctx->global_stats.config_reloads);160    APPEND_STAT("proxy_config_reload_fails", "%llu", (unsigned long long)ctx->global_stats.config_reload_fails);161    APPEND_STAT("proxy_config_cron_runs", "%llu", (unsigned long long)ctx->global_stats.config_cron_runs);162    APPEND_STAT("proxy_config_cron_fails", "%llu", (unsigned long long)ctx->global_stats.config_cron_fails);163    APPEND_STAT("proxy_backend_total", "%llu", (unsigned long long)ctx->global_stats.backend_total);164    APPEND_STAT("proxy_backend_marked_bad", "%llu", (unsigned long long)ctx->global_stats.backend_marked_bad);165    APPEND_STAT("proxy_backend_failed", "%llu", (unsigned long long)ctx->global_stats.backend_failed);166    APPEND_STAT("proxy_request_failed_depth", "%llu", (unsigned long long)ctx->global_stats.request_failed_depth);167    STAT_UL(ctx);168}169170void process_proxy_stats(void *arg, ADD_STAT add_stats, void *c) {171    char key_str[STAT_KEY_LEN];172    struct proxy_int_stats istats = {0};173    int64_t req_limit = 0;174    uint64_t buffer_memory_limit = 0;175    uint64_t buffer_memory_used = 0;176177    if (!arg) {178        return;179    }180    proxy_ctx_t *ctx = arg;181    STAT_L(ctx);182    req_limit = ctx->active_req_limit;183    buffer_memory_limit = ctx->buffer_memory_limit;184185    // prepare aggregated counters.186    struct proxy_user_stats_entry *us = ctx->user_stats;187    int stats_num = ctx->user_stats_num;188    uint64_t counters[stats_num];189    memset(counters, 0, sizeof(counters));190191    // TODO (v3): more globals to remove and/or change API method.192    // aggregate worker thread counters.193    for (int x = 0; x < settings.num_threads; x++) {194        LIBEVENT_THREAD *t = get_worker_thread(x);195        struct proxy_user_stats *tus = t->proxy_user_stats;196        struct proxy_int_stats *is = t->proxy_int_stats;197        WSTAT_L(t);198        for (int i = 0; i < CMD_FINAL; i++) {199            istats.counters[i] += is->counters[i];200        }201        istats.vm_gc_runs += is->vm_gc_runs;202        istats.vm_memory_kb += is->vm_memory_kb;203        if (tus && tus->num_stats >= stats_num) {204            for (int i = 0; i < stats_num; i++) {205                counters[i] += tus->counters[i];206            }207        }208        WSTAT_UL(t);209        buffer_memory_used += t->proxy_buffer_memory_used;210    }211212    // return all of the user generated stats213    if (ctx->user_stats_namebuf) {214        char vbuf[INCR_MAX_STORAGE_LEN];215        char *e = NULL; // ptr into vbuf216        const char *pfx = "user_";217        const size_t pfxlen = strlen(pfx);218        for (int x = 0; x < stats_num; x++) {219            if (us[x].cname) {220                char *name = ctx->user_stats_namebuf + us[x].cname;221                size_t nlen = strlen(name);222                if (nlen > STAT_KEY_LEN-6) {223                    // impossible, but for paranoia.224                    nlen = STAT_KEY_LEN-6;225                }226                // avoiding an snprintf call for some performance ("user_%s")227                memcpy(key_str, pfx, pfxlen);228                memcpy(key_str+pfxlen, name, nlen);229                key_str[pfxlen+nlen] = '\0';230231                // APPEND_STAT() calls another snprintf, which calls our232                // add_stats argument. Lets skip yet another snprintf with233                // some unrolling.234                e = itoa_u64(counters[x], vbuf);235                *(e+1) = '\0';236                add_stats(key_str, pfxlen+nlen, vbuf, e-vbuf, c);237            }238        }239    }240241    STAT_UL(ctx);242243    if (buffer_memory_limit == UINT64_MAX) {244        buffer_memory_limit = 0;245    } else {246        buffer_memory_limit *= settings.num_threads;247    }248    if (req_limit == INT64_MAX) {249        req_limit = 0;250    } else {251        req_limit *= settings.num_threads;252    }253254    // return proxy counters255    APPEND_STAT("active_req_limit", "%lld", (long long int)req_limit);256    APPEND_STAT("buffer_memory_limit", "%llu", (unsigned long long)buffer_memory_limit);257    APPEND_STAT("buffer_memory_used", "%llu", (unsigned long long)buffer_memory_used);258    APPEND_STAT("vm_gc_runs", "%llu", (unsigned long long)istats.vm_gc_runs);259    APPEND_STAT("vm_memory_kb", "%llu", (unsigned long long)istats.vm_memory_kb);260    APPEND_STAT("cmd_mg", "%llu", (unsigned long long)istats.counters[CMD_MG]);261    APPEND_STAT("cmd_ms", "%llu", (unsigned long long)istats.counters[CMD_MS]);262    APPEND_STAT("cmd_md", "%llu", (unsigned long long)istats.counters[CMD_MD]);263    APPEND_STAT("cmd_mn", "%llu", (unsigned long long)istats.counters[CMD_MN]);264    APPEND_STAT("cmd_ma", "%llu", (unsigned long long)istats.counters[CMD_MA]);265    APPEND_STAT("cmd_me", "%llu", (unsigned long long)istats.counters[CMD_ME]);266    APPEND_STAT("cmd_get", "%llu", (unsigned long long)istats.counters[CMD_GET]);267    APPEND_STAT("cmd_gat", "%llu", (unsigned long long)istats.counters[CMD_GAT]);268    APPEND_STAT("cmd_set", "%llu", (unsigned long long)istats.counters[CMD_SET]);269    APPEND_STAT("cmd_add", "%llu", (unsigned long long)istats.counters[CMD_ADD]);270    APPEND_STAT("cmd_cas", "%llu", (unsigned long long)istats.counters[CMD_CAS]);271    APPEND_STAT("cmd_gets", "%llu", (unsigned long long)istats.counters[CMD_GETS]);272    APPEND_STAT("cmd_gats", "%llu", (unsigned long long)istats.counters[CMD_GATS]);273    APPEND_STAT("cmd_incr", "%llu", (unsigned long long)istats.counters[CMD_INCR]);274    APPEND_STAT("cmd_decr", "%llu", (unsigned long long)istats.counters[CMD_DECR]);275    APPEND_STAT("cmd_touch", "%llu", (unsigned long long)istats.counters[CMD_TOUCH]);276    APPEND_STAT("cmd_append", "%llu", (unsigned long long)istats.counters[CMD_APPEND]);277    APPEND_STAT("cmd_prepend", "%llu", (unsigned long long)istats.counters[CMD_PREPEND]);278    APPEND_STAT("cmd_delete", "%llu", (unsigned long long)istats.counters[CMD_DELETE]);279    APPEND_STAT("cmd_replace", "%llu", (unsigned long long)istats.counters[CMD_REPLACE]);280}281282void process_proxy_funcstats(void *arg, ADD_STAT add_stats, void *c) {283    char key_str[STAT_KEY_LEN];284    if (!arg) {285        return;286    }287    proxy_ctx_t *ctx = arg;288    lua_State *L = ctx->proxy_sharedvm;289    pthread_mutex_lock(&ctx->sharedvm_lock);290291    // iterate all of the named function slots292    lua_pushnil(L);293    while (lua_next(L, SHAREDVM_FGEN_IDX) != 0) {294        int n = lua_tointeger(L, -1);295        lua_pop(L, 1); // drop the value, leave the key.296        if (n != 0) {297            // reuse the key. make a copy since rawget will pop it.298            lua_pushvalue(L, -1);299            lua_rawget(L, SHAREDVM_FGENSLOT_IDX);300            int slots = lua_tointeger(L, -1);301            lua_pop(L, 1); // drop the slot count.302303            // now grab the name key.304            const char *name = lua_tostring(L, -1);305            snprintf(key_str, STAT_KEY_LEN-1, "funcs_%s", name);306            APPEND_STAT(key_str, "%d", n);307            snprintf(key_str, STAT_KEY_LEN-1, "slots_%s", name);308            APPEND_STAT(key_str, "%d", slots);309        } else {310            // TODO: It is safe to delete keys here. Slightly complex so low311            // priority.312        }313    }314315    pthread_mutex_unlock(&ctx->sharedvm_lock);316}317318void process_proxy_bestats(void *arg, ADD_STAT add_stats, void *c) {319    char key_str[STAT_KEY_LEN];320    if (!arg) {321        return;322    }323    proxy_ctx_t *ctx = arg;324    lua_State *L = ctx->proxy_sharedvm;325    pthread_mutex_lock(&ctx->sharedvm_lock);326327    // iterate all of the listed backends328    lua_pushnil(L);329    while (lua_next(L, SHAREDVM_BACKEND_IDX) != 0) {330        int n = lua_tointeger(L, -1);331        lua_pop(L, 1); // drop the value, leave the key.332        if (n != 0) {333            // now grab the name key.334            const char *name = lua_tostring(L, -1);335            snprintf(key_str, STAT_KEY_LEN-1, "bad_%s", name);336            APPEND_STAT(key_str, "%d", n);337        } else {338            // delete keys of backends that are no longer bad or no longer339            // exist to keep the table small.340            const char *name = lua_tostring(L, -1);341            lua_pushnil(L);342            lua_setfield(L, SHAREDVM_BACKEND_IDX, name);343        }344    }345346    pthread_mutex_unlock(&ctx->sharedvm_lock);347}348349// start the centralized lua state and config thread.350void *proxy_init(bool use_uring, bool proxy_memprofile) {351    proxy_ctx_t *ctx = calloc(1, sizeof(proxy_ctx_t));352    ctx->use_uring = use_uring;353    ctx->memprofile = proxy_memprofile;354355    pthread_mutex_init(&ctx->config_lock, NULL);356    pthread_cond_init(&ctx->config_cond, NULL);357    pthread_mutex_init(&ctx->worker_lock, NULL);358    pthread_cond_init(&ctx->worker_cond, NULL);359    pthread_mutex_init(&ctx->manager_lock, NULL);360    pthread_cond_init(&ctx->manager_cond, NULL);361    pthread_mutex_init(&ctx->stats_lock, NULL);362363    ctx->active_req_limit = INT64_MAX;364    ctx->buffer_memory_limit = UINT64_MAX;365366    // FIXME (v2): default defines.367    ctx->tunables.tcp_keepalive = false;368    ctx->tunables.backend_failure_limit = 3;369    ctx->tunables.connect.tv_sec = 5;370    ctx->tunables.retry.tv_sec = 3;371    ctx->tunables.read.tv_sec = 3;372    ctx->tunables.flap_backoff_ramp = 1.5;373    ctx->tunables.flap_backoff_max = 3600;374    ctx->tunables.gc_ratio = PROXY_GC_DEFAULT_RATIO;375    ctx->tunables.backend_depth_limit = 0;376    ctx->tunables.max_ustats = MAX_USTATS_DEFAULT;377    ctx->tunables.use_iothread = false;378    ctx->tunables.use_tls = false;379380    STAILQ_INIT(&ctx->manager_head);381    lua_State *L = NULL;382    if (ctx->memprofile) {383        struct mcp_memprofile *prof = calloc(1, sizeof(struct mcp_memprofile));384        prof->id = ctx->memprofile_thread_counter++;385        L = lua_newstate(mcp_profile_alloc, prof);386    } else {387        L = luaL_newstate();388    }389    ctx->proxy_state = L;390    luaL_openlibs(L);391    // NOTE: might need to differentiate the libs yes?392    proxy_register_libs(ctx, NULL, L);393    // Create the cron table.394    lua_newtable(L);395    ctx->cron_ref = luaL_ref(L, LUA_REGISTRYINDEX);396    ctx->cron_next = INT_MAX;397398    // set up the shared state VM. Used by short-lock events (counters/state)399    // for global visibility.400    pthread_mutex_init(&ctx->sharedvm_lock, NULL);401    ctx->proxy_sharedvm = luaL_newstate();402    luaL_openlibs(ctx->proxy_sharedvm);403    // we keep info tables in the top level stack so we don't have to404    // constantly fetch them from registry.405    lua_newtable(ctx->proxy_sharedvm); // fgen count406    lua_newtable(ctx->proxy_sharedvm); // fgen slot count407    lua_newtable(ctx->proxy_sharedvm); // backend down status408409    // Create/start the IO thread, which we need before servers410    // start getting created.411    proxy_event_thread_t *t = calloc(1, sizeof(proxy_event_thread_t));412    ctx->proxy_io_thread = t;413    proxy_init_event_thread(t, ctx, NULL);414415    pthread_create(&t->thread_id, NULL, proxy_event_thread, t);416    thread_setname(t->thread_id, "mc-prx-io");417418    _start_proxy_config_threads(ctx);419    return ctx;420}421422// Initialize the VM for an individual worker thread.423void proxy_thread_init(void *ctx, LIBEVENT_THREAD *thr) {424    assert(ctx != NULL);425    assert(thr != NULL);426427    // Create the hook table.428    thr->proxy_hooks = calloc(CMD_SIZE, sizeof(struct proxy_hook));429    if (thr->proxy_hooks == NULL) {430        fprintf(stderr, "Failed to allocate proxy hooks\n");431        exit(EXIT_FAILURE);432    }433    thr->proxy_int_stats = calloc(1, sizeof(struct proxy_int_stats));434    if (thr->proxy_int_stats == NULL) {435        fprintf(stderr, "Failed to allocate proxy thread stats\n");436        exit(EXIT_FAILURE);437    }438    thr->proxy_ctx = ctx;439440    // Initialize the lua state.441    proxy_ctx_t *pctx = ctx;442    lua_State *L = NULL;443    if (pctx->memprofile) {444        struct mcp_memprofile *prof = calloc(1, sizeof(struct mcp_memprofile));445        prof->id = pctx->memprofile_thread_counter++;446        L = lua_newstate(mcp_profile_alloc, prof);447    } else {448        L = luaL_newstate();449    }450451    // With smaller requests the default incremental collector appears to452    // never complete. With this simple tuning (def-1, def, def) it seems453    // fine.454    // We can't use GCGEN until we manage pools with reference counting, as455    // they may never hit GC and thus never release their connection456    // resources.457    lua_gc(L, LUA_GCINC, 199, 100, 12);458    lua_gc(L, LUA_GCSTOP); // handle GC on our own schedule.459    thr->L = L;460    luaL_openlibs(L);461    proxy_register_libs(ctx, thr, L);462    // TODO: srand on time? do we need to bother?463    for (int x = 0; x < 3; x++) {464        thr->proxy_rng[x] = rand();465    }466467    // init our internal GC checker.468    thr->proxy_vm_last_kb = lua_gc(L, LUA_GCCOUNT);469    assert(thr->proxy_vm_last_kb != 0);470    thr->proxy_gc_timer = evtimer_new(thr->base, proxy_gc_timerpoke, thr);471    // kick off the timer loop.472    proxy_gc_timerpoke(0, 0, thr);473474    // Create a proxy event thread structure to piggyback on the worker.475    proxy_event_thread_t *t = calloc(1, sizeof(proxy_event_thread_t));476    thr->proxy_event_thread = t;477    proxy_init_event_thread(t, ctx, thr->base);478}479480void proxy_submit_cb(io_queue_t *q) {481    proxy_event_thread_t *e = ((proxy_ctx_t *)q->ctx)->proxy_io_thread;482    iop_head_t head;483    be_head_t w_head; // worker local stack.484    STAILQ_INIT(&head);485    STAILQ_INIT(&w_head);486487    while (!STAILQ_EMPTY(&q->stack)) {488        mcp_backend_t *be;489        io_pending_proxy_t *p = (io_pending_proxy_t *)STAILQ_FIRST(&q->stack);490        STAILQ_REMOVE_HEAD(&q->stack, iop_next);491        P_DEBUG("%s: queueing req for backend: %p\n", __func__, (void *)p);492493        if (p->background) {494            P_DEBUG("%s: fast-returning background object: %p\n", __func__, (void *)p);495            assert(p->backend == NULL);496            // must not resume requests inline here but they can be scheduled497            // to run drive_machine() later.498            conn_io_queue_return((io_pending_t *)p);499            continue;500        }501        be = p->backend;502503        if (be->use_io_thread) {504            STAILQ_INSERT_TAIL(&head, (io_pending_t *)p, iop_next);505        } else {506            // emulate some of handler_dequeue()507            STAILQ_INSERT_TAIL(&be->iop_head, (io_pending_t *)p, iop_next);508            assert(be->depth > -1);509            be->depth++;510            if (!be->stacked) {511                be->stacked = true;512                STAILQ_INSERT_TAIL(&w_head, be, be_next);513            }514        }515    }516517    // q->stack must now be empty, so we can submit new IO's while handling518    // the existing ones.519520    if (!STAILQ_EMPTY(&head)) {521        bool do_notify = false;522        P_DEBUG("%s: submitting queue to IO thread\n", __func__);523        // Transfer request stack to event thread.524        pthread_mutex_lock(&e->mutex);525        if (STAILQ_EMPTY(&e->iop_head_in)) {526            do_notify = true;527        }528        STAILQ_CONCAT(&e->iop_head_in, &head);529        // No point in holding the lock since we're not doing a cond signal.530        pthread_mutex_unlock(&e->mutex);531532        if (do_notify) {533        // Signal to check queue.534#ifdef USE_EVENTFD535        uint64_t u = 1;536        // TODO (v2): check result? is it ever possible to get a short write/failure537        // for an eventfd?538        if (write(e->event_fd, &u, sizeof(uint64_t)) != sizeof(uint64_t)) {539            assert(1 == 0);540        }541#else542        if (write(e->notify_send_fd, "w", 1) <= 0) {543            assert(1 == 0);544        }545#endif546        }547    }548549    if (!STAILQ_EMPTY(&w_head)) {550        P_DEBUG("%s: running inline worker queue\n", __func__);551        // emulating proxy_event_handler552        proxy_run_backend_queue(&w_head);553    }554    return;555}556557// This function handles return processing for the "old style" API:558// currently just `mcp.internal()`559void proxy_return_rctx_cb(io_pending_t *pending) {560    io_pending_proxy_t *p = (io_pending_proxy_t *)pending;561    if (p->client_resp && p->client_resp->blen) {562        // FIXME: workaround for buffer memory being external to objects.563        // can't run 0 since that means something special (run the GC)564        unsigned int kb = p->client_resp->blen / 1000;565        p->thread->proxy_vm_extra_kb += kb > 0 ? kb : 1;566    }567568    mcp_rcontext_t *rctx = p->rctx;569    lua_rotate(rctx->Lc, 1, 1);570    lua_settop(rctx->Lc, 1);571    // hold the resp for a minute.572    mc_resp *resp = rctx->resp;573574    proxy_run_rcontext(rctx);575576    if (p->io_sub_type != IO_PENDING_TYPE_EXTSTORE) {577        // if we're doing an extstore subrequest, the iop needs to live until578        // resp's ->finish_cb is called.579        resp->io_pending = NULL;580        do_cache_free(p->thread->io_cache, p);581    }582}583584// This is called if resp_finish is called while an iop exists on the585// resp.586// so we need to release our iop and rctx.587// - This can't happen unless we're doing extstore fetches.588// - the request context is freed before connection processing resumes.589void proxy_finalize_rctx_cb(io_pending_t *pending) {590    io_pending_proxy_t *p = (io_pending_proxy_t *)pending;591    // TODO: need to remove from stack if subtype is p->active592593    if (p->io_sub_type == IO_PENDING_TYPE_EXTSTORE) {594        assert(p->active == false);595        if (p->hdr_it) {596            // TODO: lock once, worst case this hashes/locks twice.597            if (p->miss) {598                item_unlink(p->hdr_it);599            }600            item_remove(p->hdr_it);601        }602    }603}604605int try_read_command_proxy(conn *c) {606    char *el, *cont;607608    if (c->rbytes == 0)609        return 0;610611    el = memchr(c->rcurr, '\n', c->rbytes);612    if (!el) {613        if (c->rbytes > 1024) {614            /*615             * We didn't have a '\n' in the first k. This _has_ to be a616             * large multiget, if not we should just nuke the connection.617             */618            char *ptr = c->rcurr;619            char *end = c->rcurr + c->rbytes-6;620            while (*ptr == ' ' && ptr != end) { /* ignore leading whitespaces */621                ++ptr;622            }623624            if (ptr - c->rcurr > 100 ||625                (strncmp(ptr, "get ", 4) && strncmp(ptr, "gets ", 5))) {626627                conn_set_state(c, conn_closing);628                return 1;629            }630631            // ASCII multigets are unbound, so our fixed size rbuf may not632            // work for this particular workload... For backcompat we'll use a633            // malloc/realloc/free routine just for this.634            if (!c->rbuf_malloced) {635                if (!rbuf_switch_to_malloc(c)) {636                    conn_set_state(c, conn_closing);637                    return 1;638                }639            }640        }641642        return 0;643    }644    cont = el + 1;645646    assert(cont <= (c->rcurr + c->rbytes));647648    c->last_cmd_time = current_time;649    proxy_process_command(c, c->rcurr, cont - c->rcurr, PROCESS_NORMAL);650651    c->rbytes -= (cont - c->rcurr);652    c->rcurr = cont;653654    assert(c->rcurr <= (c->rbuf + c->rsize));655656    return 1;657658}659660// Called when a connection is closed while in nread state reading a set661// Must only be called with an active coroutine.662void proxy_cleanup_conn(conn *c) {663    assert(c->proxy_rctx);664    mcp_rcontext_t *rctx = c->proxy_rctx;665    assert(rctx->pending_reqs == 1);666    rctx->pending_reqs = 0;667668    mcp_funcgen_return_rctx(rctx);669    c->proxy_rctx = NULL;670}671672// we buffered a SET of some kind.673void complete_nread_proxy(conn *c) {674    assert(c != NULL);675676    LIBEVENT_THREAD *thr = c->thread;677    lua_State *L = thr->L;678679    if (c->proxy_rctx == NULL) {680        complete_nread_ascii(c);681        return;682    }683684    conn_set_state(c, conn_new_cmd);685686    assert(c->proxy_rctx);687    mcp_rcontext_t *rctx = c->proxy_rctx;688    c->proxy_rctx = NULL;689    mcp_request_t *rq = rctx->request;690691    if (strncmp((char *)c->item + rq->pr.vlen - 2, "\r\n", 2) != 0) {692        lua_settop(L, 0); // clear anything remaining on the main thread.693        // FIXME (v2): need to set noreply false if mset_res, but that's kind694        // of a weird hack to begin with. Evaluate how to best do that here.695        out_string(c, "CLIENT_ERROR bad data chunk");696        rctx->pending_reqs--;697        mcp_funcgen_return_rctx(rctx);698        return;699    }700701    // We move ownership of the c->item buffer from the connection to the702    // request object here. Else we can double free if the conn closes while703    // inside nread.704    rq->pr.vbuf = c->item;705    c->item = NULL;706    c->item_malloced = false;707    thr->proxy_buffer_memory_used += rq->pr.vlen;708709    conn_resp_suspend(rctx->c, rctx->resp);710    proxy_run_rcontext(rctx);711712    lua_settop(L, 0); // clear anything remaining on the main thread.713714    return;715}716717// Simple error wrapper for common failures.718// lua_error() is a jump so this function never returns719// for clarity add a 'return' after calls to this.720void proxy_lua_error(lua_State *L, const char *s) {721    lua_pushstring(L, s);722    lua_error(L);723}724725// Need a custom function so we can prefix lua strings easily.726void proxy_out_errstring(mc_resp *resp, char *type, const char *str) {727    size_t len;728    size_t prefix_len = strlen(type);729730    assert(resp != NULL);731732    resp_reset(resp);733    // avoid noreply since we're throwing important errors.734735    // Fill response object with static string.736    len = strlen(str);737    if ((len + prefix_len + 2) > WRITE_BUFFER_SIZE) {738        /* ought to be always enough. just fail for simplicity */739        str = "SERVER_ERROR output line too long";740        len = strlen(str);741    }742743    char *w = resp->wbuf;744    memcpy(w, type, prefix_len);745    w += prefix_len;746747    memcpy(w, str, len);748    w += len;749750    memcpy(w, "\r\n", 2);751    resp_add_iov(resp, resp->wbuf, len + prefix_len + 2);752    return;753}754755// NOTE: See notes in mcp_queue_io; the secondary problem with setting the756// noreply mode from the response object is that the proxy can return strings757// manually, so we have no way to obey what the original request wanted in758// that case.759static void _set_noreply_mode(mc_resp *resp, mcp_request_t *rq, mcp_resp_t *r) {760    if (rq->pr.cmd_type == CMD_TYPE_META761        && mcmc_token_has_flag_bit(&rq->pr.tok, (uint64_t)1<<48) == MCMC_OK) {762        if (r->resp.code == MCMC_CODE_END) {763            resp->skip = true;764        } else if (r->cmd != CMD_MG && r->resp.code == MCMC_CODE_OK) {765            // FIXME (v2): mcmc's parser needs to help us out a bit more766            // by maybe flagging a response as quietable.767            // This is a broken case in the protocol though; quiet mode768            // ignores HD for mutations but not get.769            resp->skip = true;770        }771    } else if (rq->pr.noreply && r->status == MCMC_OK) {772        // ascii noreply only threw egregious errors to client773        resp->skip = true;774    }775}776777static void _proxy_run_rcontext_queues(mcp_rcontext_t *rctx) {778    for (int x = 0; x < rctx->fgen->max_queues; x++) {779        mcp_run_rcontext_handle(rctx, x);780    }781}782783static void _proxy_run_tresp_to_resp(mc_resp *tresp, mc_resp *resp) {784    // The internal cache handler has created a resp we want to swap in785    // here. It would be fastest to swap *resp's position in the786    // link but if the set is deep this would instead be slow, so787    // we copy over details from this temporary resp instead.788789    // So far all we fill is the wbuf and some iov's? so just copy790    // that + the UDP info?791    memcpy(resp->wbuf, tresp->wbuf, tresp->iov[0].iov_len);792    resp->tosend = 0;793    for (int x = 0; x < tresp->iovcnt; x++) {794        resp->iov[x] = tresp->iov[x];795        resp->tosend += tresp->iov[x].iov_len;796    }797    // resp->iov[x].iov_base needs to be updated if it's798    // pointing within its wbuf.799    // FIXME: This is too fragile. we need to be able to800    // inherit details and swap resp objects around.801    if (tresp->iov[0].iov_base == tresp->wbuf) {802        resp->iov[0].iov_base = resp->wbuf;803    }804    resp->iovcnt = tresp->iovcnt;805    resp->chunked_total = tresp->chunked_total;806    resp->chunked_data_iov = tresp->chunked_data_iov;807    // copy UDP headers...808    resp->request_id = tresp->request_id;809    resp->udp_sequence = tresp->udp_sequence;810    resp->udp_total = tresp->udp_total;811    resp->request_addr = tresp->request_addr;812    resp->request_addr_size = tresp->request_addr_size;813    resp->item = tresp->item; // will be populated if not extstore fetch814    tresp->item = NULL; // move ownership of the item to resp from tresp815    resp->skip = tresp->skip;816}817818int proxy_run_rcontext(mcp_rcontext_t *rctx) {819    int nresults = 0;820    lua_State *Lc = rctx->Lc;821    assert(rctx->lua_narg != 0);822    int cores = lua_resume(Lc, NULL, rctx->lua_narg, &nresults);823    rctx->lua_narg = 1; // reset to default since not-default is uncommon.824    size_t rlen = 0;825    mc_resp *resp = rctx->resp;826827    if (cores == LUA_OK) {828        // don't touch the result object if we were a sub-context.829        if (!rctx->parent) {830            int type = lua_type(Lc, 1);831            mcp_resp_t *r = NULL;832            P_DEBUG("%s: coroutine completed. return type: %d\n", __func__, type);833            if (type == LUA_TUSERDATA && (r = luaL_testudata(Lc, 1, "mcp.response")) != NULL) {834                _set_noreply_mode(resp, rctx->request, r);835                if (r->status != MCMC_OK && r->resp.type != MCMC_RESP_ERRMSG) {836                    proxy_out_errstring(resp, PROXY_SERVER_ERROR, "backend failure");837                } else if (r->cresp) {838                    mc_resp *tresp = r->cresp;839840                    _proxy_run_tresp_to_resp(tresp, resp);841                    // hand off ownership of the result buffer if we were an842                    // extstore fetch.843                    if (!resp->item) {844                        resp->write_and_free = r->buf;845                        r->buf = NULL;846                    }847                    // we let the mcp_resp gc handler free up tresp and any848                    // associated io_pending's of its own later.849                } else if (r->buf) {850                    // response set from C.851                    resp->write_and_free = r->buf;852                    resp_add_iov(resp, r->buf, r->blen);853                    // stash the length to later remove from memory tracking854                    resp->wbytes = r->blen;855                    resp->proxy_res = true;856                    r->buf = NULL;857                } else {858                    // Empty response: used for ascii multiget emulation.859                }860861            } else if (type == LUA_TSTRING) {862                // response is a raw string from lua.863                const char *s = lua_tolstring(Lc, 1, &rlen);864                size_t l = rlen > WRITE_BUFFER_SIZE ? WRITE_BUFFER_SIZE : rlen;865                memcpy(resp->wbuf, s, l);866                resp_add_iov(resp, resp->wbuf, l);867                lua_pop(Lc, 1);868            } else {869                proxy_out_errstring(resp, PROXY_SERVER_ERROR, "bad response");870            }871872            if (rctx->ascii_multiget && !resp->skip) {873                struct iovec *iov = &resp->iov[resp->iovcnt-1];874                // look for END at end of final iov and blank it.875                // Bad hack, but centralized here now.876                if (iov->iov_len >= ENDLEN &&877                        memcmp(((char *)iov->iov_base)+(iov->iov_len-ENDLEN), ENDSTR, ENDLEN) == 0) {878                    iov->iov_len -= ENDLEN;879                    resp->tosend -= ENDLEN;880                    if (resp->tosend == 0) {881                        // some platforms don't like 0 byte iov's.882                        // may be misremembering, double check? mac os?883                        resp->skip = true;884                    }885                }886            }887            conn_resp_unsuspend(rctx->c, resp);888            rctx->c = NULL; // *conn cannot be used past this point!889            rctx->pending_reqs--;890            mcp_funcgen_return_rctx(rctx);891        } else {892            rctx->pending_reqs--;893        }894    } else if (cores == LUA_YIELD) {895        int yield_type = lua_tointeger(Lc, -1);896        P_DEBUG("%s: coroutine yielded. return type: %d\n", __func__, yield_type);897        assert(yield_type != 0);898        lua_pop(Lc, 1);899900        int res = 0;901        switch (yield_type) {902            case MCP_YIELD_INTERNAL:903                // stack should be: rq, res904                if (rctx->parent) {905                    LOGGER_LOG(NULL, LOG_PROXYEVENTS, LOGGER_PROXY_ERROR, NULL, "cannot run mcp.internal from a sub request");906                    rctx->pending_reqs--;907                    return LUA_ERRRUN;908                } else {909                    res = mcplib_internal_run(rctx);910                    if (res == 0) {911                        // stack should still be: rq, res912                        // TODO: turn this function into a for loop that re-runs on913                        // certain status codes, to avoid recursive depth here.914                        // or maybe... a goto? :P915                        proxy_run_rcontext(rctx);916                    } else if (res > 0) {917                        // internal run queued for extstore.918                    } else {919                        assert(res < 0);920                        proxy_out_errstring(resp, PROXY_SERVER_ERROR, "bad request");921                    }922                }923                break;924            case MCP_YIELD_WAITCOND:925            case MCP_YIELD_WAITHANDLE:926                // Even if we're in WAITHANDLE, we want to dispatch any queued927                // requests, so we still need to iterate the full set of qslots.928                _proxy_run_rcontext_queues(rctx);929                break;930            case MCP_YIELD_SLEEP:931                // Pause coroutine and do nothing. Alarm will resume.932                break;933            default:934                abort();935        }936937    } else {938        // Log the error where it happens, then the parent will handle a939        // result object normally.940        P_DEBUG("%s: Failed to run coroutine: %s\n", __func__, lua_tostring(Lc, -1));941        LOGGER_LOG(NULL, LOG_PROXYEVENTS, LOGGER_PROXY_ERROR, NULL, lua_tostring(Lc, -1));942        if (settings.verbose) {943            luaL_traceback(Lc, Lc, lua_tostring(Lc, -1), 1);944            fprintf(stderr, "proxy_lua_error: %s\n", lua_tostring(Lc, -1));945        }946        if (!rctx->parent) {947            proxy_out_errstring(resp, PROXY_SERVER_ERROR, "lua failure");948            conn_resp_unsuspend(rctx->c, resp);949            rctx->c = NULL; // *conn cannot be used past this point!950            rctx->pending_reqs--;951            mcp_funcgen_return_rctx(rctx);952        } else {953            rctx->pending_reqs--;954        }955    }956957    return cores;958}959960// basically any data before the first key.961// max is like 15ish plus spaces. we can be more strict about how many spaces962// to expect because any client spamming space is being deliberately stupid963// anyway.964#define MAX_CMD_PREFIX 20965966static void proxy_process_command(conn *c, char *command, size_t cmdlen, bool multiget) {967    assert(c != NULL);968    LIBEVENT_THREAD *thr = c->thread;969    struct proxy_hook *hooks = thr->proxy_hooks;970    lua_State *L = thr->L;971    proxy_ctx_t *ctx = thr->proxy_ctx;972    mcp_parser_t pr = {0};973974    // Avoid doing resp_start() here, instead do it a bit later or as-needed.975    // This allows us to hop over to the internal text protocol parser, which976    // also calls resp_start().977    // Tighter integration later should obviate the need for this, it is not a978    // permanent solution.979    int ret = process_request(&pr, command, cmdlen);980    if (ret != 0) {981        WSTAT_INCR(c->thread, proxy_conn_errors, 1);982        if (!resp_start(c)) {983            conn_set_state(c, conn_closing);984            return;985        }986        proxy_out_errstring(c->resp, PROXY_CLIENT_ERROR, "parsing request");987        if (ret == -2) {988            // Kill connection on more critical parse failure.989            conn_set_state(c, conn_closing);990        }991        return;992    }993994    struct proxy_hook *hook = &hooks[pr.command];995    struct proxy_hook_ref hook_ref = hook->ref;996    // if client came from a tagged listener, scan for a more specific hook.997    // TODO: (v2) avoiding a hash table lookup here, but maybe some other998    // datastructure would suffice. for 4-8 tags this is perfectly fast.999    if (c->tag && hook->tagged) {1000        struct proxy_hook_tagged *pht = hook->tagged;1001        while (pht->ref.lua_ref) {1002            if (c->tag == pht->tag) {1003                hook_ref = pht->ref;1004                break;1005            }1006            pht++;1007        }1008    }10091010    if (!hook_ref.lua_ref) {1011        // lets nread_proxy know we're in ascii mode.1012        c->proxy_rctx = NULL;1013        process_command_ascii(c, command, cmdlen);1014        return;1015    }10161017    // If ascii multiget, we turn this into a self-calling loop :(1018    // create new request with next key, call this func again, then advance1019    // original string.1020    // might be better to split this function; the below bits turn into a1021    // function call, then we don't re-process the above bits in the same way?1022    // The way this is detected/passed on is very fragile.1023    if (!multiget && pr.cmd_type == CMD_TYPE_GET && pr.tok.ntokens > pr.keytoken+1) {1024        uint32_t keyoff = pr.tok.tokens[pr.keytoken];1025        const char *curkey = pr.request + keyoff;1026        int klen = pr.klen;1027        int cmd_prefix_len = pr.tok.tokens[pr.keytoken];1028        size_t endlen = 0;10291030        // TODO: TEST THIS CASE1031        // check once if our prefix is too long.1032        if (cmd_prefix_len > MAX_CMD_PREFIX) {1033            if (!resp_start(c)) {1034                conn_set_state(c, conn_closing);1035                return;1036            }1037            proxy_out_errstring(c->resp, PROXY_CLIENT_ERROR, "malformed request");1038            return;1039        }10401041        assert(pr.request[pr.reqlen-1] == '\n');1042        if (pr.request[pr.reqlen-2] == '\r') {1043            endlen = pr.reqlen - 2;1044        } else {1045            endlen = pr.reqlen - 1;1046        }10471048        while (klen != 0) {1049            char temp[KEY_MAX_LENGTH + MAX_CMD_PREFIX + 30];1050            char *cur = temp;1051            // Core daemon can abort the entire command if one key is bad, but1052            // we cannot from the proxy. Instead we have to inject errors into1053            // the stream. This should, thankfully, be rare at least.1054            if (klen > KEY_MAX_LENGTH) {1055                if (!resp_start(c)) {1056                    conn_set_state(c, conn_closing);1057                    return;1058                }1059                proxy_out_errstring(c->resp, PROXY_CLIENT_ERROR, "key too long");1060            } else {1061                // copy original request up until the original key token.1062                memcpy(cur, pr.request, cmd_prefix_len);1063                cur += cmd_prefix_len;10641065                // now copy in our "current" key.1066                memcpy(cur, curkey, klen);1067                cur += klen;10681069                memcpy(cur, "\r\n", 2);1070                cur += 2;10711072                *cur = '\0';1073                P_DEBUG("%s: new multiget sub request: %s [%u/%u]\n", __func__, temp, keyoff, pr.klen);1074                proxy_process_command(c, temp, cur - temp, PROCESS_MULTIGET);1075            }10761077            // FIXME: test multigets with excess space at the end.1078            // Find the next key manually. This keeps the special cased code1079            // here instead of adding extra junk to the parser structure.1080            curkey += klen;1081            int remain = endlen - (curkey - pr.request);1082            while (remain) {1083                if (*curkey == ' ') {1084                    remain--;1085                    curkey++;1086                } else {1087                    break;1088                }1089            }10901091            if (remain) {1092                const char *s = memchr(curkey, ' ', remain);1093                if (s != NULL) {1094                    klen = s - curkey;1095                } else {1096                    klen = remain;1097                }1098            } else {1099                // no more keys1100                break;1101            }1102        }11031104        if (!resp_start(c)) {1105            conn_set_state(c, conn_closing);1106            return;1107        }11081109        // The above recursions should have created c->resp's in dispatch1110        // order.1111        // So now we add another one at the end to create the capping END1112        // string.1113        memcpy(c->resp->wbuf, ENDSTR, ENDLEN);1114        resp_add_iov(c->resp, c->resp->wbuf, ENDLEN);11151116        return;1117    }11181119    // We test the command length all the way down here because multigets can1120    // be very long, and they're chopped up by now.1121    if (cmdlen >= MCP_REQUEST_MAXLEN) {1122        WSTAT_INCR(c->thread, proxy_conn_errors, 1);1123        if (!resp_start(c)) {1124            conn_set_state(c, conn_closing);1125            return;1126        }1127        proxy_out_errstring(c->resp, PROXY_CLIENT_ERROR, "request too long");1128        conn_set_state(c, conn_closing);1129        return;1130    }11311132    if (!resp_start(c)) {1133        conn_set_state(c, conn_closing);1134        return;1135    }11361137    // Count requests handled by proxy vs local.1138    // Also batch the counts down this far so we can lock once for the active1139    // counter instead of twice.1140    struct proxy_int_stats *istats = c->thread->proxy_int_stats;1141    int64_t active_reqs = 0;1142    WSTAT_L(c->thread);1143    istats->counters[pr.command]++;1144    c->thread->stats.proxy_conn_requests++;1145    active_reqs = c->thread->stats.proxy_req_active;1146    WSTAT_UL(c->thread);11471148    if (active_reqs >= ctx->active_req_limit) {1149        proxy_out_errstring(c->resp, PROXY_SERVER_ERROR, "active request limit reached");1150        if (pr.vlen != 0) {1151            c->sbytes = pr.vlen;1152            conn_set_state(c, conn_swallow);1153        }1154        return;1155    }11561157    // hook is owned by a function generator.1158    mcp_rcontext_t *rctx = mcp_funcgen_start(L, hook_ref.ctx, &pr);1159    if (rctx == NULL) {1160        proxy_out_errstring(c->resp, PROXY_SERVER_ERROR, "lua start failure");1161        if (pr.vlen != 0) {1162            c->sbytes = pr.vlen;1163            conn_set_state(c, conn_swallow);1164        }1165        return;1166    }11671168    mcp_set_request(&pr, rctx->request, command, cmdlen);1169    rctx->ascii_multiget = multiget;1170    rctx->c = c;1171    rctx->conn_fd = c->sfd;1172    rctx->pending_reqs++; // seed counter with the "main" request1173    // remember the top level mc_resp, because further requests on the1174    // same connection will replace c->resp.1175    rctx->resp = c->resp;11761177    // for the very first call we need to place:1178    // - rctx->function_ref + rctx->request_ref1179    // I _think_ here is the right place to do that?1180    lua_rawgeti(rctx->Lc, LUA_REGISTRYINDEX, rctx->function_ref);1181    lua_rawgeti(rctx->Lc, LUA_REGISTRYINDEX, rctx->request_ref);11821183    if (pr.vlen != 0) {1184        c->item = NULL;1185        // Need to add the used memory later due to needing an extra callback1186        // handler on error during nread.1187        bool oom = proxy_bufmem_checkadd(c->thread, 0);11881189        // relying on temporary malloc's not having fragmentation1190        if (!oom) {1191            c->item = malloc(pr.vlen);1192        }1193        if (c->item == NULL) {1194            // return the RCTX1195            rctx->pending_reqs--;1196            mcp_funcgen_return_rctx(rctx);1197            // normal cleanup1198            lua_settop(L, 0);1199            proxy_out_errstring(c->resp, PROXY_SERVER_ERROR, "out of memory");1200            c->sbytes = pr.vlen;1201            conn_set_state(c, conn_swallow);1202            return;1203        }1204        c->item_malloced = true;1205        c->ritem = c->item;1206        c->rlbytes = pr.vlen;12071208        // remember the request context for later.1209        c->proxy_rctx = rctx;12101211        conn_set_state(c, conn_nread);1212        return;1213    }12141215    conn_resp_suspend(rctx->c, rctx->resp);1216    proxy_run_rcontext(rctx);12171218    lua_settop(L, 0); // clear any junk from the main thread.1219}12201221mcp_resp_t *mcp_prep_bare_resobj(lua_State *L, LIBEVENT_THREAD *t) {1222    mcp_resp_t *r = lua_newuserdatauv(L, sizeof(mcp_resp_t), 0);1223    // FIXME (v2): is this memset still necessary? I was using it for1224    // debugging.1225    memset(r, 0, sizeof(mcp_resp_t));1226    r->thread = t;1227    assert(r->thread != NULL);1228    gettimeofday(&r->start, NULL);12291230    luaL_getmetatable(L, "mcp.response");1231    lua_setmetatable(L, -2);12321233    return r;1234}12351236void mcp_set_resobj(mcp_resp_t *r, mcp_request_t *rq, mcp_backend_t *be, LIBEVENT_THREAD *t) {1237    memset(r, 0, sizeof(mcp_resp_t));1238    r->thread = t;1239    assert(r->thread != NULL);1240    gettimeofday(&r->start, NULL);1241    // Set noreply mode.1242    // TODO (v2): the response "inherits" the request's noreply mode, which isn't1243    // strictly correct; we should inherit based on the request that spawned1244    // the coroutine but the structure doesn't allow that yet.1245    // Should also be able to settle this exact mode from the parser so we1246    // don't have to re-branch here.1247    if (rq->pr.cmd_type == CMD_TYPE_META) {1248        if (mcmc_token_has_flag_bit(&rq->pr.tok, (uint64_t)1<<48) == MCMC_OK) {1249            // FIXME: should this use the new interface? re-evaluate?1250            for (int x = 2; x < rq->pr.tok.ntokens; x++) {1251                if (rq->request[rq->pr.tok.tokens[x]] == 'q') {1252                    rq->request[rq->pr.tok.tokens[x]] = ' ';1253                }1254            }1255        }1256    } else {1257        if (rq->pr.noreply) {1258            rq->request[rq->pr.reqlen - 3] = 'Y';1259        }1260    }12611262    r->cmd = rq->pr.command;1263    r->be = be;1264}12651266void mcp_resp_set_elapsed(mcp_resp_t *r) {1267    struct timeval end;1268    // stamp the elapsed time into the response object.1269    gettimeofday(&end, NULL);1270    r->elapsed = (end.tv_sec - r->start.tv_sec) * 1000000 +1271        (end.tv_usec - r->start.tv_usec);1272}12731274// Used for any cases where we're queueing requests to the IO subsystem.1275// NOTE: it's not currently possible to limit the memory used by the IO1276// object cache. So this check is redundant, and any callers may proceed1277// as though it is successful.1278io_pending_proxy_t *mcp_queue_rctx_io(mcp_rcontext_t *rctx, mcp_request_t *rq, mcp_backend_t *be, mcp_resp_t *r) {1279    conn *c = rctx->c;1280    io_queue_t *q = thread_io_queue_get(rctx->fgen->thread, IO_QUEUE_PROXY);1281    io_pending_proxy_t *p = do_cache_alloc(c->thread->io_cache);1282    if (p == NULL) {1283        WSTAT_INCR(c->thread, proxy_conn_oom, 1);1284        proxy_lua_error(rctx->Lc, "out of memory allocating from IO cache");1285        // NOTE: the error call above jumps to an error handler, so this does1286        // not actually return.1287        return NULL;1288    }12891290    // this is a re-cast structure, so assert that we never outsize it.1291    assert(sizeof(io_pending_t) >= sizeof(io_pending_proxy_t));1292    memset(p, 0, sizeof(io_pending_proxy_t));1293    // set up back references.1294    p->io_queue_type = IO_QUEUE_PROXY;1295    p->thread = c->thread;1296    p->c = c;1297    p->client_resp = r;1298    p->flushed = false;1299    p->return_cb = NULL;1300    p->finalize_cb = proxy_finalize_rctx_cb;13011302    // pass along the request context for resumption.1303    p->rctx = rctx;13041305    if (rq) {1306        // The direct backend object. Lc is holding the reference in the stack1307        p->backend = be;13081309        mcp_request_attach(rq, p);1310    }13111312    // link into the batch chain.1313    STAILQ_INSERT_TAIL(&q->stack, (io_pending_t *)p, iop_next);1314    P_DEBUG("%s: queued\n", __func__);13151316    return p;1317}13181319// DO NOT call this method frequently! globally locked!1320void mcp_sharedvm_delta(proxy_ctx_t *ctx, int tidx, const char *name, int delta) {1321    lua_State *L = ctx->proxy_sharedvm;1322    pthread_mutex_lock(&ctx->sharedvm_lock);13231324    if (lua_getfield(L, tidx, name) == LUA_TNIL) {1325        lua_pop(L, 1);1326        lua_pushinteger(L, delta);1327        lua_setfield(L, tidx, name);1328    } else {1329        lua_pushinteger(L, delta);1330        lua_arith(L, LUA_OPADD);1331        lua_setfield(L, tidx, name);1332    }13331334    pthread_mutex_unlock(&ctx->sharedvm_lock);1335}13361337void mcp_sharedvm_remove(proxy_ctx_t *ctx, int tidx, const char *name) {1338    lua_State *L = ctx->proxy_sharedvm;1339    pthread_mutex_lock(&ctx->sharedvm_lock);13401341    lua_pushnil(L);1342    lua_setfield(L, tidx, name);13431344    pthread_mutex_unlock(&ctx->sharedvm_lock);1345}13461347// Global object support code.1348// Global objects are created in the configuration VM, and referenced in1349// worker VMs via proxy objects that refer back to memory in the1350// configuration VM.1351// We manage reference counts: once all remote proxy objects are collected, we1352// signal the config thread to remove a final reference and collect garbage to1353// remove the global object.13541355static void mcp_gobj_enqueue(proxy_ctx_t *ctx, struct mcp_globalobj_s *g) {1356    pthread_mutex_lock(&ctx->manager_lock);1357    STAILQ_INSERT_TAIL(&ctx->manager_head, g, next);1358    pthread_cond_signal(&ctx->manager_cond);1359    pthread_mutex_unlock(&ctx->manager_lock);1360}13611362// References the object, initializing the self-reference if necessary.1363// Call from config thread, with global object on top of stack.1364void mcp_gobj_ref(lua_State *L, struct mcp_globalobj_s *g) {1365    pthread_mutex_lock(&g->lock);1366    if (g->self_ref == 0) {1367        // Initialization requires a small dance:1368        // - store a negative of our ref, increase refcount an extra time1369        // - then link and signal the manager thread as though we were GC'ing1370        // the object.1371        // - the manager thread will later acknowledge the initialization of1372        // this global object and negate the self_ref again1373        // - this prevents an unused proxy object from causing the global1374        // object to be reaped early while we are still copying it to worker1375        // threads, as the manager thread will block waiting for the config1376        // thread to finish its reload work.1377        g->self_ref = -luaL_ref(L, LUA_REGISTRYINDEX);1378        g->refcount++;1379        proxy_ctx_t *ctx = PROXY_GET_CTX(L);1380        mcp_gobj_enqueue(ctx, g);1381    } else {1382        lua_pop(L, 1); // drop the reference we didn't end up using.1383    }1384    g->refcount++;1385    pthread_mutex_unlock(&g->lock);1386}13871388void mcp_gobj_unref(proxy_ctx_t *ctx, struct mcp_globalobj_s *g) {1389    pthread_mutex_lock(&g->lock);1390    g->refcount--;1391    if (g->refcount == 0) {1392        mcp_gobj_enqueue(ctx, g);1393    }1394    pthread_mutex_unlock(&g->lock);1395}13961397void mcp_gobj_finalize(struct mcp_globalobj_s *g) {1398    pthread_mutex_destroy(&g->lock);1399}14001401static void *mcp_profile_alloc(void *ud, void *ptr, size_t osize,1402                                            size_t nsize) {1403    struct mcp_memprofile *prof = ud;1404    struct timespec now;1405    clock_gettime(CLOCK_MONOTONIC, &now);1406    enum mcp_memprofile_types t = mcp_memp_free;1407    if (ptr == NULL) {1408        switch (osize) {1409            case LUA_TSTRING:1410                t = mcp_memp_string;1411                //fprintf(stderr, "alloc string: %ld\n", nsize);1412                break;1413            case LUA_TTABLE:1414                t = mcp_memp_table;1415                //fprintf(stderr, "alloc table: %ld\n", nsize);1416                break;1417            case LUA_TFUNCTION:1418                t = mcp_memp_func;1419                //fprintf(stderr, "alloc func: %ld\n", nsize);1420                break;1421            case LUA_TUSERDATA:1422                t = mcp_memp_userdata;1423                //fprintf(stderr, "alloc userdata: %ld\n", nsize);1424                break;1425            case LUA_TTHREAD:1426                t = mcp_memp_thread;1427                //fprintf(stderr, "alloc thread: %ld\n", nsize);1428                break;1429            default:1430                t = mcp_memp_default;1431                //fprintf(stderr, "alloc osize: %ld nsize: %ld\n", osize, nsize);1432        }1433        prof->allocs[t]++;1434        prof->alloc_bytes[t] += nsize;1435    } else {1436        if (nsize != 0) {1437            prof->allocs[mcp_memp_realloc]++;1438            prof->alloc_bytes[mcp_memp_realloc] += nsize;1439        } else {1440            prof->allocs[mcp_memp_free]++;1441            prof->alloc_bytes[mcp_memp_free] += osize;1442        }1443        //fprintf(stderr, "realloc: osize: %ld nsize: %ld\n", osize, nsize);1444    }14451446    if (now.tv_sec != prof->last_status.tv_sec) {1447        prof->last_status.tv_sec = now.tv_sec;1448        fprintf(stderr, "MEMPROF[%d]:\tstring[%llu][%llu] table[%llu][%llu] func[%llu][%llu] udata[%llu][%llu] thr[%llu][%llu] def[%llu][%llu] realloc[%llu][%llu] free[%llu][%llu]\n",1449                prof->id,1450                (unsigned long long)prof->allocs[1],1451                (unsigned long long)prof->alloc_bytes[1],1452                (unsigned long long)prof->allocs[2],1453                (unsigned long long)prof->alloc_bytes[2],1454                (unsigned long long)prof->allocs[3],1455                (unsigned long long)prof->alloc_bytes[3],1456                (unsigned long long)prof->allocs[4],1457                (unsigned long long)prof->alloc_bytes[4],1458                (unsigned long long)prof->allocs[5],1459                (unsigned long long)prof->alloc_bytes[5],1460                (unsigned long long)prof->allocs[6],1461                (unsigned long long)prof->alloc_bytes[6],1462                (unsigned long long)prof->allocs[7],1463                (unsigned long long)prof->alloc_bytes[7],1464                (unsigned long long)prof->allocs[0],1465                (unsigned long long)prof->alloc_bytes[0]);1466        for (int x = 0; x < 8; x++) {1467            prof->allocs[x] = 0;1468            prof->alloc_bytes[x] = 0;1469        }1470    }14711472    if (nsize == 0) {1473        free(ptr);1474        return NULL;1475    } else {1476        return realloc(ptr, nsize);1477    }1478}14791480// Common lua debug command.1481__attribute__((unused)) void dump_stack(lua_State *L, const char *msg) {1482    int top = lua_gettop(L);1483    int i = 1;1484    fprintf(stderr, "--TOP OF STACK [%d] | %s\n", top, msg);1485    for (; i < top + 1; i++) {1486        int type = lua_type(L, i);1487        void *udata = NULL;1488        // lets find the metatable of this userdata to identify it.1489        if (lua_getmetatable(L, i) != 0) {1490            lua_pushstring(L, "__name");1491            if (lua_rawget(L, -2) != LUA_TNIL) {1492                if (type == LUA_TUSERDATA) {1493                    udata = lua_touserdata(L, i);1494                }1495                fprintf(stderr, "--|%d| [%s] (%s) [ptr: %p]\n", i, lua_typename(L, type), lua_tostring(L, -1), udata);1496                lua_pop(L, 2);1497                continue;1498            }1499            lua_pop(L, 2);1500        }1501        if (type == LUA_TSTRING) {1502            fprintf(stderr, "--|%d| [%s] | %s\n", i, lua_typename(L, type), lua_tostring(L, i));1503        } else {1504            if (type == LUA_TUSERDATA) {1505                udata = lua_touserdata(L, i);1506            }1507            fprintf(stderr, "--|%d| [%s] [ptr: %p]\n", i, lua_typename(L, type), udata);1508        }1509    }1510    fprintf(stderr, "-----------------\n");1511}15121513// Not very pretty, but helped.1514// Nice to haves:1515// - summarize counts for each metatable (easy enough to do from logging)1516// - use a less noisy stack dump instead of calling dump_stack()1517__attribute__((unused)) void dump_registry(lua_State *L, const char *msg) {1518    int ref_size = lua_rawlen(L, LUA_REGISTRYINDEX);1519    fprintf(stderr, "--LUA REGISTRY TABLE [%d] | %s\n", ref_size, msg);1520    // walk registry1521    int ridx = lua_absindex(L, LUA_REGISTRYINDEX);1522    int udata = 0;1523    int number = 0;1524    int string = 0;1525    int function = 0;1526    int table = 0;1527    lua_pushnil(L);1528    while (lua_next(L, ridx) != 0) {1529        dump_stack(L, "===registry entry===");1530        int type = lua_type(L, -1);1531        if (type == LUA_TUSERDATA) {1532            udata++;1533        } else if (type == LUA_TNUMBER) {1534            number++;1535        } else if (type == LUA_TSTRING) {1536            string++;1537        } else if (type == LUA_TFUNCTION) {1538            function++;1539        } else if (type == LUA_TTABLE) {1540            table++;1541        }1542        lua_pop(L, 1); // drop value1543    }1544    fprintf(stderr, "SUMMARY:\n\n");1545    fprintf(stderr, "### UDATA\t[%d]\n", udata);1546    fprintf(stderr, "### NUMBER\t[%d]\n", number);1547    fprintf(stderr, "### STRING\t[%d]\n", string);1548    fprintf(stderr, "### FUNCTION\t[%d]\n", function );1549    fprintf(stderr, "### TABLE\t[%d]\n", table);1550    fprintf(stderr, "-----------------\n");1551}15521553// Searches for a function generator with a specific name attached.1554// Adding breakpoints on the print lines lets you inspect the fgen and its1555// slots.1556__attribute__((unused)) void dump_funcgen(lua_State *L, const char *name, const char *msg) {1557    int ref_size = lua_rawlen(L, LUA_REGISTRYINDEX);1558    fprintf(stderr, "--LUA FUNCGEN FINDER [%d] | %s\n", ref_size, msg);1559    // walk registry1560    int ridx = lua_absindex(L, LUA_REGISTRYINDEX);1561    lua_pushnil(L);1562    while (lua_next(L, ridx) != 0) {1563        int type = lua_type(L, -1);1564        if (type == LUA_TUSERDATA) {1565            mcp_funcgen_t *f = luaL_testudata(L, -1, "mcp.funcgen");1566            if (f != NULL && strcmp(name, f->name) == 0) {1567                fprintf(stderr, "===found funcgen [%s] [%p]===\n", f->name, (void *)f);1568                lua_getiuservalue(L, -1, 1);1569                int tidx = lua_absindex(L, -1);1570                lua_pushnil(L);1571                while (lua_next(L, tidx) != 0) {1572                    mcp_rcontext_t *rctx = lua_touserdata(L, -1);1573                    if (rctx != NULL) {1574                        fprintf(stderr, "-- slot: [%p]\n", (void *)rctx);1575                    }1576                    lua_pop(L, 1); // drop value1577                }1578                lua_pop(L, 1); // drop slot table1579            }1580        }1581        lua_pop(L, 1); // drop value1582    }1583    fprintf(stderr, "-----------------\n");1584}15851586static void dump_pool_info(mcp_pool_t *p) {1587    fprintf(stderr, "--pool: [%s] size: [%d] be_total: [%d] rc: [%d] io: [%d]\n",1588            p->beprefix, p->pool_size, p->pool_be_total, p->g.refcount, p->use_iothread);15891590    for (int x = 0; x < p->pool_be_total; x++) {1591        mcp_backend_t *be = p->pool[x].be;1592        // Dumb: pool_be_total is wrong if pool is using iothread. Why?1593        if (be != NULL) {1594            fprintf(stderr, "  --be[%d] label: [%s] name: [%s] conns: [%d] depth: [%d]\n",1595                    x, be->label, be->name, be->conncount, be->depth);1596            for (int i = 0; i < be->conncount; i++) {1597                struct mcp_backendconn_s *bec = &be->be[i];1598                fprintf(stderr, "    --bec[%d] bad: [%d] failcnt: [%d] depth: [%d] pendread: [%d] state: [%d] can_write[%d] write_event[%d]\n",1599                        i, bec->bad, bec->failed_count, bec->depth, bec->pending_read, bec->state, bec->can_write, event_pending(&bec->timeout_event, EV_WRITE, NULL));1600            }1601        }1602    }1603    fprintf(stderr, "=======\n");1604}16051606// Dumps some info about pools.1607// If given the config thread, it should find the main pools1608// If given a worker thread, it will look for the pool proxy objects and find1609// the main pools that way.1610__attribute__((unused)) void dump_pools(lua_State *L, const char *msg) {1611    int ref_size = lua_rawlen(L, LUA_REGISTRYINDEX);1612    fprintf(stderr, "--LUA POOL DUMPER [%d] | %s\n", ref_size, msg);1613    // walk registry1614    int ridx = lua_absindex(L, LUA_REGISTRYINDEX);1615    lua_pushnil(L);1616    while (lua_next(L, ridx) != 0) {1617        int type = lua_type(L, -1);1618        if (type == LUA_TUSERDATA) {1619            mcp_pool_t *p = luaL_testudata(L, -1, "mcp.pool");1620            if (p != NULL) {1621                dump_pool_info(p);1622            } else {1623                mcp_pool_proxy_t *pp = luaL_testudata(L, -1, "mcp.pool_proxy");1624                if (pp != NULL) {1625                    dump_pool_info(pp->main);1626                }1627            }1628        }1629        lua_pop(L, 1); // drop value1630    }1631    fprintf(stderr, "-----------------\n");16321633}