1/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */2// Functions related to the backend handler thread.34#include "proxy.h"5#include "proxy_tls.h"67enum proxy_be_failures {8    P_BE_FAIL_TIMEOUT = 0,9    P_BE_FAIL_DISCONNECTED,10    P_BE_FAIL_CONNECTING,11    P_BE_FAIL_CONNTIMEOUT,12    P_BE_FAIL_READVALIDATE,13    P_BE_FAIL_BADVALIDATE,14    P_BE_FAIL_WRITING,15    P_BE_FAIL_READING,16    P_BE_FAIL_PARSING,17    P_BE_FAIL_CLOSED,18    P_BE_FAIL_UNHANDLEDRES,19    P_BE_FAIL_OOM,20    P_BE_FAIL_ENDSYNC,21    P_BE_FAIL_TRAILINGDATA,22    P_BE_FAIL_INVALIDPROTOCOL,23};2425const char *proxy_be_failure_text[] = {26    [P_BE_FAIL_TIMEOUT] = "timeout",27    [P_BE_FAIL_DISCONNECTED] = "disconnected",28    [P_BE_FAIL_CONNECTING] = "connecting",29    [P_BE_FAIL_CONNTIMEOUT] = "conntimeout",30    [P_BE_FAIL_READVALIDATE] = "readvalidate",31    [P_BE_FAIL_BADVALIDATE] = "badvalidate",32    [P_BE_FAIL_WRITING] = "writing",33    [P_BE_FAIL_READING] = "reading",34    [P_BE_FAIL_PARSING] = "parsing",35    [P_BE_FAIL_CLOSED] = "closedsock",36    [P_BE_FAIL_UNHANDLEDRES] = "unhandledres",37    [P_BE_FAIL_OOM] = "outofmemory",38    [P_BE_FAIL_ENDSYNC] = "missingend",39    [P_BE_FAIL_TRAILINGDATA] = "trailingdata",40    [P_BE_FAIL_INVALIDPROTOCOL] = "invalidprotocol",41    NULL42};4344static void proxy_backend_handler(const int fd, const short which, void *arg);45static void proxy_backend_tls_handler(const int fd, const short which, void *arg);46static void proxy_beconn_handler(const int fd, const short which, void *arg);47static void proxy_beconn_tls_handler(const int fd, const short which, void *arg);48static void proxy_event_handler(evutil_socket_t fd, short which, void *arg);49static void proxy_event_beconn(evutil_socket_t fd, short which, void *arg);50static int _prep_pending_write(struct mcp_backendconn_s *be, int *count, int *bytes, bool *iov_limit);51static void _post_pending_write(struct mcp_backendconn_s *be, ssize_t sent);52static int _flush_pending_write(struct mcp_backendconn_s *be);53static int _flush_pending_tls_write(struct mcp_backendconn_s *be);54static void _cleanup_backend(mcp_backend_t *be);55static void _reset_bad_backend(struct mcp_backendconn_s *be, enum proxy_be_failures err);56static void _set_main_event(struct mcp_backendconn_s *be, struct event_base *base, int flags, struct timeval *t, event_callback_fn callback);57static void _stop_main_event(struct mcp_backendconn_s *be);58static void _start_write_event(struct mcp_backendconn_s *be);59static void _stop_write_event(struct mcp_backendconn_s *be);60static void _start_timeout_event(struct mcp_backendconn_s *be);61static void _stop_timeout_event(struct mcp_backendconn_s *be);62static int proxy_backend_drive_machine(struct mcp_backendconn_s *be);6364/* Helper routines common to io_uring and libevent modes */6566// TODO (v3): doing an inline syscall here, not ideal for uring mode.67// leaving for now since this should be extremely uncommon.68static int _beconn_send_validate(struct mcp_backendconn_s *be) {69    const char *str = "version\r\n";70    const ssize_t len = strlen(str);7172    ssize_t res = write(mcmc_fd(be->client), str, len);7374    if (res == -1) {75        return -1;76    }7778    // I'm making an opinionated statement that we should be able to write79    // "version\r\n" into a fresh socket without hitting EAGAIN.80    if (res < len) {81        return -1;82    }8384    return 1;85}8687static int _proxy_beconn_checkconnect(struct mcp_backendconn_s *be) {88    int err = 0;89    // We were connecting, now ensure we're properly connected.90    if (mcmc_check_nonblock_connect(be->client, &err) != MCMC_OK) {91        P_DEBUG("%s: backend failed to connect (%s:%s)\n", __func__, be->be_parent->name, be->be_parent->port);92        // kick the bad backend, clear the queue, retry later.93        // FIXME (v2): if a connect fails, anything currently in the queue94        // should be safe to hold up until their timeout.95        _reset_bad_backend(be, P_BE_FAIL_CONNECTING);96        return -1;97    }98    P_DEBUG("%s: backend connected [fd: %d] (%s:%s)\n", __func__, mcmc_fd(be->client), be->be_parent->name, be->be_parent->port);99    be->connecting = false;100    be->state = mcp_backend_read;101102    // seed the failure time for the flap check.103    gettimeofday(&be->last_failed, NULL);104105    be->validating = true;106    // TODO: make validation optional.107108    return 0;109}110111// Use a simple heuristic to choose a backend connection socket out of a list112// of sockets.113struct mcp_backendconn_s *proxy_choose_beconn(mcp_backend_t *be) {114    struct mcp_backendconn_s *bec = &be->be[0];115    if (be->conncount != 1) {116        int depth = INT_MAX;117        // TODO: to computationally limit + ensure each connection stays118        // somewhat warm:119        // - remember idx of last conn used.120        // - if next idx has a lower depth, use that one instead121        // - tick idx (and reset if necessary)122        // else under low loads only the first conn will ever get used (which123        // is normally good; but sometimes bad if using stateful firewalls)124        for (int x = 0; x < be->conncount; x++) {125            struct mcp_backendconn_s *bec_i = &be->be[x];126            if (bec_i->bad) {127                continue;128            }129            if (bec_i->depth == 0) {130                bec = bec_i;131                break;132            } else if (bec_i->depth < depth) {133                depth = bec_i->depth;134                bec = bec_i;135            }136        }137    }138139    return bec;140}141142static void _proxy_event_handler_dequeue(proxy_event_thread_t *t) {143    iop_head_t head;144145    STAILQ_INIT(&head);146    STAILQ_INIT(&t->be_head);147148    // Pull the entire stack of inbound into local queue.149    pthread_mutex_lock(&t->mutex);150    STAILQ_CONCAT(&head, &t->iop_head_in);151    pthread_mutex_unlock(&t->mutex);152153    while (!STAILQ_EMPTY(&head)) {154        io_pending_proxy_t *p = (io_pending_proxy_t *)STAILQ_FIRST(&head);155        p->flushed = false;156157        // _no_ mutex on backends. they are owned by the event thread.158        STAILQ_REMOVE_HEAD(&head, iop_next);159        // paranoia about moving items between lists.160        p->iop_next.stqe_next = NULL;161162        mcp_backend_t *be = p->backend;163        STAILQ_INSERT_TAIL(&be->iop_head, (io_pending_t *)p, iop_next);164        assert(be->depth > -1);165        be->depth++;166        if (!be->stacked) {167            be->stacked = true;168            STAILQ_INSERT_TAIL(&t->be_head, be, be_next);169        }170    }171}172173static void _cleanup_backend(mcp_backend_t *be) {174    if (be->use_logging) {175        if (be->logging.detail) {176            free(be->logging.detail);177            be->logging.detail = NULL;178        }179    }180181    for (int x = 0; x < be->conncount; x++) {182        struct mcp_backendconn_s *bec = &be->be[x];183        // remove any pending events.184        if (!be->tunables.down) {185            int pending = event_pending(&bec->main_event, EV_READ|EV_WRITE|EV_TIMEOUT, NULL);186            if (pending != 0) {187                event_del(&bec->main_event); // an error to call event_del() without event.188            }189            pending = event_pending(&bec->write_event, EV_READ|EV_WRITE|EV_TIMEOUT, NULL);190            if (pending != 0) {191                event_del(&bec->write_event); // an error to call event_del() without event.192            }193            pending = event_pending(&bec->timeout_event, EV_TIMEOUT, NULL);194            if (pending != 0) {195                event_del(&bec->timeout_event); // an error to call event_del() without event.196            }197198            // - assert on empty queue199            assert(STAILQ_EMPTY(&bec->iop_write));200            assert(STAILQ_EMPTY(&bec->iop_read));201202            mcp_tls_shutdown(bec);203            mcmc_disconnect(bec->client);204205            if (bec->bad) {206                mcp_sharedvm_delta(bec->event_thread->ctx, SHAREDVM_BACKEND_IDX,207                    bec->be_parent->label, -1);208            }209        }210        // - free be->client211        free(bec->client);212        // - free be->rbuf213        free(bec->rbuf);214    }215    // free once parent has had all connections closed off.216    free(be);217}218219static void _setup_backend(mcp_backend_t *be) {220    for (int x = 0; x < be->conncount; x++) {221        struct mcp_backendconn_s *bec = &be->be[x];222        if (be->tunables.down) {223            // backend is "forced" into a bad state. never connect or224            // otherwise attempt to use it.225            be->be[x].bad = true;226            continue;227        }228        // assign the initial events to the backend, so we don't have to229        // constantly check if they were initialized yet elsewhere.230        // note these events will not fire until event_add() is called.231        int status = mcmc_connect(bec->client, be->name, be->port, bec->connect_flags);232        event_callback_fn _beconn_handler = &proxy_beconn_handler;233        event_callback_fn _backend_handler = &proxy_backend_handler;234        if (be->tunables.use_tls) {235            _beconn_handler = &proxy_beconn_tls_handler;236            _backend_handler = &proxy_backend_tls_handler;237        }238        event_assign(&bec->main_event, bec->event_thread->base, mcmc_fd(bec->client), EV_WRITE|EV_TIMEOUT, _beconn_handler, bec);239        event_assign(&bec->write_event, bec->event_thread->base, mcmc_fd(bec->client), EV_WRITE|EV_TIMEOUT, _backend_handler, bec);240        event_assign(&bec->timeout_event, bec->event_thread->base, -1, EV_TIMEOUT, _backend_handler, bec);241242        if (status == MCMC_CONNECTING || status == MCMC_CONNECTED) {243            // if we're already connected for some reason, still push it244            // through the connection handler to keep the code unified. It245            // will auto-wake because the socket is writeable.246            bec->connecting = true;247            bec->can_write = false;248            // kick off the event we intialized above.249            event_add(&bec->main_event, &bec->tunables.connect);250        } else {251            _reset_bad_backend(bec, P_BE_FAIL_CONNECTING);252        }253    }254}255256// event handler for injecting backends for processing257// currently just for initiating connections the first time.258static void proxy_event_beconn(evutil_socket_t fd, short which, void *arg) {259    proxy_event_thread_t *t = arg;260261#ifdef USE_EVENTFD262    uint64_t u;263    if (read(fd, &u, sizeof(uint64_t)) != sizeof(uint64_t)) {264        // Temporary error or wasn't actually ready to read somehow.265        return;266    }267#else268    char buf[1];269    if (read(fd, buf, 1) != 1) {270        P_DEBUG("%s: pipe read failed\n", __func__);271        return;272    }273#endif274275    beconn_head_t head;276277    STAILQ_INIT(&head);278    pthread_mutex_lock(&t->mutex);279    STAILQ_CONCAT(&head, &t->beconn_head_in);280    pthread_mutex_unlock(&t->mutex);281282    // Think we should reuse this code path for manually instructing backends283    // to disable/etc but not coding for that generically. We just need to284    // check the state of the backend when it reaches here or some flags at285    // least.286    // FIXME: another ->stacked flag?287    // Either that or remove the STAILQ code and just using an array of288    // ptr's.289    mcp_backend_t *be = NULL;290    // be can be freed by the loop, so can't use STAILQ_FOREACH.291    while (!STAILQ_EMPTY(&head)) {292        be = STAILQ_FIRST(&head);293        STAILQ_REMOVE_HEAD(&head, beconn_next);294        if (be->transferred) {295            // If this object was already transferred here, we're being296            // signalled to clean it up and free.297            _cleanup_backend(be);298        } else {299            be->transferred = true;300            _setup_backend(be);301        }302    }303}304305static void _proxy_flush_backend_queue(mcp_backend_t *be) {306    io_pending_proxy_t *io = NULL;307    P_DEBUG("%s: fast failing request to bad backend (%s:%s) depth: %d\n", __func__, be->name, be->port, be->depth);308309    while (!STAILQ_EMPTY(&be->iop_head)) {310        io = (io_pending_proxy_t *)STAILQ_FIRST(&be->iop_head);311        STAILQ_REMOVE_HEAD(&be->iop_head, iop_next);312        mcp_resp_set_elapsed(io->client_resp);313        io->client_resp->status = MCMC_ERR;314        io->client_resp->resp.code = MCMC_CODE_SERVER_ERROR;315        be->depth--;316        assert(be->depth > -1);317        return_io_pending((io_pending_t *)io);318    }319}320321void proxy_run_backend_queue(be_head_t *head) {322    mcp_backend_t *be;323    STAILQ_FOREACH(be, head, be_next) {324        be->stacked = false;325        int flags = 0;326        struct mcp_backendconn_s *bec = proxy_choose_beconn(be);327328        int limit = be->tunables.backend_depth_limit;329        if (bec->bad) {330            // TODO: another counter for fast fails?331            _proxy_flush_backend_queue(be);332            continue;333        } else if (limit && bec->depth > limit) {334            proxy_ctx_t *ctx = bec->event_thread->ctx;335            STAT_INCR(ctx, request_failed_depth, be->depth);336            _proxy_flush_backend_queue(be);337            continue;338        }339340        // drop new requests onto end of conn's io-head, reset the backend one.341        STAILQ_CONCAT(&bec->iop_write, &be->iop_head);342        bec->depth += be->depth;343        be->depth = 0;344345        if (bec->connecting || bec->validating || !bec->can_write) {346            P_DEBUG("%s: deferring IO pending connecting (%s:%s)\n", __func__, be->name, be->port);347        } else {348            if (!bec->ssl) {349                flags = _flush_pending_write(bec);350            } else {351                flags = _flush_pending_tls_write(bec);352            }353354            if (flags == -1) {355                _reset_bad_backend(bec, P_BE_FAIL_WRITING);356            } else if (flags & EV_WRITE) {357                // only get here because we need to kick off the write handler358                _start_write_event(bec);359            }360361            if (bec->pending_read) {362                _start_timeout_event(bec);363            }364365        }366    }367}368369// event handler for executing backend requests370static void proxy_event_handler(evutil_socket_t fd, short which, void *arg) {371    proxy_event_thread_t *t = arg;372373#ifdef USE_EVENTFD374    uint64_t u;375    if (read(fd, &u, sizeof(uint64_t)) != sizeof(uint64_t)) {376        // Temporary error or wasn't actually ready to read somehow.377        return;378    }379#else380    char buf[1];381    // TODO (v2): This is a lot more fatal than it should be. can it fail? can382    // it blow up the server?383    // TODO (v2): a cross-platform method of speeding this up would be nice. With384    // event fds we can queue N events and wakeup once here.385    // If we're pulling one byte out of the pipe at a time here it'll just386    // wake us up too often.387    // If the pipe is O_NONBLOCK then maybe just a larger read would work?388    if (read(fd, buf, 1) != 1) {389        P_DEBUG("%s: pipe read failed\n", __func__);390        return;391    }392#endif393394    _proxy_event_handler_dequeue(t);395396    // Re-walk each backend and check set event as required.397    proxy_run_backend_queue(&t->be_head);398}399400void *proxy_event_thread(void *arg) {401    proxy_event_thread_t *t = arg;402403    logger_create(); // TODO (v2): add logger ptr to structure404    event_base_loop(t->base, 0);405    event_base_free(t->base);406407    // TODO (v2): join bt threads, free array.408409    return NULL;410}411412static void _set_main_event(struct mcp_backendconn_s *be, struct event_base *base, int flags, struct timeval *t, event_callback_fn callback) {413    int pending = event_pending(&be->main_event, EV_READ|EV_WRITE|EV_TIMEOUT, NULL);414    if (pending != 0) {415        event_del(&be->main_event); // replace existing event.416    }417418    int fd = mcmc_fd(be->client);419    if (fd == 0) {420        fd = -1; // need to pass -1 to event assign if we're not operating on421                 // a connection.422    }423    event_assign(&be->main_event, base, fd,424            flags, callback, be);425    event_add(&be->main_event, t);426}427428static void _stop_main_event(struct mcp_backendconn_s *be) {429    event_del(&be->main_event);430}431432static void _start_write_event(struct mcp_backendconn_s *be) {433    int pending = event_pending(&be->write_event, EV_WRITE|EV_TIMEOUT, NULL);434    if (pending != 0) {435        return;436    }437    // FIXME: wasn't there a write timeout?438    event_add(&be->write_event, &be->tunables.read);439}440441static void _stop_write_event(struct mcp_backendconn_s *be) {442    event_del(&be->write_event);443}444445// handle the read timeouts with a side event, so we can stick with a446// persistent listener (optimization + catch disconnects faster)447static void _start_timeout_event(struct mcp_backendconn_s *be) {448    int pending = event_pending(&be->timeout_event, EV_TIMEOUT, NULL);449    if (pending != 0) {450        return;451    }452    event_add(&be->timeout_event, &be->tunables.read);453}454455static void _stop_timeout_event(struct mcp_backendconn_s *be) {456    int pending = event_pending(&be->timeout_event, EV_TIMEOUT, NULL);457    if (pending == 0) {458        return;459    }460    event_del(&be->timeout_event);461}462463static void _drive_machine_next(struct mcp_backendconn_s *be, io_pending_proxy_t *p) {464    // set the head here. when we break the head will be correct.465    assert(!STAILQ_EMPTY(&be->iop_read));466    STAILQ_REMOVE_HEAD(&be->iop_read, iop_next);467    be->depth--;468    assert(be->depth > -1);469    be->pending_read--;470    assert(be->pending_read > -1);471472    mcp_resp_set_elapsed(p->client_resp);473    // The moment we call return_io here we474    // don't own *p anymore.475    if (!be->be_parent->use_io_thread) {476        conn_io_queue_return((io_pending_t *)p);477    } else {478        return_io_pending((io_pending_t *)p);479    }480    be->state = mcp_backend_read;481}482483// NOTES:484// - mcp_backend_read: grab req_stack_head, do things485// read -> next, want_read -> next | read_end, etc.486static int proxy_backend_drive_machine(struct mcp_backendconn_s *be) {487    bool stop = false;488    io_pending_proxy_t *p = NULL;489    int flags = 0;490491    p = (io_pending_proxy_t *)STAILQ_FIRST(&be->iop_read);492    if (p == NULL) {493        // got a read event, but nothing was queued.494        // probably means a disconnect event.495        // TODO (v2): could probably confirm this by attempting to read the496        // socket, getsockopt, or something else simply for logging or497        // statistical purposes.498        // In this case we know it's going to be a close so error.499        flags = P_BE_FAIL_CLOSED;500        P_DEBUG("%s: read event but nothing in IO queue\n", __func__);501        return flags;502    }503504    while (!stop) {505        mcp_resp_t *r;506507    switch(be->state) {508        case mcp_backend_read:509            assert(p != NULL);510            // FIXME: remove the _read state?511            be->state = mcp_backend_parse;512            break;513        case mcp_backend_parse:514            r = p->client_resp;515            r->status = mcmc_parse_buf(be->rbuf, be->rbufused, &r->resp);516517            // Quick check if we need more data.518            if (r->resp.code == MCMC_WANT_READ) {519                return 0;520            }521522            // we actually don't care about anything but the value length523            // TODO (v2): if vlen != vlen_read, pull an item and copy the data.524            int extra_space = 0;525            // if all goes well, move to the next request.526            be->state = mcp_backend_next;527            switch (r->resp.type) {528                case MCMC_RESP_GET:529                    // We're in GET mode. we only support one key per530                    // GET in the proxy backends, so we need to later check531                    // for an END.532                    extra_space = ENDLEN;533                    be->state = mcp_backend_read_end;534                    break;535                case MCMC_RESP_END:536                    // this is a MISS from a GET request537                    // or final handler from a STAT request.538                    assert(r->resp.vlen == 0);539                    break;540                case MCMC_RESP_META:541                    // we can handle meta responses easily since they're self542                    // contained.543                    break;544                case MCMC_RESP_GENERIC:545                case MCMC_RESP_NUMERIC:546                    break;547                case MCMC_RESP_ERRMSG: // received an error message548                    if (r->resp.code != MCMC_CODE_SERVER_ERROR) {549                        // Non server errors are protocol errors; can't trust550                        // the connection anymore.551                        be->state = mcp_backend_next_close;552                    }553                    break;554                case MCMC_RESP_FAIL:555                    P_DEBUG("%s: mcmc_read failed [%d]\n", __func__, r->status);556                    flags = P_BE_FAIL_PARSING;557                    stop = true;558                    break;559                // TODO (v2): No-op response?560                default:561                    P_DEBUG("%s: Unhandled response from backend: %d\n", __func__, r->resp.type);562                    // unhandled :(563                    flags = P_BE_FAIL_UNHANDLEDRES;564                    stop = true;565                    break;566            }567568            // r->resp.reslen + r->resp.vlen is the total length of the response.569            // TODO (v2): need to associate a buffer with this response...570            // for now we simply malloc, but reusable buffers should be used571            if (r->resp.vlen > INT32_MAX/2) {572                // In a real memcached the value can't be over 1G. I made a573                // huge mess out of the various places blen gets used, so574                // instead of trying to fix all those paths without breaking575                // something else, we can instead clamp the value length.576                flags = P_BE_FAIL_OOM;577                r->blen = 0;578                stop = true;579                break;580            }581582            r->blen = r->resp.reslen + r->resp.vlen;583            r->buf = malloc(r->blen + extra_space);584            if (r->buf == NULL) {585                flags = P_BE_FAIL_OOM;586                r->blen = 0;587                stop = true;588                break;589            }590591            // Only do an inline check of the memory limit if we're not on an592            // IO thread. Avoids having to wrap the memory limit with a mutex593            if (!be->be_parent->use_io_thread) {594                if (r->thread->proxy_buffer_memory_used > r->thread->proxy_buffer_memory_limit) {595                    free(r->buf);596                    flags = P_BE_FAIL_OOM;597                    r->buf = NULL;598                    r->blen = 0;599                    stop = true;600                    break;601                }602            }603604            // TODO: mcmc's parser needs to use offsets into a string instead605            // of pointers. Then all we do is swap the buffer pointer. In the606            // meantime this is the smallest possible change to fix the issue607            // of res:line() and similar returning the wrong buffer for608            // pipelined requests, in cases where the result hasn't been609            // reparsed before requesting the line.610            ptrdiff_t rebase = r->buf - be->rbuf;611            r->resp.value += rebase;612            if (r->resp.rline) r->resp.rline += rebase;613614            P_DEBUG("%s: r->status: %d, r->bread: %d, r->vlen: %lu\n", __func__, r->status, r->bread, r->resp.vlen);615            if (r->resp.vlen != r->resp.vlen_read) {616                // shouldn't be possible to have excess in buffer617                // if we're dealing with a partial value.618                assert(be->rbufused == r->resp.reslen+r->resp.vlen_read);619                P_DEBUG("%s: got a short read, moving to want_read\n", __func__);620                // copy the partial and advance mcmc's buffer digestion.621                memcpy(r->buf, be->rbuf, r->resp.reslen + r->resp.vlen_read);622                r->bread = r->resp.reslen + r->resp.vlen_read;623                be->rbufused = 0;624                be->state = mcp_backend_want_read;625                flags = 0;626                stop = true;627                break;628            } else {629                // mcmc's already counted the value as read if it fit in630                // the original buffer...631                memcpy(r->buf, be->rbuf, r->resp.reslen+r->resp.vlen_read);632            }633634            // had a response, advance the buffer.635            be->rbufused -= r->resp.reslen + r->resp.vlen_read;636            if (be->rbufused > 0) {637                memmove(be->rbuf, be->rbuf+r->resp.reslen+r->resp.vlen_read, be->rbufused);638            }639640            break;641        case mcp_backend_read_end:642            r = p->client_resp;643            // we need to ensure the next data in the stream is "END\r\n"644            // if not, the stack is desynced and we lose it.645646            if (be->rbufused >= ENDLEN) {647                if (memcmp(be->rbuf, ENDSTR, ENDLEN) != 0) {648                    flags = P_BE_FAIL_ENDSYNC;649                    stop = true;650                    break;651                } else {652                    // response is good.653                    // FIXME (v2): copy what the server actually sent?654                    memcpy(r->buf+r->blen, ENDSTR, ENDLEN);655                    r->blen += 5;656657                    // advance buffer658                    be->rbufused -= ENDLEN;659                    if (be->rbufused > 0) {660                        memmove(be->rbuf, be->rbuf+ENDLEN, be->rbufused);661                    }662                }663            } else {664                flags = 0;665                stop = true;666                break;667            }668669            be->state = mcp_backend_next;670671            break;672        case mcp_backend_want_read:673            // Continuing a read from earlier674            r = p->client_resp;675            // take bread input and see if we're done reading the value,676            // else advance, set buffers, return next.677            P_DEBUG("%s: [want_read] r->bread: %d vlen: %lu\n", __func__, r->bread, r->resp.vlen);678            assert(be->rbufused != 0);679            size_t tocopy = be->rbufused < r->blen - r->bread ?680                be->rbufused : r->blen - r->bread;681            memcpy(r->buf+r->bread, be->rbuf, tocopy);682            r->bread += tocopy;683684            if (r->bread >= r->blen) {685                // all done copying data.686                if (r->resp.type == MCMC_RESP_GET) {687                    be->state = mcp_backend_read_end;688                } else {689                    be->state = mcp_backend_next;690                }691692                // shuffle remaining buffer.693                be->rbufused -= tocopy;694                if (be->rbufused > 0) {695                    memmove(be->rbuf, be->rbuf+tocopy, be->rbufused);696                }697            } else {698                assert(tocopy == be->rbufused);699                // signal to caller to issue a read.700                be->rbufused = 0;701                flags = 0;702                stop = true;703            }704705            break;706        case mcp_backend_next:707            _drive_machine_next(be, p);708709            if (STAILQ_EMPTY(&be->iop_read)) {710                stop = true;711                // if there're no pending requests, the read buffer712                // should also be empty.713                if (be->rbufused > 0) {714                    flags = P_BE_FAIL_TRAILINGDATA;715                }716                break;717            } else {718                p = (io_pending_proxy_t *)STAILQ_FIRST(&be->iop_read);719            }720721            // if leftover, keep processing IO's.722            // if no more data in buffer, need to re-set stack head and re-set723            // event.724            P_DEBUG("%s: [next] remain: %lu\n", __func__, be->rbufused);725            if (be->rbufused != 0) {726                // data trailing in the buffer, for a different request.727                be->state = mcp_backend_parse;728            } else {729                // need to read more data, buffer is empty.730                stop = true;731            }732733            break;734        case mcp_backend_next_close:735            // we advance and return the current IO, then kill the conn.736            _drive_machine_next(be, p);737            stop = true;738            flags = P_BE_FAIL_INVALIDPROTOCOL;739740            break;741        default:742            // TODO (v2): at some point (after v1?) this should attempt to recover,743            // though we should only get here from memory corruption and744            // bailing may be the right thing to do.745            fprintf(stderr, "%s: invalid backend state: %d\n", __func__, be->state);746            assert(false);747    } // switch748    } // while749750    return flags;751}752753static void _backend_reconnect(struct mcp_backendconn_s *be) {754    int status = mcmc_connect(be->client, be->be_parent->name, be->be_parent->port, be->connect_flags);755    if (status == MCMC_CONNECTED) {756        // TODO (v2): unexpected but lets let it be here.757        be->connecting = false;758        be->can_write = true;759    } else if (status == MCMC_CONNECTING) {760        be->connecting = true;761        be->can_write = false;762    } else {763        // failed to immediately re-establish the connection.764        // need to put the BE into a bad/retry state.765        be->connecting = false;766        be->can_write = true;767    }768    // re-create the write handler for the new file descriptor.769    // the main event will be re-assigned after this call.770    event_callback_fn _backend_handler = &proxy_backend_handler;771    if (be->be_parent->tunables.use_tls) {772        _backend_handler = &proxy_backend_tls_handler;773    }774    event_assign(&be->write_event, be->event_thread->base, mcmc_fd(be->client), EV_WRITE|EV_TIMEOUT, _backend_handler, be);775    // do not need to re-assign the timer event because it's not tied to fd776}777778// All we need to do here is schedule the backend to attempt to connect again.779static void proxy_backend_retry_handler(const int fd, const short which, void *arg) {780    struct mcp_backendconn_s *be = arg;781    assert(which & EV_TIMEOUT);782    struct timeval tmp_time = be->tunables.connect;783    _backend_reconnect(be);784    event_callback_fn _backend_handler = &proxy_beconn_handler;785    if (be->be_parent->tunables.use_tls) {786        _backend_handler = &proxy_beconn_tls_handler;787    }788    _set_main_event(be, be->event_thread->base, EV_WRITE, &tmp_time, _backend_handler);789}790791// must be called after _reset_bad_backend(), so the backend is currently792// clear.793// TODO (v2): extra counter for "backend connect tries" so it's still possible794// to see dead backends exist795static void _backend_reschedule(struct mcp_backendconn_s *be) {796    bool failed = false;797    struct timeval tmp_time = {0};798    long int retry_time = be->tunables.retry.tv_sec;799    char *badtext = "markedbad";800    if (be->flap_count > be->tunables.backend_failure_limit) {801        // reduce retry frequency to avoid noise.802        float backoff = retry_time;803        for (int x = 0; x < be->flap_count; x++) {804            backoff *= be->tunables.flap_backoff_ramp;805        }806        retry_time = (uint32_t)backoff;807808        if (retry_time > be->tunables.flap_backoff_max) {809            retry_time = be->tunables.flap_backoff_max;810        }811        badtext = "markedbadflap";812        failed = true;813    } else if (be->failed_count > be->tunables.backend_failure_limit) {814        failed = true;815    }816    tmp_time.tv_sec = retry_time;817818    if (failed) {819        if (!be->bad) {820            P_DEBUG("%s: marking backend as bad\n", __func__);821            STAT_INCR(be->event_thread->ctx, backend_marked_bad, 1);822            mcp_sharedvm_delta(be->event_thread->ctx, SHAREDVM_BACKEND_IDX,823                    be->be_parent->label, 1);824            LOGGER_LOG(NULL, LOG_PROXYEVENTS, LOGGER_PROXY_BE_ERROR, NULL, badtext, be->be_parent->name, be->be_parent->port, be->be_parent->label, 0, NULL, 0, retry_time);825        }826        be->bad = true;827       _set_main_event(be, be->event_thread->base, EV_TIMEOUT, &tmp_time, proxy_backend_retry_handler);828    } else {829        struct timeval tmp_time = be->tunables.connect;830        STAT_INCR(be->event_thread->ctx, backend_failed, 1);831        _backend_reconnect(be);832        event_callback_fn _backend_handler = &proxy_beconn_handler;833        if (be->be_parent->tunables.use_tls) {834            _backend_handler = &proxy_beconn_tls_handler;835        }836        _set_main_event(be, be->event_thread->base, EV_WRITE, &tmp_time, _backend_handler);837    }838}839840static void _backend_flap_check(struct mcp_backendconn_s *be, enum proxy_be_failures err) {841    struct timeval now;842    struct timeval *flap = &be->tunables.flap;843844    switch (err) {845        case P_BE_FAIL_TIMEOUT:846        case P_BE_FAIL_DISCONNECTED:847        case P_BE_FAIL_WRITING:848        case P_BE_FAIL_READING:849            if (flap->tv_sec != 0 || flap->tv_usec != 0) {850                struct timeval delta = {0};851                int64_t subsec = 0;852                gettimeofday(&now, NULL);853                delta.tv_sec = now.tv_sec - be->last_failed.tv_sec;854                subsec = now.tv_usec - be->last_failed.tv_usec;855                if (subsec < 0) {856                    // tv_usec is specced as "at least" [-1, 1000000]857                    // so to guarantee lower negatives we need this temp var.858                    delta.tv_sec--;859                    subsec += 1000000;860                    delta.tv_usec = subsec;861                }862863                if (flap->tv_sec < delta.tv_sec ||864                    (flap->tv_sec == delta.tv_sec && flap->tv_usec < delta.tv_usec)) {865                    // delta is larger than our flap range. reset the flap counter.866                    be->flap_count = 0;867                } else {868                    // seems like we flapped again.869                    be->flap_count++;870                }871                be->last_failed = now;872            }873            break;874        default:875            // only perform a flap check on network related errors.876            break;877    }878}879880// TODO (v2): add a second argument for assigning a specific error to all pending881// IO's (ie; timeout).882// The backend has gotten into a bad state (timed out, protocol desync, or883// some other supposedly unrecoverable error: purge the queue and884// cycle the socket.885// Note that some types of errors may not require flushing the queue and886// should be fixed as they're figured out.887// _must_ be called from within the event thread.888static void _reset_bad_backend(struct mcp_backendconn_s *be, enum proxy_be_failures err) {889    io_pending_proxy_t *io = NULL;890    P_DEBUG("%s: resetting bad backend: [fd: %d] %s\n", __func__, mcmc_fd(be->client), proxy_be_failure_text[err]);891    // Can't use STAILQ_FOREACH() since r_io_p() free's the current892    // io. STAILQ_FOREACH_SAFE maybe?893    int depth = be->depth;894    while (!STAILQ_EMPTY(&be->iop_write)) {895        io = (io_pending_proxy_t *)STAILQ_FIRST(&be->iop_write);896        STAILQ_REMOVE_HEAD(&be->iop_write, iop_next);897898        mcp_resp_set_elapsed(io->client_resp);899        io->client_resp->status = MCMC_ERR;900        io->client_resp->resp.code = MCMC_CODE_SERVER_ERROR;901        be->depth--;902        assert(be->depth > -1);903        return_io_pending((io_pending_t *)io);904    }905906    while (!STAILQ_EMPTY(&be->iop_read)) {907        io = (io_pending_proxy_t *)STAILQ_FIRST(&be->iop_read);908        STAILQ_REMOVE_HEAD(&be->iop_read, iop_next);909910        mcp_resp_set_elapsed(io->client_resp);911        io->client_resp->status = MCMC_ERR;912        io->client_resp->resp.code = MCMC_CODE_SERVER_ERROR;913        be->depth--;914        assert(be->depth > -1);915        return_io_pending((io_pending_t *)io);916    }917918    STAILQ_INIT(&be->iop_write);919    STAILQ_INIT(&be->iop_read);920921    // Only log if we don't already know it's messed up.922    if (!be->bad) {923        LOGGER_LOG(NULL, LOG_PROXYEVENTS, LOGGER_PROXY_BE_ERROR, NULL, proxy_be_failure_text[err], be->be_parent->name, be->be_parent->port, be->be_parent->label, depth, be->rbuf, be->rbufused, 0);924    }925926    // reset buffer to blank state.927    be->rbufused = 0;928    be->pending_read = 0;929    // clear events so the reconnect handler can re-arm them with a few fd.930    _stop_write_event(be);931    _stop_main_event(be);932    _stop_timeout_event(be);933    mcp_tls_shutdown(be);934    mcmc_disconnect(be->client);935    // we leave the main event alone, because be_failed() always overwrites.936937    // check failure counters and schedule a retry.938    be->failed_count++;939    _backend_flap_check(be, err);940    _backend_reschedule(be);941}942943static int _prep_pending_write(struct mcp_backendconn_s *be, int *count, int *bytes, bool *iov_limit) {944    struct iovec *iovs = be->write_iovs;945    io_pending_proxy_t *io = NULL;946    int iovused = 0;947    io = (io_pending_proxy_t *)STAILQ_FIRST(&be->iop_write);948    assert(io != NULL);949    for (; io; io = (io_pending_proxy_t *)STAILQ_NEXT(io, iop_next)) {950        assert(io->flushed == false);951952        if (io->iovcnt + iovused > BE_IOV_MAX) {953            // We will need to keep writing later.954            *iov_limit = true;955            break;956        }957958        memcpy(&iovs[iovused], io->iov, sizeof(struct iovec)*io->iovcnt);959        iovused += io->iovcnt;960        *bytes += io->iovbytes;961        (*count)++;962    }963    return iovused;964}965966// returns true if any pending writes were fully flushed.967static void _post_pending_write(struct mcp_backendconn_s *be, ssize_t sent) {968    io_pending_proxy_t *io = (io_pending_proxy_t *)STAILQ_FIRST(&be->iop_write);969970    while (!STAILQ_EMPTY(&be->iop_write)) {971        io = (io_pending_proxy_t *)STAILQ_FIRST(&be->iop_write);972        bool flushed = true;973        assert(io->flushed == false);974975        if (sent >= io->iovbytes) {976            // short circuit for common case.977            sent -= io->iovbytes;978        } else {979            io->iovbytes -= sent;980            for (int x = 0; x < io->iovcnt; x++) {981                struct iovec *iov = &io->iov[x];982                if (sent >= iov->iov_len) {983                    sent -= iov->iov_len;984                    iov->iov_len = 0;985                } else {986                    iov->iov_len -= sent;987                    iov->iov_base = (char *)iov->iov_base + sent;988                    sent = 0;989                    flushed = false;990                    break;991                }992            }993        }994        io->flushed = flushed;995        if (flushed) {996            STAILQ_REMOVE_HEAD(&be->iop_write, iop_next);997            STAILQ_INSERT_TAIL(&be->iop_read, (io_pending_t *)io, iop_next);998            be->pending_read++;999        }10001001        if (sent <= 0) {1002            // really shouldn't be negative, though.1003            assert(sent >= 0);1004            break;1005        }1006    } // for1007}10081009static int _flush_pending_write(struct mcp_backendconn_s *be) {1010    int flags = 0;1011    bool iov_limit = false;1012    // Allow us to be called with an empty stack to prevent dev errors.1013    if (STAILQ_EMPTY(&be->iop_write)) {1014        return 0;1015    }10161017    int count = 0;1018    int bytes = 0;1019    int iovcnt = _prep_pending_write(be, &count, &bytes, &iov_limit);10201021    ssize_t sent = writev(mcmc_fd(be->client), be->write_iovs, iovcnt);1022    if (sent > 0) {1023        if (bytes == sent && !iov_limit) {1024            // fast path if everything's sent.1025            be->pending_read += count;1026            STAILQ_CONCAT(&be->iop_read, &be->iop_write);1027        } else {1028            _post_pending_write(be, sent);1029            // still have unflushed pending IO's, check for write and re-loop.1030            if (!STAILQ_EMPTY(&be->iop_write)) {1031                // might still be writeable, just too many IOV's.1032                be->can_write = iov_limit;1033                flags |= EV_WRITE;1034            }1035        }1036    } else if (sent == -1) {1037        if (errno == EAGAIN || errno == EWOULDBLOCK) {1038            be->can_write = false;1039            flags |= EV_WRITE;1040        } else {1041            flags = -1;1042        }1043    }10441045    return flags;1046}10471048static int _flush_pending_tls_write(struct mcp_backendconn_s *be) {1049    int flags = 0;1050    bool iov_limit = false;1051    // Allow us to be called with an empty stack to prevent dev errors.1052    if (STAILQ_EMPTY(&be->iop_write)) {1053        return 0;1054    }10551056    int count = 0;1057    int bytes = 0;1058    int iovcnt = _prep_pending_write(be, &count, &bytes, &iov_limit);10591060    int sent = mcp_tls_writev(be, iovcnt);1061    if (sent > 0) {1062        if (bytes == sent && !iov_limit) {1063            // fast path if everything's sent.1064            be->pending_read += count;1065            STAILQ_CONCAT(&be->iop_read, &be->iop_write);1066        } else {1067            _post_pending_write(be, sent);1068            // still have unflushed pending IO's, check for write and re-loop.1069            if (!STAILQ_EMPTY(&be->iop_write)) {1070                // might still be writeable, just too many IOV's.1071                be->can_write = iov_limit;1072                flags |= EV_WRITE;1073            }1074        }1075    } else if (sent == MCP_TLS_NEEDIO) {1076        // want io1077        be->can_write = false;1078        flags |= EV_WRITE;1079    } else if (sent == MCP_TLS_ERR) {1080        // hard error from tls1081        flags = -1;1082    }10831084    return flags;1085}10861087static void proxy_bevalidate_tls_handler(const int fd, const short which, void *arg) {1088    assert(arg != NULL);1089    struct mcp_backendconn_s *be = arg;1090    int flags = EV_TIMEOUT;1091    struct timeval tmp_time = be->tunables.read;10921093    if (which & EV_TIMEOUT) {1094        P_DEBUG("%s: backend timed out while connecting [fd: %d]\n", __func__, mcmc_fd(be->client));1095        if (be->connecting) {1096            _reset_bad_backend(be, P_BE_FAIL_CONNTIMEOUT);1097        } else {1098            _reset_bad_backend(be, P_BE_FAIL_READVALIDATE);1099        }1100        return;1101    }11021103    if (which & EV_READ) {1104        int read = mcp_tls_read(be);11051106        if (read > 0) {1107            mcmc_resp_t r;11081109            int status = mcmc_parse_buf(be->rbuf, be->rbufused, &r);1110            if (status == MCMC_ERR) {1111                // Needed more data for a version line, somehow. I feel like1112                // this should set off some alarms, but it is possible.1113                if (r.code == MCMC_WANT_READ) {1114                    _set_main_event(be, be->event_thread->base, EV_READ, &tmp_time, proxy_bevalidate_tls_handler);1115                    return;1116                }11171118                _reset_bad_backend(be, P_BE_FAIL_READVALIDATE);1119                return;1120            }11211122            if (r.code != MCMC_CODE_VERSION) {1123                _reset_bad_backend(be, P_BE_FAIL_BADVALIDATE);1124                return;1125            }11261127            be->validating = false;1128            be->rbufused = 0;1129        } else if (read == 0) {1130            // not connected or error.1131            _reset_bad_backend(be, P_BE_FAIL_DISCONNECTED);1132            return;1133        } else if (read == MCP_TLS_NEEDIO) {1134            // try again failure.1135            _set_main_event(be, be->event_thread->base, EV_READ, &tmp_time, proxy_bevalidate_tls_handler);1136            return;1137        } else if (read == MCP_TLS_ERR) {1138            // hard failure.1139            _reset_bad_backend(be, P_BE_FAIL_READING);1140            return;1141        }11421143        // Passed validation, don't need to re-read, flush any pending writes.1144        int res = _flush_pending_tls_write(be);1145        if (res == -1) {1146            _reset_bad_backend(be, P_BE_FAIL_WRITING);1147            return;1148        }1149        if (flags & EV_WRITE) {1150            _start_write_event(be);1151        }1152        if (be->pending_read) {1153            _start_timeout_event(be);1154        }1155    }11561157    // switch to the primary persistent read event.1158    if (!be->validating) {1159        _set_main_event(be, be->event_thread->base, EV_READ|EV_PERSIST, NULL, proxy_backend_tls_handler);11601161        // we're happily validated and switching to normal processing, so1162        // _now_ the backend is no longer "bad".1163        // If we reset the failed count earlier we then can fail the1164        // validation loop indefinitely without ever being marked bad.1165        if (be->bad) {1166            // was bad, need to mark as no longer bad in shared space.1167            mcp_sharedvm_delta(be->event_thread->ctx, SHAREDVM_BACKEND_IDX,1168                    be->be_parent->label, -1);1169        }1170        be->bad = false;1171        be->failed_count = 0;1172    }1173}11741175// Libevent handler when we're in TLS mode. Unfortunately the code is1176// different enough to warrant its own function.1177static void proxy_beconn_tls_handler(const int fd, const short which, void *arg) {1178    assert(arg != NULL);1179    struct mcp_backendconn_s *be = arg;1180    //int flags = EV_TIMEOUT;1181    struct timeval tmp_time = be->tunables.read;11821183    if (which & EV_TIMEOUT) {1184        P_DEBUG("%s: backend timed out while connecting [fd: %d]\n", __func__, mcmc_fd(be->client));1185        if (be->connecting) {1186            _reset_bad_backend(be, P_BE_FAIL_CONNTIMEOUT);1187        } else {1188            _reset_bad_backend(be, P_BE_FAIL_READVALIDATE);1189        }1190        return;1191    }11921193    if (which & EV_WRITE) {1194        be->can_write = true;11951196        if (be->connecting) {1197            if (_proxy_beconn_checkconnect(be) == -1) {1198                return;1199            }1200            // TODO: check return code.1201            mcp_tls_connect(be);1202            // fall through to handshake attempt.1203        }1204    }12051206    assert(be->validating);1207    int ret = mcp_tls_handshake(be);1208    if (ret == MCP_TLS_NEEDIO) {1209        // Need to try again.1210        _set_main_event(be, be->event_thread->base, EV_READ, &tmp_time, proxy_beconn_tls_handler);1211        return;1212    } else if (ret == 1) {1213        // handshake complete.1214        if (mcp_tls_send_validate(be) != MCP_TLS_OK) {1215            _reset_bad_backend(be, P_BE_FAIL_BADVALIDATE);1216            return;1217        }12181219        // switch to another handler for the final stage.1220        _set_main_event(be, be->event_thread->base, EV_READ, &tmp_time, proxy_bevalidate_tls_handler);1221    } else if (ret < 0) {1222        // FIXME: FAIL_HANDSHAKE1223        _reset_bad_backend(be, P_BE_FAIL_BADVALIDATE);1224        return;1225    }1226}12271228// Libevent handler for backends in a connecting state.1229static void proxy_beconn_handler(const int fd, const short which, void *arg) {1230    assert(arg != NULL);1231    struct mcp_backendconn_s *be = arg;1232    int flags = EV_TIMEOUT;1233    struct timeval tmp_time = be->tunables.read;12341235    if (which & EV_TIMEOUT) {1236        P_DEBUG("%s: backend timed out while connecting [fd: %d]\n", __func__, mcmc_fd(be->client));1237        if (be->connecting) {1238            _reset_bad_backend(be, P_BE_FAIL_CONNTIMEOUT);1239        } else {1240            _reset_bad_backend(be, P_BE_FAIL_READVALIDATE);1241        }1242        return;1243    }12441245    if (which & EV_WRITE) {1246        be->can_write = true;12471248        if (be->connecting) {1249            if (_proxy_beconn_checkconnect(be) == -1) {1250                return;1251            }1252            if (_beconn_send_validate(be) == -1) {1253                _reset_bad_backend(be, P_BE_FAIL_BADVALIDATE);1254                return;1255            }1256            _set_main_event(be, be->event_thread->base, EV_READ, &tmp_time, proxy_beconn_handler);1257        }12581259        // TODO: currently never taken, until validation is made optional.1260        if (!be->validating) {1261            int res = _flush_pending_write(be);1262            if (res == -1) {1263                _reset_bad_backend(be, P_BE_FAIL_WRITING);1264                return;1265            }1266            flags |= res;1267            // FIXME: set write event?1268        }1269    }12701271    if (which & EV_READ) {1272        assert(be->validating);12731274        int read = recv(mcmc_fd(be->client), be->rbuf + be->rbufused, READ_BUFFER_SIZE - be->rbufused, 0);1275        if (read > 0) {1276            mcmc_resp_t r;1277            be->rbufused += read;12781279            int status = mcmc_parse_buf(be->rbuf, be->rbufused, &r);1280            if (status == MCMC_ERR) {1281                // Needed more data for a version line, somehow. I feel like1282                // this should set off some alarms, but it is possible.1283                if (r.code == MCMC_WANT_READ) {1284                    _set_main_event(be, be->event_thread->base, EV_READ, &tmp_time, proxy_beconn_handler);1285                    return;1286                }12871288                _reset_bad_backend(be, P_BE_FAIL_READVALIDATE);1289                return;1290            }12911292            if (r.code != MCMC_CODE_VERSION) {1293                _reset_bad_backend(be, P_BE_FAIL_BADVALIDATE);1294                return;1295            }12961297            be->validating = false;1298            be->rbufused = 0;1299        } else if (read == 0) {1300            // not connected or error.1301            _reset_bad_backend(be, P_BE_FAIL_DISCONNECTED);1302            return;1303        } else if (read == -1) {1304            // sit on epoll again.1305            if (errno != EAGAIN && errno != EWOULDBLOCK) {1306                _reset_bad_backend(be, P_BE_FAIL_READING);1307                return;1308            }1309            _set_main_event(be, be->event_thread->base, EV_READ, &tmp_time, proxy_beconn_handler);1310            return;1311        }13121313        // Passed validation, don't need to re-read, flush any pending writes.1314        int res = _flush_pending_write(be);1315        if (res == -1) {1316            _reset_bad_backend(be, P_BE_FAIL_WRITING);1317            return;1318        }1319        if (res & EV_WRITE) {1320            _start_write_event(be);1321        }1322        if (be->pending_read) {1323            _start_timeout_event(be);1324        }1325    }13261327    // switch to the primary persistent read event.1328    if (!be->validating) {1329        _set_main_event(be, be->event_thread->base, EV_READ|EV_PERSIST, NULL, proxy_backend_handler);13301331        // we're happily validated and switching to normal processing, so1332        // _now_ the backend is no longer "bad".1333        // If we reset the failed count earlier we then can fail the1334        // validation loop indefinitely without ever being marked bad.1335        if (be->bad) {1336            // was bad, need to mark as no longer bad in shared space.1337            mcp_sharedvm_delta(be->event_thread->ctx, SHAREDVM_BACKEND_IDX,1338                    be->be_parent->label, -1);1339        }1340        be->bad = false;1341        be->failed_count = 0;1342    }1343}13441345static void proxy_backend_tls_handler(const int fd, const short which, void *arg) {1346    struct mcp_backendconn_s *be = arg;13471348    if (which & EV_TIMEOUT) {1349        P_DEBUG("%s: timeout received, killing backend queue\n", __func__);1350        _reset_bad_backend(be, P_BE_FAIL_TIMEOUT);1351        return;1352    }13531354    if (which & EV_WRITE) {1355        be->can_write = true;1356        int res = _flush_pending_tls_write(be);1357        if (res == -1) {1358            _reset_bad_backend(be, P_BE_FAIL_WRITING);1359            return;1360        }1361        if (res & EV_WRITE) {1362            _start_write_event(be);1363        }1364    }13651366    if (which & EV_READ) {1367        // got a read event, always kill the pending read timer.1368        _stop_timeout_event(be);1369        // We do the syscall here before diving into the state machine to allow a1370        // common code path for io_uring/epoll/tls/etc1371        int read = mcp_tls_read(be);1372        if (read > 0) {1373            int res = proxy_backend_drive_machine(be);1374            if (res != 0) {1375                _reset_bad_backend(be, res);1376                return;1377            }1378        } else if (read == 0) {1379            // not connected or error.1380            _reset_bad_backend(be, P_BE_FAIL_DISCONNECTED);1381            return;1382        } else if (read == MCP_TLS_NEEDIO) {1383            // sit on epoll again.1384            return;1385        } else if (read == MCP_TLS_ERR) {1386            _reset_bad_backend(be, P_BE_FAIL_READING);1387            return;1388        }13891390#ifdef PROXY_DEBUG1391        if (!STAILQ_EMPTY(&be->iop_read)) {1392            P_DEBUG("backend has leftover IOs: %d\n", be->depth);1393        }1394#endif1395    }13961397    if (be->pending_read) {1398        _start_timeout_event(be);1399    }1400}14011402// The libevent backend callback handler.1403// If we end up resetting a backend, it will get put back into a connecting1404// state.1405static void proxy_backend_handler(const int fd, const short which, void *arg) {1406    struct mcp_backendconn_s *be = arg;14071408    if (which & EV_TIMEOUT) {1409        P_DEBUG("%s: timeout received, killing backend queue\n", __func__);1410        _reset_bad_backend(be, P_BE_FAIL_TIMEOUT);1411        return;1412    }14131414    if (which & EV_WRITE) {1415        be->can_write = true;1416        int res = _flush_pending_write(be);1417        if (res == -1) {1418            _reset_bad_backend(be, P_BE_FAIL_WRITING);1419            return;1420        }1421        if (res & EV_WRITE) {1422            _start_write_event(be);1423        }1424    }14251426    if (which & EV_READ) {1427        // got a read event, always kill the pending read timer.1428        _stop_timeout_event(be);1429        // We do the syscall here before diving into the state machine to allow a1430        // common code path for io_uring/epoll1431        int read = recv(mcmc_fd(be->client), be->rbuf + be->rbufused,1432                    READ_BUFFER_SIZE - be->rbufused, 0);1433        if (read > 0) {1434            be->rbufused += read;1435            int res = proxy_backend_drive_machine(be);1436            if (res != 0) {1437                _reset_bad_backend(be, res);1438                return;1439            }1440        } else if (read == 0) {1441            // not connected or error.1442            _reset_bad_backend(be, P_BE_FAIL_DISCONNECTED);1443            return;1444        } else if (read == -1) {1445            // sit on epoll again.1446            if (errno != EAGAIN && errno != EWOULDBLOCK) {1447                _reset_bad_backend(be, P_BE_FAIL_READING);1448                return;1449            }1450        }14511452#ifdef PROXY_DEBUG1453        if (!STAILQ_EMPTY(&be->iop_read)) {1454            P_DEBUG("backend has leftover IOs: %d\n", be->depth);1455        }1456#endif1457    }14581459    if (be->pending_read) {1460        _start_timeout_event(be);1461    }1462}14631464void proxy_init_event_thread(proxy_event_thread_t *t, proxy_ctx_t *ctx, struct event_base *base) {1465    t->ctx = ctx;1466#ifdef USE_EVENTFD1467    t->event_fd = eventfd(0, EFD_NONBLOCK);1468    if (t->event_fd == -1) {1469        perror("failed to create backend notify eventfd");1470        exit(1);1471    }1472    t->be_event_fd = eventfd(0, EFD_NONBLOCK);1473    if (t->be_event_fd == -1) {1474        perror("failed to create backend notify eventfd");1475        exit(1);1476    }1477#else1478    int fds[2];1479    if (pipe(fds)) {1480        perror("can't create proxy backend notify pipe");1481        exit(1);1482    }14831484    t->notify_receive_fd = fds[0];1485    t->notify_send_fd = fds[1];14861487    if (pipe(fds)) {1488        perror("can't create proxy backend connection notify pipe");1489        exit(1);1490    }1491    t->be_notify_receive_fd = fds[0];1492    t->be_notify_send_fd = fds[1];1493#endif14941495    // incoming request queue.1496    STAILQ_INIT(&t->iop_head_in);1497    STAILQ_INIT(&t->beconn_head_in);1498    pthread_mutex_init(&t->mutex, NULL);1499    pthread_cond_init(&t->cond, NULL);15001501    // initialize the event system.15021503#ifdef HAVE_LIBURING1504    if (t->ctx->use_uring) {1505        fprintf(stderr, "Sorry, io_uring not supported right now\n");1506        abort();1507    }1508#endif15091510    if (base == NULL) {1511        struct event_config *ev_config;1512        ev_config = event_config_new();1513        event_config_set_flag(ev_config, EVENT_BASE_FLAG_NOLOCK);1514        t->base = event_base_new_with_config(ev_config);1515        event_config_free(ev_config);1516        if (! t->base) {1517            fprintf(stderr, "Can't allocate event base\n");1518            exit(1);1519        }1520    } else {1521        // reusing an event base from a worker thread.1522        t->base = base;1523    }15241525    // listen for notifications.1526    // NULL was thread_libevent_process1527    // FIXME (v2): use modern format? (event_assign)1528#ifdef USE_EVENTFD1529    event_set(&t->notify_event, t->event_fd,1530          EV_READ | EV_PERSIST, proxy_event_handler, t);1531    event_set(&t->beconn_event, t->be_event_fd,1532          EV_READ | EV_PERSIST, proxy_event_beconn, t);1533#else1534    event_set(&t->notify_event, t->notify_receive_fd,1535          EV_READ | EV_PERSIST, proxy_event_handler, t);1536    event_set(&t->beconn_event, t->be_notify_receive_fd,1537          EV_READ | EV_PERSIST, proxy_event_beconn, t);1538#endif15391540    event_base_set(t->base, &t->notify_event);1541    if (event_add(&t->notify_event, 0) == -1) {1542        fprintf(stderr, "Can't monitor libevent notify pipe\n");1543        exit(1);1544    }1545    event_base_set(t->base, &t->beconn_event);1546    if (event_add(&t->beconn_event, 0) == -1) {1547        fprintf(stderr, "Can't monitor libevent notify pipe\n");1548        exit(1);1549    }1550}