1#ifndef PROXY_H2#define PROXY_H34#include "memcached.h"5#include "extstore.h"6#include <string.h>7#include <stdlib.h>8#include <ctype.h>9#include <errno.h>1011#include <lua.h>12#include <lualib.h>13#include <lauxlib.h>1415#include "config.h"1617#if defined(__linux__)18#define USE_EVENTFD 119#include <sys/eventfd.h>20#endif2122#ifdef HAVE_LIBURING23#include <liburing.h>24#include <poll.h> // POLLOUT for liburing.25#define PRING_QUEUE_SQ_ENTRIES 204826#define PRING_QUEUE_CQ_ENTRIES 1638427#endif2829#include "vendor/mcmc/mcmc.h"30#include "proto_proxy.h"31#include "proto_text.h"32#include "proto_parser.h"33#include "queue.h"34#define XXH_INLINE_ALL // modifier for xxh3's include below35#include "xxhash.h"3637#ifdef PROXY_DEBUG38#define P_DEBUG(...) \39    do { \40        fprintf(stderr, __VA_ARGS__); \41    } while (0)42#else43#define P_DEBUG(...)44#endif4546#define WSTAT_L(t) pthread_mutex_lock(&t->stats.mutex);47#define WSTAT_UL(t) pthread_mutex_unlock(&t->stats.mutex);48#define WSTAT_INCR(t, stat, amount) { \49    pthread_mutex_lock(&t->stats.mutex); \50    t->stats.stat += amount; \51    pthread_mutex_unlock(&t->stats.mutex); \52}53#define WSTAT_DECR(t, stat, amount) { \54    pthread_mutex_lock(&t->stats.mutex); \55    t->stats.stat -= amount; \56    pthread_mutex_unlock(&t->stats.mutex); \57}58#define STAT_L(ctx) pthread_mutex_lock(&ctx->stats_lock);59#define STAT_UL(ctx) pthread_mutex_unlock(&ctx->stats_lock);60#define STAT_INCR(ctx, stat, amount) { \61        pthread_mutex_lock(&ctx->stats_lock); \62        ctx->global_stats.stat += amount; \63        pthread_mutex_unlock(&ctx->stats_lock); \64}6566#define STAT_DECR(ctx, stat, amount) { \67        pthread_mutex_lock(&ctx->stats_lock); \68        ctx->global_stats.stat -= amount; \69        pthread_mutex_unlock(&ctx->stats_lock); \70}7172enum mcp_memprofile_types {73    mcp_memp_free = 0,74    mcp_memp_string,75    mcp_memp_table,76    mcp_memp_func,77    mcp_memp_userdata,78    mcp_memp_thread,79    mcp_memp_default,80    mcp_memp_realloc,81};8283struct mcp_memprofile {84    struct timespec last_status; // for per-second prints on status85    int id;86    uint64_t allocs[8];87    uint64_t alloc_bytes[8];88};8990// for various time conversion functions91#define NANOSECONDS(x) ((x) * 1E9 + 0.5)92#define MICROSECONDS(x) ((x) * 1E6 + 0.5)9394// Note: value created from thin air. Could be shorter.95#define MCP_REQUEST_MAXLEN KEY_MAX_LENGTH * 29697#define ENDSTR "END\r\n"98#define ENDLEN (sizeof(ENDSTR)-1)99100#define MCP_BACKEND_UPVALUE 1101102#define MCP_YIELD_INTERNAL 1103#define MCP_YIELD_WAITCOND 2104#define MCP_YIELD_WAITHANDLE 3105#define MCP_YIELD_SLEEP 4106107#define SHAREDVM_FGEN_IDX 1108#define SHAREDVM_FGENSLOT_IDX 2109#define SHAREDVM_BACKEND_IDX 3110111#define P_OK 0112113typedef struct _io_pending_proxy_t io_pending_proxy_t;114typedef struct proxy_event_thread_s proxy_event_thread_t;115116#ifdef HAVE_LIBURING117// TODO: pass in cqe->res instead of cqe?118typedef void (*proxy_event_cb)(void *udata, struct io_uring_cqe *cqe);119typedef struct {120    void *udata;121    proxy_event_cb cb;122    bool set; // NOTE: not sure if necessary if code structured properly123} proxy_event_t;124125void *proxy_event_thread_ur(void *arg);126#endif127128// Note: This ends up wasting a few counters, but simplifies the rest of the129// process for handling internal worker stats.130struct proxy_int_stats {131    uint64_t vm_gc_runs;132    uint64_t vm_memory_kb;133    uint64_t counters[CMD_FINAL];134};135136struct proxy_user_stats {137    int num_stats; // number of stats, for sizing various arrays138    uint64_t *counters; // array of counters.139};140141struct proxy_user_stats_entry {142    char *name;143    unsigned int cname; // offset into compact name buffer144    bool reset; // counter must reset this cycle145};146147struct proxy_global_stats {148    uint64_t config_reloads;149    uint64_t config_reload_fails;150    uint64_t config_cron_runs;151    uint64_t config_cron_fails;152    uint64_t backend_total;153    uint64_t backend_marked_bad; // backend set to autofail154    uint64_t backend_failed; // an error caused a backend reset155    uint64_t request_failed_depth; // requests fast-failed due to be depth156};157158struct proxy_tunables {159    struct timeval connect;160    struct timeval retry; // wait time before retrying a dead backend161    struct timeval read;162    struct timeval flap; // need to stay connected this long or it's flapping163    float flap_backoff_ramp; // factorial for retry time164    float gc_ratio; // how much lua VM growth to allow before running GC165    uint32_t flap_backoff_max; // don't backoff longer than this.166    int backend_depth_limit; // requests fast fail once depth over this limit167    int backend_failure_limit;168    int max_ustats; // limit the ustats index.169    bool tcp_keepalive;170    bool use_iothread; // default for using the bg io thread.171    bool use_tls; // whether or not be should use TLS172    bool down; // backend is forced into a down/bad state.173};174175struct proxy_logging {176    unsigned int deadline; // log if slower than N us (user specifies ms)177    unsigned int rate; // sampling178    bool all_errors; // always log on an error case179    char *detail;180};181182typedef STAILQ_HEAD(globalobj_head_s, mcp_globalobj_s) globalobj_head_t;183typedef struct {184    lua_State *proxy_state; // main configuration vm185    proxy_event_thread_t *proxy_io_thread;186    int64_t active_req_limit; // max total in-flight requests187    uint64_t buffer_memory_limit; // max bytes for send/receive buffers.188#ifdef PROXY_TLS189    void *tls_ctx;190#endif191    int user_stats_num; // highest seen stat index192    struct proxy_user_stats_entry *user_stats;193    char *user_stats_namebuf; // compact linear buffer for stat names194    struct proxy_tunables tunables; // NOTE: updates covered by stats_lock195    struct proxy_global_stats global_stats;196    // less frequently used entries down here.197    void *proxy_code;198    lua_State *proxy_sharedvm; // sub VM for short-lock global events/data199    pthread_mutex_t stats_lock; // used for rare global counters200    pthread_mutex_t config_lock;201    pthread_cond_t config_cond;202    pthread_t config_tid;203    pthread_mutex_t worker_lock;204    pthread_cond_t worker_cond;205    pthread_t manager_tid; // deallocation management thread206    pthread_mutex_t manager_lock;207    pthread_cond_t manager_cond;208    pthread_mutex_t sharedvm_lock; // protect statevm above209    globalobj_head_t manager_head; // stack for pool deallocation.210    int config_generation; // counter tracking config reloads211    int cron_ref; // reference to lua cron table212    int cron_next; // next cron to sleep to / execute213    bool worker_done; // signal variable for the worker lock/cond system.214    bool worker_failed; // covered by worker_lock as well.215    bool use_uring; // use IO_URING for backend connections.216    bool loading; // bool indicating an active config load.217    bool memprofile; // indicate if we want to profile lua memory.218    uint8_t memprofile_thread_counter;219} proxy_ctx_t;220221#define PROXY_GET_THR_CTX(L) ((*(LIBEVENT_THREAD **)lua_getextraspace(L))->proxy_ctx)222#define PROXY_GET_THR(L) (*(LIBEVENT_THREAD **)lua_getextraspace(L))223// Operations from the config VM don't have a libevent thread.224#define PROXY_GET_CTX(L) (*(proxy_ctx_t **)lua_getextraspace(L))225226struct proxy_hook_ref {227    int lua_ref;228    void *ctx; // if we're a generator based function.229};230231struct proxy_hook_tagged {232    uint64_t tag;233    struct proxy_hook_ref ref;234};235236struct proxy_hook {237    struct proxy_hook_ref ref;238    int tagcount;239    struct proxy_hook_tagged *tagged; // array of possible tagged hooks.240};241242// TODO (v2): some hash functions (crc?) might require initializers. If we run into243// any the interface might need expanding.244typedef uint64_t (*key_hash_func)(const void *key, size_t len, uint64_t seed);245struct proxy_hash_func {246    key_hash_func func;247};248typedef const char *(*key_hash_filter_func)(const char *conf, const char *key, size_t klen, size_t *newlen);249typedef uint32_t (*hash_selector_func)(uint64_t hash, void *ctx);250struct proxy_hash_caller {251    hash_selector_func selector_func;252    void *ctx;253};254255enum mcp_backend_states {256    mcp_backend_read = 0, // waiting to read any response257    mcp_backend_parse, // have some buffered data to check258    mcp_backend_read_end, // looking for an "END" marker for GET259    mcp_backend_want_read, // read more data to complete command260    mcp_backend_next, // advance to the next IO261    mcp_backend_next_close, // complete current request, then close socket262};263264typedef struct mcp_cron_s mcp_cron_t;265typedef struct mcp_backend_wrap_s mcp_backend_wrap_t;266typedef struct mcp_backend_label_s mcp_backend_label_t;267typedef struct mcp_backend_s mcp_backend_t;268typedef struct mcp_request_s mcp_request_t;269typedef struct mcp_rcontext_s mcp_rcontext_t;270typedef struct mcp_funcgen_s mcp_funcgen_t;271272#define PARSER_MAX_TOKENS 24273274#define MAX_REQ_TOKENS 2275struct mcp_request_s {276    mcp_parser_t pr; // non-lua-specific parser handling.277    char request[];278};279280struct mcp_cron_s {281    uint32_t gen;282    uint32_t next;283    uint32_t every;284    bool repeat;285};286287#define MAX_LABELLEN 512288#define MAX_NAMELEN 255289#define MAX_PORTLEN 6290// TODO (v2): IOV_MAX tends to be 1000+ which would allow for more batching but we291// don't have a good temporary space and don't want to malloc/free on every292// write. transmit() uses the stack but we can't do that for uring's use case.293#if MEMCACHED_DEBUG294#define BE_IOV_MAX 128 // let bench tests trigger max condition easily295#elif (IOV_MAX > 1024)296#define BE_IOV_MAX 1024297#else298#define BE_IOV_MAX IOV_MAX299#endif300// lua descriptor object: passed to pools, which create wrappers.301struct mcp_backend_label_s {302    char name[MAX_NAMELEN+1];303    char port[MAX_PORTLEN+1];304    char label[MAX_LABELLEN+1];305    size_t llen; // cache label length for small speedup in pool creation.306    int conncount; // number of sockets to make.307    bool use_logging;308    struct proxy_tunables tunables;309    struct proxy_logging logging;310};311312// lua object wrapper meant to own a malloc'ed conn structure313// when this object is created, it ships its connection to the real owner314// (worker, IO thread, etc)315// when this object is garbage collected, it ships a notice to the owner316// thread to stop using and free the backend conn memory.317struct mcp_backend_wrap_s {318    mcp_backend_t *be;319};320321struct mcp_backendconn_s {322    mcp_backend_t *be_parent; // find the wrapper.323    int self; // our index into the parent array.324    int depth; // total number of requests in queue325    int pending_read; // number of requests written to socket, pending read.326    int failed_count; // number of fails (timeouts) in a row327    int flap_count; // number of times we've "flapped" into bad state.328    proxy_event_thread_t *event_thread; // event thread owning this backend.329    void *client; // mcmc client330#ifdef PROXY_TLS331    void *ssl;332#endif333    iop_head_t iop_write; // requests needing to be written334    iop_head_t iop_read; // requests waiting for read response335    char *rbuf; // statically allocated read buffer.336    size_t rbufused; // currently active bytes in the buffer337    struct event main_event; // libevent: changes role, mostly for main read events338    struct event write_event; // libevent: only used when socket wbuf full339    struct event timeout_event; // libevent: alarm for pending reads340    struct proxy_tunables tunables;341    struct timeval last_failed; // time the backend was last reset.342    enum mcp_backend_states state; // readback state machine343    int connect_flags; // flags to pass to mcmc_connect344    bool connecting; // in the process of an asynch connection.345    bool validating; // in process of validating a new backend connection.346    bool can_write; // recently got a WANT_WRITE or are connecting.347    bool bad; // timed out, marked as bad.348#ifndef PROXY_TLS349    bool ssl;350#endif351    struct iovec write_iovs[BE_IOV_MAX]; // iovs to stage batched writes352};353354// TODO: move depth and flags to a second top level array so we can make index355// decisions from fewer memory stalls.356struct mcp_backend_s {357    int conncount; // total number of connections managed.358    int depth; // temporary depth counter for io_head359    bool transferred; // if beconn has been shipped to owner thread.360    bool use_io_thread; // note if this backend is worker-local or not.361    bool stacked; // if backend already queued for syscalls.362    bool use_logging; // if automatic logging is enabled.363    STAILQ_ENTRY(mcp_backend_s) beconn_next; // stack for connecting conns364    STAILQ_ENTRY(mcp_backend_s) be_next; // stack for backends365    iop_head_t iop_head; // stack of inbound requests.366    struct proxy_logging logging;367    struct proxy_tunables tunables; // this gets copied a few times for speed.368    char name[MAX_NAMELEN+1];369    char port[MAX_PORTLEN+1];370    char label[MAX_LABELLEN+1];371    struct mcp_backendconn_s be[];372};373typedef STAILQ_HEAD(be_head_s, mcp_backend_s) be_head_t;374typedef STAILQ_HEAD(beconn_head_s, mcp_backend_s) beconn_head_t;375376struct proxy_event_thread_s {377    pthread_t thread_id;378    struct event_base *base;379    struct event notify_event; // listen event for the notify pipe/eventfd.380    struct event beconn_event; // listener for backends in connect state381#ifdef HAVE_LIBURING382    struct io_uring ring;383    proxy_event_t ur_notify_event; // listen on eventfd.384    proxy_event_t ur_benotify_event; // listen on eventfd for backend connections.385    eventfd_t event_counter;386    eventfd_t beevent_counter;387    bool use_uring;388#endif389#ifdef PROXY_TLS390    char *tls_wbuf;391    size_t tls_wbuf_size;392#endif393    pthread_mutex_t mutex; // covers stack.394    pthread_cond_t cond; // condition to wait on while stack drains.395    iop_head_t iop_head_in; // inbound requests to process.396    be_head_t be_head; // stack of backends for processing.397    beconn_head_t beconn_head_in; // stack of backends for connection processing.398#ifdef USE_EVENTFD399    int event_fd; // for request ingestion400    int be_event_fd; // for backend ingestion401#else402    int notify_receive_fd;403    int notify_send_fd;404    int be_notify_receive_fd;405    int be_notify_send_fd;406#endif407    proxy_ctx_t *ctx; // main context.408};409410#define RESP_CMD_MAX 8411typedef struct {412    mcmc_resp_t resp;413    mcmc_tokenizer_t tok; // optional tokenization of res414    char *buf; // response line + potentially value.415    mc_resp *cresp; // client mc_resp object during extstore fetches.416    LIBEVENT_THREAD *thread; // cresp's owner thread needed for extstore cleanup.417    mcp_backend_t *be; // backend that generated this response418    unsigned int blen; // total size of the value to read.419    struct timeval start; // time this object was created.420    long elapsed; // time elapsed once handled.421    int status; // status code from mcmc_read()422    int bread; // amount of bytes read into value so far.423    uint8_t cmd; // from parser (pr.command)424} mcp_resp_t;425426// re-cast an io_pending_t into this more descriptive structure.427// the first few items _must_ match the original struct.428struct _io_pending_proxy_t {429    uint8_t io_queue_type;430    uint8_t io_sub_type;431    uint8_t payload; // payload offset432    LIBEVENT_THREAD *thread;433    conn *c;434    mc_resp *resp;435    io_queue_cb return_cb; // called on worker thread.436    io_queue_cb finalize_cb; // called back on the worker thread.437    STAILQ_ENTRY(io_pending_t) iop_next; // queue chain.438    // original struct ends here439440    mcp_rcontext_t *rctx; // pointer to request context.441    mcp_resp_t *client_resp; // reference (currently pointing to a lua object)442    int queue_handle; // queue slot to return this result to443    union {444        // extstore IO.445        struct {446            obj_io eio;447            item *hdr_it;448            mc_resp *tresp; // temporary mc_resp for storage to fill.449            int gettype;450            int iovec_data;451            bool miss;452            bool badcrc;453            bool active;454        };455        // backend request IO456        struct {457            mcp_backend_t *backend; // backend server to request from458            struct iovec iov[2]; // request string + tail buffer459            int iovcnt; // 1 or 2...460            unsigned int iovbytes; // total bytes in the iovec461            bool flushed; // whether we've fully written this request to a backend.462            bool background; // dummy IO for backgrounded awaits463        };464    };465};466467struct mcp_globalobj_s {468    pthread_mutex_t lock; // protects refcount/object.469    STAILQ_ENTRY(mcp_globalobj_s) next;470    int refcount;471    int self_ref;472};473474// Note: does *be have to be a sub-struct? how stable are userdata pointers?475// https://stackoverflow.com/questions/38718475/lifetime-of-lua-userdata-pointers476// - says no.477typedef struct {478    int ref; // luaL_ref reference of backend_wrap_t obj.479    mcp_backend_t *be;480} mcp_pool_be_t;481482#define KEY_HASH_FILTER_MAX 5483typedef struct mcp_pool_s mcp_pool_t;484struct mcp_pool_s {485    struct proxy_hash_caller phc;486    key_hash_filter_func key_filter;487    key_hash_func key_hasher;488    proxy_ctx_t *ctx; // main context.489    char key_filter_conf[KEY_HASH_FILTER_MAX+1];490    struct mcp_globalobj_s g;491    char beprefix[MAX_LABELLEN+1]; // TODO: should probably be shorter.492    uint64_t hash_seed; // calculated from a string.493    int pool_size;494    int pool_be_total; // can be different from pool size for worker IO495    int phc_ref;496    bool use_iothread;497    mcp_pool_be_t pool[];498};499500typedef struct {501    mcp_pool_t *main; // ptr to original502    mcp_pool_be_t *pool; // ptr to main->pool starting offset for owner thread.503} mcp_pool_proxy_t;504505// utils506bool proxy_bufmem_checkadd(LIBEVENT_THREAD *t, int len);507void mcp_sharedvm_delta(proxy_ctx_t *ctx, int tidx, const char *name, int delta);508void mcp_sharedvm_remove(proxy_ctx_t *ctx, int tidx, const char *name);509510void mcp_gobj_ref(lua_State *L, struct mcp_globalobj_s *g);511void mcp_gobj_unref(proxy_ctx_t *ctx, struct mcp_globalobj_s *g);512void mcp_gobj_finalize(struct mcp_globalobj_s *g);513514// networking interface515void proxy_init_event_thread(proxy_event_thread_t *t, proxy_ctx_t *ctx, struct event_base *base);516void *proxy_event_thread(void *arg);517void proxy_run_backend_queue(be_head_t *head);518struct mcp_backendconn_s *proxy_choose_beconn(mcp_backend_t *be);519mcp_resp_t *mcp_prep_bare_resobj(lua_State *L, LIBEVENT_THREAD *t);520void mcp_resp_set_elapsed(mcp_resp_t *r);521io_pending_proxy_t *mcp_queue_rctx_io(mcp_rcontext_t *rctx, mcp_request_t *rq, mcp_backend_t *be, mcp_resp_t *r);522523// internal request interface524int mcplib_internal(lua_State *L);525int mcplib_internal_run(mcp_rcontext_t *rctx);526void *mcp_rcontext_internal(mcp_rcontext_t *rctx, mcp_request_t *rq, mcp_resp_t *r);527528// user stats interface529#define MAX_USTATS_DEFAULT 1024530int mcplib_add_stat(lua_State *L);531int mcplib_stat(lua_State *L);532size_t _process_request_next_key(mcp_parser_t *pr);533mcp_request_t *mcp_new_request(lua_State *L, mcp_parser_t *pr, const char *command, size_t cmdlen);534void mcp_set_request(mcp_parser_t *pr, mcp_request_t *r, const char *command, size_t cmdlen);535536// rate limit interfaces537int mcplib_ratelim_tbf(lua_State *L);538int mcplib_ratelim_tbf_call(lua_State *L);539int mcplib_ratelim_global_tbf(lua_State *L);540int mcplib_ratelim_proxy_tbf_call(lua_State *L);541int mcp_ratelim_proxy_tbf(lua_State *from, lua_State *to);542int mcplib_ratelim_global_tbf_gc(lua_State *L);543int mcplib_ratelim_proxy_tbf_gc(lua_State *L);544545// request function generator interface546void proxy_return_rctx_cb(io_pending_t *pending);547void proxy_finalize_rctx_cb(io_pending_t *pending);548549enum mcp_rqueue_e {550    QWAIT_IDLE = 0,551    QWAIT_ANY,552    QWAIT_OK,553    QWAIT_GOOD,554    QWAIT_FASTGOOD,555    QWAIT_HANDLE,556    QWAIT_SLEEP,557};558559#define FGEN_NAME_MAXLEN 80560struct mcp_funcgen_s {561    LIBEVENT_THREAD *thread; // worker thread that created this funcgen.562    int generator_ref; // reference to the generator function.563    int self_ref; // self-reference if we're attached anywhere564    int argument_ref; // reference to an argument to pass to generator565    int max_queues; // how many queue slots rctx's have566    int uobj_queues; // how many extra queue slots for object storage we have567    unsigned int refcount; // reference counter568    unsigned int total; // total contexts managed569    unsigned int free; // free contexts570    unsigned int free_max; // size of list below.571    unsigned int free_pressure; // "pressure" for when to early release rctx572    bool closed; // the hook holding this fgen has been replaced573    bool ready; // if we're locked down or not.574    bool is_router; // if this fgen is actually a router object.575    struct timespec free_waiter; // must be "too free" for this much time576    mcp_rcontext_t **list;577    struct mcp_rqueue_s *queue_list;578    char name[FGEN_NAME_MAXLEN+1]; // string name for the generator.579};580581enum mcp_funcgen_router_e {582    FGEN_ROUTER_NONE = 0,583    FGEN_ROUTER_CMDMAP,584    FGEN_ROUTER_SHORTSEP,585    FGEN_ROUTER_LONGSEP,586    FGEN_ROUTER_ANCHORSM,587    FGEN_ROUTER_ANCHORBIG,588};589590struct mcp_router_long_s {591    char start[KEY_HASH_FILTER_MAX+1];592    char stop[KEY_HASH_FILTER_MAX+1];593};594595// To simplify the attach/start code we wrap a funcgen with the router596// structure. This allows us to have a larger router structure without597// bloating the fgen object itself, and still benefit from letting funcgen598// new/cleanup handle most of the memory management.599struct mcp_funcgen_router {600    mcp_funcgen_t fgen_self;601    enum mcp_funcgen_router_e type;602    union {603        char sep;604        char lsep[KEY_HASH_FILTER_MAX+1];605        char anchorsm[2]; // short anchored mode.606        struct mcp_router_long_s big;607    } conf;608    int map_ref;609    mcp_funcgen_t *def_fgen; // default route610    mcp_funcgen_t *cmap[CMD_END_STORAGE]; // fallback command map611};612613#define RQUEUE_TYPE_NONE 0614#define RQUEUE_TYPE_POOL 1615#define RQUEUE_TYPE_INT  2616#define RQUEUE_TYPE_FGEN 3617#define RQUEUE_TYPE_UOBJ 4 // user tracked object types past this point618#define RQUEUE_TYPE_UOBJ_REQ 5619#define RQUEUE_TYPE_UOBJ_RES 6620#define RQUEUE_ASSIGNED (1<<0)621#define RQUEUE_R_RESUME (1<<1)622#define RQUEUE_R_GOOD (1<<3)623#define RQUEUE_R_OK (1<<4)624#define RQUEUE_R_ANY (1<<5)625#define RQUEUE_R_ERROR (1<<7)626#define RQUEUE_UOBJ_MAX UINT8_MAX627628enum mcp_rqueue_state {629    RQUEUE_IDLE = 0,630    RQUEUE_QUEUED,631    RQUEUE_ACTIVE,632    RQUEUE_COMPLETE,633    RQUEUE_WAITED634};635636struct mcp_rqueue_s {637    int obj_ref; // reference to pool/func/etc object638    int cb_ref; // if a lua callback was specified639    int req_ref; // reference to associated request object.640    int res_ref; // reference to lua response object.641    void *obj; // direct pointer to the object for fast access.642    mcp_request_t *rq; // request set to this slot643    mcp_resp_t *res_obj; // pointer to result object644    enum mcp_rqueue_state state; // queued/active/etc645    uint8_t obj_type; // what the obj_ref actually is.646    uint8_t flags; // bit flags for various states647};648649struct mcp_rcontext_s {650    int self_ref; // reference to our own object651    int request_ref; // top level request for this context.652    int function_ref; // ref to the created route function.653    int coroutine_ref; // ref to our encompassing coroutine.654    int pending_reqs; // pending requests and sub-requests655    unsigned int wait_count;656    unsigned int wait_done; // TODO: change these variables to uint8's657    int wait_handle; // waiting on a specific queue slot658    int parent_handle; // queue slot in parent rctx659    int conn_fd; // fd of the originating client, as *c can become invalid660    enum mcp_rqueue_e wait_mode;661    uint8_t lua_narg; // number of responses to push when yield resuming.662    uint8_t uobj_count; // number of extra tracked req/res objects.663    bool ascii_multiget; // ascii multiget mode. (hide errors/END)664    lua_State *Lc; // coroutine thread pointer.665    mcp_request_t *request; // ptr to the above reference.666    mcp_rcontext_t *parent; // parent rctx in the call graph667    conn *c; // associated client object.668    mc_resp *resp; // top level response object to fill.669    mcp_funcgen_t *fgen; // parent function generator context.670    struct event timeout_event; // for *_wait_timeout() and sleep() calls671    struct mcp_rqueue_s qslots[]; // queueable slots.672};673674#define mcp_is_flag_invalid(f) (f < 65 || f > 122)675676void mcp_run_rcontext_handle(mcp_rcontext_t *rctx, int handle);677void mcp_process_rctx_wait(mcp_rcontext_t *rctx, int handle);678int mcp_process_rqueue_return(mcp_rcontext_t *rctx, int handle, mcp_resp_t *res);679int mcplib_rcontext_handle_set_cb(lua_State *L);680int mcplib_rcontext_enqueue(lua_State *L);681int mcplib_rcontext_wait_cond(lua_State *L);682int mcplib_rcontext_wait_handle(lua_State *L);683int mcplib_rcontext_enqueue_and_wait(lua_State *L);684int mcplib_rcontext_res_good(lua_State *L);685int mcplib_rcontext_res_any(lua_State *L);686int mcplib_rcontext_res_ok(lua_State *L);687int mcplib_rcontext_result(lua_State *L);688int mcplib_rcontext_best_result(lua_State *L);689int mcplib_rcontext_worst_result(lua_State *L);690int mcplib_rcontext_cfd(lua_State *L);691int mcplib_rcontext_tls_peer_cn(lua_State *L);692int mcplib_rcontext_request_new(lua_State *L);693int mcplib_rcontext_response_new(lua_State *L);694int mcplib_rcontext_sleep(lua_State *L);695int mcplib_funcgenbare_new(lua_State *L);696int mcplib_funcgen_new(lua_State *L);697int mcplib_funcgen_new_handle(lua_State *L);698int mcplib_funcgen_ready(lua_State *L);699int mcplib_router_new(lua_State *L);700mcp_rcontext_t *mcp_funcgen_start(lua_State *L, mcp_funcgen_t *fgen, mcp_parser_t *pr);701mcp_rcontext_t *mcp_funcgen_get_rctx(lua_State *L, int fgen_ref, mcp_funcgen_t *fgen);702void mcp_funcgen_return_rctx(mcp_rcontext_t *rctx);703int mcplib_funcgen_gc(lua_State *L);704void mcp_funcgen_reference(lua_State *L);705void mcp_funcgen_dereference(lua_State *L, mcp_funcgen_t *fgen);706void mcp_rcontext_push_rqu_res(lua_State *L, mcp_rcontext_t *rctx, int handle);707void mcplib_rqu_log(mcp_request_t *rq, mcp_resp_t *rs, int flag, int cfd);708709int mcplib_factory_command_new(lua_State *L);710711// request interface712int mcplib_request(lua_State *L);713int mcplib_request_command(lua_State *L);714int mcplib_request_key(lua_State *L);715int mcplib_request_ltrimkey(lua_State *L);716int mcplib_request_rtrimkey(lua_State *L);717int mcplib_request_raw_line(lua_State *L);718int mcplib_request_raw_value(lua_State *L);719int mcplib_request_token(lua_State *L);720int mcplib_request_token_int(lua_State *L);721int mcplib_request_ntokens(lua_State *L);722int mcplib_request_has_flag(lua_State *L);723int mcplib_request_flag_token(lua_State *L);724int mcplib_request_flag_token_int(lua_State *L);725int mcplib_request_flag_add(lua_State *L);726int mcplib_request_flag_set(lua_State *L);727int mcplib_request_flag_replace(lua_State *L);728int mcplib_request_flag_del(lua_State *L);729int mcplib_request_gc(lua_State *L);730int mcplib_request_match_res(lua_State *L);731void mcp_request_cleanup(LIBEVENT_THREAD *t, mcp_request_t *rq);732733// response interface734int mcplib_response_elapsed(lua_State *L);735int mcplib_response_ok(lua_State *L);736int mcplib_response_hit(lua_State *L);737int mcplib_response_vlen(lua_State *L);738int mcplib_response_code(lua_State *L);739int mcplib_response_line(lua_State *L);740int mcplib_response_flag_blank(lua_State *L);741int mcplib_response_raw_string(lua_State *L);742743// inspector interface744int mcplib_req_inspector_new(lua_State *L);745int mcplib_res_inspector_new(lua_State *L);746int mcplib_inspector_gc(lua_State *L);747int mcplib_inspector_call(lua_State *L);748749// mutator interface750int mcplib_req_mutator_new(lua_State *L);751int mcplib_res_mutator_new(lua_State *L);752int mcplib_mutator_gc(lua_State *L);753int mcplib_mutator_call(lua_State *L);754755void mcp_response_cleanup(LIBEVENT_THREAD *t, mcp_resp_t *r);756void mcp_set_resobj(mcp_resp_t *r, mcp_request_t *rq, mcp_backend_t *be, LIBEVENT_THREAD *t);757int mcplib_response_gc(lua_State *L);758int mcplib_response_close(lua_State *L);759760int mcplib_open_dist_jump_hash(lua_State *L);761int mcplib_open_dist_ring_hash(lua_State *L);762763int proxy_run_rcontext(mcp_rcontext_t *rctx);764mcp_backend_t *mcplib_pool_proxy_call_helper(mcp_pool_proxy_t *pp, const char *key, size_t len);765void mcp_request_attach(mcp_request_t *rq, io_pending_proxy_t *p);766int mcp_request_render(mcp_request_t *rq, int idx, char flag, const char *tok, size_t len);767int mcp_request_append(mcp_request_t *rq, const char flag, const char *tok, size_t len);768void proxy_lua_error(lua_State *L, const char *s);769#define proxy_lua_ferror(L, fmt, ...) \770    do { \771        lua_pushfstring(L, fmt, __VA_ARGS__); \772        lua_error(L); \773    } while (0)774775#define PROXY_SERVER_ERROR "SERVER_ERROR "776#define PROXY_CLIENT_ERROR "CLIENT_ERROR "777void proxy_out_errstring(mc_resp *resp, char *type, const char *str);778int _start_proxy_config_threads(proxy_ctx_t *ctx);779int proxy_thread_loadconf(proxy_ctx_t *ctx, LIBEVENT_THREAD *thr);780781// TODO (v2): more .h files, perhaps?782int mcplib_open_hash_xxhash(lua_State *L);783784__attribute__((unused)) void dump_stack(lua_State *L, const char *msg);785__attribute__((unused)) void dump_registry(lua_State *L, const char *msg);786__attribute__((unused)) void dump_funcgen(lua_State *L, const char *name, const char *msg);787__attribute__((unused)) void dump_pools(lua_State *L, const char *msg);788#endif