1-- New style request factories and backend request handling.2--3-- First, this API adds a "request func generation" step when a new request4-- starts: if there is not already a cached function to use, call the5-- "generator" function, then use the response to run the request. This generated6-- function is reused until the parent generator is swapped out during reload.7-- This allows the user to pre-allocate and pre-calculate objects and data,8-- offering both safety and performance.9-- Future API revisions (such as stats) will rely on this generation step to10-- be more user friendly while retaining performance.11--12-- For backend IO's this unifies what was once two API's:13--  - result = pool(request): the non-async API14--  - table = mcp.await(etc)15--16-- It is now a single system governeed by a request context object (rctx).17-- This new system allows queueing a nearly arbitrary set of requests,18-- "blocking" a client on any individual response, and using callbacks to19-- make decisions on if a response is "good", to resume processing early, or20-- post-process once all responses are received.21--22-- The queueing system is now recursive: a fgen can new_handle() another fgen.23-- Meaning configurations can be assembled as call graphs. IE: If you have a24-- route function A and want to "shadow" some of its requests onto route25-- function B, instead of making A more complex you can create a third26-- function C which splits the traffic.27--28-- API docs: https://github.com/memcached/memcached/wiki/Proxy2930verbose = true31-- global for an error handling test32failgen_armed = false33failgenret_armed = false3435function say(...)36    if verbose then37        print(...)38    end39end4041function mcp_config_pools()42    local srv = mcp.backend4344    local b1 = srv('b1', '127.0.0.1', 12011)45    local b2 = srv('b2', '127.0.0.1', 12012)46    local b3 = srv('b3', '127.0.0.1', 12013)47    local b4 = srv('b4', '127.0.0.1', 12014)48    local b1z = mcp.pool({b1})49    local b2z = mcp.pool({b2})50    local b3z = mcp.pool({b3})51    local b4z = mcp.pool({b4})5253    local blog = srv({54        label = "blog",55        host = "127.0.0.1",56        port = "12015",57        log = {58            rate = 5,59            errors = true,60            deadline = 250,61            tag = "fastlog"62        }63    })64    blogz = mcp.pool({blog})6566    local p = {p = {b1z, b2z, b3z}, b = b4z, pl = blogz}6768    --return mcp.pool(b1z, { iothread = false })69    return p70end7172-- many of these factories have the same basic init pattern, so we can save73-- some code.74function new_basic_factory(arg, func)75    local fgen = mcp.funcgen_new()76    local o = { t = {}, c = 0 }7778    -- some of them have a wait, some don't.79    -- here would be a good place to do bounds checking on arguments in80    -- similar functions.81    o.wait = arg.wait82    o.timeout = arg.timeout83    o.mode = arg.mode84    for _, v in pairs(arg.list) do85        table.insert(o.t, fgen:new_handle(v))86        o.c = o.c + 187    end8889    fgen:ready({ f = func, a = o, n = arg.name})90    return fgen91end9293function new_prefix_factory(arg)94    local fgen = mcp.funcgen_new()95    local o = {}96    o.pattern = arg.pattern97    o.default = fgen:new_handle(arg.default)9899    o.map = {}100    -- get handler ids for each sub-route value101    -- convert the map.102    for k, v in pairs(arg.list) do103        o.map[k] = fgen:new_handle(v)104    end105106    fgen:ready({ f = prefix_factory_gen, a = o, n = arg.name })107    return fgen108end109110function prefix_factory_gen(rctx, arg)111    local p = arg.pattern112    local map = arg.map113    local d = arg.default114115    say("generating a prefix factory function")116117    return function(r)118        local key = r:key()119120        local handle = map[string.match(key, p)]121        if handle == nil then122            return rctx:enqueue_and_wait(r, d)123        end124        return rctx:enqueue_and_wait(r, handle)125    end126end127128function new_direct_factory(arg)129    local fgen = mcp.funcgen_new()130    local h = fgen:new_handle(arg.p)131    fgen:ready({ f = direct_factory_gen, a = h, n = arg.name })132    return fgen133end134135function direct_factory_gen(rctx, h)136    say("generating direct factory function")137138    return function(r)139        say("waiting on a single pool")140        return rctx:enqueue_and_wait(r, h)141    end142end143144function new_locality_factory(arg)145    local fgen = mcp.funcgen_new()146    local h = fgen:new_handle(arg.p)147    fgen:ready({ f = locality_factory_gen, a = h, n = arg.name })148    return fgen149end150151-- factory for proving slots have unique environmental memory.152-- we need to wait on a backend to allow the test to pipeline N requests in153-- parallel, to prove that each parallel slot has a unique lua environment.154function locality_factory_gen(rctx, h)155    say("generating locality factory function")156    local x = 0157158    return function(r)159        x = x + 1160        say("returning from locality: " .. x)161        local res = rctx:enqueue_and_wait(r, h)162        return "HD t" .. x .. "\r\n"163    end164end165166-- waits for only the _first_ queued handle to return.167-- ie; position 1 in the table.168-- we do a numeric for loop in the returned function to avoid allocations done169-- by a call to pairs()170function first_factory_gen(rctx, arg)171    say("generating first factory function")172    local t = arg.t173    local count = arg.c174175    return function(r)176        say("waiting on first of " .. count .. " pools")177        for x=1, count do178            rctx:enqueue(r, t[x])179        end180181        return rctx:wait_handle(t[1])182    end183end184185-- wait on x out of y186function partial_factory_gen(rctx, arg)187    say("generating partial factory function")188    local t = arg.t189    local count = arg.c190    local wait = arg.wait191192    return function(r)193        say("waiting on first " .. wait .. " out of " .. count)194        for x=1, count do195            rctx:enqueue(r, t[x])196        end197198        local done = rctx:wait_cond(wait)199        for x=1, count do200            -- :good will only return the result object if the handle's201            -- response was considered "good"202            local res = rctx:res_good(t[x])203            if res ~= nil then204                say("found a result")205                return res206            end207            -- TODO: tally up responses and send summary for test.208        end209        say("found nothing")210        -- didn't return anything good, so return one at random.211        for x=1, count do212            local res = rctx:res_any(t[x])213            if res ~= nil then214                return res215            end216        end217    end218end219220-- wait on all pool arguments221function all_factory_gen(rctx, arg)222    say("generating all factory function")223    local t = arg.t224    local count = arg.c225    if arg.wait ~= nil then226        count = arg.wait227    end228    -- should be a minor speedup avoiding the table lookup.229    local mode = mcp.WAIT_ANY230231    return function(r)232        say("all_factory waiting on " .. count)233234        rctx:enqueue(r, t)235        local done = rctx:wait_cond(count, mode)236        -- :any will give us the result object for that handle, regardless237        -- of return code/status.238        local res = rctx:res_any(t[1])239240        -- TODO: tally up the responses and return summary for test.241        return res242    end243end244245-- wait on the first good or N of total246function fastgood_factory_gen(rctx, arg)247    say("generating fastgood factory function")248    local t = arg.t249    local count = arg.c250    local wait = arg.wait251252    local cb = function(res)253        say("running in a callback!")254        if res:hit() then255            say("was a hit!")256            -- return an extra arg telling us to shortcut the wait count257            return mcp.WAIT_GOOD, mcp.WAIT_RESUME258        end259        -- default return code is mcp.WAIT_ANY260    end261262    for _, v in pairs(t) do263        rctx:handle_set_cb(v, cb)264    end265266    return function(r)267        say("first good or wait for N")268269        rctx:enqueue(r, t)270        local done = rctx:wait_cond(wait, mcp.WAIT_GOOD)271        say("fastgood done:", done)272273        if done == 1 then274            -- if we just got one "good", we're probably happy.275            for x=1, count do276                -- loop to find the good handle.277                local res = rctx:res_good(t[x])278                if res ~= nil then279                    return res280                end281            end282        else283            -- else we had to wait and now need to decide if it was a miss or284            -- network error.285            -- but for this test we'll just return the first result.286            for x=1, count do287                local res = rctx:res_any(t[x])288                if res ~= nil then289                    return res290                end291            end292        end293    end294end295296-- fastgood implemented using internal fastgood state297function fastgoodint_factory_gen(rctx, arg)298    local t = arg.t299    local count = arg.c300    local wait = arg.wait301302    return function(r)303        rctx:enqueue(r, t)304        say("enqueing fastgood:", wait)305        local done = rctx:wait_cond(wait, mcp.WAIT_FASTGOOD)306        say("fastgoodint done:", done)307308        local final = nil309        for x=1, count do310            local res, mode = rctx:result(t[x])311            if mode == mcp.WAIT_GOOD then312                return res313            elseif res ~= nil then314                final = res315            end316        end317        -- if no good found, return anything.318        return final319    end320end321322function new_blocker_factory(arg)323    local fgen = mcp.funcgen_new()324    local o = { c = 0, t = {} }325    o.b = fgen:new_handle(arg.blocker)326327    for _, v in pairs(arg.list) do328        table.insert(o.t, fgen:new_handle(v))329        o.c = o.c + 1330    end331332    fgen:ready({ f = blocker_factory_gen, a = o, n = arg.name })333    return fgen334end335336-- queue a bunch, but shortcut if a special auxiliary handle fails337function blocker_factory_gen(rctx, arg)338    say("generating blocker factory function")339    local t = arg.t340    local count = arg.c341    local blocker = arg.b342    local was_blocked = false343344    local cb = function(res)345        -- check the response or tokens or anything special to indicate346        -- success.347        -- for this test we just check if it was a hit.348        if res:hit() then349            was_blocked = false350            return mcp.WAIT_GOOD351        else352            was_blocked = true353            return mcp.WAIT_ANY354        end355    end356357    rctx:handle_set_cb(blocker, cb)358359    return function(r)360        say("function blocker test")361362        -- queue up the real queries we wanted to run.363        rctx:enqueue(r, t)364365        -- any wait command will execute all queued queries at once, but here366        -- we only wait for the blocker to complete.367        local bres = rctx:enqueue_and_wait(r, blocker)368369        -- another way of doing this is to ask:370        -- local res = rctx:res_good(blocker)371        -- if a result was returned, the callback had returned WAIT_GOOD372        if was_blocked == false then373            -- our blocker is happy...374            -- wait for the rest of the handles to come in and make a decision375            -- on what to return to the client.376            local done = rctx:wait_cond(count, mcp.WAIT_ANY)377            return rctx:res_any(t[1])378        else379            return "SERVER_ERROR blocked\r\n"380        end381    end382end383384-- log on all callbacks, even if waiting for 1385function logall_factory_gen(rctx, arg)386    say("generating logall factory function")387    local t = arg.t388389    local cb = function(res, req)390        say("received a response, logging...")391        mcp.log("received a response: " .. tostring(res:ok()))392        mcp.log_req(req, res, "even more logs", rctx:cfd())393        return mcp.WAIT_ANY394    end395396    for _, v in pairs(t) do397        rctx:handle_set_cb(v, cb)398    end399400    return function(r)401        rctx:enqueue(r, t)402        return rctx:wait_handle(t[1])403    end404end405406-- log a summary after all callbacks run407function summary_factory_gen(rctx, arg)408    say("generating summary factory function")409    local t = arg.t410    local count = arg.c411412    local todo = 0413    local cb = function(res)414        say("responses TODO: " .. todo)415        todo = todo - 1416        if todo == 0 then417            mcp.log("received all responses")418        end419    end420421    for _, v in pairs(t) do422        rctx:handle_set_cb(v, cb)423    end424425    return function(r)426        -- re-seed the todo value that the callback uses427        todo = count428429        rctx:enqueue(r, t)430        -- we're just waiting for a single response, but we queue all of the431        -- handles. the callback uses data from the shared environment and a432        -- summary is logged.433        return rctx:wait_handle(t[1])434    end435end436437-- testing various waitfor conditions.438function waitfor_factory_gen(rctx, arg)439    say("generating background factory function")440    local t = arg.t441    local count = arg.c442443    return function(r)444        local key = r:key()445        if key == "waitfor/a" then446            rctx:enqueue(r, t)447            rctx:wait_cond(0) -- issue the requests in the background448            return "HD t1\r\n" -- return whatever to the client449        elseif key == "waitfor/b" then450            rctx:enqueue(r, t)451            rctx:wait_cond(0) -- issue requests and resume452            -- now go back into wait mode, but we've already dispatched453            local done = rctx:wait_cond(2)454            if done ~= 2 then455                return "SERVER_ERROR invalid wait"456            end457            -- TODO: bonus points, count the goods or check that everyone's t458            -- flag is right.459            for x=1, count do460                local res = rctx:res_good(x)461                if res ~= nil then462                    return res463                end464                return "SERVER_ERROR no good response"465            end466        elseif key == "waitfor/c" then467            rctx:enqueue(r, t[1])468            rctx:wait_cond(0) -- issue the first queued request469            -- queue two more470            rctx:enqueue(r, t[2])471            rctx:enqueue(r, t[3])472            -- wait explicitly for the first queued one.473            return rctx:wait_handle(t[1])474        elseif key == "waitfor/d" then475            -- queue two then wait on each individually476            rctx:enqueue(r, t[1])477            rctx:enqueue(r, t[2])478            rctx:wait_handle(t[1])479            return rctx:wait_handle(t[2])480        end481    end482end483484-- try "primary zone" and then fail over to secondary zones.485-- using simplified code that just treats the first pool as the primary zone.486function failover_factory_gen(rctx, arg)487    say("generating failover factory function")488    local t = {}489    local count = arg.c490    local first = arg.t[1]491492    for x=2, count do493        table.insert(t, arg.t[x])494    end495496    return function(r)497        -- first try local498        local fres = rctx:enqueue_and_wait(r, first)499500        if fres == nil or fres:hit() == false then501            -- failed to get a local hit, queue all "far" zones.502            rctx:enqueue(r, t)503            -- wait for one.504            local done = rctx:wait_cond(1, mcp.WAIT_GOOD)505            -- find the good from the second set.506            for x=1, count-1 do507                local res = rctx:res_good(t[x])508                if res ~= nil then509                    say("found a result")510                    return res511                end512            end513            -- got nothing from second set, just return anything.514            return rctx:res_any(first)515        else516            return fres517        end518    end519end520521function new_msg_factory(msg, name)522    local fgen = mcp.funcgen_new()523    fgen:ready({ n = name, f = function(rctx)524        return function(r)525            return msg526        end527    end})528    return fgen529end530531function new_error_factory(func, name)532    local fgen = mcp.funcgen_new()533    fgen:ready({ f = func, n = name })534    return fgen535end536537function errors_factory_gen(rctx)538    say("generating errors factory")539540    return function(r)541        local key = r:key()542        -- failure scenarios that require a top-level request context543        if key == "errors/reterror" then544            error("test error")545        elseif key == "errors/retnil" then546            return nil547        elseif key == "errors/retint" then548            return 5549        elseif key == "errors/retnone" then550            return551        end552    end553end554555function suberrors_factory_gen(rctx)556    say("generating suberrors factory function")557558    return function(r)559        local key = r:key()560        if key == "suberrors/error" then561            error("test error")562        elseif key == "suberrors/nil" then563            return nil564        elseif key == "suberrors/int" then565            return 5566        elseif key == "suberrors/none" then567            return568        elseif key == "suberrors/resume" then569            rctx:sleep(0.25)570            error("error after resuming rctx")571        elseif key == "suberrors/string" then572            -- non error but immediate return scenario.573            return "SERVER_ERROR suberror/string\r\n"574        end575576    end577end578579function new_split_factory(arg)580    local fgen = mcp.funcgen_new()581    local o = {}582    o.a = fgen:new_handle(arg.a)583    o.b = fgen:new_handle(arg.b)584    fgen:ready({ f = split_factory_gen, a = o, n = name })585    return fgen586end587588-- example of a factory that takes two other factories and copies traffic589-- across them.590-- If an additional API's for hashing to numerics are added, keys can be591-- hashed to allow "1/n" of keys to copy to one of the splits. This allows592-- shadowing traffic to new/experimental pools, slow-warming traffic, etc.593function split_factory_gen(rctx, arg)594    say("generating split factory function")595    local a = arg.a596    local b = arg.b597598    return function(r)599        say("splitting traffic")600        -- b is the split path.601        rctx:enqueue(r, b)602603        -- a is the main path. so we only explicitly wait on and return a.604        return rctx:enqueue_and_wait(r, a)605    end606end607608-- test handling of failure to generate a function slot609function failgen_factory_gen(rctx)610    if failgen_armed then611        say("throwing failgen error")612        error("failgen")613    end614    say("arming failgen")615    failgen_armed = true616617    return function(r)618        return "NF\r\n"619    end620end621622function failgenret_factory_gen(rctx)623    if failgenret_armed then624        return nil625    end626    failgenret_armed = true627628    return function(r)629        return "NF\r\n"630    end631end632633function badreturn_gen(rctx)634    -- returning a userdata that isn't the correct kind of userdata.635    -- shouldn't crash the daemon!636    return function(r)637        return rctx638    end639end640641-- TODO: should really make the wait time ignored if timeout is nil? throws an642-- error but means more lua to deal with optional waits...643function bestres_factory_gen(rctx, arg)644    local handles = arg.t645    local timeout = nil646    if arg.timeout then647        timeout = arg.timeout648    end649    return function(r)650        rctx:enqueue(r, handles)651        if timeout then652            rctx:wait_cond(#handles, mcp.WAIT_ANY, timeout)653        else654            rctx:wait_cond(#handles)655        end656        local res, tag = rctx:best_result(handles)657        return res658    end659end660661function worstres_factory_gen(rctx, arg)662    local handles = arg.t663    return function(r)664        rctx:enqueue(r, handles)665        rctx:wait_cond(#handles)666        local res, tag = rctx:worst_result(handles)667        return res668    end669end670671function timeout_factory_gen(rctx, arg)672    local handles = arg.t673    local wait = #handles674    local timeout = nil675    local mode = mcp.WAIT_GOOD676    if arg.wait then677        wait = arg.wait678    end679    if arg.timeout then680        timeout = arg.timeout681    end682    if arg.mode then683        mode = arg.mode684    end685    return function(r)686        rctx:enqueue(r, handles)687        if timeout then688            rctx:wait_cond(wait, mode, timeout)689        else690            rctx:wait_cond(wait, mode)691        end692        local res, tag = rctx:best_result(handles)693        return res694    end695end696697-- TODO: this might be supported only in a later update.698-- new queue after parent return699-- - do an immediate return + cb queue, queue from that callback700-- - should still work but requires worker lua vm701-- requires removing the need of having an active client socket object to702-- queue new requests for processing.703function postreturn_factory(rctx, arg)704705end706707-- TODO: demonstrate a split call graph708-- ie; an all split into two single709710function mcp_config_routes(p)711    local b_pool = p.b712    local pl = p.pl713    p = p.p714    local single = new_direct_factory({ p = p[1], name = "single" })715    -- use the typically unused backend.716    local singletwo = new_direct_factory({ p = b_pool, name = "singletwo" })717718    local first = new_basic_factory({ list = p, name = "first" }, first_factory_gen)719    local partial = new_basic_factory({ list = p, wait = 2, name = "partial" }, partial_factory_gen)720    local all = new_basic_factory({ list = p, name = "all" }, all_factory_gen)721    local fastgood = new_basic_factory({ list = p, wait = 2, name = "fastgood" }, fastgood_factory_gen)722    local fastgoodint = new_basic_factory({ list = p, wait = 2, name = "fastgoodint" }, fastgoodint_factory_gen)723    local blocker = new_blocker_factory({ blocker = b_pool, list = p, name = "blocker" })724    local logall = new_basic_factory({ list = p, name = "logall" }, logall_factory_gen)725    local fastlog = new_direct_factory({ p = pl, name = "fastlog" })726    local subfastlog = new_direct_factory({ p = fastlog, name = "subfastlog" })727    local summary = new_basic_factory({ list = p, name = "summary" }, summary_factory_gen)728    local waitfor = new_basic_factory({ list = p, name = "waitfor" }, waitfor_factory_gen)729    local failover = new_basic_factory({ list = p, name = "failover" }, failover_factory_gen)730    local locality = new_locality_factory({ p = p[1], name = "locality" })731732    local errors = new_error_factory(errors_factory_gen, "errors")733    local suberrors = new_error_factory(suberrors_factory_gen, "suberrors")734    local suberr_wrap = new_direct_factory({ p = suberrors, name = "suberrwrap" })735    local badreturn = new_error_factory(badreturn_gen, "badreturn")736737    local bestres = new_basic_factory({ list = p, name = "bestres" }, bestres_factory_gen)738    local bestrestime = new_basic_factory({ list = p, timeout = 0.5, name = "bestres" }, bestres_factory_gen)739    local worstres = new_basic_factory({ list = p, name = "worstres" }, worstres_factory_gen)740741    -- for testing traffic splitting.742    local split = new_split_factory({ a = single, b = singletwo, name = "split" })743    local splitfailover = new_split_factory({ a = failover, b = singletwo, name = "splitfailover" })744745    -- test timeout via subrctx's that themselves timeout746    local timesubone = new_basic_factory({ list = { p[1] }, timeout = 0.25, name = "timesubone" }, timeout_factory_gen)747    local timesubtwo = new_basic_factory({ list = { p[2] }, timeout = 0.25, name = "timesubone" }, timeout_factory_gen)748    local timesubthr = new_basic_factory({ list = { p[3] }, timeout = 0.25, name = "timesubone" }, timeout_factory_gen)749    local timetop = new_basic_factory({ list = { timesubone, timesubtwo, timesubthr }, wait = 1, name = "timetop" }, timeout_factory_gen)750    local timefgtop = new_basic_factory({ list = { timesubone, timesubtwo, timesubthr }, wait = 2, mode = mcp.WAIT_FASTGOOD, name = "timefgtop" }, timeout_factory_gen)751752    -- complex stacking:753    -- - parent754    -- - sub with direct child755    -- - child that splits requests, waits756    -- - one sub-child with 3 subrctx children, each with a pool757    -- - the sub-mid child throws fatal error after enqueueing requests.758    -- bug was pending request refcount for final child increasing before759    -- dispatching instead of after dispatching.760    local complex_childa = new_direct_factory({ p = p[1], name = "cmpchilda" })761    local complex_childb = new_direct_factory({ p = p[2], name = "cmpchildb" })762    local complex_childc = new_direct_factory({ p = p[3], name = "cmpchildc" })763    local complex_msg = new_msg_factory("SERVER_ERROR toast\r\n", "cmpmsg")764    local complex_fastgoodint = new_basic_factory({ list = { complex_childa, complex_childb, complex_childc }, name = "cmpfastgoodint" }, fastgoodint_factory_gen)765    local complex_mid = new_basic_factory({ list = { complex_childa, complex_childb, complex_fastgoodint }, wait = 0, name = "cmpmid" }, all_factory_gen)766    local complex_top = new_direct_factory({ p = complex_mid, name = "complex" })767768    local map = {769        ["single"] = single,770        ["first"] = first,771        ["partial"] = partial,772        ["all"] = all,773        ["fastgood"] = fastgood,774        ["fastgoodint"] = fastgoodint,775        ["blocker"] = blocker,776        ["logall"] = logall,777        ["fastlog"] = fastlog,778        ["subfastlog"] = subfastlog,779        ["summary"] = summary,780        ["waitfor"] = waitfor,781        ["failover"] = failover,782        ["suberrors"] = suberr_wrap,783        ["errors"] = errors,784        ["split"] = split,785        ["splitfailover"] = splitfailover,786        ["locality"] = locality,787        ["badreturn"] = badreturn,788        ["bestres"] = bestres,789        ["bestrestime"] = bestrestime,790        ["worstres"] = worstres,791        ["timetop"] = timetop,792        ["timefgtop"] = timefgtop,793        ["complex"] = complex_top,794    }795796    local parg = {797        default = single,798        list = map,799        pattern = "^/(%a+)/"800    }801802    local failgen = new_error_factory(failgen_factory_gen, "failgen")803    local failgenret = new_error_factory(failgenret_factory_gen, "failgenret")804805    local mapfail = {806        ["failgen"] = failgen,807        ["failgenret"] = failgenret,808    }809    local farg = {810        default = single,811        list = mapfail,812        pattern = "^(%a+)/",813        name = "prefixfail"814    }815816    local pfx = mcp.router_new({ map = map })817    local pfxfail = new_prefix_factory(farg)818819    mcp.attach(mcp.CMD_ANY_STORAGE, pfx)820    -- TODO: might need to move this fail stuff to another test file.821    mcp.attach(mcp.CMD_MS, pfxfail)822    mcp.attach(mcp.CMD_MD, pfxfail)823end