PageRenderTime 158ms CodeModel.GetById 30ms app.highlight 94ms RepoModel.GetById 17ms app.codeStats 0ms

/redis-load.c

http://github.com/antirez/redis-tools
C | 604 lines | 509 code | 64 blank | 31 comment | 170 complexity | cd3acbfd0c659a4a97ff347e8cb3316c MD5 | raw file
  1/* Redis load utility.
  2 *
  3 * Copyright (c) 2009-2010, Salvatore Sanfilippo <antirez at gmail dot com>
  4 * All rights reserved.
  5 *
  6 * This software is NOT released under a free software license.
  7 * It is a commercial tool, under the terms of the license you can find in
  8 * the COPYING file in the Redis-Tools distribution.
  9 */
 10
 11#include "fmacros.h"
 12
 13#include <stdio.h>
 14#include <string.h>
 15#include <stdlib.h>
 16#include <unistd.h>
 17#include <errno.h>
 18#include <signal.h>
 19#include <assert.h>
 20#include <signal.h>
 21#include <math.h>
 22#include <limits.h>
 23
 24#include "hiredis.h"
 25#include "adapters/ae.h"
 26#include "adlist.h"
 27#include "zmalloc.h"
 28#include "rc4rand.h"
 29#include "utils.h"
 30
 31#define REDIS_IDLE 0
 32#define REDIS_GET 1
 33#define REDIS_SET 2
 34#define REDIS_DEL 3
 35#define REDIS_SWAPIN 4
 36#define REDIS_LPUSH 6
 37#define REDIS_LPOP 7
 38#define REDIS_HSET 8
 39#define REDIS_HGET 9
 40#define REDIS_HGETALL 10
 41
 42#define MAX_LATENCY 5000
 43#define DEFAULT_KEYSPACE 100000 /* 100k */
 44#define DEFAULT_HASHKEYSPACE 1000 /* 1k */
 45
 46#define REDIS_NOTUSED(V) ((void) V)
 47
 48static struct config {
 49    aeEventLoop *el;
 50    int debug;
 51    int done;
 52    list *clients;
 53    int num_clients;
 54    int num_requests;
 55    int issued_requests;
 56
 57    int datasize_min;
 58    int datasize_max;
 59    unsigned char *databuf;
 60
 61    int keyspace;
 62    int hashkeyspace;
 63
 64    int set_perc;
 65    int del_perc;
 66    int swapin_perc;
 67    int lpush_perc;
 68    int lpop_perc;
 69    int hset_perc;
 70    int hget_perc;
 71    int hgetall_perc;
 72
 73    int check;
 74    int rand;
 75    int longtail;
 76    int longtail_order;
 77    char *hostip;
 78    int hostport;
 79    int keepalive;
 80    long long start;
 81    long long totlatency;
 82    int *latency;
 83    int quiet;
 84    int loop;
 85    int idlemode;
 86    int ctrlc; /* Ctrl + C pressed */
 87    unsigned int prngseed;
 88    /* The following "optab" array is used in order to randomize the different
 89     * kind of operations, like GET, SET, LPUSH, LPOP, SADD, and so forth.
 90     * For every query the client will pick a random bucket from 0 to 99
 91     * and will check what is the kind of operation to perform, also it will
 92     * use the bucket ID in order to make sure this kind of operation is
 93     * always executed against the right data type, so for instance if bucket
 94     * 7 is a LPUSH the operation will be performed against key xxxxxx07
 95     * and so forth. */
 96    unsigned char optab[100];
 97} config;
 98
 99typedef struct _client {
100    redisAsyncContext *context;
101    int state;
102    int reqtype;        /* request type. REDIS_GET, REDIS_SET, ... */
103    long long start;    /* start time in milliseconds */
104    long keyid;          /* the key name for this request is "key:<keyid>" */
105} *client;
106
107/* Prototypes */
108static void issueRequest(client c);
109static void createMissingClients(void);
110
111/* Return a pseudo random number between min and max both inclusive */
112static long randbetween(long min, long max) {
113    return min+(random()%(max-min+1));
114}
115
116/* PRNG biased accordingly to the power law (Long Tail alike) */
117unsigned long longtailprng(unsigned long min, unsigned long max, int n) {
118    unsigned long pl;
119    double r = (double)(random()&(INT_MAX-1))/INT_MAX;
120
121    max += 1;
122    pl = pow((pow(max,n+1) - pow(min,n+1))*r + pow(min,n+1),1.0/(n+1));
123    return (max-1-pl)+min;
124}
125
126static void clientDisconnected(const redisAsyncContext *context, int status) {
127    listNode *ln;
128    client c = (client)context->data;
129
130    if (status != REDIS_OK) {
131        fprintf(stderr,"Disconnected: %s\n",c->context->errstr);
132        exit(1);
133    }
134
135    ln = listSearchKey(config.clients,c);
136    assert(ln != NULL);
137    listDelNode(config.clients,ln);
138    zfree(c);
139
140    /* The run was not done, create new client(s). */
141    if (!config.done) {
142        createMissingClients();
143    }
144
145    /* Stop the event loop when all clients were disconnected */
146    if (!listLength(config.clients)) {
147        aeStop(config.el);
148    }
149}
150
151static client createClient(void) {
152    client c = zmalloc(sizeof(struct _client));
153
154    c->context = redisAsyncConnect(config.hostip,config.hostport);
155    c->context->data = c;
156    redisAsyncSetDisconnectCallback(c->context,clientDisconnected);
157    if (c->context->err) {
158        fprintf(stderr,"Connect: %s\n",c->context->errstr);
159        exit(1);
160    }
161
162    redisAeAttach(config.el,c->context);
163    listAddNodeTail(config.clients,c);
164    issueRequest(c);
165    return c;
166}
167
168static void createMissingClients(void) {
169    while(listLength(config.clients) < (size_t)config.num_clients) {
170        createClient();
171    }
172}
173
174static void checkDataIntegrity(client c, redisReply *reply) {
175    if (c->reqtype == REDIS_GET && reply->type == REDIS_REPLY_STRING) {
176        unsigned char *data;
177        unsigned int datalen;
178
179        rc4rand_seed(c->keyid);
180        datalen = rc4rand_between(config.datasize_min,config.datasize_max);
181        data = zmalloc(datalen);
182        rc4rand_set(data,datalen);
183
184        if (reply->len != (int)datalen) {
185            fprintf(stderr, "*** Len mismatch for KEY key:%ld\n", c->keyid);
186            fprintf(stderr, "*** %d instead of %d\n", reply->len, datalen);
187            fprintf(stderr, "*** '%s' instead of '%s'\n", reply->str, data);
188            exit(1);
189        }
190        if (memcmp(reply->str,data,datalen) != 0) {
191            fprintf(stderr, "*** Data mismatch for KEY key:%ld\n", c->keyid);
192            fprintf(stderr, "*** '%s' instead of '%s'\n", reply->str, data);
193            exit(1);
194        }
195        zfree(data);
196    }
197}
198
199static void handleReply(redisAsyncContext *context, void *_reply, void *privdata) {
200    REDIS_NOTUSED(privdata);
201    redisReply *reply = (redisReply*)_reply;
202    client c = (client)context->data;
203    long long latency = (microseconds() - c->start) / 1000;
204
205    if (reply == NULL && context->err) {
206        fprintf(stderr,"Error: %s\n", context->errstr);
207        exit(1);
208    } else {
209        assert(reply != NULL);
210        if (reply->type == REDIS_REPLY_ERROR) {
211            fprintf(stderr,"Error: %s\n", reply->str);
212            exit(1);
213        }
214    }
215
216    if (latency > MAX_LATENCY) latency = MAX_LATENCY;
217    config.latency[latency]++;
218
219    if (config.check) checkDataIntegrity(c,reply);
220    freeReplyObject(reply);
221
222    if (config.done || config.ctrlc) {
223        redisAsyncDisconnect(c->context);
224        return;
225    }
226
227    if (config.keepalive) {
228        issueRequest(c);
229    } else {
230        /* createMissingClients will be called in the disconnection callback */
231        redisAsyncDisconnect(c->context);
232    }
233}
234
235static unsigned long randomData(long seed) {
236    unsigned long datalen;
237
238    /* We use the key number as seed of the PRNG, so we'll be able to check if
239     * a given key contains the right data later, without the use of additional
240     * memory. */
241    if (config.check) {
242        rc4rand_seed(seed);
243        datalen = rc4rand_between(config.datasize_min,config.datasize_max);
244        rc4rand_set(config.databuf,datalen);
245    } else {
246        datalen = randbetween(config.datasize_min,config.datasize_max);
247        if (config.rand) {
248            rc4rand_seed(seed);
249            rc4rand_set(config.databuf,datalen);
250        } else {
251            memset(config.databuf,'x',datalen);
252        }
253    }
254
255    return datalen;
256}
257
258static void issueRequest(client c) {
259    int op = config.optab[random() % 100];
260    long key, hashkey;
261    unsigned long datalen;
262
263    config.issued_requests++;
264    if (config.issued_requests == config.num_requests) config.done = 1;
265
266    c->start = microseconds();
267    if (config.longtail) {
268        key = longtailprng(0,config.keyspace-1,config.longtail_order);
269        hashkey = longtailprng(0,config.hashkeyspace-1,config.longtail_order);
270    } else {
271        key = random() % config.keyspace;
272        hashkey = random() % config.hashkeyspace;
273    }
274
275    c->keyid = key;
276    c->reqtype = op;
277
278    if (op == REDIS_IDLE) {
279        /* Idle */
280    } else if (op == REDIS_SET) {
281        datalen = randomData(key);
282        redisAsyncCommand(c->context,handleReply,NULL,"SET string:%ld %b",key,config.databuf,datalen);
283    } else if (op == REDIS_GET) {
284        redisAsyncCommand(c->context,handleReply,NULL,"GET string:%ld",key);
285    } else if (op == REDIS_DEL) {
286        redisAsyncCommand(c->context,handleReply,NULL,"DEL string:%ld list:%ld hash:%ld",key,key,key);
287    } else if (op == REDIS_LPUSH) {
288        datalen = randomData(key);
289        redisAsyncCommand(c->context,handleReply,NULL,"LPUSH list:%ld %b",key,config.databuf,datalen);
290    } else if (op == REDIS_LPOP) {
291        redisAsyncCommand(c->context,handleReply,NULL,"LPOP list:%ld",key);
292    } else if (op == REDIS_HSET) {
293        datalen = randomData(key);
294        redisAsyncCommand(c->context,handleReply,NULL,"HSET hash:%ld key:%ld %b",key,hashkey,config.databuf,datalen);
295    } else if (op == REDIS_HGET) {
296        redisAsyncCommand(c->context,handleReply,NULL,"HGET hash:%ld key:%ld",key,hashkey);
297    } else if (op == REDIS_HGETALL) {
298        redisAsyncCommand(c->context,handleReply,NULL,"HGETALL hash:%ld",key);
299    } else if (op == REDIS_SWAPIN) {
300        /* Only accepts a single argument, so for now only works with string keys. */
301        redisAsyncCommand(c->context,handleReply,NULL,"DEBUG SWAPIN string:%ld",key);
302    } else {
303        assert(NULL);
304    }
305}
306
307static void showLatencyReport(void) {
308    int j, seen = 0;
309    float perc, reqpersec;
310
311    reqpersec = (float)config.issued_requests/((float)config.totlatency/1000);
312    if (!config.quiet) {
313        printf("====== Report ======\n");
314        printf("  %d requests in %.3f seconds\n", config.issued_requests,
315            (float)config.totlatency/1000);
316        printf("  %.2f requests per second\n", reqpersec);
317        printf("  %d parallel clients\n", config.num_clients);
318        printf("  payload: %d..%d bytes\n", config.datasize_min, config.datasize_max);
319        printf("  keep alive: %d\n", config.keepalive);
320        printf("\n");
321        for (j = 0; j <= MAX_LATENCY; j++) {
322            if (config.latency[j]) {
323                seen += config.latency[j];
324                perc = ((float)seen*100)/config.issued_requests;
325                printf("%6.2f%% < %d ms\n", perc, j+1);
326            }
327        }
328    } else {
329        printf("%.2f requests per second\n", reqpersec);
330    }
331}
332
333static void prepareForBenchmark(void) {
334    memset(config.latency,0,sizeof(int)*(MAX_LATENCY+1));
335    config.start = microseconds();
336    config.issued_requests = 0;
337}
338
339static void endBenchmark(void) {
340    config.totlatency = (microseconds()-config.start)/1000;
341    showLatencyReport();
342}
343
344static void usage(char *wrong) {
345    if (wrong)
346        printf("Wrong option '%s' or option argument missing\n\n",wrong);
347    printf(
348"Usage: redis-load ... options ...\n\n"
349" host <hostname>      Server hostname (default 127.0.0.1)\n"
350" port <hostname>      Server port (default 6379)\n"
351" clients <clients>    Number of parallel connections (default 50)\n"
352" requests <requests>  Total number of requests (default 10k)\n"
353" mindatasize <size>   Min data size of string values in bytes (default 1)\n"
354" maxdatasize <size>   Min data size of string values in bytes (default 64)\n"
355" datasize <size>      Set both min and max data size to the same value\n"
356" keepalive            1=keep alive 0=reconnect (default 1)\n"
357" keyspace             The number of different keys to use (default 100k)\n"
358" rand                 Use random data payload (incompressible)\n"
359" check                Check integrity where reading data back (implies rand)\n"
360" longtail             Use long tail alike key access pattern distribution\n"
361" longtailorder        A value of 2: 20%% keys get 49%% accesses.\n"
362"                                 3: 20%% keys get 59%% accesses.\n"
363"                                 4: 20%% keys get 67%% accesses.\n"
364"                                 5: 20%% keys get 74%% accesses.\n"
365"                                 6: 20%% keys get 79%% accesses (default).\n"
366"                                 7: 20%% keys get 83%% accesses.\n"
367"                                 8: 20%% keys get 86%% accesses.\n"
368"                                 9: 20%% keys get 89%% accesses.\n"
369"                                10: 20%% keys get 91%% accesses.\n"
370"                                20: 20%% keys get 99%% accesses.\n"
371" seed <seed>          PRNG seed for deterministic load\n"
372" big                  alias for keyspace 1000000 requests 1000000\n"
373" verybig              alias for keyspace 10000000 requests 10000000\n"
374" quiet                Quiet mode, less verbose\n"
375" loop                 Loop. Run the tests forever\n"
376" idle                 Idle mode. Just open N idle connections and wait.\n"
377" debug                Debug mode. more verbose.\n"
378"\n"
379"Type of operations (use percentages without trailing %%):\n"
380"\n"
381" set <percentage>     Percentage of SETs (default 50)\n"
382" del <percentage>     Percentage of DELs (default 0)\n"
383" lpush <percentage>   Percentage of LPUSHs (default 0)\n"
384" lpop <percentage>    Percentage of LPOPs (default 0)\n"
385" hset <percentage>    Percentage of HSETs (default 0)\n"
386" hget <percentage>    Percentage of HGETs (default 0)\n"
387" hgetall <percentage> Percentage of HGETs (default 0)\n"
388" swapin <percentage>  Percentage of DEBUG SWAPINs (default 0)\n"
389"\n"
390" All the free percantege (in order to reach 100%%) will be used for GETs\n"
391);
392    exit(1);
393}
394
395static void parseOptions(int argc, char **argv) {
396    int i;
397
398    for (i = 1; i < argc; i++) {
399        int lastarg = i==argc-1;
400        
401        if (!strcmp(argv[i],"clients") && !lastarg) {
402            config.num_clients = atoi(argv[i+1]);
403            i++;
404        } else if (!strcmp(argv[i],"requests") && !lastarg) {
405            config.num_requests = atoi(argv[i+1]);
406            i++;
407        } else if (!strcmp(argv[i],"set") && !lastarg) {
408            config.set_perc = atoi(argv[i+1]);
409            i++;
410        } else if (!strcmp(argv[i],"del") && !lastarg) {
411            config.del_perc = atoi(argv[i+1]);
412            i++;
413        } else if (!strcmp(argv[i],"swapin") && !lastarg) {
414            config.swapin_perc = atoi(argv[i+1]);
415            i++;
416        } else if (!strcmp(argv[i],"lpush") && !lastarg) {
417            config.lpush_perc = atoi(argv[i+1]);
418            i++;
419        } else if (!strcmp(argv[i],"lpop") && !lastarg) {
420            config.lpop_perc = atoi(argv[i+1]);
421            i++;
422        } else if (!strcmp(argv[i],"hset") && !lastarg) {
423            config.hset_perc = atoi(argv[i+1]);
424            i++;
425        } else if (!strcmp(argv[i],"hget") && !lastarg) {
426            config.hget_perc = atoi(argv[i+1]);
427            i++;
428        } else if (!strcmp(argv[i],"hgetall") && !lastarg) {
429            config.hgetall_perc = atoi(argv[i+1]);
430            i++;
431        } else if (!strcmp(argv[i],"keepalive") && !lastarg) {
432            config.keepalive = atoi(argv[i+1]);
433            i++;
434        } else if (!strcmp(argv[i],"host") && !lastarg) {
435            config.hostip = argv[i+1];
436            i++;
437        } else if (!strcmp(argv[i],"port") && !lastarg) {
438            config.hostport = atoi(argv[i+1]);
439            i++;
440        } else if (!strcmp(argv[i],"datasize") && !lastarg) {
441            config.datasize_max = config.datasize_min = atoi(argv[i+1]);
442            i++;
443        } else if (!strcmp(argv[i],"mindatasize") && !lastarg) {
444            config.datasize_min = atoi(argv[i+1]);
445            i++;
446        } else if (!strcmp(argv[i],"maxdatasize") && !lastarg) {
447            config.datasize_max = atoi(argv[i+1]);
448            i++;
449        } else if (!strcmp(argv[i],"keyspace") && !lastarg) {
450            config.keyspace = atoi(argv[i+1]);
451            i++;
452        } else if (!strcmp(argv[i],"hashkeyspace") && !lastarg) {
453            config.hashkeyspace = atoi(argv[i+1]);
454            i++;
455        } else if (!strcmp(argv[i],"seed") && !lastarg) {
456            config.prngseed = strtol(argv[i+1],NULL,10);
457            i++;
458        } else if (!strcmp(argv[i],"big")) {
459            config.keyspace = 1000000;
460            config.num_requests = 1000000;
461        } else if (!strcmp(argv[i],"verybig")) {
462            config.keyspace = 10000000;
463            config.num_requests = 10000000;
464        } else if (!strcmp(argv[i],"quiet")) {
465            config.quiet = 1;
466        } else if (!strcmp(argv[i],"check")) {
467            config.check = 1;
468        } else if (!strcmp(argv[i],"rand")) {
469            config.rand = 1;
470        } else if (!strcmp(argv[i],"longtail")) {
471            config.longtail = 1;
472        } else if (!strcmp(argv[i],"longtailorder") && !lastarg) {
473            config.longtail_order = atoi(argv[i+1]);
474            i++;
475            if (config.longtail_order < 2 || config.longtail_order > 100) {
476                printf("Value out of range for 'longtailorder' option");
477                exit(1);
478            }
479        } else if (!strcmp(argv[i],"loop")) {
480            config.loop = 1;
481        } else if (!strcmp(argv[i],"debug")) {
482            config.debug = 1;
483        } else if (!strcmp(argv[i],"idle")) {
484            config.idlemode = 1;
485        } else if (!strcmp(argv[i],"help")) {
486            usage(NULL);
487        } else {
488            usage(argv[i]);
489        }
490    }
491    /* Sanitize options */
492    if (config.datasize_min < 1) config.datasize_min = 1;
493    if (config.datasize_min > 1024*1024) config.datasize_min = 1024*1024;
494    if (config.datasize_max < 1) config.datasize_max = 1;
495    if (config.datasize_max > 1024*1024) config.datasize_max = 1024*1024;
496    if (config.keyspace < 1) config.keyspace = DEFAULT_KEYSPACE;
497    if (config.hashkeyspace < 1) config.hashkeyspace = DEFAULT_HASHKEYSPACE;
498}
499
500static void ctrlc(int sig) {
501    REDIS_NOTUSED(sig);
502
503    if (config.idlemode) {
504        exit(1);
505    } else {
506        config.ctrlc++;
507        if (config.ctrlc == 1) {
508            config.done = 1;
509            printf("\nWaiting for pending requests to complete...\n");
510        } else {
511            printf("\nForcing exit...\n");
512            exit(1);
513        }
514    }
515}
516
517static void fillOpTab(int *i, int op, int perc) {
518    int j;
519
520    for (j = 0; j < perc; j++) {
521        if (*i < 100) {
522            config.optab[*i] = op;
523            (*i)++;
524        }
525    }
526}
527
528int main(int argc, char **argv) {
529    int i = 0;
530
531    signal(SIGHUP, SIG_IGN);
532    signal(SIGPIPE, SIG_IGN);
533
534    config.el = aeCreateEventLoop();
535    config.debug = 0;
536    config.done = 0;
537    config.clients = listCreate();
538    config.num_clients = 50;
539    config.num_requests = 10000;
540    config.issued_requests = 0;
541
542    config.keepalive = 1;
543    config.set_perc = 50;
544    config.del_perc = 0;
545    config.swapin_perc = 0;
546    config.lpush_perc = 0;
547    config.lpop_perc = 0;
548    config.hset_perc = 0;
549    config.hget_perc = 0;
550    config.hgetall_perc = 0;
551    config.datasize_min = 1;
552    config.datasize_max = 64;
553    config.keyspace = DEFAULT_KEYSPACE; /* 100k */
554    config.hashkeyspace = DEFAULT_HASHKEYSPACE; /* 1k */
555    config.check = 0;
556    config.rand = 0;
557    config.longtail = 0;
558    config.quiet = 0;
559    config.loop = 0;
560    config.idlemode = 0;
561    config.latency = NULL;
562    config.latency = zmalloc(sizeof(int)*(MAX_LATENCY+1));
563    config.ctrlc = 0;
564    config.prngseed = (unsigned int) (microseconds()^getpid());
565
566    config.hostip = "127.0.0.1";
567    config.hostport = 6379;
568
569    parseOptions(argc,argv);
570    config.databuf = zmalloc(config.datasize_max);
571
572    if (config.keepalive == 0) {
573        printf("WARNING: keepalive disabled, you probably need 'echo 1 > /proc/sys/net/ipv4/tcp_tw_reuse' for Linux and 'sudo sysctl -w net.inet.tcp.msl=1000' for Mac OS X in order to use a lot of clients/requests\n");
574    }
575
576    if (config.idlemode) {
577        printf("Creating %d idle connections and waiting forever (Ctrl+C when done)\n", config.num_clients);
578        memset(config.optab,REDIS_IDLE,100);
579    } else {
580        /* Setup the operation table. Start with a table with just GET
581         * operations and overwrite it with others as needed. */
582        memset(config.optab,REDIS_GET,100);
583        fillOpTab(&i,REDIS_SET,config.set_perc);
584        fillOpTab(&i,REDIS_DEL,config.del_perc);
585        fillOpTab(&i,REDIS_LPUSH,config.lpush_perc);
586        fillOpTab(&i,REDIS_LPOP,config.lpop_perc);
587        fillOpTab(&i,REDIS_HSET,config.hset_perc);
588        fillOpTab(&i,REDIS_HGET,config.hget_perc);
589        fillOpTab(&i,REDIS_HGETALL,config.hgetall_perc);
590        fillOpTab(&i,REDIS_SWAPIN,config.swapin_perc);
591    }
592
593    signal(SIGINT,ctrlc);
594    srandom(config.prngseed);
595    printf("PRNG seed is: %u - use the 'seed' option to reproduce the same sequence\n", config.prngseed);
596    do {
597        prepareForBenchmark();
598        createMissingClients();
599        aeMain(config.el);
600        endBenchmark();
601    } while(config.loop);
602
603    return 0;
604}