/redis-load.c
C | 604 lines | 509 code | 64 blank | 31 comment | 170 complexity | cd3acbfd0c659a4a97ff347e8cb3316c MD5 | raw file
Possible License(s): BSD-3-Clause
- /* Redis load utility.
- *
- * Copyright (c) 2009-2010, Salvatore Sanfilippo <antirez at gmail dot com>
- * All rights reserved.
- *
- * This software is NOT released under a free software license.
- * It is a commercial tool, under the terms of the license you can find in
- * the COPYING file in the Redis-Tools distribution.
- */
- #include "fmacros.h"
- #include <stdio.h>
- #include <string.h>
- #include <stdlib.h>
- #include <unistd.h>
- #include <errno.h>
- #include <signal.h>
- #include <assert.h>
- #include <signal.h>
- #include <math.h>
- #include <limits.h>
- #include "hiredis.h"
- #include "adapters/ae.h"
- #include "adlist.h"
- #include "zmalloc.h"
- #include "rc4rand.h"
- #include "utils.h"
- #define REDIS_IDLE 0
- #define REDIS_GET 1
- #define REDIS_SET 2
- #define REDIS_DEL 3
- #define REDIS_SWAPIN 4
- #define REDIS_LPUSH 6
- #define REDIS_LPOP 7
- #define REDIS_HSET 8
- #define REDIS_HGET 9
- #define REDIS_HGETALL 10
- #define MAX_LATENCY 5000
- #define DEFAULT_KEYSPACE 100000 /* 100k */
- #define DEFAULT_HASHKEYSPACE 1000 /* 1k */
- #define REDIS_NOTUSED(V) ((void) V)
- static struct config {
- aeEventLoop *el;
- int debug;
- int done;
- list *clients;
- int num_clients;
- int num_requests;
- int issued_requests;
- int datasize_min;
- int datasize_max;
- unsigned char *databuf;
- int keyspace;
- int hashkeyspace;
- int set_perc;
- int del_perc;
- int swapin_perc;
- int lpush_perc;
- int lpop_perc;
- int hset_perc;
- int hget_perc;
- int hgetall_perc;
- int check;
- int rand;
- int longtail;
- int longtail_order;
- char *hostip;
- int hostport;
- int keepalive;
- long long start;
- long long totlatency;
- int *latency;
- int quiet;
- int loop;
- int idlemode;
- int ctrlc; /* Ctrl + C pressed */
- unsigned int prngseed;
- /* The following "optab" array is used in order to randomize the different
- * kind of operations, like GET, SET, LPUSH, LPOP, SADD, and so forth.
- * For every query the client will pick a random bucket from 0 to 99
- * and will check what is the kind of operation to perform, also it will
- * use the bucket ID in order to make sure this kind of operation is
- * always executed against the right data type, so for instance if bucket
- * 7 is a LPUSH the operation will be performed against key xxxxxx07
- * and so forth. */
- unsigned char optab[100];
- } config;
- typedef struct _client {
- redisAsyncContext *context;
- int state;
- int reqtype; /* request type. REDIS_GET, REDIS_SET, ... */
- long long start; /* start time in milliseconds */
- long keyid; /* the key name for this request is "key:<keyid>" */
- } *client;
- /* Prototypes */
- static void issueRequest(client c);
- static void createMissingClients(void);
- /* Return a pseudo random number between min and max both inclusive */
- static long randbetween(long min, long max) {
- return min+(random()%(max-min+1));
- }
- /* PRNG biased accordingly to the power law (Long Tail alike) */
- unsigned long longtailprng(unsigned long min, unsigned long max, int n) {
- unsigned long pl;
- double r = (double)(random()&(INT_MAX-1))/INT_MAX;
- max += 1;
- pl = pow((pow(max,n+1) - pow(min,n+1))*r + pow(min,n+1),1.0/(n+1));
- return (max-1-pl)+min;
- }
- static void clientDisconnected(const redisAsyncContext *context, int status) {
- listNode *ln;
- client c = (client)context->data;
- if (status != REDIS_OK) {
- fprintf(stderr,"Disconnected: %s\n",c->context->errstr);
- exit(1);
- }
- ln = listSearchKey(config.clients,c);
- assert(ln != NULL);
- listDelNode(config.clients,ln);
- zfree(c);
- /* The run was not done, create new client(s). */
- if (!config.done) {
- createMissingClients();
- }
- /* Stop the event loop when all clients were disconnected */
- if (!listLength(config.clients)) {
- aeStop(config.el);
- }
- }
- static client createClient(void) {
- client c = zmalloc(sizeof(struct _client));
- c->context = redisAsyncConnect(config.hostip,config.hostport);
- c->context->data = c;
- redisAsyncSetDisconnectCallback(c->context,clientDisconnected);
- if (c->context->err) {
- fprintf(stderr,"Connect: %s\n",c->context->errstr);
- exit(1);
- }
- redisAeAttach(config.el,c->context);
- listAddNodeTail(config.clients,c);
- issueRequest(c);
- return c;
- }
- static void createMissingClients(void) {
- while(listLength(config.clients) < (size_t)config.num_clients) {
- createClient();
- }
- }
- static void checkDataIntegrity(client c, redisReply *reply) {
- if (c->reqtype == REDIS_GET && reply->type == REDIS_REPLY_STRING) {
- unsigned char *data;
- unsigned int datalen;
- rc4rand_seed(c->keyid);
- datalen = rc4rand_between(config.datasize_min,config.datasize_max);
- data = zmalloc(datalen);
- rc4rand_set(data,datalen);
- if (reply->len != (int)datalen) {
- fprintf(stderr, "*** Len mismatch for KEY key:%ld\n", c->keyid);
- fprintf(stderr, "*** %d instead of %d\n", reply->len, datalen);
- fprintf(stderr, "*** '%s' instead of '%s'\n", reply->str, data);
- exit(1);
- }
- if (memcmp(reply->str,data,datalen) != 0) {
- fprintf(stderr, "*** Data mismatch for KEY key:%ld\n", c->keyid);
- fprintf(stderr, "*** '%s' instead of '%s'\n", reply->str, data);
- exit(1);
- }
- zfree(data);
- }
- }
- static void handleReply(redisAsyncContext *context, void *_reply, void *privdata) {
- REDIS_NOTUSED(privdata);
- redisReply *reply = (redisReply*)_reply;
- client c = (client)context->data;
- long long latency = (microseconds() - c->start) / 1000;
- if (reply == NULL && context->err) {
- fprintf(stderr,"Error: %s\n", context->errstr);
- exit(1);
- } else {
- assert(reply != NULL);
- if (reply->type == REDIS_REPLY_ERROR) {
- fprintf(stderr,"Error: %s\n", reply->str);
- exit(1);
- }
- }
- if (latency > MAX_LATENCY) latency = MAX_LATENCY;
- config.latency[latency]++;
- if (config.check) checkDataIntegrity(c,reply);
- freeReplyObject(reply);
- if (config.done || config.ctrlc) {
- redisAsyncDisconnect(c->context);
- return;
- }
- if (config.keepalive) {
- issueRequest(c);
- } else {
- /* createMissingClients will be called in the disconnection callback */
- redisAsyncDisconnect(c->context);
- }
- }
- static unsigned long randomData(long seed) {
- unsigned long datalen;
- /* We use the key number as seed of the PRNG, so we'll be able to check if
- * a given key contains the right data later, without the use of additional
- * memory. */
- if (config.check) {
- rc4rand_seed(seed);
- datalen = rc4rand_between(config.datasize_min,config.datasize_max);
- rc4rand_set(config.databuf,datalen);
- } else {
- datalen = randbetween(config.datasize_min,config.datasize_max);
- if (config.rand) {
- rc4rand_seed(seed);
- rc4rand_set(config.databuf,datalen);
- } else {
- memset(config.databuf,'x',datalen);
- }
- }
- return datalen;
- }
- static void issueRequest(client c) {
- int op = config.optab[random() % 100];
- long key, hashkey;
- unsigned long datalen;
- config.issued_requests++;
- if (config.issued_requests == config.num_requests) config.done = 1;
- c->start = microseconds();
- if (config.longtail) {
- key = longtailprng(0,config.keyspace-1,config.longtail_order);
- hashkey = longtailprng(0,config.hashkeyspace-1,config.longtail_order);
- } else {
- key = random() % config.keyspace;
- hashkey = random() % config.hashkeyspace;
- }
- c->keyid = key;
- c->reqtype = op;
- if (op == REDIS_IDLE) {
- /* Idle */
- } else if (op == REDIS_SET) {
- datalen = randomData(key);
- redisAsyncCommand(c->context,handleReply,NULL,"SET string:%ld %b",key,config.databuf,datalen);
- } else if (op == REDIS_GET) {
- redisAsyncCommand(c->context,handleReply,NULL,"GET string:%ld",key);
- } else if (op == REDIS_DEL) {
- redisAsyncCommand(c->context,handleReply,NULL,"DEL string:%ld list:%ld hash:%ld",key,key,key);
- } else if (op == REDIS_LPUSH) {
- datalen = randomData(key);
- redisAsyncCommand(c->context,handleReply,NULL,"LPUSH list:%ld %b",key,config.databuf,datalen);
- } else if (op == REDIS_LPOP) {
- redisAsyncCommand(c->context,handleReply,NULL,"LPOP list:%ld",key);
- } else if (op == REDIS_HSET) {
- datalen = randomData(key);
- redisAsyncCommand(c->context,handleReply,NULL,"HSET hash:%ld key:%ld %b",key,hashkey,config.databuf,datalen);
- } else if (op == REDIS_HGET) {
- redisAsyncCommand(c->context,handleReply,NULL,"HGET hash:%ld key:%ld",key,hashkey);
- } else if (op == REDIS_HGETALL) {
- redisAsyncCommand(c->context,handleReply,NULL,"HGETALL hash:%ld",key);
- } else if (op == REDIS_SWAPIN) {
- /* Only accepts a single argument, so for now only works with string keys. */
- redisAsyncCommand(c->context,handleReply,NULL,"DEBUG SWAPIN string:%ld",key);
- } else {
- assert(NULL);
- }
- }
- static void showLatencyReport(void) {
- int j, seen = 0;
- float perc, reqpersec;
- reqpersec = (float)config.issued_requests/((float)config.totlatency/1000);
- if (!config.quiet) {
- printf("====== Report ======\n");
- printf(" %d requests in %.3f seconds\n", config.issued_requests,
- (float)config.totlatency/1000);
- printf(" %.2f requests per second\n", reqpersec);
- printf(" %d parallel clients\n", config.num_clients);
- printf(" payload: %d..%d bytes\n", config.datasize_min, config.datasize_max);
- printf(" keep alive: %d\n", config.keepalive);
- printf("\n");
- for (j = 0; j <= MAX_LATENCY; j++) {
- if (config.latency[j]) {
- seen += config.latency[j];
- perc = ((float)seen*100)/config.issued_requests;
- printf("%6.2f%% < %d ms\n", perc, j+1);
- }
- }
- } else {
- printf("%.2f requests per second\n", reqpersec);
- }
- }
- static void prepareForBenchmark(void) {
- memset(config.latency,0,sizeof(int)*(MAX_LATENCY+1));
- config.start = microseconds();
- config.issued_requests = 0;
- }
- static void endBenchmark(void) {
- config.totlatency = (microseconds()-config.start)/1000;
- showLatencyReport();
- }
- static void usage(char *wrong) {
- if (wrong)
- printf("Wrong option '%s' or option argument missing\n\n",wrong);
- printf(
- "Usage: redis-load ... options ...\n\n"
- " host <hostname> Server hostname (default 127.0.0.1)\n"
- " port <hostname> Server port (default 6379)\n"
- " clients <clients> Number of parallel connections (default 50)\n"
- " requests <requests> Total number of requests (default 10k)\n"
- " mindatasize <size> Min data size of string values in bytes (default 1)\n"
- " maxdatasize <size> Min data size of string values in bytes (default 64)\n"
- " datasize <size> Set both min and max data size to the same value\n"
- " keepalive 1=keep alive 0=reconnect (default 1)\n"
- " keyspace The number of different keys to use (default 100k)\n"
- " rand Use random data payload (incompressible)\n"
- " check Check integrity where reading data back (implies rand)\n"
- " longtail Use long tail alike key access pattern distribution\n"
- " longtailorder A value of 2: 20%% keys get 49%% accesses.\n"
- " 3: 20%% keys get 59%% accesses.\n"
- " 4: 20%% keys get 67%% accesses.\n"
- " 5: 20%% keys get 74%% accesses.\n"
- " 6: 20%% keys get 79%% accesses (default).\n"
- " 7: 20%% keys get 83%% accesses.\n"
- " 8: 20%% keys get 86%% accesses.\n"
- " 9: 20%% keys get 89%% accesses.\n"
- " 10: 20%% keys get 91%% accesses.\n"
- " 20: 20%% keys get 99%% accesses.\n"
- " seed <seed> PRNG seed for deterministic load\n"
- " big alias for keyspace 1000000 requests 1000000\n"
- " verybig alias for keyspace 10000000 requests 10000000\n"
- " quiet Quiet mode, less verbose\n"
- " loop Loop. Run the tests forever\n"
- " idle Idle mode. Just open N idle connections and wait.\n"
- " debug Debug mode. more verbose.\n"
- "\n"
- "Type of operations (use percentages without trailing %%):\n"
- "\n"
- " set <percentage> Percentage of SETs (default 50)\n"
- " del <percentage> Percentage of DELs (default 0)\n"
- " lpush <percentage> Percentage of LPUSHs (default 0)\n"
- " lpop <percentage> Percentage of LPOPs (default 0)\n"
- " hset <percentage> Percentage of HSETs (default 0)\n"
- " hget <percentage> Percentage of HGETs (default 0)\n"
- " hgetall <percentage> Percentage of HGETs (default 0)\n"
- " swapin <percentage> Percentage of DEBUG SWAPINs (default 0)\n"
- "\n"
- " All the free percantege (in order to reach 100%%) will be used for GETs\n"
- );
- exit(1);
- }
- static void parseOptions(int argc, char **argv) {
- int i;
- for (i = 1; i < argc; i++) {
- int lastarg = i==argc-1;
-
- if (!strcmp(argv[i],"clients") && !lastarg) {
- config.num_clients = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"requests") && !lastarg) {
- config.num_requests = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"set") && !lastarg) {
- config.set_perc = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"del") && !lastarg) {
- config.del_perc = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"swapin") && !lastarg) {
- config.swapin_perc = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"lpush") && !lastarg) {
- config.lpush_perc = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"lpop") && !lastarg) {
- config.lpop_perc = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"hset") && !lastarg) {
- config.hset_perc = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"hget") && !lastarg) {
- config.hget_perc = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"hgetall") && !lastarg) {
- config.hgetall_perc = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"keepalive") && !lastarg) {
- config.keepalive = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"host") && !lastarg) {
- config.hostip = argv[i+1];
- i++;
- } else if (!strcmp(argv[i],"port") && !lastarg) {
- config.hostport = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"datasize") && !lastarg) {
- config.datasize_max = config.datasize_min = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"mindatasize") && !lastarg) {
- config.datasize_min = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"maxdatasize") && !lastarg) {
- config.datasize_max = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"keyspace") && !lastarg) {
- config.keyspace = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"hashkeyspace") && !lastarg) {
- config.hashkeyspace = atoi(argv[i+1]);
- i++;
- } else if (!strcmp(argv[i],"seed") && !lastarg) {
- config.prngseed = strtol(argv[i+1],NULL,10);
- i++;
- } else if (!strcmp(argv[i],"big")) {
- config.keyspace = 1000000;
- config.num_requests = 1000000;
- } else if (!strcmp(argv[i],"verybig")) {
- config.keyspace = 10000000;
- config.num_requests = 10000000;
- } else if (!strcmp(argv[i],"quiet")) {
- config.quiet = 1;
- } else if (!strcmp(argv[i],"check")) {
- config.check = 1;
- } else if (!strcmp(argv[i],"rand")) {
- config.rand = 1;
- } else if (!strcmp(argv[i],"longtail")) {
- config.longtail = 1;
- } else if (!strcmp(argv[i],"longtailorder") && !lastarg) {
- config.longtail_order = atoi(argv[i+1]);
- i++;
- if (config.longtail_order < 2 || config.longtail_order > 100) {
- printf("Value out of range for 'longtailorder' option");
- exit(1);
- }
- } else if (!strcmp(argv[i],"loop")) {
- config.loop = 1;
- } else if (!strcmp(argv[i],"debug")) {
- config.debug = 1;
- } else if (!strcmp(argv[i],"idle")) {
- config.idlemode = 1;
- } else if (!strcmp(argv[i],"help")) {
- usage(NULL);
- } else {
- usage(argv[i]);
- }
- }
- /* Sanitize options */
- if (config.datasize_min < 1) config.datasize_min = 1;
- if (config.datasize_min > 1024*1024) config.datasize_min = 1024*1024;
- if (config.datasize_max < 1) config.datasize_max = 1;
- if (config.datasize_max > 1024*1024) config.datasize_max = 1024*1024;
- if (config.keyspace < 1) config.keyspace = DEFAULT_KEYSPACE;
- if (config.hashkeyspace < 1) config.hashkeyspace = DEFAULT_HASHKEYSPACE;
- }
- static void ctrlc(int sig) {
- REDIS_NOTUSED(sig);
- if (config.idlemode) {
- exit(1);
- } else {
- config.ctrlc++;
- if (config.ctrlc == 1) {
- config.done = 1;
- printf("\nWaiting for pending requests to complete...\n");
- } else {
- printf("\nForcing exit...\n");
- exit(1);
- }
- }
- }
- static void fillOpTab(int *i, int op, int perc) {
- int j;
- for (j = 0; j < perc; j++) {
- if (*i < 100) {
- config.optab[*i] = op;
- (*i)++;
- }
- }
- }
- int main(int argc, char **argv) {
- int i = 0;
- signal(SIGHUP, SIG_IGN);
- signal(SIGPIPE, SIG_IGN);
- config.el = aeCreateEventLoop();
- config.debug = 0;
- config.done = 0;
- config.clients = listCreate();
- config.num_clients = 50;
- config.num_requests = 10000;
- config.issued_requests = 0;
- config.keepalive = 1;
- config.set_perc = 50;
- config.del_perc = 0;
- config.swapin_perc = 0;
- config.lpush_perc = 0;
- config.lpop_perc = 0;
- config.hset_perc = 0;
- config.hget_perc = 0;
- config.hgetall_perc = 0;
- config.datasize_min = 1;
- config.datasize_max = 64;
- config.keyspace = DEFAULT_KEYSPACE; /* 100k */
- config.hashkeyspace = DEFAULT_HASHKEYSPACE; /* 1k */
- config.check = 0;
- config.rand = 0;
- config.longtail = 0;
- config.quiet = 0;
- config.loop = 0;
- config.idlemode = 0;
- config.latency = NULL;
- config.latency = zmalloc(sizeof(int)*(MAX_LATENCY+1));
- config.ctrlc = 0;
- config.prngseed = (unsigned int) (microseconds()^getpid());
- config.hostip = "127.0.0.1";
- config.hostport = 6379;
- parseOptions(argc,argv);
- config.databuf = zmalloc(config.datasize_max);
- if (config.keepalive == 0) {
- printf("WARNING: keepalive disabled, you probably need 'echo 1 > /proc/sys/net/ipv4/tcp_tw_reuse' for Linux and 'sudo sysctl -w net.inet.tcp.msl=1000' for Mac OS X in order to use a lot of clients/requests\n");
- }
- if (config.idlemode) {
- printf("Creating %d idle connections and waiting forever (Ctrl+C when done)\n", config.num_clients);
- memset(config.optab,REDIS_IDLE,100);
- } else {
- /* Setup the operation table. Start with a table with just GET
- * operations and overwrite it with others as needed. */
- memset(config.optab,REDIS_GET,100);
- fillOpTab(&i,REDIS_SET,config.set_perc);
- fillOpTab(&i,REDIS_DEL,config.del_perc);
- fillOpTab(&i,REDIS_LPUSH,config.lpush_perc);
- fillOpTab(&i,REDIS_LPOP,config.lpop_perc);
- fillOpTab(&i,REDIS_HSET,config.hset_perc);
- fillOpTab(&i,REDIS_HGET,config.hget_perc);
- fillOpTab(&i,REDIS_HGETALL,config.hgetall_perc);
- fillOpTab(&i,REDIS_SWAPIN,config.swapin_perc);
- }
- signal(SIGINT,ctrlc);
- srandom(config.prngseed);
- printf("PRNG seed is: %u - use the 'seed' option to reproduce the same sequence\n", config.prngseed);
- do {
- prepareForBenchmark();
- createMissingClients();
- aeMain(config.el);
- endBenchmark();
- } while(config.loop);
- return 0;
- }