PageRenderTime 1366ms CodeModel.GetById 221ms app.highlight 818ms RepoModel.GetById 182ms app.codeStats 2ms

/src/grntest.c

https://github.com/darashi/groonga
C | 3166 lines | 2783 code | 301 blank | 82 comment | 599 complexity | 50263db86204365b5f486ae06e68be0a MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1/* -*- c-basic-offset: 2 -*- */
   2/*
   3  Copyright(C) 2010-2011 Brazil
   4
   5  This library is free software; you can redistribute it and/or
   6  modify it under the terms of the GNU Lesser General Public
   7  License version 2.1 as published by the Free Software Foundation.
   8
   9  This library is distributed in the hope that it will be useful,
  10  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  Lesser General Public License for more details.
  13
  14  You should have received a copy of the GNU Lesser General Public
  15  License along with this library; if not, write to the Free Software
  16  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  17*/
  18
  19#ifdef HAVE_CONFIG_H
  20#include "config.h"
  21#endif /* HAVE_CONFIG_H */
  22
  23#include <stdio.h>
  24#include <stdlib.h>
  25#include <string.h>
  26#include <errno.h>
  27#include <sys/types.h>
  28#include <sys/stat.h>
  29
  30#ifdef HAVE_SYS_WAIT_H
  31#include <sys/wait.h>
  32#endif /* HAVE_SYS_WAIT_H */
  33#ifdef HAVE_SYS_SOCKET_H
  34#include <sys/socket.h>
  35#endif /* HAVE_SYS_SOCKET_H */
  36#ifdef HAVE_NETINET_IN_H
  37#include <netinet/in.h>
  38#endif /* HAVE_NETINET_IN_H */
  39
  40#include "lib/str.h"
  41#include "lib/com.h"
  42#include "lib/db.h"
  43
  44#ifdef WIN32
  45#include <windows.h>
  46#include <stddef.h>
  47#else
  48#include <sys/param.h>
  49#include <sys/utsname.h>
  50#include <sys/statvfs.h>
  51#include <libgen.h>
  52#endif /* WIN32 */
  53
  54/*
  55#define DEBUG_FTP
  56#define DEBUG_HTTP
  57*/
  58
  59#define FTPUSER "anonymous"
  60#define FTPPASSWD "grntest"
  61#define FTPSERVER "ftp.groonga.org"
  62#define FTPBUF 20000
  63#define DEFAULT_PORT 10041
  64#define DEFAULT_DEST "localhost"
  65
  66#define OUT_JSON 0
  67#define OUT_TSV  1
  68
  69static int grntest_outtype = OUT_JSON;
  70
  71static grn_critical_section grntest_cs;
  72
  73static int grntest_stop_flag = 0;
  74static int grntest_detail_on = 0;
  75static int grntest_remote_mode = 0;
  76static int grntest_localonly_mode = 0;
  77static int grntest_owndb_mode = 0;
  78static int grntest_onmemory_mode = 0;
  79static grn_bool grntest_ftp_mode = GRN_FALSE;
  80#define TMPFILE "_grntest.tmp"
  81
  82static grn_ctx grntest_server_context;
  83static FILE *grntest_log_file;
  84
  85#define OS_LINUX64   "LINUX64"
  86#define OS_LINUX32   "LINUX32"
  87#define OS_WINDOWS64 "WINDOWS64"
  88#define OS_WINDOWS32 "WINDOWS32"
  89
  90#ifdef WIN32
  91typedef SOCKET socket_t;
  92#define SOCKETERROR INVALID_SOCKET
  93#define socketclose closesocket
  94static const char *groonga_path = "groonga.exe";
  95static PROCESS_INFORMATION grntest_pi;
  96#else
  97static pid_t grntest_server_id = 0;
  98typedef int socket_t;
  99#define socketclose close
 100#define SOCKETERROR -1
 101static const char *groonga_path = "groonga";
 102#endif /* WIN32 */
 103
 104static const char *groonga_protocol = "gqtp";
 105static char *grntest_osinfo;
 106static int grntest_sigint = 0;
 107
 108
 109
 110static grn_obj *grntest_db = NULL;
 111
 112#define MAX_CON_JOB 10
 113#define MAX_CON 64
 114
 115#define BUF_LEN 1024
 116#define MAX_PATH_LEN 256
 117
 118#define J_DO_LOCAL  1  /* do_local */
 119#define J_DO_GQTP   2  /* do_gqtp */
 120#define J_DO_HTTP   3  /* do_http */
 121#define J_REP_LOCAL 4  /* rep_local */
 122#define J_REP_GQTP  5  /* rep_gqtp */
 123#define J_REP_HTTP  6  /* rep_http */
 124#define J_OUT_LOCAL 7  /* out_local */
 125#define J_OUT_GQTP  8  /* out_gqtp */
 126#define J_OUT_HTTP  9  /* out_http */
 127#define J_TEST_LOCAL 10  /* test_local */
 128#define J_TEST_GQTP  11  /* test_gqtp */
 129#define J_TEST_HTTP  12  /* test_http */
 130
 131static char grntest_username[BUF_LEN];
 132static char grntest_scriptname[BUF_LEN];
 133static char grntest_date[BUF_LEN];
 134static char grntest_serverhost[BUF_LEN];
 135static int grntest_serverport;
 136static const char *grntest_dbpath;
 137
 138struct job {
 139  char jobname[BUF_LEN];
 140  char commandfile[BUF_LEN];
 141  int qnum;
 142  int jobtype;
 143  int concurrency;
 144  int ntimes;
 145  int done;
 146  long long int max;
 147  long long int min;
 148  FILE *outputlog;
 149  FILE *inputlog;
 150  char logfile[BUF_LEN];
 151};
 152
 153struct task {
 154  char *file;
 155  grn_obj *commands;
 156  int jobtype;
 157  int ntimes;
 158  int qnum;
 159  int job_id;
 160  long long int max;
 161  long long int min;
 162  socket_t http_socket;
 163  grn_obj http_response;
 164};
 165
 166static struct task grntest_task[MAX_CON];
 167static struct job grntest_job[MAX_CON];
 168static int grntest_jobdone;
 169static int grntest_jobnum;
 170static grn_ctx grntest_ctx[MAX_CON];
 171static grn_obj *grntest_owndb[MAX_CON];
 172
 173static grn_obj grntest_starttime, grntest_jobs_start;
 174
 175static int
 176grntest_atoi(const char *str, const char *end, const char **rest)
 177{
 178  while (grn_isspace(str, GRN_ENC_UTF8) == 1) {
 179    str++;
 180  }
 181  return grn_atoi(str, end, rest);
 182}
 183
 184static int
 185out_p(int jobtype)
 186{
 187  if (jobtype == J_OUT_LOCAL) {
 188    return 1;
 189  }
 190  if (jobtype == J_OUT_GQTP) {
 191    return 1;
 192  }
 193  if (jobtype == J_OUT_HTTP) {
 194    return 1;
 195  }
 196  return 0;
 197}
 198
 199static int
 200test_p(int jobtype)
 201{
 202  if (jobtype == J_TEST_LOCAL) {
 203    return 1;
 204  }
 205  if (jobtype == J_TEST_GQTP) {
 206    return 1;
 207  }
 208  if (jobtype == J_TEST_HTTP) {
 209    return 1;
 210  }
 211  return 0;
 212}
 213
 214static int
 215report_p(int jobtype)
 216{
 217  if (jobtype == J_REP_LOCAL) {
 218    return 1;
 219  }
 220  if (jobtype == J_REP_GQTP) {
 221    return 1;
 222  }
 223  if (jobtype == J_REP_HTTP) {
 224    return 1;
 225  }
 226  return 0;
 227}
 228
 229static int
 230gqtp_p(int jobtype)
 231{
 232  if (jobtype == J_DO_GQTP) {
 233    return 1;
 234  }
 235  if (jobtype == J_REP_GQTP) {
 236    return 1;
 237  }
 238  if (jobtype == J_OUT_GQTP) {
 239    return 1;
 240  }
 241  if (jobtype == J_TEST_GQTP) {
 242    return 1;
 243  }
 244  return 0;
 245}
 246
 247static int
 248http_p(int jobtype)
 249{
 250  if (jobtype == J_DO_HTTP) {
 251    return 1;
 252  }
 253  if (jobtype == J_REP_HTTP) {
 254    return 1;
 255  }
 256  if (jobtype == J_OUT_HTTP) {
 257    return 1;
 258  }
 259  if (jobtype == J_TEST_HTTP) {
 260    return 1;
 261  }
 262  return 0;
 263}
 264
 265static int
 266error_exit_in_thread(intptr_t code)
 267{
 268  fprintf(stderr,
 269          "Fatal error! Check script file or database!: %ld\n", (long)code);
 270  fflush(stderr);
 271  CRITICAL_SECTION_ENTER(grntest_cs);
 272  grntest_stop_flag = 1;
 273  CRITICAL_SECTION_LEAVE(grntest_cs);
 274#ifdef WIN32
 275  _endthreadex(code);
 276#else
 277  pthread_exit((void *)code);
 278#endif /* WIN32 */
 279  return 0;
 280}
 281
 282
 283static void
 284escape_command(grn_ctx *ctx, char *in, int ilen, grn_obj *escaped_command)
 285{
 286  int i = 0;
 287
 288  while (i < ilen) {
 289    if ((in[i] == '\\') || (in[i] == '\"') || (in[i] == '/')) {
 290      GRN_TEXT_PUTC(ctx, escaped_command, '\\');
 291      GRN_TEXT_PUTC(ctx, escaped_command, in[i]);
 292      i++;
 293    } else {
 294      switch (in[i]) {
 295      case '\b':
 296        GRN_TEXT_PUTS(ctx, escaped_command, "\\b");
 297        i++;
 298        break;
 299      case '\f':
 300        GRN_TEXT_PUTS(ctx, escaped_command, "\\f");
 301        i++;
 302        break;
 303      case '\n':
 304        GRN_TEXT_PUTS(ctx, escaped_command, "\\n");
 305        i++;
 306        break;
 307      case '\r':
 308        GRN_TEXT_PUTS(ctx, escaped_command, "\\r");
 309        i++;
 310        break;
 311      case '\t':
 312        GRN_TEXT_PUTS(ctx, escaped_command, "\\t");
 313        i++;
 314        break;
 315      default:
 316        GRN_TEXT_PUTC(ctx, escaped_command, in[i]);
 317        i++;
 318        break;
 319      }
 320    }
 321  }
 322  GRN_TEXT_PUTC(ctx, escaped_command, '\0');
 323}
 324
 325static int
 326report_command(grn_ctx *ctx, char *command, char *ret, int task_id,
 327               grn_obj *start_time, grn_obj *end_time)
 328{
 329  int i, len, clen;
 330  long long int start, end;
 331  grn_obj result, escaped_command;
 332
 333  GRN_TEXT_INIT(&result, 0);
 334  if (strncmp(ret, "[[", 2) == 0) {
 335    i = 2;
 336    len = 1;
 337    while (ret[i] != ']') {
 338      i++;
 339      len++;
 340      if (ret[i] == '\0') {
 341        fprintf(stderr, "Error results:command=[%s]\n", command);
 342        error_exit_in_thread(3);
 343      }
 344    }
 345    len++;
 346    grn_text_esc(ctx, &result, ret + 1, len);
 347  } else {
 348    grn_text_esc(ctx, &result, ret, strlen(ret));
 349  }
 350
 351  start = GRN_TIME_VALUE(start_time) - GRN_TIME_VALUE(&grntest_starttime);
 352  end = GRN_TIME_VALUE(end_time) - GRN_TIME_VALUE(&grntest_starttime);
 353  clen = strlen(command);
 354  GRN_TEXT_INIT(&escaped_command, 0);
 355  escape_command(ctx, command, clen, &escaped_command);
 356  if (grntest_outtype == OUT_TSV) {
 357    fprintf(grntest_log_file, "report\t%d\t%s\t%" GRN_FMT_LLD "\t%" GRN_FMT_LLD "\t%.*s\n",
 358            task_id, GRN_TEXT_VALUE(&escaped_command), start, end,
 359            (int)GRN_TEXT_LEN(&result), GRN_TEXT_VALUE(&result));
 360  } else {
 361    fprintf(grntest_log_file, "[%d, \"%s\", %" GRN_FMT_LLD ", %" GRN_FMT_LLD ", %.*s],\n",
 362            task_id, GRN_TEXT_VALUE(&escaped_command), start, end,
 363            (int)GRN_TEXT_LEN(&result), GRN_TEXT_VALUE(&result));
 364  }
 365  fflush(grntest_log_file);
 366  GRN_OBJ_FIN(ctx, &escaped_command);
 367  GRN_OBJ_FIN(ctx, &result);
 368  return 0;
 369}
 370
 371static int
 372output_result_final(grn_ctx *ctx, int qnum)
 373{
 374  grn_obj end_time;
 375  long long int latency, self;
 376  double sec, qps;
 377
 378  GRN_TIME_INIT(&end_time, 0);
 379  GRN_TIME_NOW(ctx, &end_time);
 380
 381  latency = GRN_TIME_VALUE(&end_time) - GRN_TIME_VALUE(&grntest_starttime);
 382  self = latency;
 383  sec = self / (double)1000000;
 384  qps = (double)qnum / sec;
 385  if (grntest_outtype == OUT_TSV) {
 386    fprintf(grntest_log_file, "total\t%" GRN_FMT_LLD "\t%f\t%d\n", latency, qps, qnum);
 387  } else {
 388    fprintf(grntest_log_file,
 389           "{\"total\": %" GRN_FMT_LLD ", \"qps\": %f, \"queries\": %d}]\n", latency, qps, qnum);
 390  }
 391  grn_obj_close(ctx, &end_time);
 392  return 0;
 393}
 394
 395static int
 396output_sysinfo(char *sysinfo)
 397{
 398  if (grntest_outtype == OUT_TSV) {
 399    fprintf(grntest_log_file, "%s", sysinfo);
 400  } else {
 401    fprintf(grntest_log_file, "[%s\n", sysinfo);
 402  }
 403  return 0;
 404}
 405
 406/* #define ENABLE_ERROR_REPORT 1 */
 407#ifdef ENABLE_ERROR_REPORT
 408static int
 409error_command(grn_ctx *ctx, char *command, int task_id)
 410{
 411  fprintf(stderr, "error!:command=[%s] task_id = %d\n", command, task_id);
 412  fflush(stderr);
 413  error_exit_in_thread(1);
 414  return 0;
 415}
 416#endif
 417
 418static void
 419normalize_output(char *output, int length,
 420                 char **normalized_output, int *normalized_length)
 421{
 422  int i;
 423
 424  *normalized_output = NULL;
 425  *normalized_length = length;
 426  for (i = 0; i < length; i++) {
 427    if (!strncmp(output + i, "],", 2)) {
 428      *normalized_output = output + i + 2;
 429      *normalized_length -= i + 2;
 430      break;
 431    }
 432  }
 433
 434  if (!*normalized_output) {
 435    if (length > 2 && strncmp(output + length - 2, "]]", 2)) {
 436      *normalized_output = output + length;
 437      *normalized_length = 0;
 438    } else {
 439      *normalized_output = output;
 440    }
 441  }
 442}
 443
 444static grn_bool
 445same_result_p(char *expect, int expected_length, char *result, int result_length)
 446{
 447  char *normalized_expected, *normalized_result;
 448  int normalized_expected_length, normalized_result_length;
 449
 450  normalize_output(expect, expected_length,
 451                   &normalized_expected, &normalized_expected_length);
 452  normalize_output(result, result_length,
 453                   &normalized_result, &normalized_result_length);
 454
 455  return((normalized_expected_length == normalized_result_length) &&
 456         strncmp(normalized_expected, normalized_result,
 457                 normalized_expected_length) == 0);
 458}
 459
 460static socket_t
 461open_socket(char *host, int port)
 462{
 463  socket_t sock;
 464  struct hostent *servhost;
 465  struct sockaddr_in server;
 466  u_long inaddr;
 467  int ret;
 468
 469  servhost = gethostbyname(host);
 470  if (servhost == NULL){
 471    fprintf(stderr, "Bad hostname [%s]\n", host);
 472    return -1;
 473  }
 474  inaddr = *(u_long*)(servhost->h_addr_list[0]);
 475
 476  memset(&server, 0, sizeof(struct sockaddr_in));
 477  server.sin_family = AF_INET;
 478  server.sin_port = htons(port);
 479  server.sin_addr = *(struct in_addr*)&inaddr;
 480
 481  sock = socket(AF_INET, SOCK_STREAM, 0);
 482  if (sock == -1) {
 483    fprintf(stderr, "socket error\n");
 484    return -1;
 485  }
 486  ret = connect(sock, (struct sockaddr *)&server, sizeof(struct sockaddr_in));
 487  if (ret == -1) {
 488    fprintf(stderr, "connect error\n");
 489    return -1;
 490  }
 491  return sock;
 492}
 493
 494static int
 495write_to_server(socket_t socket, char *buf)
 496{
 497#ifdef DEBUG_FTP
 498  fprintf(stderr, "send:%s", buf);
 499#endif
 500  send(socket, buf, strlen(buf), 0);
 501  return 0;
 502}
 503
 504#define OUTPUT_TYPE "output_type"
 505#define OUTPUT_TYPE_LEN (sizeof(OUTPUT_TYPE) - 1)
 506
 507static void
 508command_line_to_uri_path(grn_ctx *ctx, grn_obj *uri, char *command)
 509{
 510  char tok_type;
 511  int offset = 0, have_key = 0;
 512  const char *p, *e, *v;
 513  grn_obj buf, *expr = NULL;
 514  grn_expr_var *vars;
 515  unsigned nvars;
 516
 517  GRN_TEXT_INIT(&buf, 0);
 518  p = command;
 519  e = command + strlen(command);
 520  p = grn_text_unesc_tok(ctx, &buf, p, e, &tok_type);
 521  if ((expr = grn_ctx_get(ctx, GRN_TEXT_VALUE(&buf), GRN_TEXT_LEN(&buf)))) {
 522    grn_obj params, output_type;
 523
 524    GRN_TEXT_INIT(&params, 0);
 525    GRN_TEXT_INIT(&output_type, 0);
 526    vars = ((grn_proc *)expr)->vars;
 527    nvars = ((grn_proc *)expr)->nvars;
 528    GRN_TEXT_PUTS(ctx, uri, "/d/");
 529    GRN_TEXT_PUT(ctx, uri, GRN_TEXT_VALUE(&buf), GRN_TEXT_LEN(&buf));
 530    while (p < e) {
 531      GRN_BULK_REWIND(&buf);
 532      p = grn_text_unesc_tok(ctx, &buf, p, e, &tok_type);
 533      v = GRN_TEXT_VALUE(&buf);
 534      switch (tok_type) {
 535      case GRN_TOK_VOID :
 536        p = e;
 537        break;
 538      case GRN_TOK_SYMBOL :
 539        if (GRN_TEXT_LEN(&buf) > 2 && v[0] == '-' && v[1] == '-') {
 540          int l = GRN_TEXT_LEN(&buf) - 2;
 541          v += 2;
 542          if (l == OUTPUT_TYPE_LEN && !memcmp(v, OUTPUT_TYPE, OUTPUT_TYPE_LEN)) {
 543            GRN_BULK_REWIND(&output_type);
 544            p = grn_text_unesc_tok(ctx, &output_type, p, e, &tok_type);
 545            break;
 546          }
 547          if (GRN_TEXT_LEN(&params)) {
 548            GRN_TEXT_PUTS(ctx, &params, "&");
 549          }
 550          grn_text_urlenc(ctx, &params, v, l);
 551          have_key = 1;
 552          break;
 553        }
 554        /* fallthru */
 555      case GRN_TOK_STRING :
 556      case GRN_TOK_QUOTE :
 557        if (!have_key) {
 558          if (offset < nvars) {
 559            if (GRN_TEXT_LEN(&params)) {
 560              GRN_TEXT_PUTS(ctx, &params, "&");
 561            }
 562            grn_text_urlenc(ctx, &params,
 563                            vars[offset].name, vars[offset].name_size);
 564            offset++;
 565          }
 566        }
 567        GRN_TEXT_PUTS(ctx, &params, "=");
 568        grn_text_urlenc(ctx, &params, GRN_TEXT_VALUE(&buf), GRN_TEXT_LEN(&buf));
 569        have_key = 0;
 570        break;
 571      }
 572    }
 573    GRN_TEXT_PUTS(ctx, uri, ".");
 574    if (GRN_TEXT_LEN(&output_type)) {
 575      GRN_TEXT_PUT(ctx, uri,
 576                   GRN_TEXT_VALUE(&output_type), GRN_TEXT_LEN(&output_type));
 577    } else {
 578      GRN_TEXT_PUTS(ctx, uri, "json");
 579    }
 580    if (GRN_TEXT_LEN(&params) > 0) {
 581      GRN_TEXT_PUTS(ctx, uri, "?");
 582      GRN_TEXT_PUT(ctx, uri, GRN_TEXT_VALUE(&params), GRN_TEXT_LEN(&params));
 583    }
 584    GRN_OBJ_FIN(ctx, &params);
 585    GRN_OBJ_FIN(ctx, &output_type);
 586  }
 587  GRN_OBJ_FIN(ctx, &buf);
 588}
 589
 590static void
 591command_send_http(grn_ctx *ctx, char *command, int type, int task_id)
 592{
 593  socket_t http_socket;
 594  grn_obj buf;
 595
 596  http_socket = open_socket(grntest_serverhost, grntest_serverport);
 597  if (http_socket == SOCKETERROR) {
 598    fprintf(stderr, "failed to connect to groonga at %s:%d via HTTP: ",
 599            grntest_serverhost, grntest_serverport);
 600#ifdef WIN32
 601    fprintf(stderr, "%d\n", GetLastError());
 602#else
 603    fprintf(stderr, "%s\n", strerror(errno));
 604#endif
 605    error_exit_in_thread(100);
 606  }
 607  grntest_task[task_id].http_socket = http_socket;
 608  GRN_BULK_REWIND(&grntest_task[task_id].http_response);
 609
 610  GRN_TEXT_INIT(&buf, 0);
 611  GRN_TEXT_PUTS(ctx, &buf, "GET ");
 612  if (strncmp(command, "/d/", 3) == 0) {
 613    GRN_TEXT_PUTS(ctx, &buf, command);
 614  } else {
 615    command_line_to_uri_path(ctx, &buf, command);
 616  }
 617#ifdef DEBUG_HTTP
 618  fprintf(stderr, "command: <%s>\n", command);
 619  fprintf(stderr, "path:    <%.*s>\n",
 620          (int)GRN_TEXT_LEN(&buf), GRN_TEXT_VALUE(&buf));
 621#endif
 622  GRN_TEXT_PUTS(ctx, &buf, " HTTP/1.1\r\n");
 623  GRN_TEXT_PUTS(ctx, &buf, "Host: ");
 624  GRN_TEXT_PUTS(ctx, &buf, grntest_serverhost);
 625  GRN_TEXT_PUTS(ctx, &buf, "\r\n");
 626  GRN_TEXT_PUTS(ctx, &buf, "User-Agent: grntest/");
 627  GRN_TEXT_PUTS(ctx, &buf, grn_get_version());
 628  GRN_TEXT_PUTS(ctx, &buf, "\r\n");
 629  GRN_TEXT_PUTS(ctx, &buf, "Connection: close\r\n");
 630  GRN_TEXT_PUTS(ctx, &buf, "\r\n");
 631  GRN_TEXT_PUTC(ctx, &buf, '\0');
 632  write_to_server(http_socket, GRN_TEXT_VALUE(&buf));
 633  GRN_OBJ_FIN(ctx, &buf);
 634}
 635
 636static void
 637command_send_ctx(grn_ctx *ctx, char *command, int type, int task_id)
 638{
 639  grn_ctx_send(ctx, command, strlen(command), 0);
 640/* fix me.
 641   when command fails, ctx->rc is not 0 in local mode!
 642  if (ctx->rc) {
 643    fprintf(stderr, "ctx_send:rc=%d:command:%s\n", ctx->rc, command);
 644    error_exit_in_thread(1);
 645  }
 646*/
 647}
 648
 649static void
 650command_send(grn_ctx *ctx, char *command, int type, int task_id)
 651{
 652  if (http_p(type)) {
 653    command_send_http(ctx, command, type, task_id);
 654  } else {
 655    command_send_ctx(ctx, command, type, task_id);
 656  }
 657}
 658
 659static void
 660command_recv_http(grn_ctx *ctx, int type, int task_id,
 661                  char **res, int *res_len, int *flags)
 662{
 663  int len;
 664  char buf[BUF_LEN];
 665  char *p, *e;
 666  socket_t http_socket;
 667  grn_obj *http_response;
 668
 669  http_socket = grntest_task[task_id].http_socket;
 670  http_response = &grntest_task[task_id].http_response;
 671  while ((len = recv(http_socket, buf, BUF_LEN - 1, 0))) {
 672#ifdef DEBUG_HTTP
 673    fprintf(stderr, "receive: <%.*s>\n", len, buf);
 674#endif
 675    GRN_TEXT_PUT(ctx, http_response, buf, len);
 676  }
 677
 678  p = GRN_TEXT_VALUE(http_response);
 679  e = p + GRN_TEXT_LEN(http_response);
 680  while (p < e) {
 681    if (p[0] != '\r') {
 682      p++;
 683      continue;
 684    }
 685    if (e - p >= 4) {
 686      if (!memcmp(p, "\r\n\r\n", 4)) {
 687        *res = p + 4;
 688        *res_len = e - *res;
 689#ifdef DEBUG_HTTP
 690        fprintf(stderr, "body: <%.*s>\n", *res_len, *res);
 691#endif
 692        break;
 693      }
 694      p += 4;
 695    } else {
 696      *res = NULL;
 697      *res_len = 0;
 698      break;
 699    }
 700  }
 701
 702  socketclose(http_socket);
 703  grntest_task[task_id].http_socket = 0;
 704}
 705
 706static void
 707command_recv_ctx(grn_ctx *ctx, int type, int task_id,
 708                 char **res, int *res_len, int *flags)
 709{
 710  grn_ctx_recv(ctx, res, res_len, flags);
 711  if (ctx->rc) {
 712    fprintf(stderr, "ctx_recv:rc=%d\n", ctx->rc);
 713    error_exit_in_thread(1);
 714  }
 715}
 716
 717static void
 718command_recv(grn_ctx *ctx, int type, int task_id,
 719             char **res, int *res_len, int *flags)
 720{
 721  if (http_p(type)) {
 722    command_recv_http(ctx, type, task_id, res, res_len, flags);
 723  } else {
 724    command_recv_ctx(ctx, type, task_id, res, res_len, flags);
 725  }
 726}
 727
 728static int
 729shutdown_server(void)
 730{
 731  char *res;
 732  int flags, res_len;
 733  int job_type;
 734  int task_id = 0;
 735
 736  if (grntest_remote_mode) {
 737    return 0;
 738  }
 739  job_type = grntest_task[task_id].jobtype;
 740  command_send(&grntest_server_context, "shutdown", job_type, task_id);
 741  if (grntest_server_context.rc) {
 742    fprintf(stderr, "ctx_send:rc=%d\n", grntest_server_context.rc);
 743    exit(1);
 744  }
 745  command_recv(&grntest_server_context, job_type, task_id,
 746               &res, &res_len, &flags);
 747
 748  return 0;
 749}
 750
 751static int
 752do_load_command(grn_ctx *ctx, char *command, int type, int task_id,
 753                long long int *load_start)
 754{
 755  char *res;
 756  int res_len, flags, ret;
 757  grn_obj start_time, end_time;
 758
 759  GRN_TIME_INIT(&start_time, 0);
 760  if (*load_start == 0) {
 761    GRN_TIME_NOW(ctx, &start_time);
 762    *load_start = GRN_TIME_VALUE(&start_time);
 763  } else {
 764    GRN_TIME_SET(ctx, &start_time, *load_start);
 765  }
 766
 767  command_send(ctx, command, type, task_id);
 768  do {
 769    command_recv(ctx, type, task_id, &res, &res_len, &flags);
 770    if (res_len) {
 771      long long int self;
 772      GRN_TIME_INIT(&end_time, 0);
 773      GRN_TIME_NOW(ctx, &end_time);
 774
 775      self = GRN_TIME_VALUE(&end_time) - *load_start;
 776
 777      if (grntest_task[task_id].max < self) {
 778        grntest_task[task_id].max = self;
 779      }
 780      if (grntest_task[task_id].min > self) {
 781        grntest_task[task_id].min = self;
 782      }
 783
 784      if (report_p(grntest_task[task_id].jobtype)) {
 785        unsigned char tmpbuf[BUF_LEN];
 786
 787        if (res_len < BUF_LEN) {
 788          strncpy(tmpbuf, res, res_len);
 789          tmpbuf[res_len] = '\0';
 790        } else {
 791          strncpy(tmpbuf, res, BUF_LEN - 2);
 792          tmpbuf[BUF_LEN -2] = '\0';
 793        }
 794        report_command(ctx, "load", tmpbuf, task_id, &start_time, &end_time);
 795      }
 796      if (out_p(grntest_task[task_id].jobtype)) {
 797        fwrite(res, 1, res_len, grntest_job[grntest_task[task_id].job_id].outputlog);
 798        fputc('\n', grntest_job[grntest_task[task_id].job_id].outputlog);
 799        fflush(grntest_job[grntest_task[task_id].job_id].outputlog);
 800      }
 801      if (test_p(grntest_task[task_id].jobtype)) {
 802        grn_obj log;
 803        FILE *input;
 804        FILE *output;
 805        GRN_TEXT_INIT(&log, 0);
 806        input = grntest_job[grntest_task[task_id].job_id].inputlog;
 807        output = grntest_job[grntest_task[task_id].job_id].outputlog;
 808        if (grn_text_fgets(ctx, &log, input) != GRN_SUCCESS) {
 809          GRN_LOG(ctx, GRN_ERROR, "Cannot get input-log");
 810          error_exit_in_thread(55);
 811        }
 812        if (GRN_TEXT_VALUE(&log)[GRN_TEXT_LEN(&log) - 1] == '\n') {
 813          grn_bulk_truncate(ctx, &log, GRN_TEXT_LEN(&log) - 1);
 814        }
 815
 816        if (!same_result_p(GRN_TEXT_VALUE(&log), GRN_TEXT_LEN(&log),
 817                           res, res_len)) {
 818          fprintf(output, "DIFF:command:%s\n", command);
 819          fprintf(output, "DIFF:result:");
 820          fwrite(res, 1, res_len, output);
 821          fputc('\n', output);
 822          fprintf(output, "DIFF:expect:%.*s\n",
 823                  (int)GRN_TEXT_LEN(&log), GRN_TEXT_VALUE(&log));
 824          fflush(output);
 825        }
 826        GRN_OBJ_FIN(ctx, &log);
 827      }
 828      grn_obj_close(ctx, &end_time);
 829      ret = 1;
 830      break;
 831    } else {
 832      ret = 0;
 833      break;
 834    }
 835  } while ((flags & GRN_CTX_MORE));
 836    grn_obj_close(ctx, &start_time);
 837
 838  return ret;
 839}
 840
 841
 842static int
 843do_command(grn_ctx *ctx, char *command, int type, int task_id)
 844{
 845  char *res;
 846  int res_len, flags;
 847  grn_obj start_time, end_time;
 848
 849  GRN_TIME_INIT(&start_time, 0);
 850  GRN_TIME_NOW(ctx, &start_time);
 851
 852  command_send(ctx, command, type, task_id);
 853  do {
 854    command_recv(ctx, type, task_id, &res, &res_len, &flags);
 855    if (res_len) {
 856      long long int self;
 857      GRN_TIME_INIT(&end_time, 0);
 858      GRN_TIME_NOW(ctx, &end_time);
 859
 860      self = GRN_TIME_VALUE(&end_time) - GRN_TIME_VALUE(&start_time);
 861
 862      if (grntest_task[task_id].max < self) {
 863        grntest_task[task_id].max = self;
 864      }
 865      if (grntest_task[task_id].min > self) {
 866        grntest_task[task_id].min = self;
 867      }
 868
 869      if (report_p(grntest_task[task_id].jobtype)) {
 870        unsigned char tmpbuf[BUF_LEN];
 871
 872        if (res_len < BUF_LEN) {
 873          strncpy(tmpbuf, res, res_len);
 874          tmpbuf[res_len] = '\0';
 875        } else {
 876          strncpy(tmpbuf, res, BUF_LEN - 2);
 877          tmpbuf[BUF_LEN -2] = '\0';
 878        }
 879        report_command(ctx, command, tmpbuf, task_id, &start_time, &end_time);
 880      }
 881      if (out_p(grntest_task[task_id].jobtype)) {
 882        fwrite(res, 1, res_len, grntest_job[grntest_task[task_id].job_id].outputlog);
 883        fputc('\n', grntest_job[grntest_task[task_id].job_id].outputlog);
 884        fflush(grntest_job[grntest_task[task_id].job_id].outputlog);
 885      }
 886      if (test_p(grntest_task[task_id].jobtype)) {
 887        grn_obj log;
 888        FILE *input;
 889        FILE *output;
 890        GRN_TEXT_INIT(&log, 0);
 891        input = grntest_job[grntest_task[task_id].job_id].inputlog;
 892        output = grntest_job[grntest_task[task_id].job_id].outputlog;
 893        if (grn_text_fgets(ctx, &log, input) != GRN_SUCCESS) {
 894          GRN_LOG(ctx, GRN_ERROR, "Cannot get input-log");
 895          error_exit_in_thread(55);
 896        }
 897        if (GRN_TEXT_VALUE(&log)[GRN_TEXT_LEN(&log) - 1] == '\n') {
 898          grn_bulk_truncate(ctx, &log, GRN_TEXT_LEN(&log) - 1);
 899        }
 900
 901        if (!same_result_p(GRN_TEXT_VALUE(&log), GRN_TEXT_LEN(&log),
 902                           res, res_len)) {
 903          fprintf(output, "DIFF:command:%s\n", command);
 904          fprintf(output, "DIFF:result:");
 905          fwrite(res, 1, res_len, output);
 906          fputc('\n', output);
 907          fprintf(output, "DIFF:expect:%.*s\n",
 908                  (int)GRN_TEXT_LEN(&log), GRN_TEXT_VALUE(&log));
 909          fflush(output);
 910        }
 911        GRN_OBJ_FIN(ctx, &log);
 912      }
 913      grn_obj_close(ctx, &end_time);
 914      break;
 915    } else {
 916#ifdef ENABLE_ERROR_REPORT
 917      error_command(ctx, command, task_id);
 918#endif
 919    }
 920  } while ((flags & GRN_CTX_MORE));
 921
 922  grn_obj_close(ctx, &start_time);
 923
 924  return 0;
 925}
 926
 927static int
 928comment_p(char *command)
 929{
 930  if (command[0] == '#') {
 931    return 1;
 932  }
 933  return 0;
 934}
 935
 936static int
 937load_command_p(char *command)
 938{
 939  int i = 0;
 940
 941  while (grn_isspace(&command[i], GRN_ENC_UTF8) == 1) {
 942    i++;
 943  }
 944  if (command[i] == '\0') {
 945    return 0;
 946  }
 947  if (!strncmp(&command[i], "load", 4)) {
 948    return 1;
 949  }
 950  return 0;
 951}
 952
 953static int
 954worker_sub(grn_ctx *ctx, grn_obj *log, int task_id)
 955{
 956  int i, load_mode, load_count;
 957  grn_obj end_time;
 958  long long int total_elapsed_time, job_elapsed_time;
 959  double sec, qps;
 960  long long int load_start;
 961  struct task *task;
 962  struct job *job;
 963
 964  task = &(grntest_task[task_id]);
 965  task->max = 0LL;
 966  task->min = 9223372036854775807LL;
 967  task->qnum = 0;
 968
 969  for (i = 0; i < task->ntimes; i++) {
 970    if (task->file != NULL) {
 971      FILE *fp;
 972      grn_obj line;
 973      fp = fopen(task->file, "r");
 974      if (!fp) {
 975        fprintf(stderr, "Cannot open %s\n",grntest_task[task_id].file);
 976        error_exit_in_thread(1);
 977      }
 978      load_mode = 0;
 979      load_count = 0;
 980      load_start = 0LL;
 981      GRN_TEXT_INIT(&line, 0);
 982      while (grn_text_fgets(ctx, &line, fp) == GRN_SUCCESS) {
 983        if (GRN_TEXT_VALUE(&line)[GRN_TEXT_LEN(&line) - 1] == '\n') {
 984          grn_bulk_truncate(ctx, &line, GRN_TEXT_LEN(&line) - 1);
 985        }
 986        if (GRN_TEXT_LEN(&line) == 0) {
 987          GRN_BULK_REWIND(&line);
 988          continue;
 989        }
 990        GRN_TEXT_PUTC(ctx, &line, '\0');
 991        if (comment_p(GRN_TEXT_VALUE(&line))) {
 992          GRN_BULK_REWIND(&line);
 993          continue;
 994        }
 995        if (load_command_p(GRN_TEXT_VALUE(&line))) {
 996          load_mode = 1;
 997          load_count = 1;
 998        }
 999        if (load_mode == 1) {
1000          if (do_load_command(&grntest_ctx[task_id], GRN_TEXT_VALUE(&line),
1001                              task->jobtype,
1002                              task_id, &load_start)) {
1003            task->qnum += load_count;
1004            load_mode = 0;
1005            load_count = 0;
1006            load_start = 0LL;
1007          }
1008          load_count++;
1009          GRN_BULK_REWIND(&line);
1010          continue;
1011        }
1012        do_command(&grntest_ctx[task_id], GRN_TEXT_VALUE(&line),
1013                   task->jobtype,
1014                   task_id);
1015        task->qnum++;
1016        GRN_BULK_REWIND(&line);
1017        if (grntest_sigint) {
1018          goto exit;
1019        }
1020      }
1021      GRN_OBJ_FIN(ctx, &line);
1022      fclose(fp);
1023    } else {
1024      int i, n_commands;
1025      grn_obj *commands;
1026      commands = task->commands;
1027      if (!commands) {
1028        error_exit_in_thread(1);
1029      }
1030      load_mode = 0;
1031      n_commands = GRN_BULK_VSIZE(commands) / sizeof(grn_obj *);
1032      for (i = 0; i < n_commands; i++) {
1033        grn_obj *command;
1034        command = GRN_PTR_VALUE_AT(commands, i);
1035        if (load_command_p(GRN_TEXT_VALUE(command))) {
1036          load_mode = 1;
1037        }
1038        if (load_mode == 1) {
1039          if (do_load_command(&grntest_ctx[task_id],
1040                              GRN_TEXT_VALUE(command),
1041                              task->jobtype, task_id, &load_start)) {
1042            load_mode = 0;
1043            load_start = 0LL;
1044            task->qnum++;
1045          }
1046          continue;
1047        }
1048        do_command(&grntest_ctx[task_id],
1049                   GRN_TEXT_VALUE(command),
1050                   task->jobtype, task_id);
1051        task->qnum++;
1052        if (grntest_sigint) {
1053          goto exit;
1054        }
1055      }
1056    }
1057  }
1058
1059exit:
1060  GRN_TIME_INIT(&end_time, 0);
1061  GRN_TIME_NOW(&grntest_ctx[task_id], &end_time);
1062  total_elapsed_time = GRN_TIME_VALUE(&end_time) - GRN_TIME_VALUE(&grntest_starttime);
1063  job_elapsed_time = GRN_TIME_VALUE(&end_time) - GRN_TIME_VALUE(&grntest_jobs_start);
1064
1065  CRITICAL_SECTION_ENTER(grntest_cs);
1066  job = &(grntest_job[task->job_id]);
1067  if (job->max < task->max) {
1068    job->max = task->max;
1069  }
1070  if (job->min > task->min) {
1071    job->min = task->min;
1072  }
1073
1074  job->qnum += task->qnum;
1075  job->done++;
1076  if (job->done == job->concurrency) {
1077    char tmpbuf[BUF_LEN];
1078    sec = job_elapsed_time / (double)1000000;
1079    qps = (double)job->qnum/ sec;
1080    grntest_jobdone++;
1081    if (grntest_outtype == OUT_TSV) {
1082      sprintf(tmpbuf,
1083              "job\t"
1084              "%s\t"
1085              "%" GRN_FMT_LLD "\t"
1086              "%" GRN_FMT_LLD "\t"
1087              "%f\t"
1088              "%" GRN_FMT_LLD "\t"
1089              "%" GRN_FMT_LLD "\t"
1090              "%d\n",
1091              job->jobname,
1092              total_elapsed_time,
1093              job_elapsed_time,
1094              qps,
1095              job->min,
1096              job->max,
1097              job->qnum);
1098    } else {
1099      sprintf(tmpbuf,
1100              "{\"job\": \"%s\", "
1101              "\"total_elapsed_time\": %" GRN_FMT_LLD ", "
1102              "\"job_elapsed_time\": %" GRN_FMT_LLD ", "
1103              "\"qps\": %f, "
1104              "\"min\": %" GRN_FMT_LLD ", "
1105              "\"max\": %" GRN_FMT_LLD ", "
1106              "\"queries\": %d}",
1107              job->jobname,
1108              total_elapsed_time,
1109              job_elapsed_time,
1110              qps,
1111              job->min,
1112              job->max,
1113              job->qnum);
1114      if (grntest_jobdone < grntest_jobnum) {
1115        strcat(tmpbuf, ",");
1116      }
1117    }
1118    GRN_TEXT_PUTS(ctx, log, tmpbuf);
1119    if (grntest_jobdone == grntest_jobnum) {
1120      if (grntest_outtype == OUT_TSV) {
1121        fprintf(grntest_log_file, "%.*s",
1122                (int)GRN_TEXT_LEN(log), GRN_TEXT_VALUE(log));
1123      } else {
1124        if (grntest_detail_on) {
1125          fseek(grntest_log_file, -2, SEEK_CUR);
1126          fprintf(grntest_log_file, "],\n");
1127        }
1128        fprintf(grntest_log_file, "\"summary\": [");
1129        fprintf(grntest_log_file, "%.*s",
1130                (int)GRN_TEXT_LEN(log), GRN_TEXT_VALUE(log));
1131        fprintf(grntest_log_file, "]");
1132      }
1133      fflush(grntest_log_file);
1134    }
1135  }
1136  grn_obj_close(&grntest_ctx[task_id], &end_time);
1137  CRITICAL_SECTION_LEAVE(grntest_cs);
1138
1139  return 0;
1140}
1141
1142typedef struct _grntest_worker {
1143  grn_ctx *ctx;
1144  grn_obj log;
1145  int task_id;
1146} grntest_worker;
1147
1148#ifdef WIN32
1149static int
1150__stdcall
1151worker(void *val)
1152{
1153  grntest_worker *worker = val;
1154  worker_sub(worker->ctx, &worker->log, worker->task_id);
1155  return 0;
1156}
1157#else
1158static void *
1159worker(void *val)
1160{
1161  grntest_worker *worker = val;
1162  worker_sub(worker->ctx, &worker->log, worker->task_id);
1163  return NULL;
1164}
1165#endif /* WIN32 */
1166
1167#ifdef WIN32
1168static int
1169thread_main(grn_ctx *ctx, int num)
1170{
1171  int  i;
1172  int  ret;
1173  HANDLE pthread[MAX_CON];
1174  grntest_worker *workers[MAX_CON];
1175
1176  for (i = 0; i < num; i++) {
1177    workers[i] = GRN_MALLOC(sizeof(grntest_worker));
1178    workers[i]->ctx = &grntest_ctx[i];
1179    GRN_TEXT_INIT(&workers[i]->log, 0);
1180    workers[i]->task_id = i;
1181    pthread[i] = (HANDLE)_beginthreadex(NULL, 0, worker, (void *)workers[i],
1182                                        0, NULL);
1183    if (pthread[i]== (HANDLE)0) {
1184       fprintf(stderr, "thread failed:%d\n", i);
1185       error_exit_in_thread(1);
1186    }
1187  }
1188
1189  ret = WaitForMultipleObjects(num, pthread, TRUE, INFINITE);
1190  if (ret == WAIT_TIMEOUT) {
1191     fprintf(stderr, "timeout\n");
1192     error_exit_in_thread(1);
1193  }
1194
1195  for (i = 0; i < num; i++) {
1196    CloseHandle(pthread[i]);
1197    GRN_OBJ_FIN(workers[i]->ctx, &workers[i]->log);
1198    GRN_FREE(workers[i]);
1199  }
1200  return 0;
1201}
1202#else
1203static int
1204thread_main(grn_ctx *ctx, int num)
1205{
1206  intptr_t i;
1207  int ret;
1208  pthread_t pthread[MAX_CON];
1209  grntest_worker *workers[MAX_CON];
1210
1211  for (i = 0; i < num; i++) {
1212    workers[i] = GRN_MALLOC(sizeof(grntest_worker));
1213    workers[i]->ctx = &grntest_ctx[i];
1214    GRN_TEXT_INIT(&workers[i]->log, 0);
1215    workers[i]->task_id = i;
1216    ret = pthread_create(&pthread[i], NULL, worker, (void *)workers[i]);
1217    if (ret) {
1218      fprintf(stderr, "Cannot create thread:ret=%d\n", ret);
1219      error_exit_in_thread(1);
1220    }
1221  }
1222
1223  for (i = 0; i < num; i++) {
1224    ret = pthread_join(pthread[i], NULL);
1225    GRN_OBJ_FIN(workers[i]->ctx, &workers[i]->log);
1226    GRN_FREE(workers[i]);
1227    if (ret) {
1228      fprintf(stderr, "Cannot join thread:ret=%d\n", ret);
1229      error_exit_in_thread(1);
1230    }
1231  }
1232  return 0;
1233}
1234#endif
1235
1236static int
1237error_exit(grn_ctx *ctx, int ret)
1238{
1239  fflush(stderr);
1240  shutdown_server();
1241  grn_ctx_fin(ctx);
1242  grn_fin();
1243  exit(ret);
1244}
1245
1246static int
1247get_sysinfo(const char *path, char *result, int olen)
1248{
1249  char tmpbuf[256];
1250
1251#ifdef WIN32
1252  int cinfo[4];
1253  ULARGE_INTEGER dinfo;
1254  char cpustring[64];
1255  SYSTEM_INFO sinfo;
1256  MEMORYSTATUSEX minfo;
1257  OSVERSIONINFO osinfo;
1258
1259  if (grntest_outtype == OUT_TSV) {
1260    result[0] = '\0';
1261    sprintf(tmpbuf, "script\t%s\n", grntest_scriptname);
1262    strcat(result, tmpbuf);
1263    sprintf(tmpbuf, "user\t%s\n", grntest_username);
1264    strcat(result, tmpbuf);
1265    sprintf(tmpbuf, "date\t%s\n", grntest_date);
1266    strcat(result, tmpbuf);
1267  } else {
1268    strcpy(result, "{");
1269    sprintf(tmpbuf, "\"script\": \"%s.scr\",\n", grntest_scriptname);
1270    strcat(result, tmpbuf);
1271    sprintf(tmpbuf, "  \"user\": \"%s\",\n", grntest_username);
1272    strcat(result, tmpbuf);
1273    sprintf(tmpbuf, "  \"date\": \"%s\",\n", grntest_date);
1274    strcat(result, tmpbuf);
1275  }
1276
1277  memset(cpustring, 0, 64);
1278#ifndef __GNUC__
1279  __cpuid(cinfo, 0x80000002);
1280  memcpy(cpustring, cinfo, 16);
1281  __cpuid(cinfo, 0x80000003);
1282  memcpy(cpustring+16, cinfo, 16);
1283  __cpuid(cinfo, 0x80000004);
1284  memcpy(cpustring+32, cinfo, 16);
1285#endif
1286
1287  if (grntest_outtype == OUT_TSV) {
1288    sprintf(tmpbuf, "%s\n", cpustring);
1289  } else {
1290    sprintf(tmpbuf, "  \"CPU\": \"%s\",\n", cpustring);
1291  }
1292  strcat(result, tmpbuf);
1293
1294  if (sizeof(int *) == 8) {
1295    grntest_osinfo = OS_WINDOWS64;
1296    if (grntest_outtype == OUT_TSV) {
1297      sprintf(tmpbuf, "64BIT\n");
1298    } else {
1299      sprintf(tmpbuf, "  \"BIT\": 64,\n");
1300    }
1301  } else {
1302    grntest_osinfo = OS_WINDOWS32;
1303    if (grntest_outtype == OUT_TSV) {
1304      sprintf(tmpbuf, "32BIT\n");
1305    } else {
1306      sprintf(tmpbuf, "  \"BIT\": 32,\n");
1307    }
1308  }
1309  strcat(result, tmpbuf);
1310
1311  GetSystemInfo(&sinfo);
1312  if (grntest_outtype == OUT_TSV) {
1313    sprintf(tmpbuf, "CORE\t%d\n", sinfo.dwNumberOfProcessors);
1314  } else {
1315    sprintf(tmpbuf, "  \"CORE\": %d,\n", sinfo.dwNumberOfProcessors);
1316  }
1317  strcat(result, tmpbuf);
1318
1319  minfo.dwLength = sizeof(MEMORYSTATUSEX);
1320  GlobalMemoryStatusEx(&minfo);
1321  if (grntest_outtype == OUT_TSV) {
1322    sprintf(tmpbuf, "RAM\t%I64dMByte\n", minfo.ullTotalPhys/(1024*1024));
1323  } else {
1324    sprintf(tmpbuf, "  \"RAM\": \"%I64dMByte\",\n", minfo.ullTotalPhys/(1024*1024));
1325  }
1326  strcat(result, tmpbuf);
1327
1328  GetDiskFreeSpaceEx(NULL, NULL, &dinfo, NULL);
1329  if (grntest_outtype == OUT_TSV) {
1330    sprintf(tmpbuf, "HDD\t%I64dKBytes\n", dinfo.QuadPart/1024 );
1331  } else {
1332    sprintf(tmpbuf, "  \"HDD\": \"%I64dKBytes\",\n", dinfo.QuadPart/1024 );
1333  }
1334  strcat(result, tmpbuf);
1335
1336  osinfo.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); GetVersionEx(&osinfo);
1337  if (grntest_outtype == OUT_TSV) {
1338    sprintf(tmpbuf, "Windows %d.%d\n", osinfo.dwMajorVersion, osinfo.dwMinorVersion);
1339  } else {
1340    sprintf(tmpbuf, "  \"OS\": \"Windows %d.%d\",\n", osinfo.dwMajorVersion,
1341            osinfo.dwMinorVersion);
1342  }
1343  strcat(result, tmpbuf);
1344
1345  if (grntest_outtype == OUT_TSV) {
1346    sprintf(tmpbuf, "%s\n", grntest_serverhost);
1347  } else {
1348    sprintf(tmpbuf, "  \"HOST\": \"%s\",\n", grntest_serverhost);
1349  }
1350  strcat(result, tmpbuf);
1351
1352  if (grntest_outtype == OUT_TSV) {
1353    sprintf(tmpbuf, "%d\n", grntest_serverport);
1354  } else {
1355    sprintf(tmpbuf, "  \"PORT\": \"%d\",\n", grntest_serverport);
1356  }
1357  strcat(result, tmpbuf);
1358
1359  if (grntest_outtype == OUT_TSV) {
1360    sprintf(tmpbuf, "%s\"\n", grn_get_version());
1361  } else {
1362    sprintf(tmpbuf, "  \"VERSION\": \"%s\"\n", grn_get_version());
1363  }
1364
1365  strcat(result, tmpbuf);
1366  if (grntest_outtype != OUT_TSV) {
1367    strcat(result, "}");
1368  }
1369
1370#else /* linux only */
1371  FILE *fp;
1372  int ret;
1373  int cpunum = 0;
1374  int minfo = 0;
1375  int unevictable = 0;
1376  int mlocked = 0;
1377  char cpustring[256];
1378  struct utsname ubuf;
1379  struct statvfs vfsbuf;
1380
1381  if (grntest_outtype == OUT_TSV) {
1382    result[0] = '\0';
1383    sprintf(tmpbuf, "sctipt\t%s\n", grntest_scriptname);
1384    strcat(result, tmpbuf);
1385    sprintf(tmpbuf, "user\t%s\n", grntest_username);
1386    strcat(result, tmpbuf);
1387    sprintf(tmpbuf, "date\t%s\n", grntest_date);
1388    strcat(result, tmpbuf);
1389  } else {
1390    strcpy(result, "{");
1391    sprintf(tmpbuf, "\"script\": \"%s.scr\",\n", grntest_scriptname);
1392    strcat(result, tmpbuf);
1393    sprintf(tmpbuf, "  \"user\": \"%s\",\n", grntest_username);
1394    strcat(result, tmpbuf);
1395    sprintf(tmpbuf, "  \"date\": \"%s\",\n", grntest_date);
1396    strcat(result, tmpbuf);
1397  }
1398
1399  fp = fopen("/proc/cpuinfo", "r");
1400  if (!fp) {
1401    fprintf(stderr, "Cannot open cpuinfo\n");
1402    exit(1);
1403  }
1404  while (fgets(tmpbuf, 256, fp) != NULL) {
1405    tmpbuf[strlen(tmpbuf)-1] = '\0';
1406    if (!strncmp(tmpbuf, "model name\t: ", 13)) {
1407      strcpy(cpustring, &tmpbuf[13]);
1408    }
1409  }
1410  fclose(fp);
1411  cpunum = sysconf(_SC_NPROCESSORS_CONF);
1412
1413  if (grntest_outtype == OUT_TSV) {
1414    sprintf(tmpbuf, "%s\n", cpustring);
1415  } else {
1416    sprintf(tmpbuf, "  \"CPU\": \"%s\",\n", cpustring);
1417  }
1418  strcat(result, tmpbuf);
1419
1420  if (sizeof(int *) == 8) {
1421    grntest_osinfo = OS_LINUX64;
1422    if (grntest_outtype == OUT_TSV) {
1423      sprintf(tmpbuf, "64BIT\n");
1424    } else {
1425      sprintf(tmpbuf, "  \"BIT\": 64,\n");
1426    }
1427  } else {
1428    grntest_osinfo = OS_LINUX32;
1429    if (grntest_outtype == OUT_TSV) {
1430      sprintf(tmpbuf, "32BIT\n");
1431    } else {
1432      sprintf(tmpbuf, "  \"BIT\": 32,\n");
1433    }
1434  }
1435  strcat(result, tmpbuf);
1436
1437  if (grntest_outtype == OUT_TSV) {
1438    sprintf(tmpbuf, "CORE\t%d\n", cpunum);
1439  } else {
1440    sprintf(tmpbuf, "  \"CORE\": %d,\n", cpunum);
1441  }
1442  strcat(result, tmpbuf);
1443
1444  fp = fopen("/proc/meminfo", "r");
1445  if (!fp) {
1446    fprintf(stderr, "Cannot open meminfo\n");
1447    exit(1);
1448  }
1449  while (fgets(tmpbuf, 256, fp) != NULL) {
1450    tmpbuf[strlen(tmpbuf)-1] = '\0';
1451    if (!strncmp(tmpbuf, "MemTotal:", 9)) {
1452      minfo = grntest_atoi(&tmpbuf[10], &tmpbuf[10] + 40, NULL);
1453    }
1454    if (!strncmp(tmpbuf, "Unevictable:", 12)) {
1455      unevictable = grntest_atoi(&tmpbuf[13], &tmpbuf[13] + 40, NULL);
1456    }
1457    if (!strncmp(tmpbuf, "Mlocked:", 8)) {
1458      mlocked = grntest_atoi(&tmpbuf[9], &tmpbuf[9] + 40, NULL);
1459    }
1460  }
1461  fclose(fp);
1462  if (grntest_outtype == OUT_TSV) {
1463    sprintf(tmpbuf, "%dMBytes\n", minfo/1024);
1464    strcat(result, tmpbuf);
1465    sprintf(tmpbuf, "%dMBytes_Unevictable\n", unevictable/1024);
1466    strcat(result, tmpbuf);
1467    sprintf(tmpbuf, "%dMBytes_Mlocked\n", mlocked/1024);
1468    strcat(result, tmpbuf);
1469  } else {
1470    sprintf(tmpbuf, "  \"RAM\": \"%dMBytes\",\n", minfo/1024);
1471    strcat(result, tmpbuf);
1472    sprintf(tmpbuf, "  \"Unevictable\": \"%dMBytes\",\n", unevictable/1024);
1473    strcat(result, tmpbuf);
1474    sprintf(tmpbuf, "  \"Mlocked\": \"%dMBytes\",\n", mlocked/1024);
1475    strcat(result, tmpbuf);
1476  }
1477
1478  ret = statvfs(path, &vfsbuf);
1479  if (ret) {
1480    fprintf(stderr, "Cannot access %s\n", path);
1481    exit(1);
1482  }
1483
1484  if (grntest_outtype == OUT_TSV) {
1485    sprintf(tmpbuf, "%luKBytes\n", vfsbuf.f_blocks * 4);
1486  } else {
1487    sprintf(tmpbuf, "  \"HDD\": \"%luKBytes\",\n", vfsbuf.f_blocks * 4);
1488  }
1489  strcat(result, tmpbuf);
1490
1491  uname(&ubuf);
1492  if (grntest_outtype == OUT_TSV) {
1493    sprintf(tmpbuf, "%s %s\n", ubuf.sysname, ubuf.release);
1494  } else {
1495    sprintf(tmpbuf, "  \"OS\": \"%s %s\",\n", ubuf.sysname, ubuf.release);
1496  }
1497  strcat(result, tmpbuf);
1498
1499  if (grntest_outtype == OUT_TSV) {
1500    sprintf(tmpbuf, "%s\n", grntest_serverhost);
1501  } else {
1502    sprintf(tmpbuf, "  \"HOST\": \"%s\",\n", grntest_serverhost);
1503  }
1504  strcat(result, tmpbuf);
1505
1506  if (grntest_outtype == OUT_TSV) {
1507    sprintf(tmpbuf, "%d\n", grntest_serverport);
1508  } else {
1509    sprintf(tmpbuf, "  \"PORT\": \"%d\",\n", grntest_serverport);
1510  }
1511  strcat(result, tmpbuf);
1512
1513  if (grntest_outtype == OUT_TSV) {
1514    sprintf(tmpbuf, "%s\n", grn_get_version());
1515  } else {
1516    sprintf(tmpbuf, "  \"VERSION\": \"%s\"\n", grn_get_version());
1517  }
1518  strcat(result, tmpbuf);
1519
1520  if (grntest_outtype != OUT_TSV) {
1521    strcat(result, "},");
1522  }
1523#endif /* WIN32 */
1524  if (strlen(result) >= olen) {
1525    fprintf(stderr, "buffer overrun in get_sysinfo!\n");
1526    exit(1);
1527  }
1528  return 0;
1529}
1530
1531static int
1532start_server(const char *dbpath, int r)
1533{
1534  int ret;
1535  char optbuf[BUF_LEN];
1536#ifdef WIN32
1537  char tmpbuf[BUF_LEN];
1538
1539  STARTUPINFO si;
1540
1541  if (strlen(dbpath) > BUF_LEN - 100) {
1542    fprintf(stderr, "too long dbpath!\n");
1543    exit(1);
1544  }
1545
1546  strcpy(tmpbuf, groonga_path);
1547  strcat(tmpbuf, " -s --protocol ");
1548  strcat(tmpbuf, groonga_protocol);
1549  strcat(tmpbuf, " -p ");
1550  sprintf(optbuf, "%d ", grntest_serverport);
1551  strcat(tmpbuf, optbuf);
1552  strcat(tmpbuf, dbpath);
1553  memset(&si, 0, sizeof(STARTUPINFO));
1554  si.cb=sizeof(STARTUPINFO);
1555  ret = CreateProcess(NULL, tmpbuf, NULL, NULL, FALSE,
1556		      0, NULL, NULL, &si, &grntest_pi);
1557
1558  if (ret == 0) {
1559    fprintf(stderr, "Cannot start groonga server: <%s>: error=%d\n",
1560            groonga_path, GetLastError());
1561    exit(1);
1562  }
1563
1564#else
1565  pid_t pid;
1566  pid = fork();
1567  if (pid < 0) {
1568    fprintf(stderr, "Cannot start groonga server:Cannot fork\n");
1569    exit(1);
1570  }
1571  sprintf(optbuf, "%d", grntest_serverport);
1572  if (pid == 0) {
1573    ret = execlp(groonga_path, groonga_path,
1574                 "-s",
1575                 "--protocol", groonga_protocol,
1576                 "-p", optbuf,
1577                 dbpath, (char*)NULL);
1578    if (ret == -1) {
1579      fprintf(stderr, "Cannot start groonga server: <%s>: errno=%d\n",
1580              groonga_path, errno);
1581      exit(1);
1582    }
1583  }
1584  else {
1585    grntest_server_id = pid;
1586  }
1587
1588#endif /* WIN32 */
1589
1590  return 0;
1591}
1592
1593static int
1594parse_line(char *buf, int start, int end, int num)
1595{
1596  int i, j, error_flag = 0, out_or_test = 0;
1597  char tmpbuf[BUF_LEN];
1598
1599  grntest_job[num].concurrency = 1;
1600  grntest_job[num].ntimes = 1;
1601  grntest_job[num].done = 0;
1602  grntest_job[num].qnum = 0;
1603  grntest_job[num].max = 0LL;
1604  grntest_job[num].min = 9223372036854775807LL;
1605  grntest_job[num].outputlog = NULL;
1606  grntest_job[num].inputlog = NULL;
1607
1608  strncpy(grntest_job[num].jobname, &buf[start], end - start);
1609  grntest_job[num].jobname[end - start] = '\0';
1610  i = start;
1611  while (i < end) {
1612    if (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) {
1613      i++;
1614      continue;
1615    }
1616    if (!strncmp(&buf[i], "do_local", 8)) {
1617      grntest_job[num].jobtype = J_DO_LOCAL;
1618      i = i + 8;
1619      break;
1620    }
1621    if (!strncmp(&buf[i], "do_gqtp", 7)) {
1622      grntest_job[num].jobtype = J_DO_GQTP;
1623      i = i + 7;
1624      break;
1625    }
1626    if (!strncmp(&buf[i], "do_http", 7)) {
1627      grntest_job[num].jobtype = J_DO_HTTP;
1628      i = i + 7;
1629      break;
1630    }
1631    if (!strncmp(&buf[i], "rep_local", 9)) {
1632      grntest_job[num].jobtype = J_REP_LOCAL;
1633      i = i + 9;
1634      break;
1635    }
1636    if (!strncmp(&buf[i], "rep_gqtp", 8)) {
1637      grntest_job[num].jobtype = J_REP_GQTP;
1638      i = i + 8;
1639      break;
1640    }
1641    if (!strncmp(&buf[i], "rep_http", 8)) {
1642      grntest_job[num].jobtype = J_REP_HTTP;
1643      i = i + 8;
1644      break;
1645    }
1646    if (!strncmp(&buf[i], "out_local", 9)) {
1647      grntest_job[num].jobtype = J_OUT_LOCAL;
1648      i = i + 9;
1649      out_or_test = 1;
1650      break;
1651    }
1652    if (!strncmp(&buf[i], "out_gqtp", 8)) {
1653      grntest_job[num].jobtype = J_OUT_GQTP;
1654      i = i + 8;
1655      out_or_test = 1;
1656      break;
1657    }
1658    if (!strncmp(&buf[i], "out_http", 8)) {
1659      grntest_job[num].jobtype = J_OUT_HTTP;
1660      i = i + 8;
1661      out_or_test = 1;
1662      break;
1663    }
1664    if (!strncmp(&buf[i], "test_local", 10)) {
1665      grntest_job[num].jobtype = J_TEST_LOCAL;
1666      i = i + 10;
1667      out_or_test = 1;
1668      break;
1669    }
1670    if (!strncmp(&buf[i], "test_gqtp", 9)) {
1671      grntest_job[num].jobtype = J_TEST_GQTP;
1672      i = i + 9;
1673      out_or_test = 1;
1674      break;
1675    }
1676    if (!strncmp(&buf[i], "test_http", 9)) {
1677      grntest_job[num].jobtype = J_TEST_HTTP;
1678      i = i + 9;
1679      out_or_test = 1;
1680      break;
1681    }
1682    error_flag = 1;
1683    i++;
1684  }
1685
1686  if (error_flag) {
1687    return 3;
1688  }
1689  if (i == end) {
1690    return 1;
1691  }
1692
1693  if (grn_isspace(&buf[i], GRN_ENC_UTF8) != 1) {
1694    return 4;
1695  }
1696  i++;
1697
1698  while (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) {
1699    i++;
1700    continue;
1701  }
1702  j = 0;
1703  while (i < end) {
1704    if (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) {
1705      break;
1706    }
1707    grntest_job[num].commandfile[j] = buf[i];
1708    i++;
1709    j++;
1710    if (j > 255) {
1711      return 5;
1712    }
1713  }
1714  grntest_job[num].commandfile[j] = '\0';
1715
1716  while (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) {
1717    i++;
1718  }
1719
1720  if (i == end) {
1721    if (out_or_test) {
1722      fprintf(stderr, "log(test)_local(gqtp|http) needs log(test)_filename\n");
1723      return 11;
1724    }
1725    return 0;
1726  }
1727
1728  j = 0;
1729  while (i < end) {
1730    if (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) {
1731      break;
1732    }
1733    tmpbuf[j] = buf[i];
1734    i++;
1735    j++;
1736    if (j >= BUF_LEN) {
1737      return 6;
1738    }
1739  }
1740  tmpbuf[j] ='\0';
1741  if (out_or_test) {
1742    if (out_p(grntest_job[num].jobtype)) {
1743      grntest_job[num].outputlog = fopen(tmpbuf, "wb");
1744      if (grntest_job[num].outputlog == NULL) {
1745        fprintf(stderr, "Cannot open %s\n", tmpbuf);
1746        return 13;
1747      }
1748    } else {
1749      char outlog[BUF_LEN];
1750      grntest_job[num].inputlog = fopen(tmpbuf, "rb");
1751      if (grntest_job[num].inputlog == NULL) {
1752        fprintf(stderr, "Cannot open %s\n", tmpbuf);
1753        return 14;
1754      }
1755      sprintf(outlog, "%s.diff", tmpbuf);
1756      grntest_job[num].outputlog = fopen(outlog, "wb");
1757      if (grntest_job[num].outputlog == NULL) {
1758        fprintf(stderr, "Cannot open %s\n", outlog);
1759        return 15;
1760      }
1761    }
1762    strcpy(grntest_job[num].logfile, tmpbuf);
1763    return 0;
1764  } else {
1765    grntest_job[num].concurrency = grntest_atoi(tmpbuf, tmpbuf + j, NULL);
1766    if (grntest_job[num].concurrency == 0) {
1767      return 7;
1768    }
1769  }
1770
1771  while (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) {
1772    i++;
1773  }
1774
1775  if (i == end) {
1776    return 0;
1777  }
1778
1779  j = 0;
1780  while (i < end) {
1781    if (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) {
1782      break;
1783    }
1784    tmpbuf[j] = buf[i];
1785    i++;
1786    j++;
1787    if (j > 16) {
1788      return 8;
1789    }
1790  }
1791  tmpbuf[j] ='\0';
1792  grntest_job[num].ntimes = grntest_atoi(tmpbuf, tmpbuf + j, NULL);
1793  if (grntest_job[num].ntimes == 0) {
1794    return 9;
1795  }
1796  if (i == end) {
1797    return 0;
1798  }
1799
1800  while (i < end) {
1801    if (grn_isspace(&buf[i], GRN_ENC_UTF8) == 1) {
1802      i++;
1803      continue;
1804    }
1805    return 10;
1806  }
1807  return 0;
1808}
1809
1810static int
1811get_jobs(grn_ctx *ctx, char *buf, int line)
1812{
1813  int i, len, start, end, ret;
1814  int jnum = 0;
1815
1816  len = strlen(buf);
1817  i = 0;
1818  while (i < len) {
1819    if ((buf[i] == '#') || (buf[i] == '\r') || (buf[i] == '\n')) {
1820      buf[i] = '\0';
1821      len = i;
1822      break;
1823    }
1824    i++;
1825  }
1826
1827  i = 0;
1828  start = 0;
1829  while (i < len) {
1830    if (buf[i] == ';') {
1831      end = i;
1832      ret = parse_line(buf, start, end, jnum);
1833      if (ret) {
1834        if (ret > 1) {
1835          fprintf(stderr, "Syntax error:line=%d:ret=%d:%s\n", line, ret, buf);
1836          error_exit(ctx, 1);
1837        }
1838      } else {
1839        jnum++;
1840      }
1841      start = end + 1;
1842    }
1843    i++;
1844  }
1845  end = len;
1846  ret = parse_line(buf, start, end, jnum);
1847  if (ret) {
1848    if (ret > 1) {
1849      fprintf(stderr, "Syntax error:line=%d:ret=%d:%s\n", line, ret, buf);
1850      error_exit(ctx, 1);
1851    }
1852  } else {
1853    jnum++;
1854  }
1855  return jnum;
1856}
1857
1858static int
1859make_task_table(grn_ctx *ctx, int jobnum)
1860{
1861  int i, j;
1862  int tid = 0;
1863  FILE *fp;
1864  grn_obj *commands = NULL;
1865
1866  for (i = 0; i < jobnum; i++) {
1867    if ((grntest_job[i].concurrency == 1) && (!grntest_onmemory_mode)) {
1868      grntest_task[tid].file = grntest_job[i].commandfile;
1869      grntest_task[tid].commands = NULL;
1870      grntest_task[tid].ntimes = grntest_job[i].ntimes;
1871      grntest_task[tid].jobtype = grntest_job[i].jobtype;
1872      grntest_task[tid].job_id = i;
1873      tid++;
1874      continue;
1875    }
1876    for (j = 0; j < grntest_job[i].concurrency; j++) {
1877      if (j == 0) {
1878        grn_obj line;
1879        GRN_TEXT_INIT(&line, 0);
1880        commands = grn_obj_open(ctx, GRN_PVECTOR, 0, GRN_VOID);
1881        if (!commands) {
1882          fprintf(stderr, "Cannot alloc commands\n");
1883          error_exit(ctx, 1);
1884        }
1885        fp = fopen(grntest_job[i].commandfile, "r");
1886        if (!fp) {
1887          fprintf(stderr, "Cannot alloc commandfile:%s\n",
1888                   grntest_job[i].commandfile);
1889          error_exit(ctx, 1);
1890        }
1891        while (grn_text_fgets(ctx, &line, fp) == GRN_SUCCESS) {
1892          grn_obj *command;
1893          if (GRN_TEXT_VALUE(&line)[GRN_TEXT_LEN(&line) - 1] == '\n') {
1894            grn_bulk_truncate(ctx, &line, GRN_TEXT_LEN(&line) - 1);
1895          }
1896          if (GRN_TEXT_LEN(&line) == 0) {
1897            GRN_BULK_REWIND(&line);
1898            continue;
1899          }
1900          GRN_TEXT_PUTC(ctx, &line, '\0');
1901          if (comment_p(GRN_TEXT_VALUE(&line))) {
1902            GRN_BULK_REWIND(&line);
1903            continue;
1904          }
1905          command = grn_obj_open(ctx, GRN_BULK, 0, GRN_VOID);
1906          if (!command) {
1907            fprintf(stderr, "Cannot alloc command: %s: %s\n",
1908                    grntest_job[i].commandfile, GRN_TEXT_VALUE(&line));
1909            GRN_OBJ_FIN(ctx, &line);
1910            error_exit(ctx, 1);
1911          }
1912          GRN_TEXT_SET(ctx, command, GRN_TEXT_VALUE(&line), GRN_TEXT_LEN(&line));
1913          GRN_PTR_PUT(ctx, commands, command);
1914          GRN_BULK_REWIND(&line);
1915        }
1916        GRN_OBJ_FIN(ctx, &line);
1917      }
1918      grntest_task[tid].file = NULL;
1919      grntest_task[tid].commands = commands;
1920      grntest_task[tid].ntimes = grntest_job[i].ntimes;
1921      grntest_task[tid].jobtype = grntest_job[i].jobtype;
1922      grntest_task[tid].job_id = i;
1923      tid++;
1924    }
1925  }
1926  return tid;
1927}
1928
1929/*
1930static int
1931print_commandlist(int task_id)
1932{
1933  int i;
1934
1935  for (i = 0; i < GRN_TEXT_LEN(grntest_task[task_id].commands); i++) {
1936    grn_obj *command;
1937    command = GRN_PTR_VALUE_AT(grntest_task[task_id].commands, i);
1938    printf("%s\n", GRN_TEXT_VALUE(command));
1939  }
1940  return 0;
1941}
1942*/
1943
1944/* return num of query */
1945static int
1946do_jobs(grn_ctx *ctx, int jobnum, int line)
1947{
1948  int i, task_num, ret, qnum = 0, thread_num = 0;
1949
1950  for (i = 0; i < jobnum; i++) {
1951/*
1952printf("%d:type =%d:file=%s:con=…

Large files files are truncated, but you can click here to view the full file