/test/runner.c

  1/* Copyright Joyent, Inc. and other Node contributors. All rights reserved.
  2 *
  3 * Permission is hereby granted, free of charge, to any person obtaining a copy
  4 * of this software and associated documentation files (the "Software"), to
  5 * deal in the Software without restriction, including without limitation the
  6 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
  7 * sell copies of the Software, and to permit persons to whom the Software is
  8 * furnished to do so, subject to the following conditions:
  9 *
 10 * The above copyright notice and this permission notice shall be included in
 11 * all copies or substantial portions of the Software.
 12 *
 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 15 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 16 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 17 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 18 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 19 * IN THE SOFTWARE.
 20 */
 21
 22#include <stdio.h>
 23#include <string.h>
 24
 25#include "runner.h"
 26#include "task.h"
 27#include "uv.h"
 28
 29char executable_path[PATHMAX] = { '\0' };
 30
 31
 32static void log_progress(int total, int passed, int failed, const char* name) {
 33  if (total == 0)
 34    total = 1;
 35
 36  LOGF("[%% %3d|+ %3d|- %3d]: %s", (int) ((passed + failed) / ((double) total) * 100.0),
 37      passed, failed, name);
 38}
 39
 40
 41const char* fmt(double d) {
 42  uint64_t v;
 43  char* p;
 44
 45  p = (char *) calloc(1, 32) + 31; /* leaks memory */
 46  v = (uint64_t) d;
 47
 48#if 0 /* works but we don't care about fractional precision */
 49  if (d - v >= 0.01) {
 50    *--p = '0' + (uint64_t) (d * 100) % 10;
 51    *--p = '0' + (uint64_t) (d * 10) % 10;
 52    *--p = '.';
 53  }
 54#endif
 55
 56  if (v == 0)
 57    *--p = '0';
 58
 59  while (v) {
 60    if (v) *--p = '0' + (v % 10), v /= 10;
 61    if (v) *--p = '0' + (v % 10), v /= 10;
 62    if (v) *--p = '0' + (v % 10), v /= 10;
 63    if (v) *--p = ',';
 64  }
 65
 66  return p;
 67}
 68
 69
 70int run_tests(int timeout, int benchmark_output) {
 71  int total, passed, failed;
 72  task_entry_t* task;
 73
 74  /* Count the number of tests. */
 75  total = 0;
 76  for (task = TASKS; task->main; task++) {
 77    if (!task->is_helper) {
 78      total++;
 79    }
 80  }
 81
 82  /* Run all tests. */
 83  passed = 0;
 84  failed = 0;
 85  for (task = TASKS; task->main; task++) {
 86    if (task->is_helper) {
 87      continue;
 88    }
 89
 90    rewind_cursor();
 91    if (!benchmark_output) {
 92      log_progress(total, passed, failed, task->task_name);
 93    }
 94
 95    if (run_test(task->task_name, timeout, benchmark_output) == 0) {
 96      passed++;
 97    } else {
 98      failed++;
 99    }
100  }
101
102  rewind_cursor();
103
104  if (!benchmark_output) {
105    log_progress(total, passed, failed, "Done.\n");
106  }
107
108  return failed;
109}
110
111
112int run_test(const char* test, int timeout, int benchmark_output) {
113  char errmsg[1024] = "no error";
114  process_info_t processes[1024];
115  process_info_t *main_proc;
116  task_entry_t* task;
117  int process_count;
118  int result;
119  int status;
120  int i;
121
122  status = 255;
123  main_proc = NULL;
124  process_count = 0;
125
126#ifndef _WIN32
127  /* Clean up stale socket from previous run. */
128  remove(TEST_PIPENAME);
129#endif
130
131  /* If it's a helper the user asks for, start it directly. */
132  for (task = TASKS; task->main; task++) {
133    if (task->is_helper && strcmp(test, task->process_name) == 0) {
134      return task->main();
135    }
136  }
137
138  /* Start the helpers first. */
139  for (task = TASKS; task->main; task++) {
140    if (strcmp(test, task->task_name) != 0) {
141      continue;
142    }
143
144    /* Skip the test itself. */
145    if (!task->is_helper) {
146      continue;
147    }
148
149    if (process_start(task->task_name,
150                      task->process_name,
151                      &processes[process_count],
152                      1 /* is_helper */) == -1) {
153      snprintf(errmsg,
154               sizeof errmsg,
155               "Process `%s` failed to start.",
156               task->process_name);
157      goto out;
158    }
159
160    process_count++;
161  }
162
163  /* Give the helpers time to settle. Race-y, fix this. */
164  uv_sleep(250);
165
166  /* Now start the test itself. */
167  for (task = TASKS; task->main; task++) {
168    if (strcmp(test, task->task_name) != 0) {
169      continue;
170    }
171
172    if (task->is_helper) {
173      continue;
174    }
175
176    if (process_start(task->task_name,
177                      task->process_name,
178                      &processes[process_count],
179                      0 /* !is_helper */) == -1) {
180      snprintf(errmsg,
181               sizeof errmsg,
182               "Process `%s` failed to start.",
183               task->process_name);
184      goto out;
185    }
186
187    main_proc = &processes[process_count];
188    process_count++;
189    break;
190  }
191
192  if (main_proc == NULL) {
193    snprintf(errmsg,
194             sizeof errmsg,
195             "No test with that name: %s",
196             test);
197    goto out;
198  }
199
200  result = process_wait(main_proc, 1, timeout);
201  if (result == -1) {
202    FATAL("process_wait failed");
203  } else if (result == -2) {
204    /* Don't have to clean up the process, process_wait() has killed it. */
205    snprintf(errmsg,
206             sizeof errmsg,
207             "timeout");
208    goto out;
209  }
210
211  status = process_reap(main_proc);
212  if (status != 0) {
213    snprintf(errmsg,
214             sizeof errmsg,
215             "exit code %d",
216             status);
217    goto out;
218  }
219
220  if (benchmark_output) {
221    /* Give the helpers time to clean up their act. */
222    uv_sleep(1000);
223  }
224
225out:
226  /* Reap running processes except the main process, it's already dead. */
227  for (i = 0; i < process_count - 1; i++) {
228    process_terminate(&processes[i]);
229  }
230
231  if (process_count > 0 &&
232      process_wait(processes, process_count - 1, -1) < 0) {
233    FATAL("process_wait failed");
234  }
235
236  /* Show error and output from processes if the test failed. */
237  if (status != 0 || task->show_output) {
238    if (status != 0) {
239      LOGF("\n`%s` failed: %s\n", test, errmsg);
240    } else {
241      LOGF("\n");
242    }
243
244    for (i = 0; i < process_count; i++) {
245      switch (process_output_size(&processes[i])) {
246       case -1:
247        LOGF("Output from process `%s`: (unavailable)\n",
248             process_get_name(&processes[i]));
249        break;
250
251       case 0:
252        LOGF("Output from process `%s`: (no output)\n",
253             process_get_name(&processes[i]));
254        break;
255
256       default:
257        LOGF("Output from process `%s`:\n", process_get_name(&processes[i]));
258        process_copy_output(&processes[i], fileno(stderr));
259        break;
260      }
261    }
262    LOG("=============================================================\n");
263
264  /* In benchmark mode show concise output from the main process. */
265  } else if (benchmark_output) {
266    switch (process_output_size(main_proc)) {
267     case -1:
268      LOGF("%s: (unavailable)\n", test);
269      break;
270
271     case 0:
272      LOGF("%s: (no output)\n", test);
273      break;
274
275     default:
276      for (i = 0; i < process_count; i++) {
277        process_copy_output(&processes[i], fileno(stderr));
278      }
279      break;
280    }
281  }
282
283  /* Clean up all process handles. */
284  for (i = 0; i < process_count; i++) {
285    process_cleanup(&processes[i]);
286  }
287
288  return status;
289}
290
291
292/* Returns the status code of the task part
293 * or 255 if no matching task was not found.
294 */
295int run_test_part(const char* test, const char* part) {
296  task_entry_t* task;
297  int r;
298
299  for (task = TASKS; task->main; task++) {
300    if (strcmp(test, task->task_name) == 0 &&
301        strcmp(part, task->process_name) == 0) {
302      r = task->main();
303      return r;
304    }
305  }
306
307  LOGF("No test part with that name: %s:%s\n", test, part);
308  return 255;
309}
310
311
312static int compare_task(const void* va, const void* vb) {
313  const task_entry_t* a = va;
314  const task_entry_t* b = vb;
315  return strcmp(a->task_name, b->task_name);
316}
317
318
319static int find_helpers(const task_entry_t* task, const task_entry_t** helpers) {
320  const task_entry_t* helper;
321  int n_helpers;
322
323  for (n_helpers = 0, helper = TASKS; helper->main; helper++) {
324    if (helper->is_helper && strcmp(helper->task_name, task->task_name) == 0) {
325      *helpers++ = helper;
326      n_helpers++;
327    }
328  }
329
330  return n_helpers;
331}
332
333
334void print_tests(FILE* stream) {
335  const task_entry_t* helpers[1024];
336  const task_entry_t* task;
337  int n_helpers;
338  int n_tasks;
339  int i;
340
341  for (n_tasks = 0, task = TASKS; task->main; n_tasks++, task++);
342  qsort(TASKS, n_tasks, sizeof(TASKS[0]), compare_task);
343
344  for (task = TASKS; task->main; task++) {
345    if (task->is_helper) {
346      continue;
347    }
348
349    n_helpers = find_helpers(task, helpers);
350    if (n_helpers) {
351      printf("%-25s (helpers:", task->task_name);
352      for (i = 0; i < n_helpers; i++) {
353        printf(" %s", helpers[i]->process_name);
354      }
355      printf(")\n");
356    } else {
357      printf("%s\n", task->task_name);
358    }
359  }
360}