Blame test/runner.c

Packit b5b901
/* Copyright Joyent, Inc. and other Node contributors. All rights reserved.
Packit b5b901
 *
Packit b5b901
 * Permission is hereby granted, free of charge, to any person obtaining a copy
Packit b5b901
 * of this software and associated documentation files (the "Software"), to
Packit b5b901
 * deal in the Software without restriction, including without limitation the
Packit b5b901
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
Packit b5b901
 * sell copies of the Software, and to permit persons to whom the Software is
Packit b5b901
 * furnished to do so, subject to the following conditions:
Packit b5b901
 *
Packit b5b901
 * The above copyright notice and this permission notice shall be included in
Packit b5b901
 * all copies or substantial portions of the Software.
Packit b5b901
 *
Packit b5b901
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
Packit b5b901
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
Packit b5b901
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
Packit b5b901
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
Packit b5b901
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
Packit b5b901
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
Packit b5b901
 * IN THE SOFTWARE.
Packit b5b901
 */
Packit b5b901
Packit b5b901
#include <stdio.h>
Packit Service e08953
#include <stdlib.h>
Packit b5b901
#include <string.h>
Packit b5b901
Packit b5b901
#include "runner.h"
Packit b5b901
#include "task.h"
Packit b5b901
#include "uv.h"
Packit b5b901
Packit b5b901
char executable_path[sizeof(executable_path)];
Packit b5b901
Packit b5b901
Packit b5b901
static int compare_task(const void* va, const void* vb) {
Packit b5b901
  const task_entry_t* a = va;
Packit b5b901
  const task_entry_t* b = vb;
Packit b5b901
  return strcmp(a->task_name, b->task_name);
Packit b5b901
}
Packit b5b901
Packit b5b901
Packit b5b901
const char* fmt(double d) {
Packit b5b901
  static char buf[1024];
Packit b5b901
  static char* p;
Packit b5b901
  uint64_t v;
Packit b5b901
Packit b5b901
  if (p == NULL)
Packit b5b901
    p = buf;
Packit b5b901
Packit b5b901
  p += 31;
Packit b5b901
Packit b5b901
  if (p >= buf + sizeof(buf))
Packit b5b901
    return "<buffer too small>";
Packit b5b901
Packit b5b901
  v = (uint64_t) d;
Packit b5b901
Packit b5b901
#if 0 /* works but we don't care about fractional precision */
Packit b5b901
  if (d - v >= 0.01) {
Packit b5b901
    *--p = '0' + (uint64_t) (d * 100) % 10;
Packit b5b901
    *--p = '0' + (uint64_t) (d * 10) % 10;
Packit b5b901
    *--p = '.';
Packit b5b901
  }
Packit b5b901
#endif
Packit b5b901
Packit b5b901
  if (v == 0)
Packit b5b901
    *--p = '0';
Packit b5b901
Packit b5b901
  while (v) {
Packit b5b901
    if (v) *--p = '0' + (v % 10), v /= 10;
Packit b5b901
    if (v) *--p = '0' + (v % 10), v /= 10;
Packit b5b901
    if (v) *--p = '0' + (v % 10), v /= 10;
Packit b5b901
    if (v) *--p = ',';
Packit b5b901
  }
Packit b5b901
Packit b5b901
  return p;
Packit b5b901
}
Packit b5b901
Packit b5b901
Packit b5b901
int run_tests(int benchmark_output) {
Packit b5b901
  int actual;
Packit b5b901
  int total;
Packit b5b901
  int passed;
Packit b5b901
  int failed;
Packit b5b901
  int skipped;
Packit b5b901
  int current;
Packit b5b901
  int test_result;
Packit b5b901
  int skip;
Packit b5b901
  task_entry_t* task;
Packit b5b901
Packit b5b901
  /* Count the number of tests. */
Packit b5b901
  actual = 0;
Packit b5b901
  total = 0;
Packit b5b901
  for (task = TASKS; task->main; task++, actual++) {
Packit b5b901
    if (!task->is_helper) {
Packit b5b901
      total++;
Packit b5b901
    }
Packit b5b901
  }
Packit b5b901
Packit b5b901
  /* Keep platform_output first. */
Packit b5b901
  skip = (actual > 0 && 0 == strcmp(TASKS[0].task_name, "platform_output"));
Packit b5b901
  qsort(TASKS + skip, actual - skip, sizeof(TASKS[0]), compare_task);
Packit b5b901
Packit b5b901
  fprintf(stderr, "1..%d\n", total);
Packit b5b901
  fflush(stderr);
Packit b5b901
Packit b5b901
  /* Run all tests. */
Packit b5b901
  passed = 0;
Packit b5b901
  failed = 0;
Packit b5b901
  skipped = 0;
Packit b5b901
  current = 1;
Packit b5b901
  for (task = TASKS; task->main; task++) {
Packit b5b901
    if (task->is_helper) {
Packit b5b901
      continue;
Packit b5b901
    }
Packit b5b901
Packit b5b901
    test_result = run_test(task->task_name, benchmark_output, current);
Packit b5b901
    switch (test_result) {
Packit b5b901
    case TEST_OK: passed++; break;
Packit b5b901
    case TEST_SKIP: skipped++; break;
Packit b5b901
    default: failed++;
Packit b5b901
    }
Packit b5b901
    current++;
Packit b5b901
  }
Packit b5b901
Packit b5b901
  return failed;
Packit b5b901
}
Packit b5b901
Packit b5b901
Packit b5b901
void log_tap_result(int test_count,
Packit b5b901
                    const char* test,
Packit b5b901
                    int status,
Packit b5b901
                    process_info_t* process) {
Packit b5b901
  const char* result;
Packit b5b901
  const char* directive;
Packit b5b901
  char reason[1024];
Packit b5b901
  int reason_length;
Packit b5b901
Packit b5b901
  switch (status) {
Packit b5b901
  case TEST_OK:
Packit b5b901
    result = "ok";
Packit b5b901
    directive = "";
Packit b5b901
    break;
Packit b5b901
  case TEST_SKIP:
Packit b5b901
    result = "ok";
Packit b5b901
    directive = " # SKIP ";
Packit b5b901
    break;
Packit b5b901
  default:
Packit b5b901
    result = "not ok";
Packit b5b901
    directive = "";
Packit b5b901
  }
Packit b5b901
Packit b5b901
  if (status == TEST_SKIP && process_output_size(process) > 0) {
Packit b5b901
    process_read_last_line(process, reason, sizeof reason);
Packit b5b901
    reason_length = strlen(reason);
Packit b5b901
    if (reason_length > 0 && reason[reason_length - 1] == '\n')
Packit b5b901
      reason[reason_length - 1] = '\0';
Packit b5b901
  } else {
Packit b5b901
    reason[0] = '\0';
Packit b5b901
  }
Packit b5b901
Packit b5b901
  fprintf(stderr, "%s %d - %s%s%s\n", result, test_count, test, directive, reason);
Packit b5b901
  fflush(stderr);
Packit b5b901
}
Packit b5b901
Packit b5b901
Packit b5b901
int run_test(const char* test,
Packit b5b901
             int benchmark_output,
Packit b5b901
             int test_count) {
Packit b5b901
  char errmsg[1024] = "";
Packit b5b901
  process_info_t processes[1024];
Packit b5b901
  process_info_t *main_proc;
Packit b5b901
  task_entry_t* task;
Packit Service e08953
  int timeout_multiplier;
Packit b5b901
  int process_count;
Packit b5b901
  int result;
Packit b5b901
  int status;
Packit b5b901
  int i;
Packit b5b901
Packit b5b901
  status = 255;
Packit b5b901
  main_proc = NULL;
Packit b5b901
  process_count = 0;
Packit b5b901
Packit b5b901
#ifndef _WIN32
Packit b5b901
  /* Clean up stale socket from previous run. */
Packit b5b901
  remove(TEST_PIPENAME);
Packit b5b901
  remove(TEST_PIPENAME_2);
Packit b5b901
  remove(TEST_PIPENAME_3);
Packit b5b901
#endif
Packit b5b901
Packit b5b901
  /* If it's a helper the user asks for, start it directly. */
Packit b5b901
  for (task = TASKS; task->main; task++) {
Packit b5b901
    if (task->is_helper && strcmp(test, task->process_name) == 0) {
Packit b5b901
      return task->main();
Packit b5b901
    }
Packit b5b901
  }
Packit b5b901
Packit b5b901
  /* Start the helpers first. */
Packit b5b901
  for (task = TASKS; task->main; task++) {
Packit b5b901
    if (strcmp(test, task->task_name) != 0) {
Packit b5b901
      continue;
Packit b5b901
    }
Packit b5b901
Packit b5b901
    /* Skip the test itself. */
Packit b5b901
    if (!task->is_helper) {
Packit b5b901
      continue;
Packit b5b901
    }
Packit b5b901
Packit b5b901
    if (process_start(task->task_name,
Packit b5b901
                      task->process_name,
Packit b5b901
                      &processes[process_count],
Packit b5b901
                      1 /* is_helper */) == -1) {
Packit b5b901
      snprintf(errmsg,
Packit b5b901
               sizeof errmsg,
Packit b5b901
               "Process `%s` failed to start.",
Packit b5b901
               task->process_name);
Packit b5b901
      goto out;
Packit b5b901
    }
Packit b5b901
Packit b5b901
    process_count++;
Packit b5b901
  }
Packit b5b901
Packit b5b901
  /* Now start the test itself. */
Packit b5b901
  for (task = TASKS; task->main; task++) {
Packit b5b901
    if (strcmp(test, task->task_name) != 0) {
Packit b5b901
      continue;
Packit b5b901
    }
Packit b5b901
Packit b5b901
    if (task->is_helper) {
Packit b5b901
      continue;
Packit b5b901
    }
Packit b5b901
Packit b5b901
    if (process_start(task->task_name,
Packit b5b901
                      task->process_name,
Packit b5b901
                      &processes[process_count],
Packit b5b901
                      0 /* !is_helper */) == -1) {
Packit b5b901
      snprintf(errmsg,
Packit b5b901
               sizeof errmsg,
Packit b5b901
               "Process `%s` failed to start.",
Packit b5b901
               task->process_name);
Packit b5b901
      goto out;
Packit b5b901
    }
Packit b5b901
Packit b5b901
    main_proc = &processes[process_count];
Packit b5b901
    process_count++;
Packit b5b901
    break;
Packit b5b901
  }
Packit b5b901
Packit b5b901
  if (main_proc == NULL) {
Packit b5b901
    snprintf(errmsg,
Packit b5b901
             sizeof errmsg,
Packit b5b901
             "No test with that name: %s",
Packit b5b901
             test);
Packit b5b901
    goto out;
Packit b5b901
  }
Packit b5b901
Packit Service e08953
  timeout_multiplier = 1;
Packit Service e08953
#ifndef _WIN32
Packit Service e08953
  do {
Packit Service e08953
    const char* var;
Packit Service e08953
Packit Service e08953
    var = getenv("UV_TEST_TIMEOUT_MULTIPLIER");
Packit Service e08953
    if (var == NULL)
Packit Service e08953
      break;
Packit Service e08953
Packit Service e08953
    timeout_multiplier = atoi(var);
Packit Service e08953
    if (timeout_multiplier <= 0)
Packit Service e08953
      timeout_multiplier = 1;
Packit Service e08953
  } while (0);
Packit Service e08953
#endif
Packit Service e08953
Packit Service e08953
  result = process_wait(main_proc, 1, task->timeout * timeout_multiplier);
Packit b5b901
  if (result == -1) {
Packit b5b901
    FATAL("process_wait failed");
Packit b5b901
  } else if (result == -2) {
Packit b5b901
    /* Don't have to clean up the process, process_wait() has killed it. */
Packit b5b901
    snprintf(errmsg,
Packit b5b901
             sizeof errmsg,
Packit b5b901
             "timeout");
Packit b5b901
    goto out;
Packit b5b901
  }
Packit b5b901
Packit b5b901
  status = process_reap(main_proc);
Packit b5b901
  if (status != TEST_OK) {
Packit b5b901
    snprintf(errmsg,
Packit b5b901
             sizeof errmsg,
Packit b5b901
             "exit code %d",
Packit b5b901
             status);
Packit b5b901
    goto out;
Packit b5b901
  }
Packit b5b901
Packit b5b901
  if (benchmark_output) {
Packit b5b901
    /* Give the helpers time to clean up their act. */
Packit b5b901
    uv_sleep(1000);
Packit b5b901
  }
Packit b5b901
Packit b5b901
out:
Packit b5b901
  /* Reap running processes except the main process, it's already dead. */
Packit b5b901
  for (i = 0; i < process_count - 1; i++) {
Packit b5b901
    process_terminate(&processes[i]);
Packit b5b901
  }
Packit b5b901
Packit b5b901
  if (process_count > 0 &&
Packit b5b901
      process_wait(processes, process_count - 1, -1) < 0) {
Packit b5b901
    FATAL("process_wait failed");
Packit b5b901
  }
Packit b5b901
Packit b5b901
  log_tap_result(test_count, test, status, &processes[i]);
Packit b5b901
Packit b5b901
  /* Show error and output from processes if the test failed. */
Packit b5b901
  if ((status != TEST_OK && status != TEST_SKIP) || task->show_output) {
Packit b5b901
    if (strlen(errmsg) > 0)
Packit b5b901
      fprintf(stderr, "# %s\n", errmsg);
Packit b5b901
    fprintf(stderr, "# ");
Packit b5b901
    fflush(stderr);
Packit b5b901
Packit b5b901
    for (i = 0; i < process_count; i++) {
Packit b5b901
      switch (process_output_size(&processes[i])) {
Packit b5b901
       case -1:
Packit b5b901
        fprintf(stderr, "Output from process `%s`: (unavailable)\n",
Packit b5b901
                process_get_name(&processes[i]));
Packit b5b901
        fflush(stderr);
Packit b5b901
        break;
Packit b5b901
Packit b5b901
       case 0:
Packit b5b901
        fprintf(stderr, "Output from process `%s`: (no output)\n",
Packit b5b901
                process_get_name(&processes[i]));
Packit b5b901
        fflush(stderr);
Packit b5b901
        break;
Packit b5b901
Packit b5b901
       default:
Packit b5b901
        fprintf(stderr, "Output from process `%s`:\n", process_get_name(&processes[i]));
Packit b5b901
        fflush(stderr);
Packit b5b901
        process_copy_output(&processes[i], stderr);
Packit b5b901
        break;
Packit b5b901
      }
Packit b5b901
    }
Packit b5b901
Packit b5b901
  /* In benchmark mode show concise output from the main process. */
Packit b5b901
  } else if (benchmark_output) {
Packit b5b901
    switch (process_output_size(main_proc)) {
Packit b5b901
     case -1:
Packit b5b901
      fprintf(stderr, "%s: (unavailable)\n", test);
Packit b5b901
      fflush(stderr);
Packit b5b901
      break;
Packit b5b901
Packit b5b901
     case 0:
Packit b5b901
      fprintf(stderr, "%s: (no output)\n", test);
Packit b5b901
      fflush(stderr);
Packit b5b901
      break;
Packit b5b901
Packit b5b901
     default:
Packit b5b901
      for (i = 0; i < process_count; i++) {
Packit b5b901
        process_copy_output(&processes[i], stderr);
Packit b5b901
      }
Packit b5b901
      break;
Packit b5b901
    }
Packit b5b901
  }
Packit b5b901
Packit b5b901
  /* Clean up all process handles. */
Packit b5b901
  for (i = 0; i < process_count; i++) {
Packit b5b901
    process_cleanup(&processes[i]);
Packit b5b901
  }
Packit b5b901
Packit b5b901
  return status;
Packit b5b901
}
Packit b5b901
Packit b5b901
Packit b5b901
/* Returns the status code of the task part
Packit b5b901
 * or 255 if no matching task was not found.
Packit b5b901
 */
Packit b5b901
int run_test_part(const char* test, const char* part) {
Packit b5b901
  task_entry_t* task;
Packit b5b901
  int r;
Packit b5b901
Packit b5b901
  for (task = TASKS; task->main; task++) {
Packit b5b901
    if (strcmp(test, task->task_name) == 0 &&
Packit b5b901
        strcmp(part, task->process_name) == 0) {
Packit b5b901
      r = task->main();
Packit b5b901
      return r;
Packit b5b901
    }
Packit b5b901
  }
Packit b5b901
Packit b5b901
  fprintf(stderr, "No test part with that name: %s:%s\n", test, part);
Packit b5b901
  fflush(stderr);
Packit b5b901
  return 255;
Packit b5b901
}
Packit b5b901
Packit b5b901
Packit b5b901
Packit b5b901
static int find_helpers(const task_entry_t* task,
Packit b5b901
                        const task_entry_t** helpers) {
Packit b5b901
  const task_entry_t* helper;
Packit b5b901
  int n_helpers;
Packit b5b901
Packit b5b901
  for (n_helpers = 0, helper = TASKS; helper->main; helper++) {
Packit b5b901
    if (helper->is_helper && strcmp(helper->task_name, task->task_name) == 0) {
Packit b5b901
      *helpers++ = helper;
Packit b5b901
      n_helpers++;
Packit b5b901
    }
Packit b5b901
  }
Packit b5b901
Packit b5b901
  return n_helpers;
Packit b5b901
}
Packit b5b901
Packit b5b901
Packit b5b901
void print_tests(FILE* stream) {
Packit b5b901
  const task_entry_t* helpers[1024];
Packit b5b901
  const task_entry_t* task;
Packit b5b901
  int n_helpers;
Packit b5b901
  int n_tasks;
Packit b5b901
  int i;
Packit b5b901
Packit b5b901
  for (n_tasks = 0, task = TASKS; task->main; n_tasks++, task++);
Packit b5b901
  qsort(TASKS, n_tasks, sizeof(TASKS[0]), compare_task);
Packit b5b901
Packit b5b901
  for (task = TASKS; task->main; task++) {
Packit b5b901
    if (task->is_helper) {
Packit b5b901
      continue;
Packit b5b901
    }
Packit b5b901
Packit b5b901
    n_helpers = find_helpers(task, helpers);
Packit b5b901
    if (n_helpers) {
Packit b5b901
      printf("%-25s (helpers:", task->task_name);
Packit b5b901
      for (i = 0; i < n_helpers; i++) {
Packit b5b901
        printf(" %s", helpers[i]->process_name);
Packit b5b901
      }
Packit b5b901
      printf(")\n");
Packit b5b901
    } else {
Packit b5b901
      printf("%s\n", task->task_name);
Packit b5b901
    }
Packit b5b901
  }
Packit b5b901
}
Packit b5b901
Packit b5b901
Packit b5b901
void print_lines(const char* buffer, size_t size, FILE* stream) {
Packit b5b901
  const char* start;
Packit b5b901
  const char* end;
Packit b5b901
Packit b5b901
  start = buffer;
Packit b5b901
  while ((end = memchr(start, '\n', &buffer[size] - start))) {
Packit Service e08953
    fputs("# ", stream);
Packit Service e08953
    fwrite(start, 1, (int)(end - start), stream);
Packit Service e08953
    fputs("\n", stream);
Packit b5b901
    fflush(stream);
Packit b5b901
    start = end + 1;
Packit b5b901
  }
Packit b5b901
Packit Service e08953
  end = &buffer[size];
Packit Service e08953
  if (start < end) {
Packit Service e08953
    fputs("# ", stream);
Packit Service e08953
    fwrite(start, 1, (int)(end - start), stream);
Packit Service e08953
    fputs("\n", stream);
Packit b5b901
    fflush(stream);
Packit b5b901
  }
Packit b5b901
}