Blame test/runner.c

Packit Service 7c31a4
/* Copyright Joyent, Inc. and other Node contributors. All rights reserved.
Packit Service 7c31a4
 *
Packit Service 7c31a4
 * Permission is hereby granted, free of charge, to any person obtaining a copy
Packit Service 7c31a4
 * of this software and associated documentation files (the "Software"), to
Packit Service 7c31a4
 * deal in the Software without restriction, including without limitation the
Packit Service 7c31a4
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
Packit Service 7c31a4
 * sell copies of the Software, and to permit persons to whom the Software is
Packit Service 7c31a4
 * furnished to do so, subject to the following conditions:
Packit Service 7c31a4
 *
Packit Service 7c31a4
 * The above copyright notice and this permission notice shall be included in
Packit Service 7c31a4
 * all copies or substantial portions of the Software.
Packit Service 7c31a4
 *
Packit Service 7c31a4
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
Packit Service 7c31a4
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
Packit Service 7c31a4
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
Packit Service 7c31a4
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
Packit Service 7c31a4
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
Packit Service 7c31a4
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
Packit Service 7c31a4
 * IN THE SOFTWARE.
Packit Service 7c31a4
 */
Packit Service 7c31a4
Packit Service 7c31a4
#include <stdio.h>
Packit Service 7c31a4
#include <stdlib.h>
Packit Service 7c31a4
#include <string.h>
Packit Service 7c31a4
Packit Service 7c31a4
#include "runner.h"
Packit Service 7c31a4
#include "task.h"
Packit Service 7c31a4
#include "uv.h"
Packit Service 7c31a4
Packit Service 7c31a4
char executable_path[sizeof(executable_path)];
Packit Service 7c31a4
Packit Service 7c31a4
Packit Service 7c31a4
static int compare_task(const void* va, const void* vb) {
Packit Service 7c31a4
  const task_entry_t* a = va;
Packit Service 7c31a4
  const task_entry_t* b = vb;
Packit Service 7c31a4
  return strcmp(a->task_name, b->task_name);
Packit Service 7c31a4
}
Packit Service 7c31a4
Packit Service 7c31a4
Packit Service 7c31a4
const char* fmt(double d) {
Packit Service 7c31a4
  static char buf[1024];
Packit Service 7c31a4
  static char* p;
Packit Service 7c31a4
  uint64_t v;
Packit Service 7c31a4
Packit Service 7c31a4
  if (p == NULL)
Packit Service 7c31a4
    p = buf;
Packit Service 7c31a4
Packit Service 7c31a4
  p += 31;
Packit Service 7c31a4
Packit Service 7c31a4
  if (p >= buf + sizeof(buf))
Packit Service 7c31a4
    return "<buffer too small>";
Packit Service 7c31a4
Packit Service 7c31a4
  v = (uint64_t) d;
Packit Service 7c31a4
Packit Service 7c31a4
#if 0 /* works but we don't care about fractional precision */
Packit Service 7c31a4
  if (d - v >= 0.01) {
Packit Service 7c31a4
    *--p = '0' + (uint64_t) (d * 100) % 10;
Packit Service 7c31a4
    *--p = '0' + (uint64_t) (d * 10) % 10;
Packit Service 7c31a4
    *--p = '.';
Packit Service 7c31a4
  }
Packit Service 7c31a4
#endif
Packit Service 7c31a4
Packit Service 7c31a4
  if (v == 0)
Packit Service 7c31a4
    *--p = '0';
Packit Service 7c31a4
Packit Service 7c31a4
  while (v) {
Packit Service 7c31a4
    if (v) *--p = '0' + (v % 10), v /= 10;
Packit Service 7c31a4
    if (v) *--p = '0' + (v % 10), v /= 10;
Packit Service 7c31a4
    if (v) *--p = '0' + (v % 10), v /= 10;
Packit Service 7c31a4
    if (v) *--p = ',';
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  return p;
Packit Service 7c31a4
}
Packit Service 7c31a4
Packit Service 7c31a4
Packit Service 7c31a4
int run_tests(int benchmark_output) {
Packit Service 7c31a4
  int actual;
Packit Service 7c31a4
  int total;
Packit Service 7c31a4
  int passed;
Packit Service 7c31a4
  int failed;
Packit Service 7c31a4
  int skipped;
Packit Service 7c31a4
  int current;
Packit Service 7c31a4
  int test_result;
Packit Service 7c31a4
  int skip;
Packit Service 7c31a4
  task_entry_t* task;
Packit Service 7c31a4
Packit Service 7c31a4
  /* Count the number of tests. */
Packit Service 7c31a4
  actual = 0;
Packit Service 7c31a4
  total = 0;
Packit Service 7c31a4
  for (task = TASKS; task->main; task++, actual++) {
Packit Service 7c31a4
    if (!task->is_helper) {
Packit Service 7c31a4
      total++;
Packit Service 7c31a4
    }
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  /* Keep platform_output first. */
Packit Service 7c31a4
  skip = (actual > 0 && 0 == strcmp(TASKS[0].task_name, "platform_output"));
Packit Service 7c31a4
  qsort(TASKS + skip, actual - skip, sizeof(TASKS[0]), compare_task);
Packit Service 7c31a4
Packit Service 7c31a4
  fprintf(stderr, "1..%d\n", total);
Packit Service 7c31a4
  fflush(stderr);
Packit Service 7c31a4
Packit Service 7c31a4
  /* Run all tests. */
Packit Service 7c31a4
  passed = 0;
Packit Service 7c31a4
  failed = 0;
Packit Service 7c31a4
  skipped = 0;
Packit Service 7c31a4
  current = 1;
Packit Service 7c31a4
  for (task = TASKS; task->main; task++) {
Packit Service 7c31a4
    if (task->is_helper) {
Packit Service 7c31a4
      continue;
Packit Service 7c31a4
    }
Packit Service 7c31a4
Packit Service 7c31a4
    test_result = run_test(task->task_name, benchmark_output, current);
Packit Service 7c31a4
    switch (test_result) {
Packit Service 7c31a4
    case TEST_OK: passed++; break;
Packit Service 7c31a4
    case TEST_SKIP: skipped++; break;
Packit Service 7c31a4
    default: failed++;
Packit Service 7c31a4
    }
Packit Service 7c31a4
    current++;
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  return failed;
Packit Service 7c31a4
}
Packit Service 7c31a4
Packit Service 7c31a4
Packit Service 7c31a4
void log_tap_result(int test_count,
Packit Service 7c31a4
                    const char* test,
Packit Service 7c31a4
                    int status,
Packit Service 7c31a4
                    process_info_t* process) {
Packit Service 7c31a4
  const char* result;
Packit Service 7c31a4
  const char* directive;
Packit Service 7c31a4
  char reason[1024];
Packit Service 7c31a4
  int reason_length;
Packit Service 7c31a4
Packit Service 7c31a4
  switch (status) {
Packit Service 7c31a4
  case TEST_OK:
Packit Service 7c31a4
    result = "ok";
Packit Service 7c31a4
    directive = "";
Packit Service 7c31a4
    break;
Packit Service 7c31a4
  case TEST_SKIP:
Packit Service 7c31a4
    result = "ok";
Packit Service 7c31a4
    directive = " # SKIP ";
Packit Service 7c31a4
    break;
Packit Service 7c31a4
  default:
Packit Service 7c31a4
    result = "not ok";
Packit Service 7c31a4
    directive = "";
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  if (status == TEST_SKIP && process_output_size(process) > 0) {
Packit Service 7c31a4
    process_read_last_line(process, reason, sizeof reason);
Packit Service 7c31a4
    reason_length = strlen(reason);
Packit Service 7c31a4
    if (reason_length > 0 && reason[reason_length - 1] == '\n')
Packit Service 7c31a4
      reason[reason_length - 1] = '\0';
Packit Service 7c31a4
  } else {
Packit Service 7c31a4
    reason[0] = '\0';
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  fprintf(stderr, "%s %d - %s%s%s\n", result, test_count, test, directive, reason);
Packit Service 7c31a4
  fflush(stderr);
Packit Service 7c31a4
}
Packit Service 7c31a4
Packit Service 7c31a4
Packit Service 7c31a4
int run_test(const char* test,
Packit Service 7c31a4
             int benchmark_output,
Packit Service 7c31a4
             int test_count) {
Packit Service 7c31a4
  char errmsg[1024] = "";
Packit Service 7c31a4
  process_info_t processes[1024];
Packit Service 7c31a4
  process_info_t *main_proc;
Packit Service 7c31a4
  task_entry_t* task;
Packit Service 7c31a4
  int timeout_multiplier;
Packit Service 7c31a4
  int process_count;
Packit Service 7c31a4
  int result;
Packit Service 7c31a4
  int status;
Packit Service 7c31a4
  int i;
Packit Service 7c31a4
Packit Service 7c31a4
  status = 255;
Packit Service 7c31a4
  main_proc = NULL;
Packit Service 7c31a4
  process_count = 0;
Packit Service 7c31a4
Packit Service 7c31a4
#ifndef _WIN32
Packit Service 7c31a4
  /* Clean up stale socket from previous run. */
Packit Service 7c31a4
  remove(TEST_PIPENAME);
Packit Service 7c31a4
  remove(TEST_PIPENAME_2);
Packit Service 7c31a4
  remove(TEST_PIPENAME_3);
Packit Service 7c31a4
#endif
Packit Service 7c31a4
Packit Service 7c31a4
  /* If it's a helper the user asks for, start it directly. */
Packit Service 7c31a4
  for (task = TASKS; task->main; task++) {
Packit Service 7c31a4
    if (task->is_helper && strcmp(test, task->process_name) == 0) {
Packit Service 7c31a4
      return task->main();
Packit Service 7c31a4
    }
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  /* Start the helpers first. */
Packit Service 7c31a4
  for (task = TASKS; task->main; task++) {
Packit Service 7c31a4
    if (strcmp(test, task->task_name) != 0) {
Packit Service 7c31a4
      continue;
Packit Service 7c31a4
    }
Packit Service 7c31a4
Packit Service 7c31a4
    /* Skip the test itself. */
Packit Service 7c31a4
    if (!task->is_helper) {
Packit Service 7c31a4
      continue;
Packit Service 7c31a4
    }
Packit Service 7c31a4
Packit Service 7c31a4
    if (process_start(task->task_name,
Packit Service 7c31a4
                      task->process_name,
Packit Service 7c31a4
                      &processes[process_count],
Packit Service 7c31a4
                      1 /* is_helper */) == -1) {
Packit Service 7c31a4
      snprintf(errmsg,
Packit Service 7c31a4
               sizeof errmsg,
Packit Service 7c31a4
               "Process `%s` failed to start.",
Packit Service 7c31a4
               task->process_name);
Packit Service 7c31a4
      goto out;
Packit Service 7c31a4
    }
Packit Service 7c31a4
Packit Service 7c31a4
    process_count++;
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  /* Now start the test itself. */
Packit Service 7c31a4
  for (task = TASKS; task->main; task++) {
Packit Service 7c31a4
    if (strcmp(test, task->task_name) != 0) {
Packit Service 7c31a4
      continue;
Packit Service 7c31a4
    }
Packit Service 7c31a4
Packit Service 7c31a4
    if (task->is_helper) {
Packit Service 7c31a4
      continue;
Packit Service 7c31a4
    }
Packit Service 7c31a4
Packit Service 7c31a4
    if (process_start(task->task_name,
Packit Service 7c31a4
                      task->process_name,
Packit Service 7c31a4
                      &processes[process_count],
Packit Service 7c31a4
                      0 /* !is_helper */) == -1) {
Packit Service 7c31a4
      snprintf(errmsg,
Packit Service 7c31a4
               sizeof errmsg,
Packit Service 7c31a4
               "Process `%s` failed to start.",
Packit Service 7c31a4
               task->process_name);
Packit Service 7c31a4
      goto out;
Packit Service 7c31a4
    }
Packit Service 7c31a4
Packit Service 7c31a4
    main_proc = &processes[process_count];
Packit Service 7c31a4
    process_count++;
Packit Service 7c31a4
    break;
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  if (main_proc == NULL) {
Packit Service 7c31a4
    snprintf(errmsg,
Packit Service 7c31a4
             sizeof errmsg,
Packit Service 7c31a4
             "No test with that name: %s",
Packit Service 7c31a4
             test);
Packit Service 7c31a4
    goto out;
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  timeout_multiplier = 1;
Packit Service 7c31a4
#ifndef _WIN32
Packit Service 7c31a4
  do {
Packit Service 7c31a4
    const char* var;
Packit Service 7c31a4
Packit Service 7c31a4
    var = getenv("UV_TEST_TIMEOUT_MULTIPLIER");
Packit Service 7c31a4
    if (var == NULL)
Packit Service 7c31a4
      break;
Packit Service 7c31a4
Packit Service 7c31a4
    timeout_multiplier = atoi(var);
Packit Service 7c31a4
    if (timeout_multiplier <= 0)
Packit Service 7c31a4
      timeout_multiplier = 1;
Packit Service 7c31a4
  } while (0);
Packit Service 7c31a4
#endif
Packit Service 7c31a4
Packit Service 7c31a4
  result = process_wait(main_proc, 1, task->timeout * timeout_multiplier);
Packit Service 7c31a4
  if (result == -1) {
Packit Service 7c31a4
    FATAL("process_wait failed");
Packit Service 7c31a4
  } else if (result == -2) {
Packit Service 7c31a4
    /* Don't have to clean up the process, process_wait() has killed it. */
Packit Service 7c31a4
    snprintf(errmsg,
Packit Service 7c31a4
             sizeof errmsg,
Packit Service 7c31a4
             "timeout");
Packit Service 7c31a4
    goto out;
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  status = process_reap(main_proc);
Packit Service 7c31a4
  if (status != TEST_OK) {
Packit Service 7c31a4
    snprintf(errmsg,
Packit Service 7c31a4
             sizeof errmsg,
Packit Service 7c31a4
             "exit code %d",
Packit Service 7c31a4
             status);
Packit Service 7c31a4
    goto out;
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  if (benchmark_output) {
Packit Service 7c31a4
    /* Give the helpers time to clean up their act. */
Packit Service 7c31a4
    uv_sleep(1000);
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
out:
Packit Service 7c31a4
  /* Reap running processes except the main process, it's already dead. */
Packit Service 7c31a4
  for (i = 0; i < process_count - 1; i++) {
Packit Service 7c31a4
    process_terminate(&processes[i]);
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  if (process_count > 0 &&
Packit Service 7c31a4
      process_wait(processes, process_count - 1, -1) < 0) {
Packit Service 7c31a4
    FATAL("process_wait failed");
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  log_tap_result(test_count, test, status, &processes[i]);
Packit Service 7c31a4
Packit Service 7c31a4
  /* Show error and output from processes if the test failed. */
Packit Service 7c31a4
  if ((status != TEST_OK && status != TEST_SKIP) || task->show_output) {
Packit Service 7c31a4
    if (strlen(errmsg) > 0)
Packit Service 7c31a4
      fprintf(stderr, "# %s\n", errmsg);
Packit Service 7c31a4
    fprintf(stderr, "# ");
Packit Service 7c31a4
    fflush(stderr);
Packit Service 7c31a4
Packit Service 7c31a4
    for (i = 0; i < process_count; i++) {
Packit Service 7c31a4
      switch (process_output_size(&processes[i])) {
Packit Service 7c31a4
       case -1:
Packit Service 7c31a4
        fprintf(stderr, "Output from process `%s`: (unavailable)\n",
Packit Service 7c31a4
                process_get_name(&processes[i]));
Packit Service 7c31a4
        fflush(stderr);
Packit Service 7c31a4
        break;
Packit Service 7c31a4
Packit Service 7c31a4
       case 0:
Packit Service 7c31a4
        fprintf(stderr, "Output from process `%s`: (no output)\n",
Packit Service 7c31a4
                process_get_name(&processes[i]));
Packit Service 7c31a4
        fflush(stderr);
Packit Service 7c31a4
        break;
Packit Service 7c31a4
Packit Service 7c31a4
       default:
Packit Service 7c31a4
        fprintf(stderr, "Output from process `%s`:\n", process_get_name(&processes[i]));
Packit Service 7c31a4
        fflush(stderr);
Packit Service 7c31a4
        process_copy_output(&processes[i], stderr);
Packit Service 7c31a4
        break;
Packit Service 7c31a4
      }
Packit Service 7c31a4
    }
Packit Service 7c31a4
Packit Service 7c31a4
  /* In benchmark mode show concise output from the main process. */
Packit Service 7c31a4
  } else if (benchmark_output) {
Packit Service 7c31a4
    switch (process_output_size(main_proc)) {
Packit Service 7c31a4
     case -1:
Packit Service 7c31a4
      fprintf(stderr, "%s: (unavailable)\n", test);
Packit Service 7c31a4
      fflush(stderr);
Packit Service 7c31a4
      break;
Packit Service 7c31a4
Packit Service 7c31a4
     case 0:
Packit Service 7c31a4
      fprintf(stderr, "%s: (no output)\n", test);
Packit Service 7c31a4
      fflush(stderr);
Packit Service 7c31a4
      break;
Packit Service 7c31a4
Packit Service 7c31a4
     default:
Packit Service 7c31a4
      for (i = 0; i < process_count; i++) {
Packit Service 7c31a4
        process_copy_output(&processes[i], stderr);
Packit Service 7c31a4
      }
Packit Service 7c31a4
      break;
Packit Service 7c31a4
    }
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  /* Clean up all process handles. */
Packit Service 7c31a4
  for (i = 0; i < process_count; i++) {
Packit Service 7c31a4
    process_cleanup(&processes[i]);
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  return status;
Packit Service 7c31a4
}
Packit Service 7c31a4
Packit Service 7c31a4
Packit Service 7c31a4
/* Returns the status code of the task part
Packit Service 7c31a4
 * or 255 if no matching task was not found.
Packit Service 7c31a4
 */
Packit Service 7c31a4
int run_test_part(const char* test, const char* part) {
Packit Service 7c31a4
  task_entry_t* task;
Packit Service 7c31a4
  int r;
Packit Service 7c31a4
Packit Service 7c31a4
  for (task = TASKS; task->main; task++) {
Packit Service 7c31a4
    if (strcmp(test, task->task_name) == 0 &&
Packit Service 7c31a4
        strcmp(part, task->process_name) == 0) {
Packit Service 7c31a4
      r = task->main();
Packit Service 7c31a4
      return r;
Packit Service 7c31a4
    }
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  fprintf(stderr, "No test part with that name: %s:%s\n", test, part);
Packit Service 7c31a4
  fflush(stderr);
Packit Service 7c31a4
  return 255;
Packit Service 7c31a4
}
Packit Service 7c31a4
Packit Service 7c31a4
Packit Service 7c31a4
Packit Service 7c31a4
static int find_helpers(const task_entry_t* task,
Packit Service 7c31a4
                        const task_entry_t** helpers) {
Packit Service 7c31a4
  const task_entry_t* helper;
Packit Service 7c31a4
  int n_helpers;
Packit Service 7c31a4
Packit Service 7c31a4
  for (n_helpers = 0, helper = TASKS; helper->main; helper++) {
Packit Service 7c31a4
    if (helper->is_helper && strcmp(helper->task_name, task->task_name) == 0) {
Packit Service 7c31a4
      *helpers++ = helper;
Packit Service 7c31a4
      n_helpers++;
Packit Service 7c31a4
    }
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  return n_helpers;
Packit Service 7c31a4
}
Packit Service 7c31a4
Packit Service 7c31a4
Packit Service 7c31a4
void print_tests(FILE* stream) {
Packit Service 7c31a4
  const task_entry_t* helpers[1024];
Packit Service 7c31a4
  const task_entry_t* task;
Packit Service 7c31a4
  int n_helpers;
Packit Service 7c31a4
  int n_tasks;
Packit Service 7c31a4
  int i;
Packit Service 7c31a4
Packit Service 7c31a4
  for (n_tasks = 0, task = TASKS; task->main; n_tasks++, task++);
Packit Service 7c31a4
  qsort(TASKS, n_tasks, sizeof(TASKS[0]), compare_task);
Packit Service 7c31a4
Packit Service 7c31a4
  for (task = TASKS; task->main; task++) {
Packit Service 7c31a4
    if (task->is_helper) {
Packit Service 7c31a4
      continue;
Packit Service 7c31a4
    }
Packit Service 7c31a4
Packit Service 7c31a4
    n_helpers = find_helpers(task, helpers);
Packit Service 7c31a4
    if (n_helpers) {
Packit Service 7c31a4
      printf("%-25s (helpers:", task->task_name);
Packit Service 7c31a4
      for (i = 0; i < n_helpers; i++) {
Packit Service 7c31a4
        printf(" %s", helpers[i]->process_name);
Packit Service 7c31a4
      }
Packit Service 7c31a4
      printf(")\n");
Packit Service 7c31a4
    } else {
Packit Service 7c31a4
      printf("%s\n", task->task_name);
Packit Service 7c31a4
    }
Packit Service 7c31a4
  }
Packit Service 7c31a4
}
Packit Service 7c31a4
Packit Service 7c31a4
Packit Service 7c31a4
void print_lines(const char* buffer, size_t size, FILE* stream) {
Packit Service 7c31a4
  const char* start;
Packit Service 7c31a4
  const char* end;
Packit Service 7c31a4
Packit Service 7c31a4
  start = buffer;
Packit Service 7c31a4
  while ((end = memchr(start, '\n', &buffer[size] - start))) {
Packit Service 7c31a4
    fputs("# ", stream);
Packit Service 7c31a4
    fwrite(start, 1, (int)(end - start), stream);
Packit Service 7c31a4
    fputs("\n", stream);
Packit Service 7c31a4
    fflush(stream);
Packit Service 7c31a4
    start = end + 1;
Packit Service 7c31a4
  }
Packit Service 7c31a4
Packit Service 7c31a4
  end = &buffer[size];
Packit Service 7c31a4
  if (start < end) {
Packit Service 7c31a4
    fputs("# ", stream);
Packit Service 7c31a4
    fwrite(start, 1, (int)(end - start), stream);
Packit Service 7c31a4
    fputs("\n", stream);
Packit Service 7c31a4
    fflush(stream);
Packit Service 7c31a4
  }
Packit Service 7c31a4
}