Blame tools/marisa-build.cc

Packit de3218
#ifdef _WIN32
Packit de3218
 #include <fcntl.h>
Packit de3218
 #include <io.h>
Packit de3218
 #include <stdio.h>
Packit de3218
#endif  // _WIN32
Packit de3218
Packit de3218
#include <cstdlib>
Packit de3218
#include <fstream>
Packit de3218
#include <iostream>
Packit de3218
#include <string>
Packit de3218
Packit de3218
#include <marisa.h>
Packit de3218
Packit de3218
#include "cmdopt.h"
Packit de3218
Packit de3218
namespace {
Packit de3218
Packit de3218
int param_num_tries = MARISA_DEFAULT_NUM_TRIES;
Packit de3218
marisa::TailMode param_tail_mode = MARISA_DEFAULT_TAIL;
Packit de3218
marisa::NodeOrder param_node_order = MARISA_DEFAULT_ORDER;
Packit de3218
marisa::CacheLevel param_cache_level = MARISA_DEFAULT_CACHE;
Packit de3218
const char *output_filename = NULL;
Packit de3218
Packit de3218
void print_help(const char *cmd) {
Packit de3218
  std::cerr << "Usage: " << cmd << " [OPTION]... [FILE]...\n\n"
Packit de3218
      "Options:\n"
Packit de3218
      "  -n, --num-tries=[N]  limit the number of tries"
Packit de3218
      " [" << MARISA_MIN_NUM_TRIES << ", " << MARISA_MAX_NUM_TRIES
Packit de3218
      << "] (default: 3)\n"
Packit de3218
      "  -t, --text-tail      build a dictionary with text TAIL (default)\n"
Packit de3218
      "  -b, --binary-tail    build a dictionary with binary TAIL\n"
Packit de3218
      "  -w, --weight-order   arrange siblings in weight order (default)\n"
Packit de3218
      "  -l, --label-order    arrange siblings in label order\n"
Packit de3218
      "  -c, --cache-level=[N]    specify the cache size"
Packit de3218
      " [1, 5] (default: 3)\n"
Packit de3218
      "  -o, --output=[FILE]  write tries to FILE (default: stdout)\n"
Packit de3218
      "  -h, --help           print this help\n"
Packit de3218
      << std::endl;
Packit de3218
}
Packit de3218
Packit de3218
void read_keys(std::istream &input, marisa::Keyset *keyset) {
Packit de3218
  std::string line;
Packit de3218
  while (std::getline(input, line)) {
Packit de3218
    const std::string::size_type delim_pos = line.find_last_of('\t');
Packit de3218
    float weight = 1.0F;
Packit de3218
    if (delim_pos != line.npos) {
Packit de3218
      char *end_of_value;
Packit de3218
      weight = (float)std::strtod(&line[delim_pos + 1], &end_of_value);
Packit de3218
      if (*end_of_value == '\0') {
Packit de3218
        line.resize(delim_pos);
Packit de3218
      }
Packit de3218
    }
Packit de3218
    keyset->push_back(line.c_str(), line.length(), weight);
Packit de3218
  }
Packit de3218
}
Packit de3218
Packit de3218
int build(const char * const *args, std::size_t num_args) {
Packit de3218
  marisa::Keyset keyset;
Packit de3218
  if (num_args == 0) try {
Packit de3218
    read_keys(std::cin, &keyset);
Packit de3218
  } catch (const marisa::Exception &ex) {
Packit de3218
    std::cerr << ex.what() << ": failed to read keys" << std::endl;
Packit de3218
    return 10;
Packit de3218
  }
Packit de3218
Packit de3218
  for (std::size_t i = 0; i < num_args; ++i) try {
Packit de3218
    std::ifstream input_file(args[i], std::ios::binary);
Packit de3218
    if (!input_file) {
Packit de3218
      std::cerr << "error: failed to open: " << args[i] << std::endl;
Packit de3218
      return 11;
Packit de3218
    }
Packit de3218
    read_keys(input_file, &keyset);
Packit de3218
  } catch (const marisa::Exception &ex) {
Packit de3218
    std::cerr << ex.what() << ": failed to read keys" << std::endl;
Packit de3218
    return 12;
Packit de3218
  }
Packit de3218
Packit de3218
  marisa::Trie trie;
Packit de3218
  try {
Packit de3218
    trie.build(keyset, param_num_tries | param_tail_mode | param_node_order |
Packit de3218
        param_cache_level);
Packit de3218
  } catch (const marisa::Exception &ex) {
Packit de3218
    std::cerr << ex.what() << ": failed to build a dictionary" << std::endl;
Packit de3218
    return 20;
Packit de3218
  }
Packit de3218
Packit de3218
  std::cerr << "#keys: " << trie.num_keys() << std::endl;
Packit de3218
  std::cerr << "#nodes: " << trie.num_nodes() << std::endl;
Packit de3218
  std::cerr << "size: " << trie.io_size() << std::endl;
Packit de3218
Packit de3218
  if (output_filename != NULL) {
Packit de3218
    try {
Packit de3218
      trie.save(output_filename);
Packit de3218
    } catch (const marisa::Exception &ex) {
Packit de3218
      std::cerr << ex.what() << ": failed to write a dictionary to file: "
Packit de3218
          << output_filename << std::endl;
Packit de3218
      return 30;
Packit de3218
    }
Packit de3218
  } else {
Packit de3218
#ifdef _WIN32
Packit de3218
    const int stdout_fileno = ::_fileno(stdout);
Packit de3218
    if (stdout_fileno < 0) {
Packit de3218
      std::cerr << "error: failed to get the file descriptor of "
Packit de3218
          "standard output" << std::endl;
Packit de3218
      return 31;
Packit de3218
    }
Packit de3218
    if (::_setmode(stdout_fileno, _O_BINARY) == -1) {
Packit de3218
      std::cerr << "error: failed to set binary mode" << std::endl;
Packit de3218
      return 32;
Packit de3218
    }
Packit de3218
#endif  // _WIN32
Packit de3218
    try {
Packit de3218
      std::cout << trie;
Packit de3218
    } catch (const marisa::Exception &ex) {
Packit de3218
      std::cerr << ex.what()
Packit de3218
          << ": failed to write a dictionary to standard output" << std::endl;
Packit de3218
      return 33;
Packit de3218
    }
Packit de3218
  }
Packit de3218
  return 0;
Packit de3218
}
Packit de3218
Packit de3218
}  // namespace
Packit de3218
Packit de3218
int main(int argc, char *argv[]) {
Packit de3218
  std::ios::sync_with_stdio(false);
Packit de3218
Packit de3218
  ::cmdopt_option long_options[] = {
Packit de3218
    { "max-num-tries", 1, NULL, 'n' },
Packit de3218
    { "text-tail", 0, NULL, 't' },
Packit de3218
    { "binary-tail", 0, NULL, 'b' },
Packit de3218
    { "weight-order", 0, NULL, 'w' },
Packit de3218
    { "label-order", 0, NULL, 'l' },
Packit de3218
    { "cache-level", 1, NULL, 'c' },
Packit de3218
    { "output", 1, NULL, 'o' },
Packit de3218
    { "help", 0, NULL, 'h' },
Packit de3218
    { NULL, 0, NULL, 0 }
Packit de3218
  };
Packit de3218
  ::cmdopt_t cmdopt;
Packit de3218
  ::cmdopt_init(&cmdopt, argc, argv, "n:tbwlc:o:h", long_options);
Packit de3218
  int label;
Packit de3218
  while ((label = ::cmdopt_get(&cmdopt)) != -1) {
Packit de3218
    switch (label) {
Packit de3218
      case 'n': {
Packit de3218
        char *end_of_value;
Packit de3218
        const long value = std::strtol(cmdopt.optarg, &end_of_value, 10);
Packit de3218
        if ((*end_of_value != '\0') || (value <= 0) ||
Packit de3218
            (value > MARISA_MAX_NUM_TRIES)) {
Packit de3218
          std::cerr << "error: option `-n' with an invalid argument: "
Packit de3218
              << cmdopt.optarg << std::endl;
Packit de3218
          return 1;
Packit de3218
        }
Packit de3218
        param_num_tries = (int)value;
Packit de3218
        break;
Packit de3218
      }
Packit de3218
      case 't': {
Packit de3218
        param_tail_mode = MARISA_TEXT_TAIL;
Packit de3218
        break;
Packit de3218
      }
Packit de3218
      case 'b': {
Packit de3218
        param_tail_mode = MARISA_BINARY_TAIL;
Packit de3218
        break;
Packit de3218
      }
Packit de3218
      case 'w': {
Packit de3218
        param_node_order = MARISA_WEIGHT_ORDER;
Packit de3218
        break;
Packit de3218
      }
Packit de3218
      case 'l': {
Packit de3218
        param_node_order = MARISA_LABEL_ORDER;
Packit de3218
        break;
Packit de3218
      }
Packit de3218
      case 'c': {
Packit de3218
        char *end_of_value;
Packit de3218
        const long value = std::strtol(cmdopt.optarg, &end_of_value, 10);
Packit de3218
        if ((*end_of_value != '\0') || (value < 1) || (value > 5)) {
Packit de3218
          std::cerr << "error: option `-c' with an invalid argument: "
Packit de3218
              << cmdopt.optarg << std::endl;
Packit de3218
          return 2;
Packit de3218
        } else if (value == 1) {
Packit de3218
          param_cache_level = MARISA_TINY_CACHE;
Packit de3218
        } else if (value == 2) {
Packit de3218
          param_cache_level = MARISA_SMALL_CACHE;
Packit de3218
        } else if (value == 3) {
Packit de3218
          param_cache_level = MARISA_NORMAL_CACHE;
Packit de3218
        } else if (value == 4) {
Packit de3218
          param_cache_level = MARISA_LARGE_CACHE;
Packit de3218
        } else if (value == 5) {
Packit de3218
          param_cache_level = MARISA_HUGE_CACHE;
Packit de3218
        }
Packit de3218
        break;
Packit de3218
      }
Packit de3218
      case 'o': {
Packit de3218
        output_filename = cmdopt.optarg;
Packit de3218
        break;
Packit de3218
      }
Packit de3218
      case 'h': {
Packit de3218
        print_help(argv[0]);
Packit de3218
        return 0;
Packit de3218
      }
Packit de3218
      default: {
Packit de3218
        return 1;
Packit de3218
      }
Packit de3218
    }
Packit de3218
  }
Packit de3218
  return build(cmdopt.argv + cmdopt.optind, cmdopt.argc - cmdopt.optind);
Packit de3218
}