/* Debuginfo-over-http server.

   Copyright (C) 2019-2020 Red Hat, Inc.

   This file is part of elfutils.

   This file is free software; you can redistribute it and/or modify

   it under the terms of the GNU General Public License as published by

   the Free Software Foundation; either version 3 of the License, or

   (at your option) any later version.

   elfutils is distributed in the hope that it will be useful, but

   WITHOUT ANY WARRANTY; without even the implied warranty of

   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License

   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */

/* cargo-cult from libdwfl linux-kernel-modules.c */

/* In case we have a bad fts we include this before config.h because it

   can't handle _FILE_OFFSET_BITS.

   Everything we need here is fine if its declarations just come first.

   Also, include sys/types.h before fts. On some systems fts.h is not self

   contained. */

#ifdef BAD_FTS

  #include <sys/types.h>

  #include <fts.h>

#endif

#ifdef HAVE_CONFIG_H

  #include "config.h"

#endif

extern "C" {

#include "printversion.h"

}

#include "debuginfod.h"

#include <dwarf.h>

#include <argp.h>

#ifdef __GNUC__

#undef __attribute__ /* glibc bug - rhbz 1763325 */

#endif

#include <unistd.h>

#include <stdlib.h>

#include <error.h>

// #include <libintl.h> // not until it supports C++ << better

#include <locale.h>

#include <pthread.h>

#include <signal.h>

#include <sys/stat.h>

#include <sys/time.h>

#include <sys/vfs.h>

#include <unistd.h>

#include <fcntl.h>

#include <netdb.h>

/* If fts.h is included before config.h, its indirect inclusions may not

   give us the right LFS aliases of these functions, so map them manually.  */

#ifdef BAD_FTS

  #ifdef _FILE_OFFSET_BITS

    #define open open64

    #define fopen fopen64

  #endif

#else

  #include <sys/types.h>

  #include <fts.h>

#endif

#include <cstring>

#include <vector>

#include <set>

#include <map>

#include <string>

#include <iostream>

#include <iomanip>

#include <ostream>

#include <sstream>

#include <mutex>

#include <deque>

#include <condition_variable>

#include <thread>

// #include <regex> // on rhel7 gcc 4.8, not competent

#include <regex.h>

// #include <algorithm>

using namespace std;

#include <gelf.h>

#include <libdwelf.h>

#include <microhttpd.h>

#if MHD_VERSION >= 0x00097002

// libmicrohttpd 0.9.71 broke API

#define MHD_RESULT enum MHD_Result

#else

#define MHD_RESULT int

#endif

#include <curl/curl.h>

#include <archive.h>

#include <archive_entry.h>

#include <sqlite3.h>

#ifdef __linux__

#include <sys/syscall.h>

#endif

#ifdef __linux__

#define tid() syscall(SYS_gettid)

#else

#define tid() pthread_self()

#endif

inline bool

string_endswith(const string& haystack, const string& needle)

{

  return (haystack.size() >= needle.size() &&

	  equal(haystack.end()-needle.size(), haystack.end(),

                needle.begin()));

}

// Roll this identifier for every sqlite schema incompatiblity.

#define BUILDIDS "buildids9"

#if SQLITE_VERSION_NUMBER >= 3008000

#define WITHOUT_ROWID "without rowid"

#else

#define WITHOUT_ROWID ""

#endif

static const char DEBUGINFOD_SQLITE_DDL[] =

  "pragma foreign_keys = on;\n"

  "pragma synchronous = 0;\n" // disable fsync()s - this cache is disposable across a machine crash

  "pragma journal_mode = wal;\n" // https://sqlite.org/wal.html

  "pragma wal_checkpoint = truncate;\n" // clean out any preexisting wal file

  "pragma journal_size_limit = 0;\n" // limit steady state file (between grooming, which also =truncate's)

  "pragma auto_vacuum = incremental;\n" // https://sqlite.org/pragma.html

  "pragma busy_timeout = 1000;\n" // https://sqlite.org/pragma.html

  // NB: all these are overridable with -D option

  // Normalization table for interning file names

  "create table if not exists " BUILDIDS "_files (\n"

  "        id integer primary key not null,\n"

  "        name text unique not null\n"

  "        );\n"

  // Normalization table for interning buildids

  "create table if not exists " BUILDIDS "_buildids (\n"

  "        id integer primary key not null,\n"

  "        hex text unique not null);\n"

  // Track the completion of scanning of a given file & sourcetype at given time

  "create table if not exists " BUILDIDS "_file_mtime_scanned (\n"

  "        mtime integer not null,\n"

  "        file integer not null,\n"

  "        size integer not null,\n" // in bytes

  "        sourcetype text(1) not null\n"

  "            check (sourcetype IN ('F', 'R')),\n"

  "        foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"

  "        primary key (file, mtime, sourcetype)\n"

  "        ) " WITHOUT_ROWID ";\n"

  "create table if not exists " BUILDIDS "_f_de (\n"

  "        buildid integer not null,\n"

  "        debuginfo_p integer not null,\n"

  "        executable_p integer not null,\n"

  "        file integer not null,\n"

  "        mtime integer not null,\n"

  "        foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"

  "        foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"

  "        primary key (buildid, file, mtime)\n"

  "        ) " WITHOUT_ROWID ";\n"

  "create table if not exists " BUILDIDS "_f_s (\n"

  "        buildid integer not null,\n"

  "        artifactsrc integer not null,\n"

  "        file integer not null,\n" // NB: not necessarily entered into _mtime_scanned

  "        mtime integer not null,\n"

  "        foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"

  "        foreign key (artifactsrc) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"

  "        foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"

  "        primary key (buildid, artifactsrc, file, mtime)\n"

  "        ) " WITHOUT_ROWID ";\n"

  "create table if not exists " BUILDIDS "_r_de (\n"

  "        buildid integer not null,\n"

  "        debuginfo_p integer not null,\n"

  "        executable_p integer not null,\n"

  "        file integer not null,\n"

  "        mtime integer not null,\n"

  "        content integer not null,\n"

  "        foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"

  "        foreign key (content) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"

  "        foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"

  "        primary key (buildid, debuginfo_p, executable_p, file, content, mtime)\n"

  "        ) " WITHOUT_ROWID ";\n"

  "create table if not exists " BUILDIDS "_r_sref (\n" // outgoing dwarf sourcefile references from rpm

  "        buildid integer not null,\n"

  "        artifactsrc integer not null,\n"

  "        foreign key (artifactsrc) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"

  "        foreign key (buildid) references " BUILDIDS "_buildids(id) on update cascade on delete cascade,\n"

  "        primary key (buildid, artifactsrc)\n"

  "        ) " WITHOUT_ROWID ";\n"

  "create table if not exists " BUILDIDS "_r_sdef (\n" // rpm contents that may satisfy sref

  "        file integer not null,\n"

  "        mtime integer not null,\n"

  "        content integer not null,\n"

  "        foreign key (file) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"

  "        foreign key (content) references " BUILDIDS "_files(id) on update cascade on delete cascade,\n"

  "        primary key (content, file, mtime)\n"

  "        ) " WITHOUT_ROWID ";\n"

  // create views to glue together some of the above tables, for webapi D queries

  "create view if not exists " BUILDIDS "_query_d as \n"

  "select\n"

  "        b.hex as buildid, n.mtime, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1\n"

  "        from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_f_de n\n"

  "        where b.id = n.buildid and f0.id = n.file and n.debuginfo_p = 1\n"

  "union all select\n"

  "        b.hex as buildid, n.mtime, 'R' as sourcetype, f0.name as source0, n.mtime as mtime, f1.name as source1\n"

  "        from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_r_de n\n"

  "        where b.id = n.buildid and f0.id = n.file and f1.id = n.content and n.debuginfo_p = 1\n"

  ";"

  // ... and for E queries

  "create view if not exists " BUILDIDS "_query_e as \n"

  "select\n"

  "        b.hex as buildid, n.mtime, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1\n"

  "        from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_f_de n\n"

  "        where b.id = n.buildid and f0.id = n.file and n.executable_p = 1\n"

  "union all select\n"

  "        b.hex as buildid, n.mtime, 'R' as sourcetype, f0.name as source0, n.mtime as mtime, f1.name as source1\n"

  "        from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_r_de n\n"

  "        where b.id = n.buildid and f0.id = n.file and f1.id = n.content and n.executable_p = 1\n"

  ";"

  // ... and for S queries

  "create view if not exists " BUILDIDS "_query_s as \n"

  "select\n"

  "        b.hex as buildid, fs.name as artifactsrc, 'F' as sourcetype, f0.name as source0, n.mtime as mtime, null as source1, null as source0ref\n"

  "        from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files fs, " BUILDIDS "_f_s n\n"

  "        where b.id = n.buildid and f0.id = n.file and fs.id = n.artifactsrc\n"

  "union all select\n"

  "        b.hex as buildid, f1.name as artifactsrc, 'R' as sourcetype, f0.name as source0, sd.mtime as mtime, f1.name as source1, fsref.name as source0ref\n"

  "        from " BUILDIDS "_buildids b, " BUILDIDS "_files f0, " BUILDIDS "_files f1, " BUILDIDS "_files fsref, "

  "        " BUILDIDS "_r_sdef sd, " BUILDIDS "_r_sref sr, " BUILDIDS "_r_de sde\n"

  "        where b.id = sr.buildid and f0.id = sd.file and fsref.id = sde.file and f1.id = sd.content\n"

  "        and sr.artifactsrc = sd.content and sde.buildid = sr.buildid\n"

  ";"

  // and for startup overview counts

  "drop view if exists " BUILDIDS "_stats;\n"

  "create view if not exists " BUILDIDS "_stats as\n"

  "          select 'file d/e' as label,count(*) as quantity from " BUILDIDS "_f_de\n"

  "union all select 'file s',count(*) from " BUILDIDS "_f_s\n"

  "union all select 'archive d/e',count(*) from " BUILDIDS "_r_de\n"

  "union all select 'archive sref',count(*) from " BUILDIDS "_r_sref\n"

  "union all select 'archive sdef',count(*) from " BUILDIDS "_r_sdef\n"

  "union all select 'buildids',count(*) from " BUILDIDS "_buildids\n"

  "union all select 'filenames',count(*) from " BUILDIDS "_files\n"

  "union all select 'files scanned (#)',count(*) from " BUILDIDS "_file_mtime_scanned\n"

  "union all select 'files scanned (mb)',coalesce(sum(size)/1024/1024,0) from " BUILDIDS "_file_mtime_scanned\n"

#if SQLITE_VERSION_NUMBER >= 3016000

  "union all select 'index db size (mb)',page_count*page_size/1024/1024 as size FROM pragma_page_count(), pragma_page_size()\n"

#endif

  ";\n"

// schema change history & garbage collection

//

// XXX: we could have migration queries here to bring prior-schema

// data over instead of just dropping it.

//

// buildids9: widen the mtime_scanned table

  "" // <<< we are here

// buildids8: slim the sref table

  "drop table if exists buildids8_f_de;\n"

  "drop table if exists buildids8_f_s;\n"

  "drop table if exists buildids8_r_de;\n"

  "drop table if exists buildids8_r_sref;\n"

  "drop table if exists buildids8_r_sdef;\n"

  "drop table if exists buildids8_file_mtime_scanned;\n"

  "drop table if exists buildids8_files;\n"

  "drop table if exists buildids8_buildids;\n"

// buildids7: separate _norm table into dense subtype tables

  "drop table if exists buildids7_f_de;\n"

  "drop table if exists buildids7_f_s;\n"

  "drop table if exists buildids7_r_de;\n"

  "drop table if exists buildids7_r_sref;\n"

  "drop table if exists buildids7_r_sdef;\n"

  "drop table if exists buildids7_file_mtime_scanned;\n"

  "drop table if exists buildids7_files;\n"

  "drop table if exists buildids7_buildids;\n"

// buildids6: drop bolo/rfolo again, represent sources / rpmcontents in main table

  "drop table if exists buildids6_norm;\n"

  "drop table if exists buildids6_files;\n"

  "drop table if exists buildids6_buildids;\n"

  "drop view if exists buildids6;\n"

// buildids5: redefine srcfile1 column to be '.'-less (for rpms)

  "drop table if exists buildids5_norm;\n"

  "drop table if exists buildids5_files;\n"

  "drop table if exists buildids5_buildids;\n"

  "drop table if exists buildids5_bolo;\n"

  "drop table if exists buildids5_rfolo;\n"

  "drop view if exists buildids5;\n"

// buildids4: introduce rpmfile RFOLO

  "drop table if exists buildids4_norm;\n"

  "drop table if exists buildids4_files;\n"

  "drop table if exists buildids4_buildids;\n"

  "drop table if exists buildids4_bolo;\n"

  "drop table if exists buildids4_rfolo;\n"

  "drop view if exists buildids4;\n"

// buildids3*: split out srcfile BOLO

  "drop table if exists buildids3_norm;\n"

  "drop table if exists buildids3_files;\n"

  "drop table if exists buildids3_buildids;\n"

  "drop table if exists buildids3_bolo;\n"

  "drop view if exists buildids3;\n"

// buildids2: normalized buildid and filenames into interning tables;

  "drop table if exists buildids2_norm;\n"

  "drop table if exists buildids2_files;\n"

  "drop table if exists buildids2_buildids;\n"

  "drop view if exists buildids2;\n"

  // buildids1: made buildid and artifacttype NULLable, to represent cached-negative

//           lookups from sources, e.g. files or rpms that contain no buildid-indexable content

  "drop table if exists buildids1;\n"

// buildids: original

  "drop table if exists buildids;\n"

  ;

static const char DEBUGINFOD_SQLITE_CLEANUP_DDL[] =

  "pragma wal_checkpoint = truncate;\n" // clean out any preexisting wal file

  ;

/* Name and version of program.  */

/* ARGP_PROGRAM_VERSION_HOOK_DEF = print_version; */ // not this simple for C++

/* Bug report address.  */

ARGP_PROGRAM_BUG_ADDRESS_DEF = PACKAGE_BUGREPORT;

/* Definitions of arguments for argp functions.  */

static const struct argp_option options[] =

  {

   { NULL, 0, NULL, 0, "Scanners:", 1 },

   { "scan-file-dir", 'F', NULL, 0, "Enable ELF/DWARF file scanning.", 0 },

   { "scan-rpm-dir", 'R', NULL, 0, "Enable RPM scanning.", 0 },

   { "scan-deb-dir", 'U', NULL, 0, "Enable DEB scanning.", 0 },

   { "scan-archive", 'Z', "EXT=CMD", 0, "Enable arbitrary archive scanning.", 0 },

   // "source-oci-imageregistry"  ...

   { NULL, 0, NULL, 0, "Options:", 2 },

   { "logical", 'L', NULL, 0, "Follow symlinks, default=ignore.", 0 },

   { "rescan-time", 't', "SECONDS", 0, "Number of seconds to wait between rescans, 0=disable.", 0 },

   { "groom-time", 'g', "SECONDS", 0, "Number of seconds to wait between database grooming, 0=disable.", 0 },

   { "maxigroom", 'G', NULL, 0, "Run a complete database groom/shrink pass at startup.", 0 },

   { "concurrency", 'c', "NUM", 0, "Limit scanning thread concurrency to NUM.", 0 },

   { "include", 'I', "REGEX", 0, "Include files matching REGEX, default=all.", 0 },

   { "exclude", 'X', "REGEX", 0, "Exclude files matching REGEX, default=none.", 0 },

   { "port", 'p', "NUM", 0, "HTTP port to listen on, default 8002.", 0 },

   { "database", 'd', "FILE", 0, "Path to sqlite database.", 0 },

   { "ddl", 'D', "SQL", 0, "Apply extra sqlite ddl/pragma to connection.", 0 },

   { "verbose", 'v', NULL, 0, "Increase verbosity.", 0 },

#define ARGP_KEY_FDCACHE_FDS 0x1001

   { "fdcache-fds", ARGP_KEY_FDCACHE_FDS, "NUM", 0, "Maximum number of archive files to keep in fdcache.", 0 },

#define ARGP_KEY_FDCACHE_MBS 0x1002

   { "fdcache-mbs", ARGP_KEY_FDCACHE_MBS, "MB", 0, "Maximum total size of archive file fdcache.", 0 },

#define ARGP_KEY_FDCACHE_PREFETCH 0x1003

   { "fdcache-prefetch", ARGP_KEY_FDCACHE_PREFETCH, "NUM", 0, "Number of archive files to prefetch into fdcache.", 0 },

   { NULL, 0, NULL, 0, NULL, 0 }

  };

/* Short description of program.  */

static const char doc[] = "Serve debuginfo-related content across HTTP from files under PATHs.";

/* Strings for arguments in help texts.  */

static const char args_doc[] = "[PATH ...]";

/* Prototype for option handler.  */

static error_t parse_opt (int key, char *arg, struct argp_state *state);

/* Data structure to communicate with argp functions.  */

static struct argp argp =

  {

   options, parse_opt, args_doc, doc, NULL, NULL, NULL

  };

static string db_path;

static sqlite3 *db; // single connection, serialized across all our threads!

static unsigned verbose;

static volatile sig_atomic_t interrupted = 0;

static volatile sig_atomic_t forced_rescan_count = 0;

static volatile sig_atomic_t sigusr1 = 0;

static volatile sig_atomic_t forced_groom_count = 0;

static volatile sig_atomic_t sigusr2 = 0;

static unsigned http_port = 8002;

static unsigned rescan_s = 300;

static unsigned groom_s = 86400;

static bool maxigroom = false;

static unsigned concurrency = std::thread::hardware_concurrency() ?: 1;

static set<string> source_paths;

static bool scan_files = false;

static map<string,string> scan_archives;

static vector<string> extra_ddl;

static regex_t file_include_regex;

static regex_t file_exclude_regex;

static bool traverse_logical;

static long fdcache_fds;

static long fdcache_mbs;

static long fdcache_prefetch;

static string tmpdir;

static void set_metric(const string& key, int64_t value);

// static void inc_metric(const string& key);

static void set_metric(const string& metric,

                       const string& lname, const string& lvalue,

                       int64_t value);

static void inc_metric(const string& metric,

                       const string& lname, const string& lvalue);

static void add_metric(const string& metric,

                       const string& lname, const string& lvalue,

                       int64_t value);

// static void add_metric(const string& metric, int64_t value);

/* Handle program arguments.  */

static error_t

parse_opt (int key, char *arg,

	   struct argp_state *state __attribute__ ((unused)))

{

  int rc;

  switch (key)

    {

    case 'v': verbose ++; break;

    case 'd': db_path = string(arg); break;

    case 'p': http_port = (unsigned) atoi(arg);

      if (http_port == 0 || http_port > 65535)

        argp_failure(state, 1, EINVAL, "port number");

      break;

    case 'F': scan_files = true; break;

    case 'R':

      scan_archives[".rpm"]="cat"; // libarchive groks rpm natively

      break;

    case 'U':

      if (access("/usr/bin/dpkg-deb", X_OK) == 0)

        {

          scan_archives[".deb"]="dpkg-deb --fsys-tarfile";

          scan_archives[".ddeb"]="dpkg-deb --fsys-tarfile";

        }

      else

        {

          scan_archives[".deb"]="(bsdtar -O -x -f - data.tar.xz)<";

          scan_archives[".ddeb"]="(bsdtar -O -x -f - data.tar.xz)<";

        }

      // .udeb too?

      break;

    case 'Z':

      {

        char* extension = strchr(arg, '=');

        if (arg[0] == '\0')

          argp_failure(state, 1, EINVAL, "missing EXT");

        else if (extension)

          scan_archives[string(arg, (extension-arg))]=string(extension+1);

        else

          scan_archives[string(arg)]=string("cat");

      }

      break;

    case 'L':

      traverse_logical = true;

      break;

    case 'D': extra_ddl.push_back(string(arg)); break;

    case 't':

      rescan_s = (unsigned) atoi(arg);

      break;

    case 'g':

      groom_s = (unsigned) atoi(arg);

      break;

    case 'G':

      maxigroom = true;

      break;

    case 'c':

      concurrency = (unsigned) atoi(arg);

      if (concurrency < 1) concurrency = 1;

      break;

    case 'I':

      // NB: no problem with unconditional free here - an earlier failed regcomp would exit program

      regfree (&file_include_regex);

      rc = regcomp (&file_include_regex, arg, REG_EXTENDED|REG_NOSUB);

      if (rc != 0)

        argp_failure(state, 1, EINVAL, "regular expession");

      break;

    case 'X':

      regfree (&file_exclude_regex);

      rc = regcomp (&file_exclude_regex, arg, REG_EXTENDED|REG_NOSUB);

      if (rc != 0)

        argp_failure(state, 1, EINVAL, "regular expession");

      break;

    case ARGP_KEY_FDCACHE_FDS:

      fdcache_fds = atol (arg);

      break;

    case ARGP_KEY_FDCACHE_MBS:

      fdcache_mbs = atol (arg);

      break;

    case ARGP_KEY_FDCACHE_PREFETCH:

      fdcache_prefetch = atol (arg);

      break;

    case ARGP_KEY_ARG:

      source_paths.insert(string(arg));

      break;

      // case 'h': argp_state_help (state, stderr, ARGP_HELP_LONG|ARGP_HELP_EXIT_OK);

    default: return ARGP_ERR_UNKNOWN;

    }

  return 0;

}

////////////////////////////////////////////////////////////////////////

// represent errors that may get reported to an ostream and/or a libmicrohttpd connection

struct reportable_exception

{

  int code;

  string message;

  reportable_exception(int c, const string& m): code(c), message(m) {}

  reportable_exception(const string& m): code(503), message(m) {}

  reportable_exception(): code(503), message() {}

  void report(ostream& o) const; // defined under obatched() class below

  MHD_RESULT mhd_send_response(MHD_Connection* c) const {

    MHD_Response* r = MHD_create_response_from_buffer (message.size(),

                                                       (void*) message.c_str(),

                                                       MHD_RESPMEM_MUST_COPY);

    MHD_add_response_header (r, "Content-Type", "text/plain");

    MHD_RESULT rc = MHD_queue_response (c, code, r);

    MHD_destroy_response (r);

    return rc;

  }

};

struct sqlite_exception: public reportable_exception

{

  sqlite_exception(int rc, const string& msg):

    reportable_exception(string("sqlite3 error: ") + msg + ": " + string(sqlite3_errstr(rc) ?: "?")) {}

};

struct libc_exception: public reportable_exception

{

  libc_exception(int rc, const string& msg):

    reportable_exception(string("libc error: ") + msg + ": " + string(strerror(rc) ?: "?")) {

    inc_metric("error_count","libc",strerror(rc));

  }

};

struct archive_exception: public reportable_exception

{

  archive_exception(const string& msg):

    reportable_exception(string("libarchive error: ") + msg) {

      inc_metric("error_count","libarchive",msg);

  }

  archive_exception(struct archive* a, const string& msg):

    reportable_exception(string("libarchive error: ") + msg + ": " + string(archive_error_string(a) ?: "?")) {

    inc_metric("error_count","libarchive",msg);

  }

};

struct elfutils_exception: public reportable_exception

{

  elfutils_exception(int rc, const string& msg):

    reportable_exception(string("elfutils error: ") + msg + ": " + string(elf_errmsg(rc) ?: "?")) {

    inc_metric("error_count","elfutils",elf_errmsg(rc));

  }

};

////////////////////////////////////////////////////////////////////////

template <typename Payload>

class workq

{

  set<Payload> q; // eliminate duplicates

  mutex mtx;

  condition_variable cv;

  bool dead;

  unsigned idlers;

public:

  workq() { dead = false; idlers = 0; }

  ~workq() {}

  void push_back(const Payload& p)

  {

    unique_lock<mutex> lock(mtx);

    q.insert (p);

    set_metric("thread_work_pending","role","scan", q.size());

    cv.notify_all();

  }

  // kill this workqueue, wake up all idlers / scanners

  void nuke() {

    unique_lock<mutex> lock(mtx);

    // optional: q.clear();

    dead = true;

    cv.notify_all();

  }

  // clear the workqueue, when scanning is interrupted with USR2

  void clear() {

    unique_lock<mutex> lock(mtx);

    q.clear();

    set_metric("thread_work_pending","role","scan", q.size());

    cv.notify_all(); // maybe wake up waiting idlers

  }

  // block this scanner thread until there is work to do and no active

  bool wait_front (Payload& p)

  {

    unique_lock<mutex> lock(mtx);

    while (!dead && (q.size() == 0 || idlers > 0))

      cv.wait(lock);

    if (dead)

      return false;

    else

      {

        p = * q.begin();

        q.erase (q.begin());

        set_metric("thread_work_pending","role","scan", q.size());

        if (q.size() == 0)

          cv.notify_all(); // maybe wake up waiting idlers

        return true;

      }

  }

  // block this idler thread until there is no work to do

  void wait_idle ()

  {

    unique_lock<mutex> lock(mtx);

    cv.notify_all(); // maybe wake up waiting scanners

    while (!dead && (q.size() != 0))

      cv.wait(lock);

    idlers ++;

  }

  void done_idle ()

  {

    unique_lock<mutex> lock(mtx);

    idlers --;

    cv.notify_all(); // maybe wake up waiting scanners, but probably not (shutting down)

  }

};

typedef struct stat stat_t;

typedef pair<string,stat_t> scan_payload;

inline bool operator< (const scan_payload& a, const scan_payload& b)

{

  return a.first < b.first; // don't bother compare the stat fields

}

static workq<scan_payload> scanq; // just a single one

// producer & idler: thread_main_fts_source_paths()

// consumer: thread_main_scanner()

// idler: thread_main_groom()

////////////////////////////////////////////////////////////////////////

// Print a standard timestamp.

static ostream&

timestamp (ostream &o)

{

  char datebuf[80];

  char *now2 = NULL;

  time_t now_t = time(NULL);

  struct tm *now = gmtime (&now_t);

  if (now)

    {

      (void) strftime (datebuf, sizeof (datebuf), "%c", now);

      now2 = datebuf;

    }

  return o << "[" << (now2 ? now2 : "") << "] "

           << "(" << getpid () << "/" << tid() << "): ";

}

// A little class that impersonates an ostream to the extent that it can

// take << streaming operations.  It batches up the bits into an internal

// stringstream until it is destroyed; then flushes to the original ostream.

// It adds a timestamp

class obatched

{

private:

  ostream& o;

  stringstream stro;

  static mutex lock;

public:

  obatched(ostream& oo, bool timestamp_p = true): o(oo)

  {

    if (timestamp_p)

      timestamp(stro);

  }

  ~obatched()

  {

    unique_lock<mutex> do_not_cross_the_streams(obatched::lock);

    o << stro.str();

    o.flush();

  }

  operator ostream& () { return stro; }

  template <typename T> ostream& operator << (const T& t) { stro << t; return stro; }

};

mutex obatched::lock; // just the one, since cout/cerr iostreams are not thread-safe

void reportable_exception::report(ostream& o) const {

  obatched(o) << message << endl;

}

////////////////////////////////////////////////////////////////////////

// RAII style sqlite prepared-statement holder that matches { } block lifetime

struct sqlite_ps

{

private:

  sqlite3* db;

  const string nickname;

  const string sql;

  sqlite3_stmt *pp;

  sqlite_ps(const sqlite_ps&); // make uncopyable

  sqlite_ps& operator=(const sqlite_ps &); // make unassignable

public:

  sqlite_ps (sqlite3* d, const string& n, const string& s): db(d), nickname(n), sql(s) {

    if (verbose > 4)

      obatched(clog) << nickname << " prep " << sql << endl;

    int rc = sqlite3_prepare_v2 (db, sql.c_str(), -1 /* to \0 */, & this->pp, NULL);

    if (rc != SQLITE_OK)

      throw sqlite_exception(rc, "prepare " + sql);

  }

  sqlite_ps& reset()

  {

    sqlite3_reset(this->pp);

    return *this;

  }

  sqlite_ps& bind(int parameter, const string& str)

  {

    if (verbose > 4)

      obatched(clog) << nickname << " bind " << parameter << "=" << str << endl;

    int rc = sqlite3_bind_text (this->pp, parameter, str.c_str(), -1, SQLITE_TRANSIENT);

    if (rc != SQLITE_OK)

      throw sqlite_exception(rc, "sqlite3 bind");

    return *this;

  }

  sqlite_ps& bind(int parameter, int64_t value)

  {

    if (verbose > 4)

      obatched(clog) << nickname << " bind " << parameter << "=" << value << endl;

    int rc = sqlite3_bind_int64 (this->pp, parameter, value);

    if (rc != SQLITE_OK)

      throw sqlite_exception(rc, "sqlite3 bind");

    return *this;

  }

  sqlite_ps& bind(int parameter)

  {

    if (verbose > 4)

      obatched(clog) << nickname << " bind " << parameter << "=" << "NULL" << endl;

    int rc = sqlite3_bind_null (this->pp, parameter);

    if (rc != SQLITE_OK)

      throw sqlite_exception(rc, "sqlite3 bind");

    return *this;

  }

  void step_ok_done() {

    int rc = sqlite3_step (this->pp);

    if (verbose > 4)

      obatched(clog) << nickname << " step-ok-done(" << sqlite3_errstr(rc) << ") " << sql << endl;

    if (rc != SQLITE_OK && rc != SQLITE_DONE && rc != SQLITE_ROW)

      throw sqlite_exception(rc, "sqlite3 step");

    (void) sqlite3_reset (this->pp);

  }

  int step() {

    int rc = sqlite3_step (this->pp);

    if (verbose > 4)

      obatched(clog) << nickname << " step(" << sqlite3_errstr(rc) << ") " << sql << endl;

    return rc;

  }

  ~sqlite_ps () { sqlite3_finalize (this->pp); }

  operator sqlite3_stmt* () { return this->pp; }

};

////////////////////////////////////////////////////////////////////////

// RAII style templated autocloser

template <class Payload, class Ignore>

struct defer_dtor

{

public:

  typedef Ignore (*dtor_fn) (Payload);

private:

  Payload p;

  dtor_fn fn;

public:

  defer_dtor(Payload _p, dtor_fn _fn): p(_p), fn(_fn) {}

  ~defer_dtor() { (void) (*fn)(p); }

private:

  defer_dtor(const defer_dtor<Payload,Ignore>&); // make uncopyable

  defer_dtor& operator=(const defer_dtor<Payload,Ignore> &); // make unassignable

};

////////////////////////////////////////////////////////////////////////

static string

header_censor(const string& str)

{

  string y;

  for (auto&& x : str)

    {

      if (isalnum(x) || x == '/' || x == '.' || x == ',' || x == '_' || x == ':')

        y += x;

    }

  return y;

}

static string

conninfo (struct MHD_Connection * conn)

{

  char hostname[256]; // RFC1035

  char servname[256];

  int sts = -1;

  if (conn == 0)

    return "internal";

  /* Look up client address data. */

  const union MHD_ConnectionInfo *u = MHD_get_connection_info (conn,

                                                               MHD_CONNECTION_INFO_CLIENT_ADDRESS);

  struct sockaddr *so = u ? u->client_addr : 0;

  if (so && so->sa_family == AF_INET) {

    sts = getnameinfo (so, sizeof (struct sockaddr_in), hostname, sizeof (hostname), servname,

                       sizeof (servname), NI_NUMERICHOST | NI_NUMERICSERV);

  } else if (so && so->sa_family == AF_INET6) {

    sts = getnameinfo (so, sizeof (struct sockaddr_in6), hostname, sizeof (hostname),

                       servname, sizeof (servname), NI_NUMERICHOST | NI_NUMERICSERV);

  }

  if (sts != 0) {

    hostname[0] = servname[0] = '\0';

  }

  // extract headers relevant to administration

  const char* user_agent = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "User-Agent") ?: "";

  const char* x_forwarded_for = MHD_lookup_connection_value (conn, MHD_HEADER_KIND, "X-Forwarded-For") ?: "";