From c036609d34dfbfded9891be83d5e43db0a9feae2 Mon Sep 17 00:00:00 2001 From: Jakub Filak Date: Mon, 7 Jul 2014 18:11:13 +0200 Subject: [PATCH 4/9] koops: dump oopses from systemd-journal Resolves rhbz#1059724 --- configure.ac | 3 + doc/Makefile.am | 1 + doc/abrt-dump-journal-oops.txt | 68 ++++++ init-scripts/abrt-oops.service | 4 +- po/POTFILES.in | 2 + src/include/libabrt.h | 19 ++ src/lib/kernel.c | 103 +++++---- src/plugins/Makefile.am | 23 ++ src/plugins/abrt-dump-journal-oops.c | 394 +++++++++++++++++++++++++++++++++++ src/plugins/abrt-dump-oops.c | 313 ++++------------------------ src/plugins/abrt-journal.c | 295 ++++++++++++++++++++++++++ src/plugins/abrt-journal.h | 110 ++++++++++ src/plugins/oops-utils.c | 279 +++++++++++++++++++++++++ src/plugins/oops-utils.h | 48 +++++ 15 files changed, 1353 insertions(+), 310 deletions(-) create mode 100644 doc/abrt-dump-journal-oops.txt create mode 100644 src/plugins/abrt-dump-journal-oops.c create mode 100644 src/plugins/abrt-journal.c create mode 100644 src/plugins/abrt-journal.h create mode 100644 src/plugins/oops-utils.c create mode 100644 src/plugins/oops-utils.h diff --git a/configure.ac b/configure.ac index c051ec5..03882e9 100644 --- a/configure.ac +++ b/configure.ac @@ -140,6 +140,7 @@ PKG_CHECK_MODULES([LIBREPORT_GTK], [libreport-gtk]) PKG_CHECK_MODULES([POLKIT], [polkit-gobject-1]) PKG_CHECK_MODULES([GIO], [gio-2.0]) PKG_CHECK_MODULES([SATYR], [satyr]) +PKG_CHECK_MODULES([SYSTEMD_JOURNAL], [libsystemd-journal]) PKG_PROG_PKG_CONFIG AC_ARG_WITH([systemdsystemunitdir], @@ -167,6 +168,7 @@ AC_CHECK_HEADERS([locale.h]) CONF_DIR='${sysconfdir}/${PACKAGE_NAME}' DEFAULT_CONF_DIR='${datadir}/${PACKAGE_NAME}/conf.d' VAR_RUN='${localstatedir}/run' +VAR_STATE='${localstatedir}/lib/${PACKAGE_NAME}' PLUGINS_CONF_DIR='${sysconfdir}/${PACKAGE_NAME}/plugins' DEFAULT_PLUGINS_CONF_DIR='${datadir}/${PACKAGE_NAME}/conf.d/plugins' EVENTS_DIR='${datadir}/libreport/events' @@ -255,6 +257,7 @@ AC_ARG_ENABLE([native-unwinder], AC_SUBST(CONF_DIR) AC_SUBST(DEFAULT_CONF_DIR) AC_SUBST(VAR_RUN) +AC_SUBST(VAR_STATE) AC_SUBST(PLUGINS_CONF_DIR) AC_SUBST(DEFAULT_PLUGINS_CONF_DIR) AC_SUBST(EVENTS_CONF_DIR) diff --git a/doc/Makefile.am b/doc/Makefile.am index 55cb0f3..064e2ba 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -19,6 +19,7 @@ MAN1_TXT += abrt-action-perform-ccpp-analysis.txt MAN1_TXT += abrt-action-notify.txt MAN1_TXT += abrt-applet.txt MAN1_TXT += abrt-dump-oops.txt +MAN1_TXT += abrt-dump-journal-oops.txt MAN1_TXT += abrt-dump-xorg.txt MAN1_TXT += abrt-auto-reporting.txt MAN1_TXT += abrt-retrace-client.txt diff --git a/doc/abrt-dump-journal-oops.txt b/doc/abrt-dump-journal-oops.txt new file mode 100644 index 0000000..e0b8d79 --- /dev/null +++ b/doc/abrt-dump-journal-oops.txt @@ -0,0 +1,68 @@ +abrt-dump-journal-oops(1) +========================= + +NAME +---- +abrt-dump-journal-oops - Extract oops from systemd-journal + +SYNOPSIS +-------- +'abrt-dump-journal-oops' [-vsoxtf] [-e]/[-c CURSOR] [-d DIR]/[-D] + +DESCRIPTION +----------- +This tool creates problem directory from oops extracted from systemd-journal. +The tool can follow systemd-journal and extract oopses in time of their +occurrence. + +The following start from the last seen cursor. If the last seen cursor file +does not exist, the following start by scanning the entire sytemd-journal or +from the end if '-e' option is specified. + +FILES +----- +/etc/abrt/plugins/oops.conf:: + Configuration file where user can disable detection of non-fatal MCEs + +/var/lib/abrt/abrt-dump-journal-oops.state:: + State file where systemd-journal cursor to the last seen message is saved + +OPTIONS +------- +-v, --verbose:: + Be more verbose. Can be given multiple times. + +-s:: + Log to syslog + +-o:: + Print found oopses on standard output + +-d DIR:: + Create new problem directory in DIR for every oops found + +-D:: + Same as -d DumpLocation, DumpLocation is specified in abrt.conf + +-s CURSOR:: + Starts scannig systemd-journal from CURSOR + +-e:: + Starts following systemd-journal from the end + +-x:: + Make the problem directory world readable. Usable only with -d/-D + +-t:: + Throttle problem directory creation to 1 per second + +-f:: + Follow systemd-journal + +SEE ALSO +-------- +abrt.conf(5) + +AUTHORS +------- +* ABRT team diff --git a/init-scripts/abrt-oops.service b/init-scripts/abrt-oops.service index d8ac028..69aaaa9 100644 --- a/init-scripts/abrt-oops.service +++ b/init-scripts/abrt-oops.service @@ -4,8 +4,8 @@ After=abrtd.service Requisite=abrtd.service [Service] -# TODO: do we really need absolute paths here? -ExecStart=/bin/sh -c '/bin/dmesg | /usr/bin/abrt-dump-oops -xD; exec /usr/bin/abrt-watch-log -F "`/usr/bin/abrt-dump-oops -m`" /var/log/messages -- /usr/bin/abrt-dump-oops -xtD' +# systemd requires absolute paths to executables +ExecStart=/usr/bin/abrt-dump-journal-oops -fxtD [Install] WantedBy=multi-user.target diff --git a/po/POTFILES.in b/po/POTFILES.in index ff9b97a..160cd8b 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -35,12 +35,14 @@ src/plugins/abrt-action-trim-files.c src/plugins/abrt-gdb-exploitable src/plugins/abrt-watch-log.c src/plugins/abrt-dump-oops.c +src/plugins/abrt-dump-journal-oops.c src/plugins/abrt-dump-xorg.c src/plugins/abrt-retrace-client.c src/plugins/analyze_LocalGDB.xml.in src/plugins/analyze_RetraceServer.xml.in src/plugins/collect_xsession_errors.xml.in src/plugins/https-utils.c +src/plugins/oops-utils.c src/plugins/bodhi.c src/hooks/abrt-merge-pstoreoops.c diff --git a/src/include/libabrt.h b/src/include/libabrt.h index d6eb4a5..37704dd 100644 --- a/src/include/libabrt.h +++ b/src/include/libabrt.h @@ -109,8 +109,27 @@ char *kernel_tainted_long(const char *tainted_short); int koops_hash_str_ext(char hash_str[SHA1_RESULT_LEN*2 + 1], const char *oops_buf, int frame_count, int duphas_flags); #define koops_hash_str abrt_koops_hash_str int koops_hash_str(char hash_str[SHA1_RESULT_LEN*2 + 1], const char *oops_buf); + + +#define koops_line_skip_level abrt_koops_line_skip_level +int koops_line_skip_level(const char **c); +#define koops_line_skip_jiffies abrt_koops_line_skip_jiffies +void koops_line_skip_jiffies(const char **c); + +/* + * extract_oops tries to find oops signatures in a log + */ +struct abrt_koops_line_info { + char *ptr; + int level; +}; + +#define koops_extract_oopses_from_lines abrt_koops_extract_oopses_from_lines +void koops_extract_oopses_from_lines(GList **oops_list, const struct abrt_koops_line_info *lines_info, int lines_info_size); #define koops_extract_oopses abrt_koops_extract_oopses void koops_extract_oopses(GList **oops_list, char *buffer, size_t buflen); +#define koops_suspicious_strings_list abrt_koops_suspicious_strings_list +GList *koops_suspicious_strings_list(void); #define koops_print_suspicious_strings abrt_koops_print_suspicious_strings void koops_print_suspicious_strings(void); /** diff --git a/src/lib/kernel.c b/src/lib/kernel.c index b2d72b6..be80cbc 100644 --- a/src/lib/kernel.c +++ b/src/lib/kernel.c @@ -22,21 +22,12 @@ #define _GNU_SOURCE 1 /* for strcasestr */ #include "libabrt.h" -/* - * extract_oops tries to find oops signatures in a log - */ - -struct line_info { - char *ptr; - char level; -}; - /* Used to be 100, but some MCE oopses are short: * "CPU 0: Machine Check Exception: 0000000000000007" */ #define SANE_MIN_OOPS_LEN 30 -static void record_oops(GList **oops_list, struct line_info* lines_info, int oopsstart, int oopsend) +static void record_oops(GList **oops_list, const struct abrt_koops_line_info* lines_info, int oopsstart, int oopsend) { int q; int len; @@ -161,6 +152,15 @@ void koops_print_suspicious_strings(void) koops_print_suspicious_strings_filtered(NULL); } +GList *koops_suspicious_strings_list(void) +{ + GList *strings = NULL; + for (const char *const *str = s_koops_suspicious_strings; *str; ++str) + strings = g_list_prepend(strings, (gpointer)*str); + + return strings; +} + static bool match_any(const regex_t **res, const char *str) { for (const regex_t **r = res; *r != NULL; ++r) @@ -189,12 +189,57 @@ void koops_print_suspicious_strings_filtered(const regex_t **filterout) } } + +void koops_line_skip_jiffies(const char **c) +{ + /* remove jiffies time stamp counter if present + * jiffies are unsigned long, so it can be 2^64 long, which is + * 20 decimal digits + */ + if (**c == '[') + { + const char *c2 = strchr(*c, '.'); + const char *c3 = strchr(*c, ']'); + if (c2 && c3 && (c2 < c3) && (c3-*c) < 21) + { + *c = c3 + 1; + if (**c == ' ') + (*c)++; + } + } +} + +int koops_line_skip_level(const char **c) +{ + int linelevel = 0; + if (**c == '<') + { + const char *ptr = *c + 1; + while (isdigit(*ptr)) + ++ptr; + + if (*ptr == '>' && (ptr - *c > 1)) + { + const char *const bck = ptr + 1; + unsigned exp = 1; + while (--ptr != *c) + { + linelevel += (*ptr - '0') * exp; + exp *= 10; + } + *c = bck; + } + } + + return linelevel; +} + void koops_extract_oopses(GList **oops_list, char *buffer, size_t buflen) { char *c; int linecount = 0; int lines_info_size = 0; - struct line_info *lines_info = NULL; + struct abrt_koops_line_info *lines_info = NULL; /* Split buffer into lines */ @@ -254,28 +299,10 @@ void koops_extract_oopses(GList **oops_list, char *buffer, size_t buflen) c = kernel_str + sizeof("kernel: ")-1; } - linelevel = 0; /* store and remove kernel log level */ - if (*c == '<' && c[1] && c[2] == '>') - { - linelevel = c[1]; - c += 3; - } - /* remove jiffies time stamp counter if present - * jiffies are unsigned long, so it can be 2^64 long, which is - * 20 decimal digits - */ - if (*c == '[') - { - char *c2 = strchr(c, '.'); - char *c3 = strchr(c, ']'); - if (c2 && c3 && (c2 < c3) && (c3-c) < 21) - { - c = c3 + 1; - if (*c == ' ') - c++; - } - } + linelevel = koops_line_skip_level((const char **)&c); + koops_line_skip_jiffies((const char **)&c); + if ((lines_info_size & 0xfff) == 0) { lines_info = xrealloc(lines_info, (lines_info_size + 0x1000) * sizeof(lines_info[0])); @@ -287,6 +314,12 @@ next_line: c = c9 + 1; } + koops_extract_oopses_from_lines(oops_list, lines_info, lines_info_size); + free(lines_info); +} + +void koops_extract_oopses_from_lines(GList **oops_list, const struct abrt_koops_line_info *lines_info, int lines_info_size) +{ /* Analyze lines */ int i; @@ -323,8 +356,6 @@ next_line: { /* debug information */ log_debug("Found oops at line %d: '%s'", oopsstart, lines_info[oopsstart].ptr); - if (oopsstart != i) - log_debug("Trigger line is %d: '%s'", i, c); /* try to find the end marker */ int i2 = i + 1; while (i2 < lines_info_size && i2 < (i+50)) @@ -471,10 +502,7 @@ next_line: record_oops(oops_list, lines_info, oopsstart, oopsstart); } } - - free(lines_info); } - int koops_hash_str_ext(char result[SHA1_RESULT_LEN*2 + 1], const char *oops_buf, int frame_count, int duphash_flags) { char *hash_str = NULL, *error = NULL; @@ -507,6 +535,7 @@ int koops_hash_str_ext(char result[SHA1_RESULT_LEN*2 + 1], const char *oops_buf, else log("Nothing useful for duphash"); + free(hash_str); } diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am index 14b6fe0..a804f82 100644 --- a/src/plugins/Makefile.am +++ b/src/plugins/Makefile.am @@ -15,6 +15,7 @@ bin_SCRIPTS = \ bin_PROGRAMS = \ abrt-watch-log \ abrt-dump-oops \ + abrt-dump-journal-oops \ abrt-dump-xorg \ abrt-action-analyze-c \ abrt-action-analyze-python \ @@ -97,6 +98,8 @@ EXTRA_DIST = \ abrt-action-ureport \ abrt-gdb-exploitable \ https-utils.h \ + oops-utils.h \ + abrt-journal.h \ post_report.xml.in \ abrt-action-analyze-ccpp-local.in @@ -120,6 +123,7 @@ abrt_watch_log_LDADD = \ ../lib/libabrt.la abrt_dump_oops_SOURCES = \ + oops-utils.c \ abrt-dump-oops.c abrt_dump_oops_CPPFLAGS = \ -I$(srcdir)/../include \ @@ -133,6 +137,25 @@ abrt_dump_oops_LDADD = \ $(LIBREPORT_LIBS) \ ../lib/libabrt.la +abrt_dump_journal_oops_SOURCES = \ + oops-utils.c \ + abrt-journal.c \ + abrt-dump-journal-oops.c +abrt_dump_journal_oops_CPPFLAGS = \ + -I$(srcdir)/../include \ + -I$(srcdir)/../lib \ + $(GLIB_CFLAGS) \ + $(LIBREPORT_CFLAGS) \ + $(SYSTEMD_JOURNAL_CFLAGS) \ + -DDEFAULT_DUMP_DIR_MODE=$(DEFAULT_DUMP_DIR_MODE) \ + -DVAR_STATE=\"$(VAR_STATE)\" \ + -D_GNU_SOURCE +abrt_dump_journal_oops_LDADD = \ + $(GLIB_LIBS) \ + $(LIBREPORT_LIBS) \ + $(SYSTEMD_JOURNAL_LIBS) \ + ../lib/libabrt.la + abrt_dump_xorg_SOURCES = \ abrt-dump-xorg.c abrt_dump_xorg_CPPFLAGS = \ diff --git a/src/plugins/abrt-dump-journal-oops.c b/src/plugins/abrt-dump-journal-oops.c new file mode 100644 index 0000000..3f1f419 --- /dev/null +++ b/src/plugins/abrt-dump-journal-oops.c @@ -0,0 +1,394 @@ +/* + * Copyright (C) 2014 ABRT team + * Copyright (C) 2014 RedHat Inc + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include "libabrt.h" +#include "abrt-journal.h" +#include "oops-utils.h" + +#define ABRT_JOURNAL_WATCH_STATE_FILE VAR_STATE"/abrt-dump-journal-oops.state" +#define ABRT_JOURNAL_WATCH_STATE_FILE_MODE 0600 +#define ABRT_JOURNAL_WATCH_STATE_FILE_MAX_SZ (4 * 1024) + +/* Limit number of buffered lines */ +#define ABRT_JOURNAL_MAX_READ_LINES (1024 * 1024) + +/* Forward declarations */ +static void save_abrt_journal_watch_position(abrt_journal_t *journal, const char *file_name); + +/* + * Koops extractor + */ + +static GList* abrt_journal_extract_kernel_oops(abrt_journal_t *journal) +{ + size_t lines_info_count = 0; + size_t lines_info_size = 32; + struct abrt_koops_line_info *lines_info = xmalloc(lines_info_size * sizeof(lines_info[0])); + + do + { + const char *line = NULL; + if (abrt_journal_get_log_line(journal, &line) < 0) + error_msg_and_die(_("Cannot read journal data.")); + + if (lines_info_count == lines_info_size) + { + lines_info_size *= 2; + lines_info = xrealloc(lines_info, lines_info_size * sizeof(lines_info[0])); + } + + lines_info[lines_info_count].level = koops_line_skip_level(&line); + koops_line_skip_jiffies(&line); + + lines_info[lines_info_count].ptr = xstrdup(line); + + ++lines_info_count; + } + while (lines_info_count < ABRT_JOURNAL_MAX_READ_LINES + && abrt_journal_next(journal) > 0); + + GList *oops_list = NULL; + koops_extract_oopses_from_lines(&oops_list, lines_info, lines_info_count); + + log_debug("Extracted: %d oopses", g_list_length(oops_list)); + + for (size_t i = 0; i < lines_info_count; ++i) + free(lines_info[i].ptr); + + free(lines_info); + + return oops_list; +} + +/* + * An adatapter of abrt_journal_extract_kernel_oops for abrt_journal_watch_callback + */ +struct watch_journald_settings +{ + const char *dump_location; + int oops_utils_flags; +}; + +static void abrt_journal_watch_extract_kernel_oops(abrt_journal_watch_t *watch, void *data) +{ + const struct watch_journald_settings *conf = (const struct watch_journald_settings *)data; + + abrt_journal_t *journal = abrt_journal_watch_get_journal(watch); + + /* Give systemd-journal one second to suck in all kernel's strings */ + if (abrt_oops_signaled_sleep(1) > 0) + { + abrt_journal_watch_stop(watch); + return; + } + + GList *oopses = abrt_journal_extract_kernel_oops(journal); + abrt_oops_process_list(oopses, conf->dump_location, conf->oops_utils_flags); + g_list_free_full(oopses, (GDestroyNotify)free); + + /* Skip stuff which appeared while processing oops as it is not necessary */ + /* to catch all consecutive oopses (anyway such oopses are almost */ + /* certainly duplicates of the already extracted ones) */ + abrt_journal_seek_tail(journal); + + /* In case of disaster, lets make sure we won't read the journal messages */ + /* again. */ + save_abrt_journal_watch_position(journal, ABRT_JOURNAL_WATCH_STATE_FILE); + + if (g_abrt_oops_sleep_woke_up_on_signal > 0) + abrt_journal_watch_stop(watch); +} + +/* + * Koops extractor end + */ + +static void try_restore_abrt_journal_watch_position(abrt_journal_t *journal, const char *file_name) +{ + struct stat buf; + if (lstat(file_name, &buf) < 0) + { + if (errno == ENOENT) + { + /* Only notice because this is expected */ + log_notice(_("Not restoring journal watch's position: file '%s' does not exist"), file_name); + return; + } + + perror_msg(_("Cannot restore journal watch's position form file '%s'"), file_name); + return; + } + + if (!(buf.st_mode & S_IFREG)) + { + error_msg(_("Cannot restore journal watch's position: path '%s' is not regular file"), file_name); + return; + } + + if (buf.st_size > ABRT_JOURNAL_WATCH_STATE_FILE_MAX_SZ) + { + error_msg(_("Cannot restore journal watch's position: file '%s' exceeds %dB size limit"), + file_name, ABRT_JOURNAL_WATCH_STATE_FILE_MAX_SZ); + return; + } + + int state_fd = open(file_name, O_RDONLY | O_NOFOLLOW); + if (state_fd < 0) + { + perror_msg(_("Cannot restore journal watch's position: open('%s')"), file_name); + return; + } + + char *crsr = xmalloc(buf.st_size + 1); + + const int sz = full_read(state_fd, crsr, buf.st_size); + if (sz != buf.st_size) + { + error_msg(_("Cannot restore journal watch's position: cannot read entire file '%s'"), file_name); + close(state_fd); + return; + } + + crsr[sz] = '\0'; + close(state_fd); + + const int r = abrt_journal_set_cursor(journal, crsr); + if (r < 0) + { + /* abrt_journal_set_cursor() prints error message in verbose mode */ + error_msg(_("Failed to move the journal to a cursor from file '%s'"), file_name); + return; + } + + free(crsr); +} + +static void save_abrt_journal_watch_position(abrt_journal_t *journal, const char *file_name) +{ + char *crsr = NULL; + const int r = abrt_journal_get_cursor(journal, &crsr); + + if (r < 0) + { + /* abrt_journal_set_cursor() prints error message in verbose mode */ + error_msg(_("Cannot save journal watch's position")); + return; + } + + int state_fd = open(file_name, + O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW, + ABRT_JOURNAL_WATCH_STATE_FILE_MODE); + + if (state_fd < 0) + { + perror_msg(_("Cannot save journal watch's position: open('%s')"), file_name); + return; + } + + full_write_str(state_fd, crsr); + close(state_fd); + + free(crsr); +} + +static void watch_journald(abrt_journal_t *journal, const char *dump_location, int flags) +{ + GList *koops_strings = koops_suspicious_strings_list(); + + char *oops_string_filter_regex = abrt_oops_string_filter_regex(); + if (oops_string_filter_regex) + { + regex_t filter_re; + if (regcomp(&filter_re, oops_string_filter_regex, REG_NOSUB) != 0) + perror_msg_and_die(_("Failed to compile regex")); + + GList *iter = koops_strings; + while(iter != NULL) + { + GList *next = g_list_next(iter); + + const int reti = regexec(&filter_re, (const char *)iter->data, 0, NULL, 0); + if (reti == 0) + koops_strings = g_list_delete_link(koops_strings, iter); + else if (reti != REG_NOMATCH) + { + char msgbuf[100]; + regerror(reti, &filter_re, msgbuf, sizeof(msgbuf)); + error_msg_and_die("Regex match failed: %s", msgbuf); + } + + iter = next; + } + + regfree(&filter_re); + free(oops_string_filter_regex); + } + + struct watch_journald_settings watch_conf = { + .dump_location = dump_location, + .oops_utils_flags = flags, + }; + + struct abrt_journal_watch_notify_strings notify_strings_conf = { + .decorated_cb = abrt_journal_watch_extract_kernel_oops, + .decorated_cb_data = &watch_conf, + .strings = koops_strings, + }; + + abrt_journal_watch_t *watch = NULL; + if (abrt_journal_watch_new(&watch, journal, abrt_journal_watch_notify_strings, ¬ify_strings_conf) < 0) + error_msg_and_die(_("Failed to initialize systemd-journal watch")); + + abrt_journal_watch_run_sync(watch); + abrt_journal_watch_free(watch); + + g_list_free(koops_strings); +} + +int main(int argc, char *argv[]) +{ + /* I18n */ + setlocale(LC_ALL, ""); +#if ENABLE_NLS + bindtextdomain(PACKAGE, LOCALEDIR); + textdomain(PACKAGE); +#endif + + abrt_init(argv); + + /* Can't keep these strings/structs static: _() doesn't support that */ + const char *program_usage_string = _( + "& [-vsoxtf] [-e]/[-c CURSOR] [-d DIR]/[-D]\n" + "\n" + "Extract oops from systemd-journal\n" + "\n" + "-c and -e options conflicts because both specifies the first read message.\n" + "\n" + "-e is useful only for -f because the following of journal starts by reading \n" + "the entire journal if the last seen possition is not available.\n" + "\n" + "The last seen position is saved in "ABRT_JOURNAL_WATCH_STATE_FILE"\n" + ); + enum { + OPT_v = 1 << 0, + OPT_s = 1 << 1, + OPT_o = 1 << 2, + OPT_d = 1 << 3, + OPT_D = 1 << 4, + OPT_x = 1 << 5, + OPT_t = 1 << 6, + OPT_c = 1 << 7, + OPT_e = 1 << 8, + OPT_f = 1 << 9, + }; + + char *cursor = NULL; + char *dump_location = NULL; + + /* Keep enum above and order of options below in sync! */ + struct options program_options[] = { + OPT__VERBOSE(&g_verbose), + OPT_BOOL( 's', NULL, NULL, _("Log to syslog")), + OPT_BOOL( 'o', NULL, NULL, _("Print found oopses on standard output")), + /* oopses don't contain any sensitive info, and even + * the old koops app was showing the oopses to all users + */ + OPT_STRING('d', NULL, &dump_location, "DIR", _("Create new problem directory in DIR for every oops found")), + OPT_BOOL( 'D', NULL, NULL, _("Same as -d DumpLocation, DumpLocation is specified in abrt.conf")), + OPT_BOOL( 'x', NULL, NULL, _("Make the problem directory world readable")), + OPT_BOOL( 't', NULL, NULL, _("Throttle problem directory creation to 1 per second")), + OPT_STRING('c', NULL, &cursor, "CURSOR", _("Start reading systemd-journal from the CURSOR position")), + OPT_BOOL( 'e', NULL, NULL, _("Start reading systemd-journal from the end")), + OPT_BOOL( 'f', NULL, NULL, _("Follow systemd-journal from the last seen position (if available)")), + OPT_END() + }; + unsigned opts = parse_opts(argc, argv, program_options, program_usage_string); + + export_abrt_envvars(0); + + msg_prefix = g_progname; + if ((opts & OPT_s) || getenv("ABRT_SYSLOG")) + { + logmode = LOGMODE_JOURNAL; + } + + if ((opts & OPT_c) && (opts & OPT_e)) + error_msg_and_die(_("You need to specify either -c CURSOR or -e")); + + if (opts & OPT_D) + { + if (opts & OPT_d) + show_usage_and_die(program_usage_string, program_options); + load_abrt_conf(); + dump_location = g_settings_dump_location; + g_settings_dump_location = NULL; + free_abrt_conf_data(); + } + + int oops_utils_flags = 0; + if ((opts & OPT_x)) + oops_utils_flags |= ABRT_OOPS_WORLD_READABLE; + + if ((opts & OPT_t)) + oops_utils_flags |= ABRT_OOPS_THROTTLE_CREATION; + + if ((opts & OPT_o)) + oops_utils_flags |= ABRT_OOPS_PRINT_STDOUT; + + const char *const env_journal_filter = getenv("ABRT_DUMP_JOURNAL_OOPS_DEBUG_FILTER"); + static const char *kernel_journal_filter[2] = { 0 }; + kernel_journal_filter[0] = (env_journal_filter ? env_journal_filter : "SYSLOG_IDENTIFIER=kernel"); + log_debug("Using journal match: '%s'", kernel_journal_filter[0]); + + abrt_journal_t *journal = NULL; + if (abrt_journal_new(&journal)) + error_msg_and_die(_("Cannot open systemd-journal")); + + if (abrt_journal_set_journal_filter(journal, kernel_journal_filter) < 0) + error_msg_and_die(_("Cannot filter systemd-journal to kernel data only")); + + if ((opts & OPT_e) && abrt_journal_seek_tail(journal) < 0) + error_msg_and_die(_("Cannot seek to the end of journal")); + + if ((opts & OPT_f)) + { + if (!cursor) + try_restore_abrt_journal_watch_position(journal, ABRT_JOURNAL_WATCH_STATE_FILE); + else if(abrt_journal_set_cursor(journal, cursor)) + error_msg_and_die(_("Failed to start watch from cursor '%s'"), cursor); + + watch_journald(journal, dump_location, oops_utils_flags); + + save_abrt_journal_watch_position(journal, ABRT_JOURNAL_WATCH_STATE_FILE); + } + else + { + if (cursor && abrt_journal_set_cursor(journal, cursor)) + error_msg_and_die(_("Failed to set systemd-journal cursor '%s'"), cursor); + + /* Compatibility hack, a watch's callback gets the journal already moved + * to a next message.*/ + abrt_journal_next(journal); + + GList *oopses = abrt_journal_extract_kernel_oops(journal); + const int errors = abrt_oops_process_list(oopses, dump_location, oops_utils_flags); + g_list_free_full(oopses, (GDestroyNotify)free); + + return errors; + } + + abrt_journal_free(journal); + + return EXIT_SUCCESS; +} diff --git a/src/plugins/abrt-dump-oops.c b/src/plugins/abrt-dump-oops.c index 9f0dc87..b1031ea 100644 --- a/src/plugins/abrt-dump-oops.c +++ b/src/plugins/abrt-dump-oops.c @@ -1,6 +1,6 @@ /* - Copyright (C) 2011 ABRT team - Copyright (C) 2011 RedHat Inc + Copyright (C) 2011,2014 ABRT team + Copyright (C) 2011,2014 RedHat Inc This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -18,16 +18,7 @@ */ #include #include "libabrt.h" - -/* How many problem dirs to create at most? - * Also causes cooldown sleep with -t if exceeded - - * useful when called from a log watcher. - */ -#define MAX_DUMPED_DD_COUNT 5 - -static bool world_readable_dump = false; -static bool throttle_dd_creation = false; -static const char *debug_dumps_dir = "."; +#include "oops-utils.h" #define MAX_SCAN_BLOCK (4*1024*1024) #define READ_AHEAD (10*1024) @@ -69,175 +60,6 @@ static void scan_syslog_file(GList **oops_list, int fd) free(buffer); } -static char *list_of_tainted_modules(const char *proc_modules) -{ - struct strbuf *result = strbuf_new(); - - const char *p = proc_modules; - for (;;) - { - const char *end = strchrnul(p, '\n'); - const char *paren = strchrnul(p, '('); - /* We look for a line with this format: - * "kvm_intel 126289 0 - Live 0xf829e000 (taint_flags)" - * where taint_flags have letters - * (flags '+' and '-' indicate (un)loading, we must ignore them). - */ - while (++paren < end) - { - if ((unsigned)(toupper(*paren) - 'A') <= 'Z'-'A') - { - strbuf_append_strf(result, result->len == 0 ? "%.*s" : ",%.*s", - (int)(strchrnul(p,' ') - p), p - ); - break; - } - if (*paren == ')') - break; - } - - if (*end == '\0') - break; - p = end + 1; - } - - if (result->len == 0) - { - strbuf_free(result); - return NULL; - } - return strbuf_free_nobuf(result); -} - -static void save_oops_data_in_dump_dir(struct dump_dir *dd, char *oops, const char *proc_modules) -{ - char *first_line = oops; - char *second_line = (char*)strchr(first_line, '\n'); /* never NULL */ - *second_line++ = '\0'; - - if (first_line[0]) - dd_save_text(dd, FILENAME_KERNEL, first_line); - dd_save_text(dd, FILENAME_BACKTRACE, second_line); - - /* check if trace doesn't have line: 'Your BIOS is broken' */ - if (strstr(second_line, "Your BIOS is broken")) - dd_save_text(dd, FILENAME_NOT_REPORTABLE, - _("A kernel problem occurred because of broken BIOS. " - "Unfortunately, such problems are not fixable by kernel maintainers.")); - /* check if trace doesn't have line: 'Your hardware is unsupported' */ - else if (strstr(second_line, "Your hardware is unsupported")) - dd_save_text(dd, FILENAME_NOT_REPORTABLE, - _("A kernel problem occurred, but your hardware is unsupported, " - "therefore kernel maintainers are unable to fix this problem.")); - else - { - char *tainted_short = kernel_tainted_short(second_line); - if (tainted_short) - { - log_notice("Kernel is tainted '%s'", tainted_short); - dd_save_text(dd, FILENAME_TAINTED_SHORT, tainted_short); - - char *tnt_long = kernel_tainted_long(tainted_short); - dd_save_text(dd, FILENAME_TAINTED_LONG, tnt_long); - free(tnt_long); - - struct strbuf *reason = strbuf_new(); - const char *fmt = _("A kernel problem occurred, but your kernel has been " - "tainted (flags:%s). Kernel maintainers are unable to " - "diagnose tainted reports."); - strbuf_append_strf(reason, fmt, tainted_short); - - char *modlist = !proc_modules ? NULL : list_of_tainted_modules(proc_modules); - if (modlist) - { - strbuf_append_strf(reason, _(" Tainted modules: %s."), modlist); - free(modlist); - } - - dd_save_text(dd, FILENAME_NOT_REPORTABLE, reason->buf); - strbuf_free(reason); - free(tainted_short); - } - } - - // TODO: add "Kernel oops: " prefix, so that all oopses have recognizable FILENAME_REASON? - // kernel oops 1st line may look quite puzzling otherwise... - strchrnul(second_line, '\n')[0] = '\0'; - dd_save_text(dd, FILENAME_REASON, second_line); -} - -/* returns number of errors */ -static unsigned create_oops_dump_dirs(GList *oops_list, unsigned oops_cnt) -{ - unsigned countdown = MAX_DUMPED_DD_COUNT; /* do not report hundreds of oopses */ - - log_notice("Saving %u oopses as problem dirs", oops_cnt >= countdown ? countdown : oops_cnt); - - char *cmdline_str = xmalloc_fopen_fgetline_fclose("/proc/cmdline"); - char *fips_enabled = xmalloc_fopen_fgetline_fclose("/proc/sys/crypto/fips_enabled"); - char *proc_modules = xmalloc_open_read_close("/proc/modules", /*maxsize:*/ NULL); - char *suspend_stats = xmalloc_open_read_close("/sys/kernel/debug/suspend_stats", /*maxsize:*/ NULL); - - time_t t = time(NULL); - const char *iso_date = iso_date_string(&t); - /* dump should be readable by all if we're run with -x */ - uid_t my_euid = (uid_t)-1L; - mode_t mode = DEFAULT_DUMP_DIR_MODE | S_IROTH; - /* and readable only for the owner otherwise */ - if (!world_readable_dump) - { - mode = DEFAULT_DUMP_DIR_MODE; - my_euid = geteuid(); - } - - pid_t my_pid = getpid(); - unsigned idx = 0; - unsigned errors = 0; - while (idx < oops_cnt) - { - char base[sizeof("oops-YYYY-MM-DD-hh:mm:ss-%lu-%lu") + 2 * sizeof(long)*3]; - sprintf(base, "oops-%s-%lu-%lu", iso_date, (long)my_pid, (long)idx); - char *path = concat_path_file(debug_dumps_dir, base); - - struct dump_dir *dd = dd_create(path, /*uid:*/ my_euid, mode); - if (dd) - { - dd_create_basic_files(dd, /*uid:*/ my_euid, NULL); - save_oops_data_in_dump_dir(dd, (char*)g_list_nth_data(oops_list, idx++), proc_modules); - dd_save_text(dd, FILENAME_ABRT_VERSION, VERSION); - dd_save_text(dd, FILENAME_ANALYZER, "Kerneloops"); - dd_save_text(dd, FILENAME_TYPE, "Kerneloops"); - if (cmdline_str) - dd_save_text(dd, FILENAME_CMDLINE, cmdline_str); - if (proc_modules) - dd_save_text(dd, "proc_modules", proc_modules); - if (fips_enabled && strcmp(fips_enabled, "0") != 0) - dd_save_text(dd, "fips_enabled", fips_enabled); - if (suspend_stats) - dd_save_text(dd, "suspend_stats", suspend_stats); - dd_close(dd); - notify_new_path(path); - } - else - errors++; - - free(path); - - if (--countdown == 0) - break; - - if (dd && throttle_dd_creation) - sleep(1); - } - - free(cmdline_str); - free(proc_modules); - free(fips_enabled); - free(suspend_stats); - - return errors; -} - int main(int argc, char **argv) { /* I18n */ @@ -267,6 +89,7 @@ int main(int argc, char **argv) OPT_m = 1 << 8, }; char *problem_dir = NULL; + char *dump_location = NULL; /* Keep enum above and order of options below in sync! */ struct options program_options[] = { OPT__VERBOSE(&g_verbose), @@ -275,7 +98,7 @@ int main(int argc, char **argv) /* oopses don't contain any sensitive info, and even * the old koops app was showing the oopses to all users */ - OPT_STRING('d', NULL, &debug_dumps_dir, "DIR", _("Create new problem directory in DIR for every oops found")), + OPT_STRING('d', NULL, &dump_location, "DIR", _("Create new problem directory in DIR for every oops found")), OPT_BOOL( 'D', NULL, NULL, _("Same as -d DumpLocation, DumpLocation is specified in abrt.conf")), OPT_STRING('u', NULL, &problem_dir, "PROBLEM", _("Save the extracted information in PROBLEM")), OPT_BOOL( 'x', NULL, NULL, _("Make the problem directory world readable")), @@ -295,26 +118,19 @@ int main(int argc, char **argv) if (opts & OPT_m) { - map_string_t *settings = new_map_string(); - - load_abrt_plugin_conf_file("oops.conf", settings); - - int only_fatal_mce = 1; - try_get_map_string_item_as_bool(settings, "OnlyFatalMCE", &only_fatal_mce); - - free_map_string(settings); - - if (only_fatal_mce) + char *oops_string_filter_regex = abrt_oops_string_filter_regex(); + if (oops_string_filter_regex) { - regex_t mce_re; - if (regcomp(&mce_re, "^Machine .*$", REG_NOSUB) != 0) + regex_t filter_re; + if (regcomp(&filter_re, oops_string_filter_regex, REG_NOSUB) != 0) perror_msg_and_die(_("Failed to compile regex")); - const regex_t *filter[] = { &mce_re, NULL }; + const regex_t *filter[] = { &filter_re, NULL }; koops_print_suspicious_strings_filtered(filter); - regfree(&mce_re); + regfree(&filter_re); + free(oops_string_filter_regex); } else koops_print_suspicious_strings(); @@ -327,100 +143,55 @@ int main(int argc, char **argv) if (opts & OPT_d) show_usage_and_die(program_usage_string, program_options); load_abrt_conf(); - debug_dumps_dir = g_settings_dump_location; + dump_location = g_settings_dump_location; g_settings_dump_location = NULL; free_abrt_conf_data(); } + int oops_utils_flags = 0; + if ((opts & OPT_x)) + oops_utils_flags |= ABRT_OOPS_WORLD_READABLE; + + if ((opts & OPT_t)) + oops_utils_flags |= ABRT_OOPS_THROTTLE_CREATION; + + if ((opts & OPT_o)) + oops_utils_flags |= ABRT_OOPS_PRINT_STDOUT; + argv += optind; if (argv[0]) xmove_fd(xopen(argv[0], O_RDONLY), STDIN_FILENO); - world_readable_dump = (opts & OPT_x); - throttle_dd_creation = (opts & OPT_t); - unsigned errors = 0; GList *oops_list = NULL; scan_syslog_file(&oops_list, STDIN_FILENO); - int oops_cnt = g_list_length(oops_list); - if (oops_cnt != 0) + unsigned errors = 0; + if (opts & OPT_u) { - log("Found oopses: %d", oops_cnt); - if (opts & OPT_o) - { - int i = 0; - while (i < oops_cnt) - { - char *kernel_bt = (char*)g_list_nth_data(oops_list, i++); - char *tainted_short = kernel_tainted_short(kernel_bt); - if (tainted_short) - log("Kernel is tainted '%s'", tainted_short); - - free(tainted_short); - printf("\nVersion: %s", kernel_bt); - } - } - if (opts & (OPT_d|OPT_D)) - { - if (opts & OPT_D) - { - load_abrt_conf(); - debug_dumps_dir = g_settings_dump_location; - } - - log("Creating problem directories"); - errors = create_oops_dump_dirs(oops_list, oops_cnt); - if (errors) - log("%d errors while dumping oopses", errors); - /* - * This marker in syslog file prevents us from - * re-parsing old oopses. The only problem is that we - * can't be sure here that the file we are watching - * is the same file where syslog(xxx) stuff ends up. - */ - syslog(LOG_WARNING, - "Reported %u kernel oopses to Abrt", - oops_cnt - ); - } - if (opts & OPT_u) + log("Updating problem directory"); + switch (g_list_length(oops_list)) { - log("Updating problem directory"); - switch (oops_cnt) - { - case 1: + case 1: + { + struct dump_dir *dd = dd_opendir(problem_dir, /*open for writing*/0); + if (dd) { - struct dump_dir *dd = dd_opendir(problem_dir, /*open for writing*/0); - if (dd) - { - save_oops_data_in_dump_dir(dd, (char *)oops_list->data, /*no proc modules*/NULL); - dd_close(dd); - } + abrt_oops_save_data_in_dump_dir(dd, (char *)oops_list->data, /*no proc modules*/NULL); + dd_close(dd); } - break; - default: - error_msg(_("Can't update the problem: more than one oops found")); - break; - } + } + break; + default: + error_msg(_("Can't update the problem: more than one oops found")); + errors = 1; + break; } } + else + errors = abrt_oops_process_list(oops_list, dump_location, oops_utils_flags); + list_free_with_free(oops_list); //oops_list = NULL; - /* If we are run by a log watcher, this delays log rescan - * (because log watcher waits to us to terminate) - * and possibly prevents dreaded "abrt storm". - */ - int unreported_cnt = oops_cnt - MAX_DUMPED_DD_COUNT; - if (unreported_cnt > 0 && throttle_dd_creation) - { - /* Quadratic throttle time growth, but careful to not overflow in "n*n" */ - int n = unreported_cnt > 30 ? 30 : unreported_cnt; - n = n * n; - if (n > 9) - log(_("Sleeping for %d seconds"), n); - sleep(n); /* max 15 mins */ - } - return errors; } diff --git a/src/plugins/abrt-journal.c b/src/plugins/abrt-journal.c new file mode 100644 index 0000000..472357d --- /dev/null +++ b/src/plugins/abrt-journal.c @@ -0,0 +1,295 @@ +/* + * Copyright (C) 2014 ABRT team + * Copyright (C) 2014 RedHat Inc + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include +#include +#include +#include +#include +#include + +#include "abrt-journal.h" + +#include + + +struct abrt_journal +{ + sd_journal *j; +}; + +int abrt_journal_new(abrt_journal_t **journal) +{ + sd_journal *j; + const int r = sd_journal_open(&j, SD_JOURNAL_LOCAL_ONLY); + if (r < 0) + { + log_notice("Failed to open journal: %s", strerror(-r)); + return r; + } + + *journal = xzalloc(sizeof(**journal)); + (*journal)->j = j; + + return 0; +} + +void abrt_journal_free(abrt_journal_t *journal) +{ + sd_journal_close(journal->j); + journal->j = (void *)0xDEADBEAF; + + free(journal); +} + +int abrt_journal_set_journal_filter(abrt_journal_t *journal, const char *const *journal_filter_list) +{ + const char *const *cursor = journal_filter_list; + + while (*cursor) + { + const int r = sd_journal_add_match(journal->j, *cursor, strlen(*cursor)); + if (r < 0) + { + log_notice("Failed to set journal filter: %s", strerror(-r)); + return r; + } + + ++cursor; + } + + return 0; +} + +int abrt_journal_get_field(abrt_journal_t *journal, const char *field, const void **value, size_t *value_len) +{ + const int r = sd_journal_get_data(journal->j, field, value, value_len); + if (r < 0) + { + log_notice("Failed to read '%s' field: %s", field, strerror(-r)); + return r; + } + + return 0; +} + +int abrt_journal_get_string_field(abrt_journal_t *journal, const char *field, const char **value) +{ + size_t value_len; + const int r = abrt_journal_get_field(journal, field, (const void **)value, &value_len); + if (r < 0) + { + return r; + } + + const size_t pfx_len = strlen(field) + 1; + if (value_len < pfx_len) + { + error_msg("Invalid data format from journal: field data are not prefixed with field name"); + return -EBADMSG; + } + + *value += pfx_len; + return 0; +} + +int abrt_journal_get_log_line(abrt_journal_t *journal, const char **line) +{ + const int r = abrt_journal_get_string_field(journal, "MESSAGE", line); + if (r < 0) + log_notice("Cannot read journal data. Exiting"); + + return r; +} + +int abrt_journal_get_cursor(abrt_journal_t *journal, char **cursor) +{ + const int r = sd_journal_get_cursor(journal->j, cursor); + + if (r < 0) + { + log_notice("Could not get journal cursor: '%s'", strerror(-r)); + return r; + } + + return 0; +} + +int abrt_journal_set_cursor(abrt_journal_t *journal, const char *cursor) +{ + const int r = sd_journal_seek_cursor(journal->j, cursor); + if (r < 0) + { + log_notice("Failed to seek journal to cursor '%s': %s\n", cursor, strerror(-r)); + return r; + } + + return 0; +} + +int abrt_journal_seek_tail(abrt_journal_t *journal) +{ + const int r = sd_journal_seek_tail(journal->j); + if (r < 0) + { + log_notice("Failed to seek journal to the end: %s\n", strerror(-r)); + return r; + } + + /* BUG: https://bugzilla.redhat.com/show_bug.cgi?id=979487 */ + sd_journal_previous_skip(journal->j, 1); + return 0; +} + +int abrt_journal_next(abrt_journal_t *journal) +{ + const int r = sd_journal_next(journal->j); + if (r < 0) + log_notice("Failed to iterate to next entry: %s", strerror(-r)); + return r; +} + +/* + * ABRT systemd-journal wrapper end + */ + +static volatile int s_loop_terminated; +void signal_loop_to_terminate(int signum) +{ + signum = signum; + s_loop_terminated = 1; +} + +enum abrt_journal_watch_state +{ + ABRT_JOURNAL_WATCH_READY, + ABRT_JOURNAL_WATCH_STOPPED, +}; + +struct abrt_journal_watch +{ + abrt_journal_t *j; + int state; + + abrt_journal_watch_callback callback; + void *callback_data; +}; + +int abrt_journal_watch_new(abrt_journal_watch_t **watch, abrt_journal_t *journal, abrt_journal_watch_callback callback, void *callback_data) +{ + assert(callback != NULL || !"ABRT watch needs valid callback ptr"); + + *watch = xzalloc(sizeof(**watch)); + (*watch)->j = journal; + (*watch)->callback = callback; + (*watch)->callback_data = callback_data; + + return 0; +} + +void abrt_journal_watch_free(abrt_journal_watch_t *watch) +{ + watch->j = (void *)0xDEADBEAF; + free(watch); +} + +abrt_journal_t *abrt_journal_watch_get_journal(abrt_journal_watch_t *watch) +{ + return watch->j; +} + +int abrt_journal_watch_run_sync(abrt_journal_watch_t *watch) +{ + sigset_t mask; + sigfillset(&mask); + + /* Exit gracefully: */ + /* services usually exit on SIGTERM and SIGHUP */ + sigdelset(&mask, SIGTERM); + signal(SIGTERM, signal_loop_to_terminate); + sigdelset(&mask, SIGHUP); + signal(SIGHUP, signal_loop_to_terminate); + /* Ctrl-C for easier debugging */ + sigdelset(&mask, SIGINT); + signal(SIGINT, signal_loop_to_terminate); + + /* Die on kill $PID */ + sigdelset(&mask, SIGKILL); + + struct pollfd pollfd; + pollfd.fd = sd_journal_get_fd(watch->j->j); + pollfd.events = sd_journal_get_events(watch->j->j); + + int r = 0; + + while (!s_loop_terminated && watch->state == ABRT_JOURNAL_WATCH_READY) + { + r = sd_journal_next(watch->j->j); + if (r < 0) + { + log_warning("Failed to iterate to next entry: %s", strerror(-r)); + break; + } + else if (r == 0) + { + ppoll(&pollfd, 1, NULL, &mask); + r = sd_journal_process(watch->j->j); + if (r < 0) + { + log_warning("Failed to get journal changes: %s\n", strerror(-r)); + break; + } + continue; + } + + watch->callback(watch, watch->callback_data); + } + + return r; +} + +void abrt_journal_watch_stop(abrt_journal_watch_t *watch) +{ + watch->state = ABRT_JOURNAL_WATCH_STOPPED; +} + +/* + * ABRT systemd-journal watch - end + */ + +void abrt_journal_watch_notify_strings(abrt_journal_watch_t *watch, void *data) +{ + struct abrt_journal_watch_notify_strings *conf = (struct abrt_journal_watch_notify_strings *)data; + + const char *message = NULL; + + if (abrt_journal_get_string_field(abrt_journal_watch_get_journal(watch), "MESSAGE", &message) < 0) + error_msg_and_die("Cannot read journal data."); + + GList *cur = conf->strings; + while (cur) + { + if (strstr(message, cur->data) != NULL) + break; + + cur = g_list_next(cur); + } + + if (cur) + conf->decorated_cb(watch, conf->decorated_cb_data); +} + +/* + * ABRT systemd-journal strings notifier - end + */ diff --git a/src/plugins/abrt-journal.h b/src/plugins/abrt-journal.h new file mode 100644 index 0000000..219cf60 --- /dev/null +++ b/src/plugins/abrt-journal.h @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2014 ABRT team + * Copyright (C) 2014 RedHat Inc + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#ifndef _ABRT_JOURNAL_H_ +#define _ABRT_JOURNAL_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * A systemd-journal wrapper + * (isolates systemd API in a single compile unit) + */ +struct abrt_journal; +typedef struct abrt_journal abrt_journal_t; + +int abrt_journal_new(abrt_journal_t **journal); + +void abrt_journal_free(abrt_journal_t *journal); + +int abrt_journal_set_journal_filter(abrt_journal_t *journal, + const char *const *journal_filter_list); + +int abrt_journal_get_field(abrt_journal_t *journal, + const char *field, + const void **value, + size_t *value_len); + +int abrt_journal_get_string_field(abrt_journal_t *journal, + const char *field, + const char **value); + +int abrt_journal_get_log_line(abrt_journal_t *journal, const char **line); + +int abrt_journal_get_cursor(abrt_journal_t *journal, char **cursor); + +int abrt_journal_set_cursor(abrt_journal_t *journal, const char *cursor); + +int abrt_journal_seek_tail(abrt_journal_t *journal); + +int abrt_journal_next(abrt_journal_t *journal); + +/* + * A systemd-journal listener which waits for new messages a loop and notifies + * them via a call back + */ +struct abrt_journal_watch; +typedef struct abrt_journal_watch abrt_journal_watch_t; + +typedef void (* abrt_journal_watch_callback)(struct abrt_journal_watch *watch, + void *data); + +int abrt_journal_watch_new(abrt_journal_watch_t **watch, + abrt_journal_t *journal, + abrt_journal_watch_callback callback, + void *callback_data); + +void abrt_journal_watch_free(abrt_journal_watch_t *watch); + +/* + * Returns the watched journal. + */ +abrt_journal_t *abrt_journal_watch_get_journal(abrt_journal_watch_t *watch); + +/* + * Starts reading journal messages and waiting for new messages in a loop. + * + * SIGTERM and SIGINT terminates the loop gracefully. + */ +int abrt_journal_watch_run_sync(abrt_journal_watch_t *watch); + +/* + * Can be used to terminate the loop in abrt_journal_watch_run_sync() + */ +void abrt_journal_watch_stop(abrt_journal_watch_t *watch); + + +/* + * A decorator for abrt_journal_watch call backs which calls the decorated call + * back in case where journal message contains a string from the interested + * list. + */ +struct abrt_journal_watch_notify_strings +{ + abrt_journal_watch_callback decorated_cb; + void *decorated_cb_data; + GList *strings; +}; + +void abrt_journal_watch_notify_strings(abrt_journal_watch_t *watch, void *data); + +#ifdef __cplusplus +} +#endif + +#endif /*_ABRT_JOURNAL_H_*/ diff --git a/src/plugins/oops-utils.c b/src/plugins/oops-utils.c new file mode 100644 index 0000000..9e2355e --- /dev/null +++ b/src/plugins/oops-utils.c @@ -0,0 +1,279 @@ +/* + * Copyright (C) 2014 ABRT team + * Copyright (C) 2014 RedHat Inc + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include "oops-utils.h" +#include "libabrt.h" + +int abrt_oops_process_list(GList *oops_list, const char *dump_location, int flags) +{ + unsigned errors = 0; + + int oops_cnt = g_list_length(oops_list); + if (oops_cnt != 0) + { + log("Found oopses: %d", oops_cnt); + if ((flags & ABRT_OOPS_PRINT_STDOUT)) + { + int i = 0; + while (i < oops_cnt) + { + char *kernel_bt = (char*)g_list_nth_data(oops_list, i++); + char *tainted_short = kernel_tainted_short(kernel_bt); + if (tainted_short) + log("Kernel is tainted '%s'", tainted_short); + + free(tainted_short); + printf("\nVersion: %s", kernel_bt); + } + } + if (dump_location != NULL) + { + log("Creating problem directories"); + errors = abrt_oops_create_dump_dirs(oops_list, dump_location, flags); + if (errors) + log("%d errors while dumping oopses", errors); + /* + * This marker in syslog file prevents us from + * re-parsing old oopses. The only problem is that we + * can't be sure here that the file we are watching + * is the same file where syslog(xxx) stuff ends up. + */ + syslog(LOG_WARNING, + "Reported %u kernel oopses to Abrt", + oops_cnt + ); + } + } + + /* If we are run by a log watcher, this delays log rescan + * (because log watcher waits to us to terminate) + * and possibly prevents dreaded "abrt storm". + */ + int unreported_cnt = oops_cnt - ABRT_OOPS_MAX_DUMPED_COUNT; + if (g_abrt_oops_sleep_woke_up_on_signal <= 0 && + (unreported_cnt > 0 && (flags & ABRT_OOPS_THROTTLE_CREATION))) + { + /* Quadratic throttle time growth, but careful to not overflow in "n*n" */ + int n = unreported_cnt > 30 ? 30 : unreported_cnt; + n = n * n; + if (n > 9) + log(_("Sleeping for %d seconds"), n); + abrt_oops_signaled_sleep(n); /* max 15 mins */ + } + + return errors; +} + +/* returns number of errors */ +unsigned abrt_oops_create_dump_dirs(GList *oops_list, const char *dump_location, int flags) +{ + const int oops_cnt = g_list_length(oops_list); + unsigned countdown = ABRT_OOPS_MAX_DUMPED_COUNT; /* do not report hundreds of oopses */ + + log_notice("Saving %u oopses as problem dirs", oops_cnt >= countdown ? countdown : oops_cnt); + + char *cmdline_str = xmalloc_fopen_fgetline_fclose("/proc/cmdline"); + char *fips_enabled = xmalloc_fopen_fgetline_fclose("/proc/sys/crypto/fips_enabled"); + char *proc_modules = xmalloc_open_read_close("/proc/modules", /*maxsize:*/ NULL); + char *suspend_stats = xmalloc_open_read_close("/sys/kernel/debug/suspend_stats", /*maxsize:*/ NULL); + + time_t t = time(NULL); + const char *iso_date = iso_date_string(&t); + /* dump should be readable by all if we're run with -x */ + uid_t my_euid = (uid_t)-1L; + mode_t mode = DEFAULT_DUMP_DIR_MODE | S_IROTH; + /* and readable only for the owner otherwise */ + if (!(flags & ABRT_OOPS_WORLD_READABLE)) + { + mode = DEFAULT_DUMP_DIR_MODE; + my_euid = geteuid(); + } + + pid_t my_pid = getpid(); + unsigned idx = 0; + unsigned errors = 0; + while (idx < oops_cnt) + { + char base[sizeof("oops-YYYY-MM-DD-hh:mm:ss-%lu-%lu") + 2 * sizeof(long)*3]; + sprintf(base, "oops-%s-%lu-%lu", iso_date, (long)my_pid, (long)idx); + char *path = concat_path_file(dump_location, base); + + struct dump_dir *dd = dd_create(path, /*uid:*/ my_euid, mode); + if (dd) + { + dd_create_basic_files(dd, /*uid:*/ my_euid, NULL); + abrt_oops_save_data_in_dump_dir(dd, (char*)g_list_nth_data(oops_list, idx++), proc_modules); + dd_save_text(dd, FILENAME_ABRT_VERSION, VERSION); + dd_save_text(dd, FILENAME_ANALYZER, "Kerneloops"); + dd_save_text(dd, FILENAME_TYPE, "Kerneloops"); + if (cmdline_str) + dd_save_text(dd, FILENAME_CMDLINE, cmdline_str); + if (proc_modules) + dd_save_text(dd, "proc_modules", proc_modules); + if (fips_enabled && strcmp(fips_enabled, "0") != 0) + dd_save_text(dd, "fips_enabled", fips_enabled); + if (suspend_stats) + dd_save_text(dd, "suspend_stats", suspend_stats); + dd_close(dd); + notify_new_path(path); + } + else + errors++; + + free(path); + + if (--countdown == 0) + break; + + if (dd && (flags & ABRT_OOPS_THROTTLE_CREATION)) + if (abrt_oops_signaled_sleep(1) > 0) + break; + } + + free(cmdline_str); + free(proc_modules); + free(fips_enabled); + free(suspend_stats); + + return errors; +} + +static char *abrt_oops_list_of_tainted_modules(const char *proc_modules) +{ + struct strbuf *result = strbuf_new(); + + const char *p = proc_modules; + for (;;) + { + const char *end = strchrnul(p, '\n'); + const char *paren = strchrnul(p, '('); + /* We look for a line with this format: + * "kvm_intel 126289 0 - Live 0xf829e000 (taint_flags)" + * where taint_flags have letters + * (flags '+' and '-' indicate (un)loading, we must ignore them). + */ + while (++paren < end) + { + if ((unsigned)(toupper(*paren) - 'A') <= 'Z'-'A') + { + strbuf_append_strf(result, result->len == 0 ? "%.*s" : ",%.*s", + (int)(strchrnul(p,' ') - p), p + ); + break; + } + if (*paren == ')') + break; + } + + if (*end == '\0') + break; + p = end + 1; + } + + if (result->len == 0) + { + strbuf_free(result); + return NULL; + } + return strbuf_free_nobuf(result); +} + +void abrt_oops_save_data_in_dump_dir(struct dump_dir *dd, char *oops, const char *proc_modules) +{ + char *first_line = oops; + char *second_line = (char*)strchr(first_line, '\n'); /* never NULL */ + *second_line++ = '\0'; + + if (first_line[0]) + dd_save_text(dd, FILENAME_KERNEL, first_line); + dd_save_text(dd, FILENAME_BACKTRACE, second_line); + + /* check if trace doesn't have line: 'Your BIOS is broken' */ + if (strstr(second_line, "Your BIOS is broken")) + dd_save_text(dd, FILENAME_NOT_REPORTABLE, + _("A kernel problem occurred because of broken BIOS. " + "Unfortunately, such problems are not fixable by kernel maintainers.")); + /* check if trace doesn't have line: 'Your hardware is unsupported' */ + else if (strstr(second_line, "Your hardware is unsupported")) + dd_save_text(dd, FILENAME_NOT_REPORTABLE, + _("A kernel problem occurred, but your hardware is unsupported, " + "therefore kernel maintainers are unable to fix this problem.")); + else + { + char *tainted_short = kernel_tainted_short(second_line); + if (tainted_short) + { + log_notice("Kernel is tainted '%s'", tainted_short); + dd_save_text(dd, FILENAME_TAINTED_SHORT, tainted_short); + + char *tnt_long = kernel_tainted_long(tainted_short); + dd_save_text(dd, FILENAME_TAINTED_LONG, tnt_long); + free(tnt_long); + + struct strbuf *reason = strbuf_new(); + const char *fmt = _("A kernel problem occurred, but your kernel has been " + "tainted (flags:%s). Kernel maintainers are unable to " + "diagnose tainted reports."); + strbuf_append_strf(reason, fmt, tainted_short); + + char *modlist = !proc_modules ? NULL : abrt_oops_list_of_tainted_modules(proc_modules); + if (modlist) + { + strbuf_append_strf(reason, _(" Tainted modules: %s."), modlist); + free(modlist); + } + + dd_save_text(dd, FILENAME_NOT_REPORTABLE, reason->buf); + strbuf_free(reason); + free(tainted_short); + } + } + + // TODO: add "Kernel oops: " prefix, so that all oopses have recognizable FILENAME_REASON? + // kernel oops 1st line may look quite puzzling otherwise... + strchrnul(second_line, '\n')[0] = '\0'; + dd_save_text(dd, FILENAME_REASON, second_line); +} + +int abrt_oops_signaled_sleep(int seconds) +{ + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGTERM); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGHUP); + + struct timespec timeout; + timeout.tv_sec = seconds; + timeout.tv_nsec = 0; + + return g_abrt_oops_sleep_woke_up_on_signal = sigtimedwait(&set, NULL, &timeout); +} + +char *abrt_oops_string_filter_regex(void) +{ + map_string_t *settings = new_map_string(); + + load_abrt_plugin_conf_file("oops.conf", settings); + + int only_fatal_mce = 1; + try_get_map_string_item_as_bool(settings, "OnlyFatalMCE", &only_fatal_mce); + + free_map_string(settings); + + if (only_fatal_mce) + return xstrdup("^Machine .*$"); + + return NULL; +} diff --git a/src/plugins/oops-utils.h b/src/plugins/oops-utils.h new file mode 100644 index 0000000..947f652 --- /dev/null +++ b/src/plugins/oops-utils.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2014 ABRT team + * Copyright (C) 2014 RedHat Inc + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#ifndef _ABRT_OOPS_UTILS_H_ +#define _ABRT_OOPS_UTILS_H_ + +#include "libabrt.h" + +/* How many problem dirs to create at most? + * Also causes cooldown sleep with -t if exceeded - + * useful when called from a log watcher. + */ +#define ABRT_OOPS_MAX_DUMPED_COUNT 5 + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + ABRT_OOPS_THROTTLE_CREATION = 1 << 0, + ABRT_OOPS_WORLD_READABLE = 1 << 1, + ABRT_OOPS_PRINT_STDOUT = 1 << 2, +}; + +int g_abrt_oops_sleep_woke_up_on_signal; + +int abrt_oops_process_list(GList *oops_list, const char *dump_location, int flags); +unsigned abrt_oops_create_dump_dirs(GList *oops_list, const char *dump_location, int flags); +void abrt_oops_save_data_in_dump_dir(struct dump_dir *dd, char *oops, const char *proc_modules); +int abrt_oops_signaled_sleep(int seconds); +char *abrt_oops_string_filter_regex(void); + +#ifdef __cplusplus +} +#endif + +#endif /*_ABRT_OOPS_UTILS_H_*/ -- 1.9.3