/**
 * @file normalizer.c
 * @brief A small tool to normalize data.
 *
 * This is the most basic example demonstrating how to use liblognorm.
 * It loads log samples from the files specified on the command line,
 * reads to-be-normalized data from stdin and writes the normalized
 * form to stdout. Besides being an example, it also carries out useful
 * processing.
 *
 * @author Rainer Gerhards <rgerhards@adiscon.com>
 *
 *//*
 * liblognorm - a fast samples-based log normalization library
 * Copyright 2010-2016 by Rainer Gerhards and Adiscon GmbH.
 *
 * This file is part of liblognorm.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 *
 * A copy of the LGPL v2.1 can be found in the file "COPYING" in this distribution.
 */
#include "config.h"
#include <stdio.h>
#include <string.h>
#include <getopt.h>
#include <libestr.h>

#include "liblognorm.h"
#include "lognorm.h"
#include "enc.h"

/* we need to turn off this warning, as it also comes up in C99 mode, which
 * we use.
 */
#pragma GCC diagnostic ignored "-Wdeclaration-after-statement"

static ln_ctx ctx;

static int verbose = 0;
#define OUTPUT_PARSED_RECS 0x01
#define OUTPUT_UNPARSED_RECS 0x02
static int recOutput = OUTPUT_PARSED_RECS | OUTPUT_UNPARSED_RECS;
				/**< controls which records to output */
static int outputSummaryLine = 0;
static int outputNbrUnparsed = 0;
static int addErrLineNbr = 0;	/**< add line number info to unparsed events */
static int flatTags = 0;	/**< print event.tags in JSON? */
static FILE *fpDOT;
static es_str_t *encFmt = NULL; /**< a format string for encoder use */
static es_str_t *mandatoryTag = NULL; /**< tag which must be given so that mesg will
					   be output. NULL=all */
static enum { f_syslog, f_json, f_xml, f_csv, f_raw } outfmt = f_json;

static void
errCallBack(void __attribute__((unused)) *cookie, const char *msg,
	    size_t __attribute__((unused)) lenMsg)
{
	fprintf(stderr, "liblognorm error: %s\n", msg);
}

static void
dbgCallBack(void __attribute__((unused)) *cookie, const char *msg,
	    size_t __attribute__((unused)) lenMsg)
{
	fprintf(stderr, "liblognorm: %s\n", msg);
}

static void
complain(const char *errmsg)
{
	fprintf(stderr, "%s\n", errmsg);
}


/* rawmsg is, as the name says, the raw message, in case we have
 * "raw" formatter requested.
 */
static void
outputEvent(struct json_object *json, const char *const rawmsg)
{
	char *cstr = NULL;
	es_str_t *str = NULL;

	if(outfmt == f_raw) {
		printf("%s\n", rawmsg);
		return;
	}

	switch(outfmt) {
	case f_json:
		if(!flatTags) {
			json_object_object_del(json, "event.tags");
		}
		cstr = (char*)json_object_to_json_string(json);
		break;
	case f_syslog:
		ln_fmtEventToRFC5424(json, &str);
		break;
	case f_xml:
		ln_fmtEventToXML(json, &str);
		break;
	case f_csv:
		ln_fmtEventToCSV(json, &str, encFmt);
		break;
	case f_raw:
		fprintf(stderr, "program error: f_raw should not occur "
			"here (file %s, line %d)\n", __FILE__, __LINE__);
		abort();
		break;
	default:
		fprintf(stderr, "program error: default case should not occur "
			"here (file %s, line %d)\n", __FILE__, __LINE__);
		abort();
		break;
	}
	if (str != NULL)
		cstr = es_str2cstr(str, NULL);
	if(verbose > 0) fprintf(stderr, "normalized: '%s'\n", cstr);
	printf("%s\n", cstr);
	if (str != NULL)
		free(cstr);
	es_deleteStr(str);
}

/* test if the tag exists */
static int
eventHasTag(struct json_object *json, const char *tag)
{
	struct json_object *tagbucket, *tagObj;
	int i;
	const char *tagCstr;
	
	if (tag == NULL)
		return 1;
	if (json_object_object_get_ex(json, "event.tags", &tagbucket)) {
		if (json_object_get_type(tagbucket) == json_type_array) {
			for (i = json_object_array_length(tagbucket) - 1; i >= 0; i--) {
				tagObj = json_object_array_get_idx(tagbucket, i);
				tagCstr = json_object_get_string(tagObj);
				if (!strcmp(tag, tagCstr))
					return 1;
			}
		}
	}
	if (verbose > 1)
		printf("Mandatory tag '%s' has not been found\n", tag);
	return 0;
}

static void
amendLineNbr(json_object *const json, const int line_nbr)
{
	
	if(addErrLineNbr) {
		struct json_object *jval;
		jval = json_object_new_int(line_nbr);
		json_object_object_add(json, "lognormalizer.line_nbr", jval);
	}
}

#define DEFAULT_LINE_SIZE (10 * 1024)

static char *
read_line(FILE *fp)
{
	size_t line_capacity = DEFAULT_LINE_SIZE;
	char *line = NULL;
	size_t line_len = 0;
	int ch = 0;
	do {
		ch = fgetc(fp);
		if (ch == EOF) break;
		if (line == NULL) {
			line = malloc(line_capacity);
		} else if (line_len == line_capacity) {
			line_capacity *= 2;
			line = realloc(line, line_capacity);
		}
		if (line == NULL) {
			fprintf(stderr, "Couldn't allocate working-buffer for log-line\n");
			return NULL;
		}
		line[line_len++] = ch;
	} while(ch != '\n');

	if (line != NULL) {
		line[--line_len] = '\0';
		if(line_len > 0 && line[line_len - 1] == '\r')
			line[--line_len] = '\0';
	}
	return line;
}

/* normalize input data
 */
static void
normalize(void)
{
	FILE *fp = stdin;
	char *line = NULL;
	struct json_object *json = NULL;
	long long unsigned numParsed = 0;
	long long unsigned numUnparsed = 0;
	long long unsigned numWrongTag = 0;
	char *mandatoryTagCstr = NULL;
	int line_nbr = 0;	/* must be int to keep compatible with older json-c */

	if (mandatoryTag != NULL) {
		mandatoryTagCstr = es_str2cstr(mandatoryTag, NULL);
	}

	while((line = read_line(fp)) != NULL) {
		++line_nbr;
		if(verbose > 0) fprintf(stderr, "To normalize: '%s'\n", line);
		ln_normalize(ctx, line, strlen(line), &json);
		if(json != NULL) {
			if(eventHasTag(json, mandatoryTagCstr)) {
				struct json_object *dummy;
				const int parsed = !json_object_object_get_ex(json,
					"unparsed-data", &dummy);
				if(parsed) {
					numParsed++;
					if(recOutput & OUTPUT_PARSED_RECS) {
						outputEvent(json, line);
					}
				} else {
					numUnparsed++;
					amendLineNbr(json, line_nbr);
					if(recOutput & OUTPUT_UNPARSED_RECS) {
						outputEvent(json, line);
					}
				}
			} else {
				numWrongTag++;
			}
			json_object_put(json);
			json = NULL;
		}
	free(line);
	}
	if(outputNbrUnparsed && numUnparsed > 0)
		fprintf(stderr, "%llu unparsable entries\n", numUnparsed);
	if(numWrongTag > 0)
		fprintf(stderr, "%llu entries with wrong tag dropped\n", numWrongTag);
	if(outputSummaryLine) {
		fprintf(stderr, "%llu records processed, %llu parsed, %llu unparsed\n",
			numParsed+numUnparsed, numParsed, numUnparsed);
	}
	free(mandatoryTagCstr);
}


/**
 * Generate a command file for the GNU DOT tools.
 */
static void
genDOT(void)
{
	es_str_t *str;

	str = es_newStr(1024);
	ln_genDotPDAGGraph(ctx->pdag, &str);
	fwrite(es_getBufAddr(str), 1, es_strlen(str), fpDOT);
}

static
void printVersion(void)
{
	fprintf(stderr, "lognormalizer version: " VERSION "\n");
	fprintf(stderr, "liblognorm version: %s\n", ln_version());
	fprintf(stderr, "\tadvanced stats: %s\n",
		ln_hasAdvancedStats() ? "available" : "not available");
}

static void
handle_generic_option(const char* opt) {
	if (strcmp("allowRegex", opt) == 0) {
		ln_setCtxOpts(ctx, LN_CTXOPT_ALLOW_REGEX);
	} else if (strcmp("addExecPath", opt) == 0) {
		ln_setCtxOpts(ctx, LN_CTXOPT_ADD_EXEC_PATH);
	} else if (strcmp("addOriginalMsg", opt) == 0) {
		ln_setCtxOpts(ctx, LN_CTXOPT_ADD_ORIGINALMSG);
	} else if (strcmp("addRule", opt) == 0) {
		ln_setCtxOpts(ctx, LN_CTXOPT_ADD_RULE);
	} else if (strcmp("addRuleLocation", opt) == 0) {
		ln_setCtxOpts(ctx, LN_CTXOPT_ADD_RULE_LOCATION);
	} else {
		fprintf(stderr, "invalid -o option '%s'\n", opt);
		exit(1);
	}
}

static void usage(void)
{
fprintf(stderr,
	"Options:\n"
	"    -r<rulebase> Rulebase to use. This is required option\n"
	"    -H           print summary line (nbr of msgs Handled)\n"
	"    -U           print number of unparsed messages (only if non-zero)\n"
	"    -e<json|xml|csv|cee-syslog|raw>\n"
	"                 Change output format. By default, json is used\n"
	"                 Raw is exactly like the input. It is useful in combination\n"
	"                 with -p/-P options to extract known good/bad messages\n"
	"    -E<format>   Encoder-specific format (used for CSV, read docs)\n"
	"    -T           Include 'event.tags' in JSON format\n"
	"    -oallowRegex Allow regexp matching (read docs about performance penalty)\n"
	"    -oaddRule    Add a mockup of the matching rule.\n"
	"    -oaddRuleLocation Add location of matching rule to metadata\n"
	"    -oaddExecPath Add exec_path attribute to output\n"
	"    -oaddOriginalMsg Always add original message to output, not just in error case\n"
	"    -p           Print back only if the message has been parsed succesfully\n"
	"    -P           Print back only if the message has NOT been parsed succesfully\n"
	"    -L           Add source file line number information to unparsed line output\n"
	"    -t<tag>      Print back only messages matching the tag\n"
	"    -v           Print debug. When used 3 times, prints parse DAG\n"
	"    -V           Print version information\n"
	"    -d           Print DOT file to stdout and exit\n"
	"    -d<filename> Save DOT file to the filename\n"
	"    -s<filename> Print parse dag statistics and exit\n"
	"    -S<filename> Print extended parse dag statistics and exit (includes -s)\n"
	"    -x<filename> Print statistics as dot file (called only)\n"
	"\n"
	);
}

int main(int argc, char *argv[])
{
	int opt;
	char *repository = NULL;
	int usedRB = 0; /* 0=no rule; 1=rule from rulebase; 2=rule from string */
	int ret = 0;
	FILE *fpStats = NULL;
	FILE *fpStatsDOT = NULL;
	int extendedStats = 0;

	if((ctx = ln_initCtx()) == NULL) {
		complain("Could not initialize liblognorm context");
		ret = 1;
		goto exit;
	}

	while((opt = getopt(argc, argv, "d:s:S:e:r:R:E:vVpPt:To:hHULx:")) != -1) {
		switch (opt) {
		case 'V':
			printVersion();
			exit(1);
			break;
		case 'd': /* generate DOT file */
			if(!strcmp(optarg, "")) {
				fpDOT = stdout;
			} else {
				if((fpDOT = fopen(optarg, "w")) == NULL) {
					perror(optarg);
					complain("Cannot open DOT file");
					ret = 1;
					goto exit;
				}
			}
			break;
		case 'x': /* generate statistics DOT file */
			if(!strcmp(optarg, "")) {
				fpStatsDOT = stdout;
			} else {
				if((fpStatsDOT = fopen(optarg, "w")) == NULL) {
					perror(optarg);
					complain("Cannot open statistics DOT file");
					ret = 1;
					goto exit;
				}
			}
			break;
		case 'S': /* generate pdag statistic file */
			extendedStats = 1;
			/* INTENTIONALLY NO BREAK! - KEEP order! */
			/*FALLTHROUGH*/
		case 's': /* generate pdag statistic file */
			if(!strcmp(optarg, "-")) {
				fpStats = stdout;
			} else {
				if((fpStats = fopen(optarg, "w")) == NULL) {
					perror(optarg);
					complain("Cannot open parser statistics file");
					ret = 1;
					goto exit;
				}
			}
			break;
		case 'v':
			verbose++;
			break;
		case 'E': /* encoder-specific format string (will be validated by encoder) */
			encFmt = es_newStrFromCStr(optarg, strlen(optarg));
			break;
		case 'p':
			recOutput = OUTPUT_PARSED_RECS;
			break;
		case 'P':
			recOutput = OUTPUT_UNPARSED_RECS;
			break;
		case 'H':
			outputSummaryLine = 1;
			break;
		case 'U':
			outputNbrUnparsed = 1;
			break;
		case 'L':
			addErrLineNbr = 1;
			break;
		case 'T':
			flatTags = 1;
			break;
		case 'e': /* encoder to use */
			if(!strcmp(optarg, "json")) {
				outfmt = f_json;
			} else if(!strcmp(optarg, "xml")) {
				outfmt = f_xml;
			} else if(!strcmp(optarg, "cee-syslog")) {
				outfmt = f_syslog;
			} else if(!strcmp(optarg, "csv")) {
				outfmt = f_csv;
			} else if(!strcmp(optarg, "raw")) {
				outfmt = f_raw;
			}
			break;
		case 'r': /* rule base to use */
			if(usedRB != 2) {
				repository = optarg;
				usedRB = 1;
			} else {
				usedRB = -1;
			}
			break;
		case 'R':
			if(usedRB != 1) {
				repository = optarg;
				usedRB = 2;
			} else {
				usedRB = -1;
			}
			break;
		case 't': /* if given, only messages tagged with the argument
			     are output */
			mandatoryTag = es_newStrFromCStr(optarg, strlen(optarg));
			break;
		case 'o':
			handle_generic_option(optarg);
			break;
		case 'h':
		default:
			usage();
			ret = 1;
			goto exit;
			break;
		}
	}

	if(repository == NULL) {
		complain("Samples repository or String must be given (-r or -R)");
		ret = 1;
		goto exit;
	}

	if(usedRB == -1) {
		complain("Only use one rulebase (-r or -R)");
		ret = 1;
		goto exit;
	}

	ln_setErrMsgCB(ctx, errCallBack, NULL);
	if(verbose) {
		ln_setDebugCB(ctx, dbgCallBack, NULL);
		ln_enableDebug(ctx, 1);
	}

	if(usedRB == 1) {
		if(ln_loadSamples(ctx, repository)) {
			fprintf(stderr, "fatal error: cannot load rulebase\n");
			exit(1);
		}
	} else if(usedRB == 2) {
		if(ln_loadSamplesFromString(ctx, repository)) {
			fprintf(stderr, "fatal error: cannot load rule from String\n");
			exit(1);
		}
	}

	if(verbose > 0)
		fprintf(stderr, "number of tree nodes: %d\n", ctx->nNodes);

	if(fpDOT != NULL) {
		genDOT();
		ret=1;
		goto exit;
	}

	if(verbose > 2) ln_displayPDAG(ctx);

	normalize();

	if(fpStats != NULL) {
		ln_fullPdagStats(ctx, fpStats, extendedStats);
	}

	if(fpStatsDOT != NULL) {
		ln_fullPDagStatsDOT(ctx, fpStatsDOT);
	}

exit:
	if (ctx) ln_exitCtx(ctx);
	if (encFmt != NULL)
		free(encFmt);
	return ret;
}