/**
 * \file lexer.c
 * bogofilter's lexical analyzer (control routines)
 *
 * \date 2003-01-01 split out of lexer.l
 */
 
#include "common.h"
 
#include <ctype.h>
#include <stdlib.h>
#include <assert.h>
 
#include "base64.h"
#include "bogoconfig.h"
#include "bogoreader.h"
#include "charset.h"
#include "error.h"
#ifndef	DISABLE_UNICODE
#include "convert_unicode.h"
#include "iconvert.h"
#endif
#include "lexer.h"
#include "memstr.h"
#include "mime.h"
#include "msgcounts.h"
#include "qp.h"
#include "textblock.h"
#include "token.h"
#include "word.h"
#include "xmalloc.h"
 
/* Global Variables */
 
extern int yylineno;
 
bool msg_header = true;
bool have_body  = false;
lexer_t *lexer = NULL;
 
/* Local Variables */
 
static lexer_t v3_lexer = {
    yylex,
    lexer_v3_get_token
};
 
lexer_t msg_count_lexer = {
    read_msg_count_line,
    msg_count_get_token
};
 
/* Function Prototypes */
 
static int yy_get_new_line(buff_t *buff);
static int get_decoded_line(buff_t *buff);
static int skip_folded_line(buff_t *buff);
 
/* Function Definitions */
 
void lexer_init(void)
{
    mime_reset();
    token_init();
    lexer_v3_init(NULL);
    init_charset_table(charset_default);
}
 
static void lexer_display_buffer(buff_t *buff)
{
    fprintf(dbgout, "*** %2d %c%c %2ld ",
	    yylineno-1, msg_header ? 'h' : 'b', yy_get_state(),
	    (long)(buff->t.leng - buff->read));
    buff_puts(buff, 0, dbgout);
    if (buff->t.leng > 0 && buff->t.u.text[buff->t.leng-1] != '\n')
	fputc('\n', dbgout);
}
 
/**
 * Check for lines wholly composed of printable characters as they can
 * cause a scanner abort "input buffer overflow, can't enlarge buffer
 * because scanner uses REJECT"
 *
 * \bug this function must go, we need to fix the lexer
 */
static bool long_token(byte *buf, uint count)
{
    uint i;
    for (i=0; i < count; i += 1) {
	byte c = buf[i];
	/* 10/23/05 - fix SIGSEGV with msg.1023.6479.txt
	** evidently caused by 09/07/05 patch for 0.96.2
	*/
	if (c == '\0')
	    break;
	if ((iscntrl(c) || isspace(c) || ispunct(c)) && (c != '_'))
	    return false;
    }
    return true;
}
 
static int yy_get_new_line(buff_t *buff)
{
    int count = (*reader_getline)(buff);
    const byte *buf = buff->t.u.text;
 
    static size_t hdrlen = 0;
    if (hdrlen==0)
	hdrlen=strlen(spam_header_name);
 
    if (count > 0)
	yylineno += 1;
 
    if (count == EOF) {
	if (fpin == NULL || !ferror(fpin)) {
	    return YY_NULL;
	}
	else {
	    print_error(__FILE__, __LINE__, "input in flex scanner failed\n");
	    exit(EX_ERROR);
	}
    }
 
    /* Mime header check needs to be performed on raw input
    ** -- before mime decoding.  Without it, flex aborts:
    ** "fatal flex scanner internal error--end of buffer missed" */
 
    if (buff->t.leng > 2 &&
	buf[0] == '-' && buf[1] == '-' &&
	got_mime_boundary(&buff->t)) {
	yy_set_state_initial();
    }
 
    if (count >= 0 && DEBUG_LEXER(0))
	lexer_display_buffer(buff);
 
    /* skip spam_header ("X-Bogosity:") lines */
    while (msg_header
	   && count != EOF
/* don't skip if inside message/rfc822 */
	   && msg_state->parent == NULL
	   && buff->t.leng >= hdrlen
	   && memcmp(buff->t.u.text,spam_header_name,hdrlen) == 0) {
	count = skip_folded_line(buff);
    }
 
    return count;
}
 
static int get_decoded_line(buff_t *buff)
{
    int count;
    buff_t *linebuff;
    /* since msg_state might change during calls */
    bool mime_dont_decode = msg_state->mime_dont_decode;
 
#ifdef	DISABLE_UNICODE
    linebuff = buff;
#else
    if (encoding == E_RAW ||
	mime_dont_decode ) {
	linebuff = buff;
    }
    else {
	static buff_t *tempbuff = NULL;
 
	if (tempbuff == NULL)
	    tempbuff = (buff_t *) calloc(sizeof(buff_t), 1);
 
	/* UTF-8 uses up to six octets per character.  Make input buffer
	 * sufficiently small that the UTF-8 text can fit in the output
	 * buffer */
	if (tempbuff->size < buff->size / 6) {
	    xfree(tempbuff->t.u.text);
	    tempbuff->size = buff->size / 6;
	    tempbuff->t.u.text = (byte *) xmalloc(tempbuff->size+D);
	}
 
	tempbuff->t.leng = tempbuff->read = 0;
	linebuff = tempbuff;
    }
#endif
 
    /* note that this call might invoke got_mimeboundary() thus
     * changing the global msg_state variable */
    count = yy_get_new_line(linebuff);
 
    if (count == EOF) {
	if ( !ferror(fpin))
	    return YY_NULL;
	else {
	    print_error(__FILE__, __LINE__, "input in flex scanner failed\n");
	    exit(EX_ERROR);
	}
    }
 
    /* Save the text on a linked list of lines.
     * Note that we store fixed-length blocks here, not lines.
     * One very long physical line could break up into more
     * than one of these. */
 
    if (passthrough && count > 0)
	textblock_add(linebuff->t.u.text+linebuff->read, (size_t) count);
 
    if ( !msg_header && 
	 !mime_dont_decode &&
	 msg_state->mime_type != MIME_TYPE_UNKNOWN)
    {
	word_t temp;
	uint decoded_count;
 
	temp.leng = (uint) count;
	temp.u.text = linebuff->t.u.text+linebuff->read;
 
	decoded_count = mime_decode(&temp);
	/*change buffer size only if the decoding worked */
	if (decoded_count != 0 && decoded_count < (uint) count) {
	    linebuff->t.leng -= (uint) (count - decoded_count);
	    count = (int) decoded_count;
	    if (DEBUG_LEXER(1))
		lexer_display_buffer(linebuff);
	}
    }
 
#ifndef	DISABLE_UNICODE
    if (encoding == E_UNICODE &&
	!mime_dont_decode &&
        count > 0)
    {
	iconvert(linebuff, buff);
 
	/* If we return count = 0 here, the caller will think we have
	 * no more bytes left to read, even though before the iconvert
	 * call we had a positive number of bytes. This *will* lead to
	 * a message truncation which we try to avoid by simply
	 * returning another in-band error code. */
	if (buff->t.leng == 0) {
	    count = -2;
        } else {
	    /* iconvert, treating multi-byte sequences, can shrink or enlarge
	     * the output compared to its input.  Correct count. */
	    count = buff->t.leng;
	}
    }
#endif
 
#ifdef EXCESSIVE_DEBUG
    /* debug */
    fprintf(dbgout, "%d: ", count);
    buff_puts(buff, 0, dbgout);
    fprintf(dbgout, "\n");
#endif
 
    /* CRLF -> NL */
    if (count >= 2) {
	byte *buf = buff->t.u.text;
	if (memcmp(buf + count - 2, CRLF, 2) == 0) {
	    count --;
	    --buff->t.leng;
	    *(buf + count - 1) = (byte) '\n';
	}
    }
 
    if (buff->t.leng < buff->size)     /* for easier debugging - removable */
	Z(buff->t.u.text[buff->t.leng]);  /* for easier debugging - removable */
 
    return count;
}
 
static int skip_folded_line(buff_t *buff)
{
    for (;;) {
	int count;
	buff->t.leng = 0;
	count = reader_getline(buff);
	yylineno += 1;
	/* only check for LWSP-char (RFC-822) aka. WSP (RFC-2822),
	 * these only include SP and HTAB */
	if (buff->t.u.text[0] != ' ' &&
	    buff->t.u.text[0] != '\t')
	    return count;
	/* Check for empty line which terminates message header */
	if (is_eol((char *)buff->t.u.text, count))
	    return count;
    }
}
 
void yyinit(void)
{
    yylineno = 0;
 
    if ( !msg_count_file)
	lexer = &v3_lexer;
}
 
int yyinput(byte *buf, size_t used, size_t size)
/* input getter for the scanner */
{
    int cnt;
    int count = 0;
    buff_t buff;
 
    buff_init(&buff, buf, 0, (uint) size);
 
    /* After reading a line of text, check if it has special characters.
     * If not, trim some, but leave enough to match a max length token.
     * Then read more text.  This will ensure that a really long sequence
     * of alphanumerics, which bogofilter will ignore anyway, doesn't crash
     * the flex lexer.
     */
 
    while ((cnt = get_decoded_line(&buff)) != 0) {
        if (cnt > 0)
            count = buff.t.leng;
 
	/* Note: some malformed messages can cause xfgetsl() to report
	** "Invalid buffer size, exiting."  and then abort.  This
	** can happen when the parser is in html mode and there's a
	** leading '<' but no closing '>'.
	**
	** The "fix" is to check for a nearly full lexer buffer and
	** discard most of it.
	*/
 
	/* if not nearly full */
	if (used < 1000 || used < size * 10)
	    break;
 
	if (count >= MAX_TOKEN_LEN * 2 && 
	    long_token(buff.t.u.text, (uint) count)) {
	    /* Make sure not to shift bytes outside the buffer */
	    if (buff.t.leng >= (uint) count) {
		    uint start = buff.t.leng - count;
		    uint length = count - max_token_len;
		    buff_shift(&buff, start, length);
	    }
	    count = buff.t.leng;
	}
	else
	    break;
    }
 
    if (msg_state &&
	msg_state->mime_dont_decode &&
	(msg_state->mime_disposition != MIME_DISPOSITION_UNKNOWN)) {
        assert(size <= INT_MAX && count <= (int)size);
	return (count == EOF ? 0 : count);   /* not decode at all */
    }
 
#if	defined(CP866) && !defined(ENABLE_ICONV)
    /* EK -  decoding things like &#1084 and charset_table */
    count = decode_and_htmlUNICODE_to_cp866(buf, count);
#endif
 
    if (replace_nonascii_characters) {
	/* do non-ascii replacement */
	int i;
	for (i = 0; i < count; i++ )
	{
	    byte ch = buf[i];
	    buf[i] = charset_table[ch];
	}
    }
 
    if (DEBUG_LEXER(2))
	fprintf(dbgout, "*** yyinput(\"%-.*s\", %lu, %lu) = %d\n", count, buf, (unsigned long)used, (unsigned long)size, count);
 
    assert(size <= INT_MAX && count <= (int)size);
    return (count == EOF ? 0 : count);
}
 
static char *charset_as_string(const byte *txt, const size_t len)
{
    static char *charset_text = NULL;
    static unsigned short charset_leng = 0;
 
    if (charset_text == NULL)
	charset_text = (char *)xmalloc(len+D);
    else {
	if (charset_leng < len) {
	    charset_leng = len;
	    charset_text = (char *)xrealloc(charset_text, charset_leng+D);
	}
    }
 
    memcpy(charset_text, txt, len);
    Z(charset_text[len]);			/* for easier debugging - removable */
 
    return charset_text;
}
 
word_t *text_decode(word_t *w)
{
    word_t *r = w;
    byte *const beg = w->u.text;		/* base pointer, fixed */
    byte *const fin = beg + w->leng;	/* end+1 position */
 
    byte *txt = (byte *) memstr(w->u.text, w->leng, "=?");	/* input position */
    uint size = (uint) (txt - beg);				/* output offset */
 
#ifndef	DISABLE_UNICODE
    size_t max = w->leng * 4;
    static buff_t * buf = NULL;
#endif
 
    if (txt == NULL)
	return r;
 
#ifndef	DISABLE_UNICODE
    if (encoding == E_UNICODE) {
	if (buf == NULL)
	    buf = buff_new((byte *)xmalloc(max+D), 0, max);
	r = &buf->t;				/* Use buf to return unicode result */
 
	buf->t.leng = 0;
	if (buf->size < max) {
	    buf->size = max;
	    buf->t.u.text = (byte *) xrealloc(buf->t.u.text, buf->size+D);
	}
 
	buf->t.leng = size;
	memcpy(buf->t.u.text, beg, size );
	Z(buf->t.u.text[buf->t.leng]);		/* for easier debugging - removable */
    }
#endif
 
    if (DEBUG_LEXER(2)) {
	fputs("**1**  ", dbgout);
	word_puts(w, 0, dbgout);
	fputs("\n", dbgout);
    }
 
    while (txt < fin) {
	byte *typ, *tmp, *end;
	uint len;
	bool adjacent;
 
	char *charset;
 
	txt += 2;
	typ = (byte *) memchr((char *)txt+1, '?', fin-txt);	/* Encoding type - 'B' or 'Q' */
	*typ++ = '\0';						/* nul terminate */
 
	charset = charset_as_string(txt, typ - txt - 1);
 
	tmp = typ + 2;						/* start of encoded word */
	end = (byte *) memstr((char *)tmp, fin-tmp, "?=");	/* last byte of encoded word  */
	len = end - tmp;
 
	w->u.text = tmp;				/* Start of encoded word */
	w->leng = len;				/* Length of encoded word */
	Z(w->u.text[w->leng]);			/* for easier debugging - removable */
 
	if (DEBUG_LEXER(2)) {
	    fputs("**2**  ", dbgout);
	    word_puts(w, 0, dbgout);
	    fputs("\n", dbgout);
	}
 
	switch (tolower(*typ)) {		/* ... encoding type */
	case 'b':
	    if (base64_validate(w))
		len = base64_decode(w);		/* decode base64 */
	    break;
	case 'q':
	    if (qp_validate(w, RFC2047))
		len = qp_decode(w, RFC2047);	/* decode quoted-printable */
	    break;
	}
 
	/* move decoded word to where the encoded used to be */
	if (encoding == E_RAW) {
	    memmove(beg+size, w->u.text, len);
	    size += len;			/* bump output pointer */
	    Z(beg[size]);			/* for easier debugging - removable */
 
	    if (DEBUG_LEXER(3))
		fprintf(dbgout, "**3**  %s\n", beg);
	}
 
#ifndef	DISABLE_UNICODE
	if (encoding == E_UNICODE) {
	    iconv_t cd;
	    buff_t  src;
 
	    /* convert 'word_t *w' to 'buff_t src' because
	    ** iconvert_cd() needs buff_t pointers
	    */
	    src.t.u.text = w->u.text;
	    src.t.leng = len;
	    src.read   = 0;
	    src.size   = len;
 
	    cd = bf_iconv_open( charset_unicode, charset );
	    iconvert_cd(cd, &src, buf);
	    iconv_close(cd);
 
	    if (DEBUG_LEXER(3)) {
		fputs("**4**  ", dbgout);
		word_puts(&buf->t, 0, dbgout);
		fputs("\n", dbgout);
	    }
	}
#endif
 
	txt = end + 2;	/* skip ?= trailer */
	if (txt >= fin)
	    break;
 
	/* check for next encoded word */
	end = (byte *) memstr((char *)txt, fin-txt, "=?");
	adjacent = end != NULL;
 
	/* clear adjacent flag if non-whitespace character found between
	 * adjacent encoded words */
	if (adjacent) {
	    tmp = txt;
	    while (adjacent && tmp < end) {
		if (*tmp && strchr(" \t\r\n", *tmp))
		    tmp += 1;
		else
		    adjacent = false;
	    }
	}
 
	/* we have a next encoded word and we've had only whitespace
	 * between the current and the next */
	if (adjacent)
	    /* just skip whitespace */
	    txt = end;
	else
	    /* copy everything that was between the encoded words */
	    while (txt < end) {
		if (encoding == E_RAW)
		    beg[size++] = *txt++;
#ifndef	DISABLE_UNICODE
		if (encoding == E_UNICODE)
		    buf->t.u.text[buf->t.leng++] = *txt++;
#endif
	    }
    }
 
    if (encoding == E_RAW) {
	r->u.text = beg;
	r->leng = size;
    }
 
    return r;
}
 
/*
 * The following sets edit modes for GNU EMACS
 * Local Variables:
 * mode:c
 * End:
 */