#!/usr/bin/python

# -*- coding: utf-8 -*-

# make_unicode_fold_data.py

# Copyright (c) 2016-2017  K.Kosako

import sys

import re

import codecs

SOURCE_FILE = 'CaseFolding.txt'

GPERF_UNFOLD_KEY_FILE = 'unicode_unfold_key.gperf'

GPERF_FOLD_KEY_FILES  = ['unicode_fold1_key.gperf', 'unicode_fold2_key.gperf', 'unicode_fold3_key.gperf']

DataName = 'OnigUnicodeFolds'

ENCODING = 'utf-8'

LINE_REG = re.compile("([0-9A-F]{1,6}); (.); ([0-9A-F]{1,6})(?: ([0-9A-F]{1,6}))?(?: ([0-9A-F]{1,6}))?;(?:\s*#\s*)(.*)")

VERSION_REG  = re.compile("#.*-(\d+\.\d+\.\d+)\.txt")

VERSION_INFO = None

FOLDS = {}

TURKISH_FOLDS = {}

LOCALE_FOLDS  = {}

UNFOLDS = {}

TURKISH_UNFOLDS = {}

LOCALE_UNFOLDS  = {}

class Entry:

    def __init__(self, fold):

        self.fold = fold

        self.unfolds = []

        self.fold_len = len(fold)

        self.index = -1

        self.comment = None

def fold_key(fold):

    sfold = map(lambda i: "%06x" % i, fold)

    return ':'.join(sfold)

def form16(x, size):

    form = "0x%06x" if x > 0xffff else "0x%04x"

    s = form % x

    rem = size - len(s)

    if rem > 0:

        s = ' ' * rem + s

    return s

def form3bytes(x):

    x0 = x & 0xff

    x1 = (x>>8)  & 0xff

    x2 = (x>>16) & 0xff

    return "\\x%02x\\x%02x\\x%02x" % (x2, x1, x0)

def check_version_info(s):

    global VERSION_INFO

    if VERSION_INFO is None:

        m = VERSION_REG.match(s)

        if m is not None:

            VERSION_INFO = m.group(1)

def parse_line(s):

    if len(s) == 0:

        return False

    if s[0] == '#':

        check_version_info(s)

        return False

    m = LINE_REG.match(s)

    if m is None:

        print >> sys.stderr, s.encode(ENCODING)

        sys.exit(-1)

    s_unfold = m.group(1)

    s_type   = m.group(2)

    s_fold   = m.group(3)

    comment  = m.group(6)

    if s_type == 'S':

        return False;

    unfold = int(s_unfold, 16)

    f1     = int(s_fold, 16)

    fold = [f1]

    if m.group(4) is not None:

        f2 = int(m.group(4), 16)

        fold.append(f2)

        if m.group(5) is not None:

            f3 = int(m.group(5), 16)

            fold.append(f3)

    if s_type == 'T':

        dic   = TURKISH_FOLDS

        undic = TURKISH_UNFOLDS

    else:

        dic   = FOLDS

        undic = UNFOLDS

    key = fold_key(fold)

    e = dic.get(key, None)

    if e is None:

        e = Entry(fold)

        e.comment = comment

        dic[key] = e

    e.unfolds.append(unfold)

    if undic.get(unfold, None) is not None:

        print >> sys.stderr, ("unfold dup: 0x%04x %s\n" % (unfold, s_type))

    undic[unfold] = e

    return True

def parse_file(f):

    line = f.readline()

    while line:

        s = line.strip()

        parse_line(s)

        line = f.readline()

def make_locale():

    for unfold, te in TURKISH_UNFOLDS.items():

        e = UNFOLDS.get(unfold, None)

        if e is None:

            continue

        fkey = fold_key(e.fold)

        if len(e.unfolds) == 1:

            del FOLDS[fkey]

        else:

            e.unfolds.remove(unfold)

            e = Entry(e.fold)

            e.unfolds.append(unfold)

        LOCALE_FOLDS[fkey] = e

        LOCALE_UNFOLDS[unfold] = e

        del UNFOLDS[unfold]

def output_typedef(f):

    s = """\

typedef unsigned long OnigCodePoint;

"""

    print >> f, s

def divide_by_fold_len(d):

    l = d.items()

    l1 = filter(lambda (k,e):e.fold_len == 1, l)

    l2 = filter(lambda (k,e):e.fold_len == 2, l)

    l3 = filter(lambda (k,e):e.fold_len == 3, l)

    sl1 = sorted(l1, key=lambda (k,e):k)

    sl2 = sorted(l2, key=lambda (k,e):k)

    sl3 = sorted(l3, key=lambda (k,e):k)

    return (sl1, sl2, sl3)

def output_comment(f, s):

    f.write(" /* %s */" % s)

def output_data_n1(f, n, fn, c, out_comment):

    for k, e in fn:

        e.index = c

        if out_comment and n > 1 and e.comment is not None:

            output_comment(f, e.comment)

            print >> f, ''

        f.write(' ')

        f.write("/*%4d*/ " % c)

        for i in range(0, n):

            s = form16(e.fold[i], 8)

            f.write(" %s," % s)

        usize = len(e.unfolds)

        f.write("  %d," % usize)

        for u in e.unfolds:

            s = form16(u, 8)

            f.write(" %s," % s)

        if out_comment and n == 1 and e.comment is not None:

            if len(e.comment) < 35:

                s = e.comment

            else:

                s = e.comment[0:33] + '..'

            output_comment(f, s)

        f.write("\n")

        c += n + 1 + usize

    return c

def output_data_n(f, name, n, fn, lfn, out_comment):

    print >> f, "OnigCodePoint %s%d[] = {" % (name, n)

    c = 0

    c = output_data_n1(f, n,  fn, c, out_comment)

    print >> f, "#define FOLDS%d_NORMAL_END_INDEX   %d" % (n, c)

    print >> f, " /* ----- LOCALE ----- */"

    c = output_data_n1(f, n, lfn, c, out_comment)

    print >> f, "#define FOLDS%d_END_INDEX   %d" % (n, c)

    print >> f, "};"

def output_fold_data(f, name, out_comment):

    f1, f2, f3 = divide_by_fold_len(FOLDS)

    lf1, lf2, lf3 = divide_by_fold_len(LOCALE_FOLDS)

    output_data_n(f, name, 1, f1, lf1, out_comment)

    print >> f, ''

    output_data_n(f, name, 2, f2, lf2, out_comment)

    print >> f, ''

    output_data_n(f, name, 3, f3, lf3, out_comment)

    print >> f, ''

def output_macros(f, name):

    print >> f, "#define FOLDS1_FOLD(i)         (%s1 + (i))" % name

    print >> f, "#define FOLDS2_FOLD(i)         (%s2 + (i))" % name

    print >> f, "#define FOLDS3_FOLD(i)         (%s3 + (i))" % name

    print >> f, "#define FOLDS1_UNFOLDS_NUM(i)  %s1[(i)+1]" % name

    print >> f, "#define FOLDS2_UNFOLDS_NUM(i)  %s2[(i)+2]" % name

    print >> f, "#define FOLDS3_UNFOLDS_NUM(i)  %s3[(i)+3]" % name

    print >> f, "#define FOLDS1_UNFOLDS(i)      (%s1 + (i) + 2)" % name

    print >> f, "#define FOLDS2_UNFOLDS(i)      (%s2 + (i) + 3)" % name

    print >> f, "#define FOLDS3_UNFOLDS(i)      (%s3 + (i) + 4)" % name

    print >> f, "#define FOLDS1_NEXT_INDEX(i)   ((i) + 2 + %s1[(i)+1])" % name

    print >> f, "#define FOLDS2_NEXT_INDEX(i)   ((i) + 3 + %s1[(i)+2])" % name

    print >> f, "#define FOLDS3_NEXT_INDEX(i)   ((i) + 4 + %s1[(i)+3])" % name

def output_fold_source(f, out_comment):

    print >> f, "/* This file was generated by make_unicode_fold_data.py. */"

    print >> f, '#include "regenc.h"'

    print >> f, ''

    if VERSION_INFO is not None:

        print "#define CASEFOLD_VERSION  %s" % re.sub(r'[\.-]', '_', VERSION_INFO)

        print ''

    #output_macros(f, DataName)

    print >> f, ''

    #output_typedef(f)

    output_fold_data(f, DataName, out_comment)

def output_gperf_unfold_key(f):

    head = """\

%{

/* This gperf source file was generated by make_unicode_fold_data.py */

#include <string.h>

#include "regenc.h"

%}

struct ByUnfoldKey {

  OnigCodePoint code;

  short int   index;

  short int   fold_len;

};

%%

"""

    f.write(head)

    UNFOLDS.update(LOCALE_UNFOLDS)

    l = UNFOLDS.items()

    sl = sorted(l, key=lambda (k,e):(e.fold_len, e.index))

    for k, e in sl:

        f.write('"%s", /*0x%04x*/ %4d, %d\n' %

                (form3bytes(k), k, e.index, e.fold_len))

    print >> f, '%%'

def output_gperf_fold_key(f, key_len):

    head = """\

%{

/* This gperf source file was generated by make_unicode_fold_data.py */

#include <string.h>

#include "regenc.h"

%}

short int

%%

"""

    f.write(head)

    l = FOLDS.items()

    l = filter(lambda (k,e):e.fold_len == key_len, l)

    sl = sorted(l, key=lambda (k,e):e.index)

    for k, e in sl:

        skey = ''.join(map(lambda i: form3bytes(i), e.fold))

        f.write('"%s", %4d\n' % (skey, e.index))

    print >> f, '%%'

def output_gperf_source():

   with open(GPERF_UNFOLD_KEY_FILE, 'w') as f:

       output_gperf_unfold_key(f)

   FOLDS.update(LOCALE_FOLDS)

   for i in range(1, 4):

       with open(GPERF_FOLD_KEY_FILES[i-1], 'w') as f:

           output_gperf_fold_key(f, i)

## main ##

with open(SOURCE_FILE, 'r') as f:

    parse_file(f)

make_locale()

out_comment = True

output_fold_source(sys.stdout, out_comment)

output_gperf_source()