#!/usr/bin/python3

#

# Generate a Unicode conforming LC_CTYPE category from a UnicodeData file.

# Copyright (C) 2014-2018 Free Software Foundation, Inc.

# This file is part of the GNU C Library.

# Based on gen-unicode-ctype.c by Bruno Haible <haible@clisp.cons.org>, 2000.

#

# The GNU C Library is free software; you can redistribute it and/or

# modify it under the terms of the GNU Lesser General Public

# License as published by the Free Software Foundation; either

# version 2.1 of the License, or (at your option) any later version.

#

# The GNU C Library is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

# Lesser General Public License for more details.

#

# You should have received a copy of the GNU Lesser General Public

# License along with the GNU C Library; if not, see

# <http://www.gnu.org/licenses/>.

'''

Generate a Unicode conforming LC_CTYPE category from UnicodeData.txt and

DerivedCoreProperties.txt files.

To see how this script is used, call it with the “-h” option:

    $ ./gen_unicode_ctype.py -h

    … prints usage message …

'''

import argparse

import time

import re

import unicode_utils

def code_point_ranges(is_class_function):

    '''Returns a list of ranges of code points for which is_class_function

    returns True.

    Example:

    [[65, 90], [192, 214], [216, 222], [256], … ]

    '''

    cp_ranges  = []

    for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):

        if is_class_function(code_point):

            if (cp_ranges

                and cp_ranges[-1][-1] == code_point - 1):

                if len(cp_ranges[-1]) == 1:

                    cp_ranges[-1].append(code_point)

                else:

                    cp_ranges[-1][-1] = code_point

            else:

                cp_ranges.append([code_point])

    return cp_ranges

def output_charclass(i18n_file, class_name, is_class_function):

    '''Output a LC_CTYPE character class section

    Example:

    upper /

       <U0041>..<U005A>;<U00C0>..<U00D6>;<U00D8>..<U00DE>;<U0100>;<U0102>;/

       …

       <U0001D790>..<U0001D7A8>;<U0001D7CA>;<U0001F130>..<U0001F149>;/

       <U0001F150>..<U0001F169>;<U0001F170>..<U0001F189>

    '''

    cp_ranges = code_point_ranges(is_class_function)

    if cp_ranges:

        i18n_file.write('%s /\n' %class_name)

        max_column = 75

        prefix = '   '

        line = prefix

        range_string = ''

        for code_point_range in cp_ranges:

            if line.strip():

                line  += ';'

            if len(code_point_range) == 1:

                range_string = unicode_utils.ucs_symbol(code_point_range[0])

            else:

                range_string = unicode_utils.ucs_symbol_range(

                    code_point_range[0], code_point_range[-1])

            if len(line+range_string) > max_column:

                i18n_file.write(line+'/\n')

                line = prefix

            line += range_string

        if line.strip():

            i18n_file.write(line+'\n')

        i18n_file.write('\n')

def output_charmap(i18n_file, map_name, map_function):

    '''Output a LC_CTYPE character map section

    Example:

    toupper /

      (<U0061>,<U0041>);(<U0062>,<U0042>);(<U0063>,<U0043>);(<U0064>,<U0044>);/

      …

      (<U000118DC>,<U000118BC>);(<U000118DD>,<U000118BD>);/

      (<U000118DE>,<U000118BE>);(<U000118DF>,<U000118BF>)

    '''

    max_column = 75

    prefix = '   '

    line = prefix

    map_string = ''

    i18n_file.write('%s /\n' %map_name)

    for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):

        mapped = map_function(code_point)

        if code_point != mapped:

            if line.strip():

                line += ';'

            map_string = '(' \

                         + unicode_utils.ucs_symbol(code_point) \

                         + ',' \

                         + unicode_utils.ucs_symbol(mapped) \

                         + ')'

            if len(line+map_string) > max_column:

                i18n_file.write(line+'/\n')

                line = prefix

            line += map_string

    if line.strip():

        i18n_file.write(line+'\n')

    i18n_file.write('\n')

def read_input_file(filename):

    '''Reads the original glibc i18n file to get the original head

    and tail.

    We want to replace only the character classes in LC_CTYPE, and the

    date stamp. All the rest of the i18n file should stay unchanged.

    To avoid having to cut and paste the generated data into the

    original file, it is helpful to read the original file here

    to be able to generate a complete result file.

    '''

    head = tail = ''

    with open(filename, mode='r') as i18n_file:

        for line in i18n_file:

            match = re.match(

                r'^(?P<key>date\s+)(?P<value>"[0-9]{4}-[0-9]{2}-[0-9]{2}")',

                line)

            if match:

                line = match.group('key') \

                       + '"{:s}"\n'.format(time.strftime('%Y-%m-%d'))

            head = head + line

            if line.startswith('LC_CTYPE'):

                break

        for line in i18n_file:

            if line.startswith('translit_start'):

                tail = line

                break

        for line in i18n_file:

            tail = tail + line

    return (head, tail)

def output_head(i18n_file, unicode_version, head=''):

    '''Write the header of the output file, i.e. the part of the file

    before the “LC_CTYPE” line.

    '''

    if ARGS.input_file and head:

        i18n_file.write(head)

    else:

        i18n_file.write('escape_char /\n')

        i18n_file.write('comment_char %\n')

        i18n_file.write('\n')

        i18n_file.write('% Generated automatically by '

                        + 'gen_unicode_ctype.py '

                        + 'for Unicode {:s}.\n'.format(unicode_version))

        i18n_file.write('\n')

        i18n_file.write('LC_IDENTIFICATION\n')

        i18n_file.write('title     "Unicode {:s} FDCC-set"\n'.format(

            unicode_version))

        i18n_file.write('source    "UnicodeData.txt, '

                        + 'DerivedCoreProperties.txt"\n')

        i18n_file.write('address   ""\n')

        i18n_file.write('contact   ""\n')

        i18n_file.write('email     "bug-glibc-locales@gnu.org"\n')

        i18n_file.write('tel       ""\n')

        i18n_file.write('fax       ""\n')

        i18n_file.write('language  ""\n')

        i18n_file.write('territory "Earth"\n')

        i18n_file.write('revision  "{:s}"\n'.format(unicode_version))

        i18n_file.write('date      "{:s}"\n'.format(

            time.strftime('%Y-%m-%d')))

        i18n_file.write('category  "i18n:2012";LC_CTYPE\n')

        i18n_file.write('END LC_IDENTIFICATION\n')

        i18n_file.write('\n')

        i18n_file.write('LC_CTYPE\n')

def output_tail(i18n_file, tail=''):

    '''Write the tail of the output file, i.e. the part of the file

    after the last “LC_CTYPE” character class.

    '''

    if ARGS.input_file and tail:

        i18n_file.write(tail)

    else:

        i18n_file.write('END LC_CTYPE\n')

def output_tables(i18n_file, unicode_version, turkish):

    '''Write the new LC_CTYPE character classes to the output file'''

    i18n_file.write('% The following is the 14652 i18n fdcc-set '

                    + 'LC_CTYPE category.\n')

    i18n_file.write('% It covers Unicode version {:s}.\n'.format(

        unicode_version))

    i18n_file.write('% The character classes and mapping tables were '

                    + 'automatically\n')

    i18n_file.write('% generated using the gen_unicode_ctype.py '

                    + 'program.\n\n')

    i18n_file.write('% The "upper" class reflects the uppercase '

                    + 'characters of class "alpha"\n')

    output_charclass(i18n_file, 'upper', unicode_utils.is_upper)

    i18n_file.write('% The "lower" class reflects the lowercase '

                    + 'characters of class "alpha"\n')

    output_charclass(i18n_file, 'lower', unicode_utils.is_lower)

    i18n_file.write('% The "alpha" class of the "i18n" FDCC-set is '

                    + 'reflecting\n')

    i18n_file.write('% the recommendations in TR 10176 annex A\n')

    output_charclass(i18n_file, 'alpha', unicode_utils.is_alpha)

    i18n_file.write('% The "digit" class must only contain the '

                    + 'BASIC LATIN digits, says ISO C 99\n')

    i18n_file.write('% (sections 7.25.2.1.5 and 5.2.1).\n')

    output_charclass(i18n_file, 'digit', unicode_utils.is_digit)

    i18n_file.write('% The "outdigit" information is by default '

                    + '"0" to "9".  We don\'t have to\n')

    i18n_file.write('% provide it here since localedef will fill '

               + 'in the bits and it would\n')

    i18n_file.write('% prevent locales copying this file define '

                    + 'their own values.\n')

    i18n_file.write('% outdigit /\n')

    i18n_file.write('%    <U0030>..<U0039>\n\n')

    # output_charclass(i18n_file, 'outdigit', is_outdigit)

    output_charclass(i18n_file, 'space', unicode_utils.is_space)

    output_charclass(i18n_file, 'cntrl', unicode_utils.is_cntrl)

    output_charclass(i18n_file, 'punct', unicode_utils.is_punct)

    output_charclass(i18n_file, 'graph', unicode_utils.is_graph)

    output_charclass(i18n_file, 'print', unicode_utils.is_print)

    i18n_file.write('% The "xdigit" class must only contain the '

                    + 'BASIC LATIN digits and A-F, a-f,\n')

    i18n_file.write('% says ISO C 99 '

                    + '(sections 7.25.2.1.12 and 6.4.4.1).\n')

    output_charclass(i18n_file, 'xdigit', unicode_utils.is_xdigit)

    output_charclass(i18n_file, 'blank', unicode_utils.is_blank)

    if turkish:

        i18n_file.write('% The case conversions reflect '

                        + 'Turkish conventions.\n')

        output_charmap(i18n_file, 'toupper', unicode_utils.to_upper_turkish)

        output_charmap(i18n_file, 'tolower', unicode_utils.to_lower_turkish)

    else:

        output_charmap(i18n_file, 'toupper', unicode_utils.to_upper)

        output_charmap(i18n_file, 'tolower', unicode_utils.to_lower)

    output_charmap(i18n_file, 'map "totitle";', unicode_utils.to_title)

    i18n_file.write('% The "combining" class reflects ISO/IEC 10646-1 '

                    + 'annex B.1\n')

    i18n_file.write('% That is, all combining characters (level 2+3).\n')

    output_charclass(i18n_file, 'class "combining";',

                     unicode_utils.is_combining)

    i18n_file.write('% The "combining_level3" class reflects '

                    + 'ISO/IEC 10646-1 annex B.2\n')

    i18n_file.write('% That is, combining characters of level 3.\n')

    output_charclass(i18n_file, 'class "combining_level3";',

                     unicode_utils.is_combining_level3)

if __name__ == "__main__":

    PARSER = argparse.ArgumentParser(

        description='''

        Generate a Unicode conforming LC_CTYPE category from

        UnicodeData.txt and DerivedCoreProperties.txt files.

        ''')

    PARSER.add_argument(

        '-u', '--unicode_data_file',

        nargs='?',

        type=str,

        default='UnicodeData.txt',

        help=('The UnicodeData.txt file to read, '

              + 'default: %(default)s'))

    PARSER.add_argument(

        '-d', '--derived_core_properties_file',

        nargs='?',

        type=str,

        default='DerivedCoreProperties.txt',

        help=('The DerivedCoreProperties.txt file to read, '

              + 'default: %(default)s'))

    PARSER.add_argument(

        '-i', '--input_file',

        nargs='?',

        type=str,

        help='''The original glibc/localedata/locales/i18n file.''')

    PARSER.add_argument(

        '-o', '--output_file',

        nargs='?',

        type=str,

        default='i18n.new',

        help='''The file which shall contain the generated LC_CTYPE category,

        default: %(default)s.  If the original

        glibc/localedata/locales/i18n has been given

        as an option, all data from the original file

        except the newly generated LC_CTYPE character

        classes and the date stamp in

        LC_IDENTIFICATION will be copied unchanged

        into the output file.  ''')

    PARSER.add_argument(

        '--unicode_version',

        nargs='?',

        required=True,

        type=str,

        help='The Unicode version of the input files used.')

    PARSER.add_argument(

        '--turkish',

        action='store_true',

        help='Use Turkish case conversions.')

    ARGS = PARSER.parse_args()

    unicode_utils.fill_attributes(

        ARGS.unicode_data_file)

    unicode_utils.fill_derived_core_properties(

        ARGS.derived_core_properties_file)

    unicode_utils.verifications()

    HEAD = TAIL = ''

    if ARGS.input_file:

        (HEAD, TAIL) = read_input_file(ARGS.input_file)

    with open(ARGS.output_file, mode='w') as I18N_FILE:

        output_head(I18N_FILE, ARGS.unicode_version, head=HEAD)

        output_tables(I18N_FILE, ARGS.unicode_version, ARGS.turkish)

        output_tail(I18N_FILE, tail=TAIL)