#!/usr/bin/python3

# -*- coding: utf-8 -*-

# Copyright (C) 2014-2018 Free Software Foundation, Inc.

# This file is part of the GNU C Library.

#

# The GNU C Library is free software; you can redistribute it and/or

# modify it under the terms of the GNU Lesser General Public

# License as published by the Free Software Foundation; either

# version 2.1 of the License, or (at your option) any later version.

#

# The GNU C Library is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU

# Lesser General Public License for more details.

#

# You should have received a copy of the GNU Lesser General Public

# License along with the GNU C Library; if not, see

# <http://www.gnu.org/licenses/>.

'''

This script is useful for checking backward compatibility of newly

generated UTF-8 file from utf8_gen.py script

To see how this script is used, call it with the “-h” option:

    $ ./utf8_compatibility.py -h

    … prints usage message …

'''

import sys

import re

import argparse

import unicode_utils

def create_charmap_dictionary(file_name):

    '''Create a dictionary for all code points found in the CHARMAP

    section of a file

    '''

    with open(file_name, mode='r') as utf8_file:

        charmap_dictionary = {}

        for line in utf8_file:

            if line.startswith('CHARMAP'):

                break

        for line in utf8_file:

            if line.startswith('END CHARMAP'):

                return charmap_dictionary

            if line.startswith('%'):

                continue

            match = re.match(

                r'^<U(?P<codepoint1>[0-9A-F]{4,8})>'

                +r'(:?\.\.<U(?P<codepoint2>[0-9-A-F]{4,8})>)?'

                +r'\s+(?P<hexutf8>(/x[0-9a-f]{2}){1,4})',

                line)

            if not match:

                continue

            codepoint1 = match.group('codepoint1')

            codepoint2 = match.group('codepoint2')

            if not codepoint2:

                codepoint2 = codepoint1

            for i in range(int(codepoint1, 16),

                           int(codepoint2, 16) + 1):

                charmap_dictionary[i] = match.group('hexutf8')

        sys.stderr.write('No “CHARMAP” or no “END CHARMAP” found in %s\n'

                         %file_name)

        exit(1)

def check_charmap(original_file_name, new_file_name):

    '''Report differences in the CHARMAP section between the old and the

    new file

    '''

    print('************************************************************')

    print('Report on CHARMAP:')

    ocharmap = create_charmap_dictionary(original_file_name)

    ncharmap = create_charmap_dictionary(new_file_name)

    print('------------------------------------------------------------')

    print('Total removed characters in newly generated CHARMAP: %d'

          %len(set(ocharmap)-set(ncharmap)))

    if ARGS.show_missing_characters:

        for key in sorted(set(ocharmap)-set(ncharmap)):

            print('removed: {:s}     {:s} {:s}'.format(

                unicode_utils.ucs_symbol(key),

                ocharmap[key],

                unicode_utils.UNICODE_ATTRIBUTES[key]['name'] \

                if key in unicode_utils.UNICODE_ATTRIBUTES else 'None'))

    print('------------------------------------------------------------')

    changed_charmap = {}

    for key in set(ocharmap).intersection(set(ncharmap)):

        if ocharmap[key] != ncharmap[key]:

            changed_charmap[key] = (ocharmap[key], ncharmap[key])

    print('Total changed characters in newly generated CHARMAP: %d'

          %len(changed_charmap))

    if ARGS.show_changed_characters:

        for key in sorted(changed_charmap):

            print('changed: {:s}     {:s}->{:s} {:s}'.format(

                unicode_utils.ucs_symbol(key),

                changed_charmap[key][0],

                changed_charmap[key][1],

                unicode_utils.UNICODE_ATTRIBUTES[key]['name'] \

                if key in unicode_utils.UNICODE_ATTRIBUTES else 'None'))

    print('------------------------------------------------------------')

    print('Total added characters in newly generated CHARMAP: %d'

          %len(set(ncharmap)-set(ocharmap)))

    if ARGS.show_added_characters:

        for key in sorted(set(ncharmap)-set(ocharmap)):

            print('added: {:s}     {:s} {:s}'.format(

                unicode_utils.ucs_symbol(key),

                ncharmap[key],

                unicode_utils.UNICODE_ATTRIBUTES[key]['name'] \

                if key in unicode_utils.UNICODE_ATTRIBUTES else 'None'))

def create_width_dictionary(file_name):

    '''Create a dictionary for all code points found in the WIDTH

    section of a file

    '''

    with open(file_name, mode='r') as utf8_file:

        width_dictionary = {}

        for line in utf8_file:

            if line.startswith('WIDTH'):

                break

        for line in utf8_file:

            if line.startswith('END WIDTH'):

                return width_dictionary

            match = re.match(

                r'^<U(?P<codepoint1>[0-9A-F]{4,8})>'

                +r'(:?\.\.\.<U(?P<codepoint2>[0-9-A-F]{4,8})>)?'

                +r'\s+(?P<width>[02])',

                line)

            if not match:

                continue

            codepoint1 = match.group('codepoint1')

            codepoint2 = match.group('codepoint2')

            if not codepoint2:

                codepoint2 = codepoint1

            for i in range(int(codepoint1, 16),

                           int(codepoint2, 16) + 1):

                width_dictionary[i] = int(match.group('width'))

        sys.stderr.write('No “WIDTH” or no “END WIDTH” found in %s\n' %file)

def check_width(original_file_name, new_file_name):

    '''Report differences in the WIDTH section between the old and the new

    file

    '''

    print('************************************************************')

    print('Report on WIDTH:')

    owidth = create_width_dictionary(original_file_name)

    nwidth = create_width_dictionary(new_file_name)

    print('------------------------------------------------------------')

    print('Total removed characters in newly generated WIDTH: %d'

          %len(set(owidth)-set(nwidth)))

    print('(Characters not in WIDTH get width 1 by default, '

          + 'i.e. these have width 1 now.)')

    if ARGS.show_missing_characters:

        for key in sorted(set(owidth)-set(nwidth)):

            print('removed: {:s} '.format(unicode_utils.ucs_symbol(key))

                  + '{:d} : '.format(owidth[key])

                  + 'eaw={:s} '.format(

                      unicode_utils.EAST_ASIAN_WIDTHS[key]

                      if key in unicode_utils.EAST_ASIAN_WIDTHS else 'None')

                  + 'category={:2s} '.format(

                      unicode_utils.UNICODE_ATTRIBUTES[key]['category']

                      if key in unicode_utils.UNICODE_ATTRIBUTES else 'None')

                  + 'bidi={:3s} '.format(

                      unicode_utils.UNICODE_ATTRIBUTES[key]['bidi']

                      if key in unicode_utils.UNICODE_ATTRIBUTES else 'None')

                  + 'name={:s}'.format(

                      unicode_utils.UNICODE_ATTRIBUTES[key]['name']

                      if key in unicode_utils.UNICODE_ATTRIBUTES else 'None'))

    print('------------------------------------------------------------')

    changed_width = {}

    for key in set(owidth).intersection(set(nwidth)):

        if owidth[key] != nwidth[key]:

            changed_width[key] = (owidth[key], nwidth[key])

    print('Total changed characters in newly generated WIDTH: %d'

          %len(changed_width))

    if ARGS.show_changed_characters:

        for key in sorted(changed_width):

            print('changed width: {:s} '.format(unicode_utils.ucs_symbol(key))

                  + '{:d}->{:d} : '.format(changed_width[key][0],

                                          changed_width[key][1])

                  + 'eaw={:s} '.format(

                      unicode_utils.EAST_ASIAN_WIDTHS[key]

                      if key in unicode_utils.EAST_ASIAN_WIDTHS else 'None')

                  + 'category={:2s} '.format(

                      unicode_utils.UNICODE_ATTRIBUTES[key]['category']

                      if key in unicode_utils.UNICODE_ATTRIBUTES else 'None')

                  + 'bidi={:3s} '.format(

                      unicode_utils.UNICODE_ATTRIBUTES[key]['bidi']

                      if key in unicode_utils.UNICODE_ATTRIBUTES else 'None')

                  + 'name={:s}'.format(

                      unicode_utils.UNICODE_ATTRIBUTES[key]['name']

                      if key in unicode_utils.UNICODE_ATTRIBUTES else 'None'))

    print('------------------------------------------------------------')

    print('Total added characters in newly generated WIDTH: %d'

          %len(set(nwidth)-set(owidth)))

    print('(Characters not in WIDTH get width 1 by default, '

          + 'i.e. these had width 1 before.)')

    if ARGS.show_added_characters:

        for key in sorted(set(nwidth)-set(owidth)):

            print('added: {:s} '.format(unicode_utils.ucs_symbol(key))

                  + '{:d} : '.format(nwidth[key])

                  + 'eaw={:s} '.format(

                      unicode_utils.EAST_ASIAN_WIDTHS[key]

                      if key in unicode_utils.EAST_ASIAN_WIDTHS else 'None')

                  + 'category={:2s} '.format(

                      unicode_utils.UNICODE_ATTRIBUTES[key]['category']

                      if key in unicode_utils.UNICODE_ATTRIBUTES else 'None')

                  + 'bidi={:3s} '.format(

                      unicode_utils.UNICODE_ATTRIBUTES[key]['bidi']

                      if key in unicode_utils.UNICODE_ATTRIBUTES else 'None')

                  + 'name={:s}'.format(

                      unicode_utils.UNICODE_ATTRIBUTES[key]['name']

                      if key in unicode_utils.UNICODE_ATTRIBUTES else 'None'))

if __name__ == "__main__":

    PARSER = argparse.ArgumentParser(

        description='''

        Compare the contents of LC_CTYPE in two files and check for errors.

        ''')

    PARSER.add_argument(

        '-o', '--old_utf8_file',

        nargs='?',

        required=True,

        type=str,

        help='The old UTF-8 file.')

    PARSER.add_argument(

        '-n', '--new_utf8_file',

        nargs='?',

        required=True,

        type=str,

        help='The new UTF-8 file.')

    PARSER.add_argument(

        '-u', '--unicode_data_file',

        nargs='?',

        type=str,

        help='The UnicodeData.txt file to read.')

    PARSER.add_argument(

        '-e', '--east_asian_width_file',

        nargs='?',

        type=str,

        help='The EastAsianWidth.txt file to read.')

    PARSER.add_argument(

        '-a', '--show_added_characters',

        action='store_true',

        help='Show characters which were added in detail.')

    PARSER.add_argument(

        '-m', '--show_missing_characters',

        action='store_true',

        help='Show characters which were removed in detail.')

    PARSER.add_argument(

        '-c', '--show_changed_characters',

        action='store_true',

        help='Show characters whose width was changed in detail.')

    ARGS = PARSER.parse_args()

    if ARGS.unicode_data_file:

        unicode_utils.fill_attributes(ARGS.unicode_data_file)

    if ARGS.east_asian_width_file:

        unicode_utils.fill_east_asian_widths(ARGS.east_asian_width_file)

    check_charmap(ARGS.old_utf8_file, ARGS.new_utf8_file)

    check_width(ARGS.old_utf8_file, ARGS.new_utf8_file)