#!/usr/bin/python3
#
# Generate a Unicode conforming LC_CTYPE category from a UnicodeData file.
# Copyright (C) 2014-2018 Free Software Foundation, Inc.
# This file is part of the GNU C Library.
# Based on gen-unicode-ctype.c by Bruno Haible <haible@clisp.cons.org>, 2000.
#
# The GNU C Library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# The GNU C Library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with the GNU C Library; if not, see
# <http://www.gnu.org/licenses/>.
'''
Generate a Unicode conforming LC_CTYPE category from UnicodeData.txt and
DerivedCoreProperties.txt files.
To see how this script is used, call it with the “-h” option:
$ ./gen_unicode_ctype.py -h
… prints usage message …
'''
import argparse
import time
import re
import unicode_utils
def code_point_ranges(is_class_function):
'''Returns a list of ranges of code points for which is_class_function
returns True.
Example:
[[65, 90], [192, 214], [216, 222], [256], … ]
'''
cp_ranges = []
for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
if is_class_function(code_point):
if (cp_ranges
and cp_ranges[-1][-1] == code_point - 1):
if len(cp_ranges[-1]) == 1:
cp_ranges[-1].append(code_point)
else:
cp_ranges[-1][-1] = code_point
else:
cp_ranges.append([code_point])
return cp_ranges
def output_charclass(i18n_file, class_name, is_class_function):
'''Output a LC_CTYPE character class section
Example:
upper /
<U0041>..<U005A>;<U00C0>..<U00D6>;<U00D8>..<U00DE>;<U0100>;<U0102>;/
…
<U0001D790>..<U0001D7A8>;<U0001D7CA>;<U0001F130>..<U0001F149>;/
<U0001F150>..<U0001F169>;<U0001F170>..<U0001F189>
'''
cp_ranges = code_point_ranges(is_class_function)
if cp_ranges:
i18n_file.write('%s /\n' %class_name)
max_column = 75
prefix = ' '
line = prefix
range_string = ''
for code_point_range in cp_ranges:
if line.strip():
line += ';'
if len(code_point_range) == 1:
range_string = unicode_utils.ucs_symbol(code_point_range[0])
else:
range_string = unicode_utils.ucs_symbol_range(
code_point_range[0], code_point_range[-1])
if len(line+range_string) > max_column:
i18n_file.write(line+'/\n')
line = prefix
line += range_string
if line.strip():
i18n_file.write(line+'\n')
i18n_file.write('\n')
def output_charmap(i18n_file, map_name, map_function):
'''Output a LC_CTYPE character map section
Example:
toupper /
(<U0061>,<U0041>);(<U0062>,<U0042>);(<U0063>,<U0043>);(<U0064>,<U0044>);/
…
(<U000118DC>,<U000118BC>);(<U000118DD>,<U000118BD>);/
(<U000118DE>,<U000118BE>);(<U000118DF>,<U000118BF>)
'''
max_column = 75
prefix = ' '
line = prefix
map_string = ''
i18n_file.write('%s /\n' %map_name)
for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
mapped = map_function(code_point)
if code_point != mapped:
if line.strip():
line += ';'
map_string = '(' \
+ unicode_utils.ucs_symbol(code_point) \
+ ',' \
+ unicode_utils.ucs_symbol(mapped) \
+ ')'
if len(line+map_string) > max_column:
i18n_file.write(line+'/\n')
line = prefix
line += map_string
if line.strip():
i18n_file.write(line+'\n')
i18n_file.write('\n')
def read_input_file(filename):
'''Reads the original glibc i18n file to get the original head
and tail.
We want to replace only the character classes in LC_CTYPE, and the
date stamp. All the rest of the i18n file should stay unchanged.
To avoid having to cut and paste the generated data into the
original file, it is helpful to read the original file here
to be able to generate a complete result file.
'''
head = tail = ''
with open(filename, mode='r') as i18n_file:
for line in i18n_file:
match = re.match(
r'^(?P<key>date\s+)(?P<value>"[0-9]{4}-[0-9]{2}-[0-9]{2}")',
line)
if match:
line = match.group('key') \
+ '"{:s}"\n'.format(time.strftime('%Y-%m-%d'))
head = head + line
if line.startswith('LC_CTYPE'):
break
for line in i18n_file:
if line.startswith('translit_start'):
tail = line
break
for line in i18n_file:
tail = tail + line
return (head, tail)
def output_head(i18n_file, unicode_version, head=''):
'''Write the header of the output file, i.e. the part of the file
before the “LC_CTYPE” line.
'''
if ARGS.input_file and head:
i18n_file.write(head)
else:
i18n_file.write('escape_char /\n')
i18n_file.write('comment_char %\n')
i18n_file.write('\n')
i18n_file.write('% Generated automatically by '
+ 'gen_unicode_ctype.py '
+ 'for Unicode {:s}.\n'.format(unicode_version))
i18n_file.write('\n')
i18n_file.write('LC_IDENTIFICATION\n')
i18n_file.write('title "Unicode {:s} FDCC-set"\n'.format(
unicode_version))
i18n_file.write('source "UnicodeData.txt, '
+ 'DerivedCoreProperties.txt"\n')
i18n_file.write('address ""\n')
i18n_file.write('contact ""\n')
i18n_file.write('email "bug-glibc-locales@gnu.org"\n')
i18n_file.write('tel ""\n')
i18n_file.write('fax ""\n')
i18n_file.write('language ""\n')
i18n_file.write('territory "Earth"\n')
i18n_file.write('revision "{:s}"\n'.format(unicode_version))
i18n_file.write('date "{:s}"\n'.format(
time.strftime('%Y-%m-%d')))
i18n_file.write('category "i18n:2012";LC_CTYPE\n')
i18n_file.write('END LC_IDENTIFICATION\n')
i18n_file.write('\n')
i18n_file.write('LC_CTYPE\n')
def output_tail(i18n_file, tail=''):
'''Write the tail of the output file, i.e. the part of the file
after the last “LC_CTYPE” character class.
'''
if ARGS.input_file and tail:
i18n_file.write(tail)
else:
i18n_file.write('END LC_CTYPE\n')
def output_tables(i18n_file, unicode_version, turkish):
'''Write the new LC_CTYPE character classes to the output file'''
i18n_file.write('% The following is the 14652 i18n fdcc-set '
+ 'LC_CTYPE category.\n')
i18n_file.write('% It covers Unicode version {:s}.\n'.format(
unicode_version))
i18n_file.write('% The character classes and mapping tables were '
+ 'automatically\n')
i18n_file.write('% generated using the gen_unicode_ctype.py '
+ 'program.\n\n')
i18n_file.write('% The "upper" class reflects the uppercase '
+ 'characters of class "alpha"\n')
output_charclass(i18n_file, 'upper', unicode_utils.is_upper)
i18n_file.write('% The "lower" class reflects the lowercase '
+ 'characters of class "alpha"\n')
output_charclass(i18n_file, 'lower', unicode_utils.is_lower)
i18n_file.write('% The "alpha" class of the "i18n" FDCC-set is '
+ 'reflecting\n')
i18n_file.write('% the recommendations in TR 10176 annex A\n')
output_charclass(i18n_file, 'alpha', unicode_utils.is_alpha)
i18n_file.write('% The "digit" class must only contain the '
+ 'BASIC LATIN digits, says ISO C 99\n')
i18n_file.write('% (sections 7.25.2.1.5 and 5.2.1).\n')
output_charclass(i18n_file, 'digit', unicode_utils.is_digit)
i18n_file.write('% The "outdigit" information is by default '
+ '"0" to "9". We don\'t have to\n')
i18n_file.write('% provide it here since localedef will fill '
+ 'in the bits and it would\n')
i18n_file.write('% prevent locales copying this file define '
+ 'their own values.\n')
i18n_file.write('% outdigit /\n')
i18n_file.write('% <U0030>..<U0039>\n\n')
# output_charclass(i18n_file, 'outdigit', is_outdigit)
output_charclass(i18n_file, 'space', unicode_utils.is_space)
output_charclass(i18n_file, 'cntrl', unicode_utils.is_cntrl)
output_charclass(i18n_file, 'punct', unicode_utils.is_punct)
output_charclass(i18n_file, 'graph', unicode_utils.is_graph)
output_charclass(i18n_file, 'print', unicode_utils.is_print)
i18n_file.write('% The "xdigit" class must only contain the '
+ 'BASIC LATIN digits and A-F, a-f,\n')
i18n_file.write('% says ISO C 99 '
+ '(sections 7.25.2.1.12 and 6.4.4.1).\n')
output_charclass(i18n_file, 'xdigit', unicode_utils.is_xdigit)
output_charclass(i18n_file, 'blank', unicode_utils.is_blank)
if turkish:
i18n_file.write('% The case conversions reflect '
+ 'Turkish conventions.\n')
output_charmap(i18n_file, 'toupper', unicode_utils.to_upper_turkish)
output_charmap(i18n_file, 'tolower', unicode_utils.to_lower_turkish)
else:
output_charmap(i18n_file, 'toupper', unicode_utils.to_upper)
output_charmap(i18n_file, 'tolower', unicode_utils.to_lower)
output_charmap(i18n_file, 'map "totitle";', unicode_utils.to_title)
i18n_file.write('% The "combining" class reflects ISO/IEC 10646-1 '
+ 'annex B.1\n')
i18n_file.write('% That is, all combining characters (level 2+3).\n')
output_charclass(i18n_file, 'class "combining";',
unicode_utils.is_combining)
i18n_file.write('% The "combining_level3" class reflects '
+ 'ISO/IEC 10646-1 annex B.2\n')
i18n_file.write('% That is, combining characters of level 3.\n')
output_charclass(i18n_file, 'class "combining_level3";',
unicode_utils.is_combining_level3)
if __name__ == "__main__":
PARSER = argparse.ArgumentParser(
description='''
Generate a Unicode conforming LC_CTYPE category from
UnicodeData.txt and DerivedCoreProperties.txt files.
''')
PARSER.add_argument(
'-u', '--unicode_data_file',
nargs='?',
type=str,
default='UnicodeData.txt',
help=('The UnicodeData.txt file to read, '
+ 'default: %(default)s'))
PARSER.add_argument(
'-d', '--derived_core_properties_file',
nargs='?',
type=str,
default='DerivedCoreProperties.txt',
help=('The DerivedCoreProperties.txt file to read, '
+ 'default: %(default)s'))
PARSER.add_argument(
'-i', '--input_file',
nargs='?',
type=str,
help='''The original glibc/localedata/locales/i18n file.''')
PARSER.add_argument(
'-o', '--output_file',
nargs='?',
type=str,
default='i18n.new',
help='''The file which shall contain the generated LC_CTYPE category,
default: %(default)s. If the original
glibc/localedata/locales/i18n has been given
as an option, all data from the original file
except the newly generated LC_CTYPE character
classes and the date stamp in
LC_IDENTIFICATION will be copied unchanged
into the output file. ''')
PARSER.add_argument(
'--unicode_version',
nargs='?',
required=True,
type=str,
help='The Unicode version of the input files used.')
PARSER.add_argument(
'--turkish',
action='store_true',
help='Use Turkish case conversions.')
ARGS = PARSER.parse_args()
unicode_utils.fill_attributes(
ARGS.unicode_data_file)
unicode_utils.fill_derived_core_properties(
ARGS.derived_core_properties_file)
unicode_utils.verifications()
HEAD = TAIL = ''
if ARGS.input_file:
(HEAD, TAIL) = read_input_file(ARGS.input_file)
with open(ARGS.output_file, mode='w') as I18N_FILE:
output_head(I18N_FILE, ARGS.unicode_version, head=HEAD)
output_tables(I18N_FILE, ARGS.unicode_version, ARGS.turkish)
output_tail(I18N_FILE, tail=TAIL)