|
Packit |
6c4009 |
#!/usr/bin/python3
|
|
Packit |
6c4009 |
# -*- coding: utf-8 -*-
|
|
Packit |
6c4009 |
#
|
|
Packit |
6c4009 |
# Generate a translit_fraction file from a UnicodeData file.
|
|
Packit |
6c4009 |
# Copyright (C) 2015-2018 Free Software Foundation, Inc.
|
|
Packit |
6c4009 |
# This file is part of the GNU C Library.
|
|
Packit |
6c4009 |
#
|
|
Packit |
6c4009 |
# The GNU C Library is free software; you can redistribute it and/or
|
|
Packit |
6c4009 |
# modify it under the terms of the GNU Lesser General Public
|
|
Packit |
6c4009 |
# License as published by the Free Software Foundation; either
|
|
Packit |
6c4009 |
# version 2.1 of the License, or (at your option) any later version.
|
|
Packit |
6c4009 |
#
|
|
Packit |
6c4009 |
# The GNU C Library is distributed in the hope that it will be useful,
|
|
Packit |
6c4009 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit |
6c4009 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Packit |
6c4009 |
# Lesser General Public License for more details.
|
|
Packit |
6c4009 |
#
|
|
Packit |
6c4009 |
# You should have received a copy of the GNU Lesser General Public
|
|
Packit |
6c4009 |
# License along with the GNU C Library; if not, see
|
|
Packit |
6c4009 |
# <http://www.gnu.org/licenses/>.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
'''
|
|
Packit |
6c4009 |
Generate a translit_fraction file from UnicodeData.txt
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
To see how this script is used, call it with the “-h” option:
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
$ ./gen_translit_fraction -h
|
|
Packit |
6c4009 |
… prints usage message …
|
|
Packit |
6c4009 |
'''
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
import argparse
|
|
Packit |
6c4009 |
import time
|
|
Packit |
6c4009 |
import unicode_utils
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
def read_input_file(filename):
|
|
Packit |
6c4009 |
'''Reads the original glibc translit_fraction file to get the
|
|
Packit |
6c4009 |
original head and tail.
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
We want to replace only the part of the file between
|
|
Packit |
6c4009 |
“translit_start” and “translit_end”
|
|
Packit |
6c4009 |
'''
|
|
Packit |
6c4009 |
head = tail = ''
|
|
Packit |
6c4009 |
with open(filename, mode='r') as translit_file:
|
|
Packit |
6c4009 |
for line in translit_file:
|
|
Packit |
6c4009 |
head = head + line
|
|
Packit |
6c4009 |
if line.startswith('translit_start'):
|
|
Packit |
6c4009 |
break
|
|
Packit |
6c4009 |
for line in translit_file:
|
|
Packit |
6c4009 |
if line.startswith('translit_end'):
|
|
Packit |
6c4009 |
tail = line
|
|
Packit |
6c4009 |
break
|
|
Packit |
6c4009 |
for line in translit_file:
|
|
Packit |
6c4009 |
tail = tail + line
|
|
Packit |
6c4009 |
return (head, tail)
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
def output_head(translit_file, unicode_version, head=''):
|
|
Packit |
6c4009 |
'''Write the header of the output file, i.e. the part of the file
|
|
Packit |
6c4009 |
before the “translit_start” line.
|
|
Packit |
6c4009 |
'''
|
|
Packit |
6c4009 |
if ARGS.input_file and head:
|
|
Packit |
6c4009 |
translit_file.write(head)
|
|
Packit |
6c4009 |
else:
|
|
Packit |
6c4009 |
translit_file.write('escape_char /\n')
|
|
Packit |
6c4009 |
translit_file.write('comment_char %\n')
|
|
Packit |
6c4009 |
translit_file.write(unicode_utils.COMMENT_HEADER)
|
|
Packit |
6c4009 |
translit_file.write('\n')
|
|
Packit |
6c4009 |
translit_file.write('% Transliterations of fractions.\n')
|
|
Packit |
6c4009 |
translit_file.write('% Generated automatically from UnicodeData.txt '
|
|
Packit |
6c4009 |
+ 'by gen_translit_fraction.py '
|
|
Packit |
6c4009 |
+ 'on {:s} '.format(time.strftime('%Y-%m-%d'))
|
|
Packit |
6c4009 |
+ 'for Unicode {:s}.\n'.format(unicode_version))
|
|
Packit |
6c4009 |
translit_file.write('% The replacements have been surrounded ')
|
|
Packit |
6c4009 |
translit_file.write('with spaces, because fractions are\n')
|
|
Packit |
6c4009 |
translit_file.write('% often preceded by a decimal number and ')
|
|
Packit |
6c4009 |
translit_file.write('followed by a unit or a math symbol.\n')
|
|
Packit |
6c4009 |
translit_file.write('\n')
|
|
Packit |
6c4009 |
translit_file.write('LC_CTYPE\n')
|
|
Packit |
6c4009 |
translit_file.write('\n')
|
|
Packit |
6c4009 |
translit_file.write('translit_start\n')
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
def output_tail(translit_file, tail=''):
|
|
Packit |
6c4009 |
'''Write the tail of the output file'''
|
|
Packit |
6c4009 |
if ARGS.input_file and tail:
|
|
Packit |
6c4009 |
translit_file.write(tail)
|
|
Packit |
6c4009 |
else:
|
|
Packit |
6c4009 |
translit_file.write('translit_end\n')
|
|
Packit |
6c4009 |
translit_file.write('\n')
|
|
Packit |
6c4009 |
translit_file.write('END LC_CTYPE\n')
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
def special_decompose(code_point_list):
|
|
Packit |
6c4009 |
'''
|
|
Packit |
6c4009 |
Decompositions which are not in UnicodeData.txt at all but which
|
|
Packit |
6c4009 |
were used in the original translit_fraction file in glibc and
|
|
Packit |
6c4009 |
which seem to make sense. I want to keep the update of
|
|
Packit |
6c4009 |
translit_fraction close to the spirit of the original file,
|
|
Packit |
6c4009 |
therefore I added this special decomposition rules here.
|
|
Packit |
6c4009 |
'''
|
|
Packit |
6c4009 |
special_decompose_dict = {
|
|
Packit |
6c4009 |
(0x2044,): [0x002F], # ⁄ → /
|
|
Packit |
6c4009 |
}
|
|
Packit |
6c4009 |
if tuple(code_point_list) in special_decompose_dict:
|
|
Packit |
6c4009 |
return special_decompose_dict[tuple(code_point_list)]
|
|
Packit |
6c4009 |
else:
|
|
Packit |
6c4009 |
return code_point_list
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
def output_transliteration(translit_file):
|
|
Packit |
6c4009 |
'''Write the new transliteration to the output file'''
|
|
Packit |
6c4009 |
translit_file.write('\n')
|
|
Packit |
6c4009 |
for code_point in sorted(unicode_utils.UNICODE_ATTRIBUTES):
|
|
Packit |
6c4009 |
name = unicode_utils.UNICODE_ATTRIBUTES[code_point]['name']
|
|
Packit |
6c4009 |
decomposition = unicode_utils.UNICODE_ATTRIBUTES[
|
|
Packit |
6c4009 |
code_point]['decomposition']
|
|
Packit |
6c4009 |
if decomposition.startswith('<fraction>'):
|
|
Packit |
6c4009 |
decomposition = decomposition[11:]
|
|
Packit |
6c4009 |
decomposed_code_points = [[int(x, 16)
|
|
Packit |
6c4009 |
for x in decomposition.split(' ')]]
|
|
Packit |
6c4009 |
if decomposed_code_points[0]:
|
|
Packit |
6c4009 |
decomposed_code_points[0] = [0x0020] \
|
|
Packit |
6c4009 |
+ decomposed_code_points[0] \
|
|
Packit |
6c4009 |
+ [0x0020]
|
|
Packit |
6c4009 |
while True:
|
|
Packit |
6c4009 |
special_decomposed_code_points = special_decompose(
|
|
Packit |
6c4009 |
decomposed_code_points[-1])
|
|
Packit |
6c4009 |
if (special_decomposed_code_points
|
|
Packit |
6c4009 |
!= decomposed_code_points[-1]):
|
|
Packit |
6c4009 |
decomposed_code_points.append(
|
|
Packit |
6c4009 |
special_decomposed_code_points)
|
|
Packit |
6c4009 |
continue
|
|
Packit |
6c4009 |
special_decomposed_code_points = []
|
|
Packit |
6c4009 |
for decomposed_code_point in decomposed_code_points[-1]:
|
|
Packit |
6c4009 |
special_decomposed_code_points += special_decompose(
|
|
Packit |
6c4009 |
[decomposed_code_point])
|
|
Packit |
6c4009 |
if (special_decomposed_code_points
|
|
Packit |
6c4009 |
== decomposed_code_points[-1]):
|
|
Packit |
6c4009 |
break
|
|
Packit |
6c4009 |
decomposed_code_points.append(
|
|
Packit |
6c4009 |
special_decomposed_code_points)
|
|
Packit |
6c4009 |
translit_file.write('% {:s}\n'.format(name))
|
|
Packit |
6c4009 |
translit_file.write('{:s} '.format(
|
|
Packit |
6c4009 |
unicode_utils.ucs_symbol(code_point)))
|
|
Packit |
6c4009 |
for index in range(0, len(decomposed_code_points)):
|
|
Packit |
6c4009 |
if index > 0:
|
|
Packit |
6c4009 |
translit_file.write(';')
|
|
Packit |
6c4009 |
if len(decomposed_code_points[index]) > 1:
|
|
Packit |
6c4009 |
translit_file.write('"')
|
|
Packit |
6c4009 |
for decomposed_code_point in decomposed_code_points[index]:
|
|
Packit |
6c4009 |
translit_file.write('{:s}'.format(
|
|
Packit |
6c4009 |
unicode_utils.ucs_symbol(decomposed_code_point)))
|
|
Packit |
6c4009 |
if len(decomposed_code_points[index]) > 1:
|
|
Packit |
6c4009 |
translit_file.write('"')
|
|
Packit |
6c4009 |
translit_file.write('\n')
|
|
Packit |
6c4009 |
translit_file.write('\n')
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
if __name__ == "__main__":
|
|
Packit |
6c4009 |
PARSER = argparse.ArgumentParser(
|
|
Packit |
6c4009 |
description='''
|
|
Packit |
6c4009 |
Generate a translit_cjk_compat file from UnicodeData.txt.
|
|
Packit |
6c4009 |
''')
|
|
Packit |
6c4009 |
PARSER.add_argument(
|
|
Packit |
6c4009 |
'-u', '--unicode_data_file',
|
|
Packit |
6c4009 |
nargs='?',
|
|
Packit |
6c4009 |
type=str,
|
|
Packit |
6c4009 |
default='UnicodeData.txt',
|
|
Packit |
6c4009 |
help=('The UnicodeData.txt file to read, '
|
|
Packit |
6c4009 |
+ 'default: %(default)s'))
|
|
Packit |
6c4009 |
PARSER.add_argument(
|
|
Packit |
6c4009 |
'-i', '--input_file',
|
|
Packit |
6c4009 |
nargs='?',
|
|
Packit |
6c4009 |
type=str,
|
|
Packit |
6c4009 |
help=''' The original glibc/localedata/locales/translit_fraction
|
|
Packit |
6c4009 |
file.''')
|
|
Packit |
6c4009 |
PARSER.add_argument(
|
|
Packit |
6c4009 |
'-o', '--output_file',
|
|
Packit |
6c4009 |
nargs='?',
|
|
Packit |
6c4009 |
type=str,
|
|
Packit |
6c4009 |
default='translit_fraction.new',
|
|
Packit |
6c4009 |
help='''The new translit_fraction file, default: %(default)s. If the
|
|
Packit |
6c4009 |
original glibc/localedata/locales/translit_fraction file has
|
|
Packit |
6c4009 |
been given as an option, the header up to the
|
|
Packit |
6c4009 |
“translit_start” line and the tail from the “translit_end”
|
|
Packit |
6c4009 |
line to the end of the file will be copied unchanged into the
|
|
Packit |
6c4009 |
output file. ''')
|
|
Packit |
6c4009 |
PARSER.add_argument(
|
|
Packit |
6c4009 |
'--unicode_version',
|
|
Packit |
6c4009 |
nargs='?',
|
|
Packit |
6c4009 |
required=True,
|
|
Packit |
6c4009 |
type=str,
|
|
Packit |
6c4009 |
help='The Unicode version of the input files used.')
|
|
Packit |
6c4009 |
ARGS = PARSER.parse_args()
|
|
Packit |
6c4009 |
|
|
Packit |
6c4009 |
unicode_utils.fill_attributes(ARGS.unicode_data_file)
|
|
Packit |
6c4009 |
HEAD = TAIL = ''
|
|
Packit |
6c4009 |
if ARGS.input_file:
|
|
Packit |
6c4009 |
(HEAD, TAIL) = read_input_file(ARGS.input_file)
|
|
Packit |
6c4009 |
with open(ARGS.output_file, mode='w') as TRANSLIT_FILE:
|
|
Packit |
6c4009 |
output_head(TRANSLIT_FILE, ARGS.unicode_version, head=HEAD)
|
|
Packit |
6c4009 |
output_transliteration(TRANSLIT_FILE)
|
|
Packit |
6c4009 |
output_tail(TRANSLIT_FILE, tail=TAIL)
|