#!/usr/bin/env python
# -*- coding: utf-8 -*-#
#
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Generate name data for emoji resources. Currently in json format."""
from __future__ import print_function
import argparse
import collections
import glob
import json
import os
from os import path
import re
import sys
import generate_emoji_html
from nototools import tool_utils
from nototools import unicode_data
def _create_custom_gendered_seq_names():
"""The names have detail that is adequately represented by the image."""
BOY = 0x1f466
GIRL = 0x1f467
MAN = 0x1f468
WOMAN = 0x1f469
HEART = 0x2764 # Heavy Black Heart
KISS_MARK = 0x1f48b
return {
(MAN, HEART, KISS_MARK, MAN): 'Kiss',
(WOMAN, HEART, KISS_MARK, WOMAN): 'Kiss',
(WOMAN, HEART, KISS_MARK, MAN): 'Kiss',
(WOMAN, HEART, MAN): 'Couple with Heart',
(MAN, HEART, MAN): 'Couple with Heart',
(WOMAN, HEART, WOMAN): 'Couple with Heart',
(MAN, GIRL): 'Family',
(MAN, GIRL, GIRL): 'Family',
(MAN, GIRL, BOY): 'Family',
(MAN, BOY): 'Family',
(MAN, BOY, BOY): 'Family',
(MAN, WOMAN, GIRL): 'Family',
(MAN, WOMAN, GIRL, GIRL): 'Family',
(MAN, WOMAN, GIRL, BOY): 'Family',
(MAN, WOMAN, BOY): 'Family',
(MAN, WOMAN, BOY, BOY): 'Family',
(MAN, MAN, GIRL): 'Family',
(MAN, MAN, GIRL, GIRL): 'Family',
(MAN, MAN, GIRL, BOY): 'Family',
(MAN, MAN, BOY): 'Family',
(MAN, MAN, BOY, BOY): 'Family',
(WOMAN, GIRL): 'Family',
(WOMAN, GIRL, GIRL): 'Family',
(WOMAN, GIRL, BOY): 'Family',
(WOMAN, BOY): 'Family',
(WOMAN, BOY, BOY): 'Family',
(WOMAN, WOMAN, GIRL): 'Family',
(WOMAN, WOMAN, GIRL, GIRL): 'Family',
(WOMAN, WOMAN, GIRL, BOY): 'Family',
(WOMAN, WOMAN, BOY): 'Family',
(WOMAN, WOMAN, BOY, BOY): 'Family' }
def _create_custom_seq_names():
"""These have names that often are of the form 'Person xyz-ing' or 'Man Xyz.'
We opt to simplify the former to an activity name or action, and the latter to
drop the gender. This also generally makes the names shorter."""
EYE = 0x1f441
SPEECH = 0x1f5e8
WHITE_FLAG = 0x1f3f3
RAINBOW = 0x1f308
return {
(EYE, SPEECH): 'I Witness',
(WHITE_FLAG, RAINBOW): 'Rainbow Flag',
(0x2695,): 'Health Worker',
(0x2696,): 'Judge',
(0x26f7,): 'Skiing',
(0x26f9,): 'Bouncing a Ball',
(0x2708,): 'Pilot',
(0x1f33e,): 'Farmer',
(0x1f373,): 'Cook',
(0x1f393,): 'Student',
(0x1f3a4,): 'Singer',
(0x1f3a8,): 'Artist',
(0x1f3c2,): 'Snowboarding',
(0x1f3c3,): 'Running',
(0x1f3c4,): 'Surfing',
(0x1f3ca,): 'Swimming',
(0x1f3cb,): 'Weight Lifting',
(0x1f3cc,): 'Golfing',
(0x1f3eb,): 'Teacher',
(0x1f3ed,): 'Factory Worker',
(0x1f46e,): 'Police Officer',
(0x1f46f,): 'Partying',
(0x1f471,): 'Person with Blond Hair',
(0x1f473,): 'Person Wearing Turban',
(0x1f477,): 'Construction Worker',
(0x1f481,): 'Tipping Hand',
(0x1f482,): 'Guard',
(0x1f486,): 'Face Massage',
(0x1f487,): 'Haircut',
(0x1f4bb,): 'Technologist',
(0x1f4bc,): 'Office Worker',
(0x1f527,): 'Mechanic',
(0x1f52c,): 'Scientist',
(0x1f575,): 'Detective',
(0x1f645,): 'No Good Gesture',
(0x1f646,): 'OK Gesture',
(0x1f647,): 'Bowing Deeply',
(0x1f64b,): 'Raising Hand',
(0x1f64d,): 'Frowning',
(0x1f64e,): 'Pouting',
(0x1f680,): 'Astronaut',
(0x1f692,): 'Firefighter',
(0x1f6a3,): 'Rowing',
(0x1f6b4,): 'Bicycling',
(0x1f6b5,): 'Mountain Biking',
(0x1f6b6,): 'Walking',
(0x1f926,): 'Face Palm',
(0x1f937,): 'Shrug',
(0x1f938,): 'Doing a Cartwheel',
(0x1f939,): 'Juggling',
(0x1f93c,): 'Wrestling',
(0x1f93d,): 'Water Polo',
(0x1f93e,): 'Playing Handball',
(0x1f9d6,): 'Person in Steamy Room',
(0x1f9d7,): 'Climbing',
(0x1f9d8,): 'Person in Lotus Position',
(0x1f9d9,): 'Mage',
(0x1f9da,): 'Fairy',
(0x1f9db,): 'Vampire',
(0x1f9dd,): 'Elf',
(0x1f9de,): 'Genie',
(0x1f9df,): 'Zombie',
}
_CUSTOM_GENDERED_SEQ_NAMES = _create_custom_gendered_seq_names()
_CUSTOM_SEQ_NAMES = _create_custom_seq_names()
# Fixes for unusual capitalization or cases we don't care to handle in code.
# Also prevents titlecasing 'S' after apostrophe in posessives. Note we _do_
# want titlecasing after apostrophe in some cases, e.g. O'Clock.
_CUSTOM_CAPS_NAMES = {
(0x26d1,): 'Rescue Worker’s Helmet',
(0x1f170,): 'A Button (blood type)', # a Button (Blood Type)
(0x1f171,): 'B Button (blood type)', # B Button (Blood Type)
(0x1f17e,): 'O Button (blood type)', # O Button (Blood Type)
(0x1f18e,): 'AB Button (blood type)', # Ab Button (Blood Type)
(0x1f191,): 'CL Button', # Cl Button
(0x1f192,): 'COOL Button', # Cool Button
(0x1f193,): 'FREE Button', # Free Button
(0x1f194,): 'ID Button', # Id Button
(0x1f195,): 'NEW Button', # New Button
(0x1f196,): 'NG Button', # Ng Button
(0x1f197,): 'OK Button', # Ok Button
(0x1f198,): 'SOS Button', # Sos Button
(0x1f199,): 'UP! Button', # Up! Button
(0x1f19a,): 'VS Button', # Vs Button
(0x1f3e7,): 'ATM Sign', # Atm Sign
(0x1f44C,): 'OK Hand', # Ok Hand
(0x1f452,): 'Woman’s Hat',
(0x1f45a,): 'Woman’s Clothes',
(0x1f45e,): 'Man’s Shoe',
(0x1f461,): 'Woman’s Sandal',
(0x1f462,): 'Woman’s Boot',
(0x1f519,): 'BACK Arrow', # Back Arrow
(0x1f51a,): 'END Arrow', # End Arrow
(0x1f51b,): 'ON! Arrow', # On! Arrow
(0x1f51c,): 'SOON Arrow', # Soon Arrow
(0x1f51d,): 'TOP Arrow', # Top Arrow
(0x1f6b9,): 'Men’s Room',
(0x1f6ba,): 'Women’s Room',
}
# For the custom sequences we ignore ZWJ, the emoji variation selector
# and skin tone modifiers. We can't always ignore gender because
# the gendered sequences match against them, but we ignore gender in other
# cases so we define a separate set of gendered emoji to remove.
_NON_GENDER_CPS_TO_STRIP = frozenset(
[0xfe0f, 0x200d] +
range(unicode_data._FITZ_START, unicode_data._FITZ_END + 1))
_GENDER_CPS_TO_STRIP = frozenset([0x2640, 0x2642, 0x1f468, 0x1f469])
def _custom_name(seq):
"""Apply three kinds of custom names, based on the sequence."""
seq = tuple([cp for cp in seq if cp not in _NON_GENDER_CPS_TO_STRIP])
name = _CUSTOM_CAPS_NAMES.get(seq)
if name:
return name
# Single characters that participate in sequences (e.g. fire truck in the
# firefighter sequences) should not get converted. Single characters
# are in the custom caps names set but not the other sets.
if len(seq) == 1:
return None
name = _CUSTOM_GENDERED_SEQ_NAMES.get(seq)
if name:
return name
seq = tuple([cp for cp in seq if cp not in _GENDER_CPS_TO_STRIP])
name = _CUSTOM_SEQ_NAMES.get(seq)
return name
def _standard_name(seq):
"""Use the standard emoji name, with some algorithmic modifications.
We want to ignore skin-tone modifiers (but of course if the sequence _is_
the skin-tone modifier itself we keep that). So we strip these so we can
start with the generic name ignoring skin tone.
Non-emoji that are turned into emoji using the emoji VS have '(emoji) '
prepended to them, so strip that.
Regional indicator symbol names are a bit long, so shorten them.
Regional sequences are assumed to be ok as-is in terms of capitalization and
punctuation, so no modifications are applied to them.
After title-casing we make some English articles/prepositions lower-case
again. We also replace '&' with 'and'; Unicode seems rather fond of
ampersand."""
if not unicode_data.is_skintone_modifier(seq[0]):
seq = tuple([cp for cp in seq if not unicode_data.is_skintone_modifier(cp)])
name = unicode_data.get_emoji_sequence_name(seq)
if name.startswith('(emoji) '):
name = name[8:]
if len(seq) == 1 and unicode_data.is_regional_indicator(seq[0]):
return 'Regional Symbol ' + unicode_data.regional_indicator_to_ascii(seq[0])
if (unicode_data.is_regional_indicator_seq(seq) or
unicode_data.is_regional_tag_seq(seq)):
return name
name = name.title()
# Require space delimiting just in case...
name = re.sub(r'\s&\s', ' and ', name)
name = re.sub(
# not \b at start because we retain capital at start of phrase
r'(\s(:?A|And|From|In|Of|With|For))\b', lambda s: s.group(1).lower(),
name)
return name
def _name_data(seq, seq_file):
name = _custom_name(seq) or _standard_name(seq)
# we don't need canonical sequences
sequence = ''.join('&#x%x;' % cp for cp in seq if cp != 0xfe0f)
fname = path.basename(seq_file)
return fname, sequence, name
def generate_names(
src_dir, dst_dir, skip_limit=20, omit_groups=None, pretty_print=False,
verbose=False):
srcdir = tool_utils.resolve_path(src_dir)
if not path.isdir(srcdir):
print('%s is not a directory' % src_dir, file=sys.stderr)
return
if omit_groups:
unknown_groups = set(omit_groups) - set(unicode_data.get_emoji_groups())
if unknown_groups:
print('did not recognize %d group%s: %s' % (
len(unknown_groups), '' if len(unknown_groups) == 1 else 's',
', '.join('"%s"' % g for g in omit_groups if g in unknown_groups)), file=sys.stderr)
print('valid groups are:\n %s' % (
'\n '.join(g for g in unicode_data.get_emoji_groups())), file=sys.stderr)
return
print('omitting %d group%s: %s' % (
len(omit_groups), '' if len(omit_groups) == 1 else 's',
', '.join('"%s"' % g for g in omit_groups)))
else:
# might be None
print('keeping all groups')
omit_groups = []
# make sure the destination exists
dstdir = tool_utils.ensure_dir_exists(
tool_utils.resolve_path(dst_dir))
# _get_image_data returns canonical cp sequences
print('src dir:', srcdir)
seq_to_file = generate_emoji_html._get_image_data(srcdir, 'png', 'emoji_u')
print('seq to file has %d sequences' % len(seq_to_file))
# Aliases add non-gendered versions using gendered images for the most part.
# But when we display the images, we don't distinguish genders in the
# naming, we rely on the images-- so these look redundant. So we
# intentionally don't generate images for these.
# However, the alias file also includes the flag aliases, which we do want,
# and it also fails to exclude the unknown flag pua (since it doesn't
# map to anything), so we need to adjust for this.
canonical_aliases = generate_emoji_html._get_canonical_aliases()
aliases = set([
cps for cps in canonical_aliases.keys()
if not unicode_data.is_regional_indicator_seq(cps)])
aliases.add((0xfe82b,)) # unknown flag PUA
excluded = aliases | generate_emoji_html._get_canonical_excluded()
# The flag aliases have distinct names, so we _do_ want to show them
# multiple times.
to_add = {}
for seq in canonical_aliases:
if unicode_data.is_regional_indicator_seq(seq):
replace_seq = canonical_aliases[seq]
if seq in seq_to_file:
print('warning, alias %s has file %s' % (
unicode_data.regional_indicator_seq_to_string(seq),
seq_to_file[seq]))
continue
replace_file = seq_to_file.get(replace_seq)
if replace_file:
to_add[seq] = replace_file
seq_to_file.update(to_add)
data = []
last_skipped_group = None
skipcount = 0
for group in unicode_data.get_emoji_groups():
if group in omit_groups:
continue
name_data = []
for seq in unicode_data.get_emoji_in_group(group):
if seq in excluded:
continue
seq_file = seq_to_file.get(seq, None)
if seq_file is None:
skipcount += 1
if verbose:
if group != last_skipped_group:
print('group %s' % group)
last_skipped_group = group
print(' %s (%s)' % (
unicode_data.seq_to_string(seq),
', '.join(unicode_data.name(cp, 'x') for cp in seq)))
if skip_limit >= 0 and skipcount > skip_limit:
raise Exception('skipped too many items')
else:
name_data.append(_name_data(seq, seq_file))
data.append({'category': group, 'emojis': name_data})
outfile = path.join(dstdir, 'data.json')
with open(outfile, 'w') as f:
indent = 2 if pretty_print else None
separators = None if pretty_print else (',', ':')
json.dump(data, f, indent=indent, separators=separators)
print('wrote %s' % outfile)
def main():
DEFAULT_DSTDIR = '[emoji]/emoji'
DEFAULT_IMAGEDIR = '[emoji]/build/compressed_pngs'
parser = argparse.ArgumentParser()
parser.add_argument(
'-s', '--srcdir', help='directory containing images (default %s)' %
DEFAULT_IMAGEDIR, metavar='dir', default=DEFAULT_IMAGEDIR)
parser.add_argument(
'-d', '--dstdir', help='name of destination directory (default %s)' %
DEFAULT_DSTDIR, metavar='fname', default=DEFAULT_DSTDIR)
parser.add_argument(
'-p', '--pretty_print', help='pretty-print json file',
action='store_true')
parser.add_argument(
'-m', '--missing_limit', help='number of missing images before failure '
'(default 20), use -1 for no limit', metavar='n', default=20)
parser.add_argument(
'--omit_groups', help='names of groups to omit (default "Misc")',
metavar='name', default=['Misc'], nargs='*')
parser.add_argument(
'-v', '--verbose', help='print progress information to stdout',
action='store_true')
args = parser.parse_args()
generate_names(
args.srcdir, args.dstdir, args.missing_limit, args.omit_groups,
pretty_print=args.pretty_print, verbose=args.verbose)
if __name__ == "__main__":
main()