#!@PYTHON@ -s

#

# Copyright (c) 2010-2018 Shaun McCance <shaunm@gnome.org>

#

# ITS Tool program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by the

# Free Software Foundation, either version 3 of the License, or (at your

# option) any later version.

#

# ITS Tool is distributed in the hope that it will be useful, but WITHOUT

# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

# for more details.

#

# You should have received a copy of the GNU General Public License along

# with ITS Tool; if not, write to the Free Software Foundation, 59 Temple

# Place, Suite 330, Boston, MA  0211-1307  USA.

#

from __future__ import print_function

from __future__ import unicode_literals

VERSION="@VERSION@"

DATADIR="@DATADIR@"

import gettext

import hashlib

import libxml2

import optparse

import os

import os.path

import re

import sys

import time

import io

PY3 = sys.version_info[0] == 3

if PY3:

    string_types = str,

    def ustr(s, encoding=None):

        if isinstance(s, str):

            return s

        elif encoding:

            return str(s, encoding)

        else:

            return str(s)

    ustr_type = str

else:

    string_types = basestring,

    ustr = ustr_type = unicode

NS_ITS = 'http://www.w3.org/2005/11/its'

NS_ITST = 'http://itstool.org/extensions/'

NS_BLANK = 'http://itstool.org/extensions/blank/'

NS_XLINK = 'http://www.w3.org/1999/xlink'

NS_XML = 'http://www.w3.org/XML/1998/namespace'

class NoneTranslations:

    def gettext(self, message):

        return None

    def lgettext(self, message):

        return None

    def ngettext(self, msgid1, msgid2, n):

        return None

    def lngettext(self, msgid1, msgid2, n):

        return None

    def ugettext(self, message):

        return None

    def ungettext(self, msgid1, msgid2, n):

        return None

class MessageList (object):

    def __init__ (self):

        self._messages = []

        self._by_node = {}

        self._has_credits = False

    def add_message (self, message, node):

        self._messages.append (message)

        if node is not None:

            self._by_node[node] = message

    def add_credits(self):

        if self._has_credits:

            return

        msg = Message()

        msg.set_context('_')

        msg.add_text('translator-credits')

        msg.add_comment(Comment('Put one translator per line, in the form NAME <EMAIL>, YEAR1, YEAR2'))

        self._messages.append(msg)

        self._has_credits = True

    def get_message_by_node (self, node):

        return self._by_node.get(node, None)

    def get_nodes_with_messages (self):

        return list(self._by_node.keys())

    def output (self, out):

        msgs = []

        msgdict = {}

        for msg in self._messages:

            key = (msg.get_context(), msg.get_string())

            if key in msgdict:

                for source in msg.get_sources():

                    msgdict[key].add_source(source)

                for marker in msg.get_markers():

                    msgdict[key].add_marker(marker)

                for comment in msg.get_comments():

                    msgdict[key].add_comment(comment)

                for idvalue in msg.get_id_values():

                    msgdict[key].add_id_value(idvalue)

                if msg.get_preserve_space():

                    msgdict[key].set_preserve_space()

                if msg.get_locale_filter() is not None:

                    locale = msgdict[key].get_locale_filter()

                    if locale is not None:

                        msgdict[key].set_locale_filter('%s, %s' % (locale, msg.get_locale_filter()))

                    else:

                        msgdict[key].set_locale_filter(msg.get_locale_filter())

            else:

                msgs.append(msg)

                msgdict[key] = msg

        out.write('msgid ""\n')

        out.write('msgstr ""\n')

        out.write('"Project-Id-Version: PACKAGE VERSION\\n"\n')

        out.write('"POT-Creation-Date: %s\\n"\n' % time.strftime("%Y-%m-%d %H:%M%z"))

        out.write('"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n"\n')

        out.write('"Last-Translator: FULL NAME <EMAIL@ADDRESS>\\n"\n')

        out.write('"Language-Team: LANGUAGE <LL@li.org>\\n"\n')

        out.write('"MIME-Version: 1.0\\n"\n')

        out.write('"Content-Type: text/plain; charset=UTF-8\\n"\n')

        out.write('"Content-Transfer-Encoding: 8bit\\n"\n')

        out.write('\n')

        for msg in msgs:

            out.write(msg.format())

            out.write('\n')

class Comment (object):

    def __init__ (self, text):

        self._text = ustr(text)

        assert(text is not None)

        self._markers = []

    def add_marker (self, marker):

        self._markers.append(marker)

    def get_markers (self):

        return self._markers

    def get_text (self):

        return self._text

    def format (self):

        ret = ''

        markers = {}

        for marker in self._markers:

            if marker not in markers:

                ret += '#. (itstool) comment: ' + marker + '\n'

                markers[marker] = marker

        if '\n' in self._text:

            doadd = False

            for line in self._text.split('\n'):

                if line != '':

                    doadd = True

                if not doadd:

                    continue

                ret += '#. %s\n' % line

        else:

            text = self._text

            while len(text) > 72:

                j = text.rfind(' ', 0, 72)

                if j == -1:

                    j = text.find(' ')

                if j == -1:

                    break

                ret += '#. %s\n' % text[:j]

                text = text[j+1:]

            ret += '#. %s\n' % text

        return ret

class Placeholder (object):

    def __init__ (self, node):

        self.node = node

        self.name = ustr(node.name, 'utf-8')

class Message (object):

    def __init__ (self):

        self._message = []

        self._empty = True

        self._ctxt = None

        self._placeholders = []

        self._sources = []

        self._markers = []

        self._id_values = []

        self._locale_filter = None

        self._comments = []

        self._preserve = False

    def __repr__(self):

        if self._empty:

            return "Empty message"

        return self.get_string()

    def escape (self, text):

        return text.replace('\\','\\\\').replace('"', "\\\"").replace("\n","\\n").replace("\t","\\t")

    def add_text (self, text):

        if len(self._message) == 0 or not(isinstance(self._message[-1], string_types)):

            self._message.append('')

        if not isinstance(text, ustr_type):

            text = ustr(text, 'utf-8')

        self._message[-1] += text.replace('&', '&').replace('<', '<').replace('>', '>')

        if re.sub('\s+', ' ', text).strip() != '':

            self._empty = False

    def add_entity_ref (self, name):

        self._message.append('&' + name + ';')

        self._empty = False

    def add_placeholder (self, node):

        holder = Placeholder(node)

        self._placeholders.append(holder)

        self._message.append(holder)

    def get_placeholder (self, name):

        placeholder = 1

        for holder in self._placeholders:

            holdername = '%s-%i' % (holder.name, placeholder)

            if holdername == ustr(name, 'utf-8'):

                return holder

            placeholder += 1

    def add_start_tag (self, node):

        if len(self._message) == 0 or not(isinstance(self._message[-1], string_types)):

            self._message.append('')

        if node.ns() is not None and node.ns().name is not None:

            self._message[-1] += ('<%s:%s' % (ustr(node.ns().name, 'utf-8'), ustr(node.name, 'utf-8')))

        else:

            self._message[-1] += ('<%s' % ustr(node.name, 'utf-8'))

        for prop in xml_attr_iter(node):

            name = prop.name

            if prop.ns() is not None:

                name = prop.ns().name + ':' + name

            atval = prop.content

            if not isinstance(atval, ustr_type):

                atval = ustr(atval, 'utf-8')

            atval = atval.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"')

            self._message += " %s=\"%s\"" % (name, atval)

        if node.children is not None:

            self._message[-1] += '>'

        else:

            self._message[-1] += '/>'

    def add_end_tag (self, node):

        if node.children is not None:

            if len(self._message) == 0 or not(isinstance(self._message[-1], string_types)):

                self._message.append('')

            if node.ns() is not None and node.ns().name is not None:

                self._message[-1] += ('</%s:%s>' % (ustr(node.ns().name, 'utf-8'), ustr(node.name, 'utf-8')))

            else:

                self._message[-1] += ('</%s>' % ustr(node.name, 'utf-8'))

    def is_empty (self):

        return self._empty

    def get_context (self):

        return self._ctxt

    def set_context (self, ctxt):

        self._ctxt = ctxt

    def add_source (self, source):

        if not isinstance(source, ustr_type):

            source = ustr(source, 'utf-8')

        self._sources.append(source)

    def get_sources (self):

        return self._sources

    def add_marker (self, marker):

        if not isinstance(marker, ustr_type):

            marker = ustr(marker, 'utf-8')

        self._markers.append(marker)

    def get_markers (self):

        return self._markers

    def add_id_value(self, id_value):

        self._id_values.append(id_value)

    def get_id_values(self):

        return self._id_values

    def add_comment (self, comment):

        if comment is not None:

            self._comments.append(comment)

    def get_comments (self):

        return self._comments

    def get_string (self):

        message = ''

        placeholder = 1

        for msg in self._message:

            if isinstance(msg, string_types):

                message += msg

            elif isinstance(msg, Placeholder):

                message += '<_:%s-%i/>' % (msg.name, placeholder)

                placeholder += 1

        if not self._preserve:

            message = re.sub('\s+', ' ', message).strip()

        return message

    def get_preserve_space (self):

        return self._preserve

    def set_preserve_space (self, preserve=True):

        self._preserve = preserve

    def get_locale_filter(self):

        return self._locale_filter

    def set_locale_filter(self, locale):

        self._locale_filter = locale

    def format (self):

        ret = ''

        markers = {}

        for marker in self._markers:

            if marker not in markers:

                ret += '#. (itstool) path: ' + marker + '\n'

                markers[marker] = marker

        for idvalue in self._id_values:

            ret += '#. (itstool) id: ' + idvalue + '\n'

        if self._locale_filter is not None:

            ret += '#. (itstool) ' + self._locale_filter[1] + ' locale: ' + self._locale_filter[0] + '\n'

        comments = []

        commentsdict = {}

        for comment in self._comments:

            key = comment.get_text()

            if key in commentsdict:

                for marker in comment.get_markers():

                    commentsdict[key].add_marker(marker)

            else:

                comments.append(comment)

                commentsdict[key] = comment

        for i in range(len(comments)):

            if i != 0:

                ret += '#.\n'

            ret += comments[i].format()

        for source in self._sources:

            ret += '#: %s\n' % source

        if self._preserve:

            ret += '#, no-wrap\n'

        if self._ctxt is not None:

            ret += 'msgctxt "%s"\n' % self._ctxt

        message = self.get_string()

        if self._preserve:

            ret += 'msgid ""\n'

            lines = message.split('\n')

            for line, no in zip(lines, list(range(len(lines)))):

                if no == len(lines) - 1:

                    ret += '"%s"\n' % self.escape(line)

                else:

                    ret += '"%s\\n"\n' % self.escape(line)

        else:

            ret += 'msgid "%s"\n' % self.escape(message)

        ret += 'msgstr ""\n'

        return ret

def xml_child_iter (node):

    child = node.children

    while child is not None:

        yield child

        child = child.next

def xml_attr_iter (node):

    attr = node.get_properties()

    while attr is not None:

        yield attr

        attr = attr.next

def xml_is_ns_name (node, ns, name):

    if node.type != 'element':

        return False

    return node.name == name and node.ns() is not None and node.ns().content == ns

def xml_get_node_path(node):

    # The built-in nodePath() method only does numeric indexes

    # when necessary for disambiguation. For various reasons,

    # we prefer always using indexes.

    name = node.name

    if node.ns() is not None and node.ns().name is not None:

        name = node.ns().name + ':' + name

    if node.type == 'attribute':

        name = '@' + name

    name = '/' + name

    if node.type == 'element' and node.parent.type == 'element':

        count = 1

        prev = node.previousElementSibling()

        while prev is not None:

            if prev.name == node.name:

                if prev.ns() is None:

                    if node.ns() is None:

                        count += 1

                else:

                    if node.ns() is not None:

                        if prev.ns().name == node.ns().name:

                            count += 1

            prev = prev.previousElementSibling()

        name = '%s[%i]' % (name, count)

    if node.parent.type == 'element':

        name = xml_get_node_path(node.parent) + name

    return name

def xml_error_catcher(doc, error):

    doc._xml_err += " %s" % error

def fix_node_ns (node, nsdefs):

    childnsdefs = nsdefs.copy()

    nsdef = node.nsDefs()

    while nsdef is not None:

        nextnsdef = nsdef.next

        if nsdef.name in nsdefs and nsdefs[nsdef.name] == nsdef.content:

            node.removeNsDef(nsdef.content)

        else:

            childnsdefs[nsdef.name] = nsdef.content

        nsdef = nextnsdef

    for child in xml_child_iter(node):

        if child.type == 'element':

            fix_node_ns(child, childnsdefs)

class LocNote (object):

    def __init__(self, locnote=None, locnoteref=None, locnotetype=None, space=False):

        self.locnote = locnote

        self.locnoteref = locnoteref

        self.locnotetype = locnotetype

        if self.locnotetype != 'alert':

            self.locnotetype = 'description'

        self._preserve_space=space

    def __repr__(self):

        if self.locnote is not None:

            if self._preserve_space:

                return self.locnote

            else:

                return re.sub('\s+', ' ', self.locnote).strip()

        elif self.locnoteref is not None:

            return '(itstool) link: ' + re.sub('\s+', ' ', self.locnoteref).strip()

        return ''

class Document (object):

    def __init__ (self, filename, messages, load_dtd=False, keep_entities=False):

        self._xml_err = ''

        libxml2.registerErrorHandler(xml_error_catcher, self)

        try:

            ctxt = libxml2.createFileParserCtxt(filename)

        except:

            sys.stderr.write('Error: cannot open XML file %s\n' % filename)

            sys.exit(1)

        ctxt.lineNumbers(1)

        self._load_dtd = load_dtd

        self._keep_entities = keep_entities

        if load_dtd:

            ctxt.loadSubset(1)

        if keep_entities:

            ctxt.loadSubset(1)

            ctxt.ctxtUseOptions(libxml2.XML_PARSE_DTDLOAD)

            ctxt.replaceEntities(0)

        else:

            ctxt.replaceEntities(1)

        ctxt.parseDocument()

        self._filename = filename

        self._doc = ctxt.doc()

        self._localrules = []

        def pre_process (node):

            for child in xml_child_iter(node):

                if xml_is_ns_name(child, 'http://www.w3.org/2001/XInclude', 'include'):

                    if child.nsProp('parse', None) == 'text':

                        child.xincludeProcessTree()

                elif xml_is_ns_name(child, NS_ITS, 'rules'):

                    if child.hasNsProp('href', NS_XLINK):

                        href = child.nsProp('href', NS_XLINK)

                        fileref = os.path.join(os.path.dirname(filename), href)

                        if not os.path.exists(fileref):

                            if opts.itspath is not None:

                                for pathdir in opts.itspath:

                                    fileref = os.path.join(pathdir, href)

                                    if os.path.exists(fileref):

                                        break

                        if not os.path.exists(fileref):

                            sys.stderr.write('Error: Could not locate ITS file %s\n' % href)

                            sys.exit(1)

                        hctxt = libxml2.createFileParserCtxt(fileref)

                        hctxt.replaceEntities(1)

                        hctxt.parseDocument()

                        root = hctxt.doc().getRootElement()

                        version = None

                        if root.hasNsProp('version', None):

                            version = root.nsProp('version', None)

                        else:

                            sys.stderr.write('Warning: ITS file %s missing version attribute\n' %

                                             os.path.basename(href))

                        if version is not None and version not in ('1.0', '2.0'):

                            sys.stderr.write('Warning: Skipping ITS file %s with unknown version %s\n' %

                                             (os.path.basename(href), root.nsProp('version', None)))

                        else:

                            self._localrules.append(root)

                    version = None

                    if child.hasNsProp('version', None):

                        version = child.nsProp('version', None)

                    else:

                        root = child.doc.getRootElement()

                        if root.hasNsProp('version', NS_ITS):

                            version = root.nsProp('version', NS_ITS)

                        else:

                            sys.stderr.write('Warning: Local ITS rules missing version attribute\n')

                    if version is not None and version not in ('1.0', '2.0'):

                        sys.stderr.write('Warning: Skipping local ITS rules with unknown version %s\n' %

                                         version)

                    else:

                        self._localrules.append(child)

                pre_process(child)

        pre_process(self._doc)

        try:

            self._check_errors()

        except libxml2.parserError as e:

            sys.stderr.write('Error: Could not parse document:\n%s\n' % ustr(e))

            sys.exit(1)

        self._msgs = messages

        self._its_translate_nodes = {}

        self._its_within_text_nodes = {}

        self._its_locale_filters = {}

        self._its_id_values = {}

        self._its_loc_notes = {}

        self._its_preserve_space_nodes = {}

        self._itst_drop_nodes = {}

        self._itst_contexts = {}

        self._its_lang = {}

        self._itst_lang_attr = {}

        self._itst_credits = None

        self._its_externals = {}

    def __del__ (self):

        self._doc.freeDoc()

    def _check_errors(self):

        if self._xml_err:

            raise libxml2.parserError(self._xml_err)

    def register_its_params(self, xpath, rules, params={}):

        for child in xml_child_iter(rules):

            if xml_is_ns_name(child, NS_ITS, 'param'):

                name = child.nsProp('name', None)

                if name in params:

                    value = params[name]

                else:

                    value = child.getContent()

                xpath.xpathRegisterVariable(name, None, value)

    def apply_its_rule(self, rule, xpath):

        if rule.type != 'element':

            return

        if xml_is_ns_name(rule, NS_ITS, 'translateRule'):

            if rule.nsProp('selector', None) is not None:

                for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)):

                    self._its_translate_nodes[node] = rule.nsProp('translate', None)

        elif xml_is_ns_name(rule, NS_ITS, 'withinTextRule'):

            if rule.nsProp('selector', None) is not None:

                for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)):

                    self._its_within_text_nodes[node] = rule.nsProp('withinText', None)

        elif xml_is_ns_name(rule, NS_ITST, 'preserveSpaceRule'):

            if rule.nsProp('selector', None) is not None:

                for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)):

                    val = rule.nsProp('preserveSpace', None)

                    if val == 'yes':

                        self._its_preserve_space_nodes[node] = 'preserve'

        elif xml_is_ns_name(rule, NS_ITS, 'preserveSpaceRule'):

            if rule.nsProp('selector', None) is not None:

                for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)):

                    self._its_preserve_space_nodes[node] = rule.nsProp('space', None)

        elif xml_is_ns_name(rule, NS_ITS, 'localeFilterRule'):

            if rule.nsProp('selector', None) is not None:

                if rule.hasNsProp('localeFilterList', None):

                    lst = rule.nsProp('localeFilterList', None)

                else:

                    lst = '*'

                if rule.hasNsProp('localeFilterType', None):

                    typ = rule.nsProp('localeFilterType', None)

                else:

                    typ = 'include'

                for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)):

                    self._its_locale_filters[node] = (lst, typ)

        elif xml_is_ns_name(rule, NS_ITST, 'dropRule'):

            if rule.nsProp('selector', None) is not None:

                for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)):

                    self._itst_drop_nodes[node] = rule.nsProp('drop', None)

        elif xml_is_ns_name(rule, NS_ITS, 'idValueRule'):

            sel = rule.nsProp('selector', None)

            idv = rule.nsProp('idValue', None)

            if sel is not None and idv is not None:

                for node in self._try_xpath_eval(xpath, sel):

                    try:

                        oldnode = xpath.contextNode()

                    except:

                        oldnode = None

                    xpath.setContextNode(node)

                    idvalue = self._try_xpath_eval(xpath, idv)

                    if isinstance(idvalue, string_types):

                        self._its_id_values[node] = idvalue

                    else:

                        for val in idvalue:

                            self._its_id_values[node] = val.content

                            break

                    xpath.setContextNode(oldnode)

            pass

        elif xml_is_ns_name(rule, NS_ITST, 'contextRule'):

            if rule.nsProp('selector', None) is not None:

                for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)):

                    if rule.hasNsProp('context', None):

                        self._itst_contexts[node] = rule.nsProp('context', None)

                    elif rule.hasNsProp('contextPointer', None):

                        try:

                            oldnode = xpath.contextNode()

                        except:

                            oldnode = None

                        xpath.setContextNode(node)

                        ctxt = self._try_xpath_eval(xpath, rule.nsProp('contextPointer', None))

                        if isinstance(ctxt, string_types):

                            self._itst_contexts[node] = ctxt

                        else:

                            for ctxt in ctxt:

                                self._itst_contexts[node] = ctxt.content

                                break

                        xpath.setContextNode(oldnode)

        elif xml_is_ns_name(rule, NS_ITS, 'locNoteRule'):

            locnote = None

            notetype = rule.nsProp('locNoteType', None)

            for child in xml_child_iter(rule):

                if xml_is_ns_name(child, NS_ITS, 'locNote'):

                    locnote = LocNote(locnote=child.content, locnotetype=notetype)

                    break

            if locnote is None:

                if rule.hasNsProp('locNoteRef', None):

                    locnote = LocNote(locnoteref=rule.nsProp('locNoteRef', None), locnotetype=notetype)

            if rule.nsProp('selector', None) is not None:

                for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)):

                    if locnote is not None:

                        self._its_loc_notes.setdefault(node, []).append(locnote)

                    else:

                        if rule.hasNsProp('locNotePointer', None):

                            sel = rule.nsProp('locNotePointer', None)

                            ref = False

                        elif rule.hasNsProp('locNoteRefPointer', None):

                            sel = rule.nsProp('locNoteRefPointer', None)

                            ref = True

                        else:

                            continue

                        try:

                            oldnode = xpath.contextNode()

                        except:

                            oldnode = None

                        xpath.setContextNode(node)

                        note = self._try_xpath_eval(xpath, sel)

                        if isinstance(note, string_types):

                            if ref:

                                nodenote = LocNote(locnoteref=note, locnotetype=notetype)

                            else:

                                nodenote = LocNote(locnote=note, locnotetype=notetype)

                            self._its_loc_notes.setdefault(node, []).append(nodenote)

                        else:

                            for note in note:

                                if ref:

                                    nodenote = LocNote(locnoteref=note.content, locnotetype=notetype)

                                else:

                                    nodenote = LocNote(locnote=note.content, locnotetype=notetype,

                                                       space=self.get_preserve_space(note))

                                self._its_loc_notes.setdefault(node, []).append(nodenote)

                                break

                        xpath.setContextNode(oldnode)

        elif xml_is_ns_name(rule, NS_ITS, 'langRule'):

            if rule.nsProp('selector', None) is not None and rule.nsProp('langPointer', None) is not None:

                for node in self._try_xpath_eval(xpath, rule.nsProp('selector', None)):

                    try:

                        oldnode = xpath.contextNode()

                    except:

                        oldnode = None

                    xpath.setContextNode(node)

                    res = self._try_xpath_eval(xpath, rule.nsProp('langPointer', None))

                    if len(res) > 0:

                        self._its_lang[node] = res[0].content

                    # We need to construct language attributes, not just read

                    # language information. Technically, langPointer could be

                    # any XPath expression. But if it looks like an attribute

                    # accessor, just use the attribute name.

                    if rule.nsProp('langPointer', None)[0] == '@':

                        self._itst_lang_attr[node] = rule.nsProp('langPointer', None)[1:]

                    xpath.setContextNode(oldnode)

        elif xml_is_ns_name(rule, NS_ITST, 'credits'):

            if rule.nsProp('appendTo', None) is not None:

                for node in self._try_xpath_eval(xpath, rule.nsProp('appendTo', None)):

                    self._itst_credits = (node, rule)

                    break

        elif (xml_is_ns_name(rule, NS_ITS, 'externalResourceRefRule') or

              xml_is_ns_name(rule, NS_ITST, 'externalRefRule')):

            sel = rule.nsProp('selector', None)

            if xml_is_ns_name(rule, NS_ITS, 'externalResourceRefRule'):

                ptr = rule.nsProp('externalResourceRefPointer', None)

            else:

                ptr = rule.nsProp('refPointer', None)

            if sel is not None and ptr is not None:

                for node in self._try_xpath_eval(xpath, sel):

                    try:

                        oldnode = xpath.contextNode()

                    except:

                        oldnode = None

                    xpath.setContextNode(node)

                    res = self._try_xpath_eval(xpath, ptr)

                    if len(res) > 0:

                        self._its_externals[node] = res[0].content

                    xpath.setContextNode(oldnode)

    def apply_its_rules(self, builtins, params={}):

        if builtins:

            dirs = []

            ddir = os.getenv('XDG_DATA_HOME', '')

            if ddir == '':

                ddir = os.path.join(os.path.expanduser('~'), '.local', 'share')

            dirs.append(ddir)

            ddir = os.getenv('XDG_DATA_DIRS', '')

            if ddir == '':

                if DATADIR not in ('/usr/local/share', '/usr/share'):

                    ddir += DATADIR + ':'

                ddir += '/usr/local/share:/usr/share'

            dirs.extend(ddir.split(':'))

            ddone = {}

            for ddir in dirs:

                itsdir = os.path.join(ddir, 'itstool', 'its')

                if not os.path.exists(itsdir):

                    continue

                for dfile in os.listdir(itsdir):

                    if dfile.endswith('.its'):

                        if not ddone.get(dfile, False):

                            self.apply_its_file(os.path.join(itsdir, dfile), params=params)

                            ddone[dfile] = True

        self.apply_local_its_rules(params=params)

    def apply_its_file(self, filename, params={}):

        doc = libxml2.parseFile(filename)

        root = doc.getRootElement()

        if not xml_is_ns_name(root, NS_ITS, 'rules'):

            return

        version = None

        if root.hasNsProp('version', None):

            version = root.nsProp('version', None)

        else:

            sys.stderr.write('Warning: ITS file %s missing version attribute\n' %

                             os.path.basename(filename))

        if version is not None and version not in ('1.0', '2.0'):

            sys.stderr.write('Warning: Skipping ITS file %s with unknown version %s\n' %

                             (os.path.basename(filename), root.nsProp('version', None)))

            return

        matched = True

        for match in xml_child_iter(root):

            if xml_is_ns_name(match, NS_ITST, 'match'):

                matched = False

                xpath = self._doc.xpathNewContext()

                par = match

                nss = {}

                while par is not None:

                    nsdef = par.nsDefs()

                    while nsdef is not None:

                        if nsdef.name is not None:

                            if nsdef.name not in nss:

                                nss[nsdef.name] = nsdef.content

                                xpath.xpathRegisterNs(nsdef.name, nsdef.content)

                        nsdef = nsdef.next

                    par = par.parent

                if match.hasNsProp('selector', None):

                    if len(self._try_xpath_eval(xpath, match.nsProp('selector', None))) > 0:

                        matched = True

                        break

        if matched == False:

            return

        for rule in xml_child_iter(root):

            xpath = self._doc.xpathNewContext()

            par = match

            nss = {}

            while par is not None:

                nsdef = par.nsDefs()

                while nsdef is not None:

                    if nsdef.name is not None:

                        if nsdef.name not in nss:

                            nss[nsdef.name] = nsdef.content

                            xpath.xpathRegisterNs(nsdef.name, nsdef.content)

                    nsdef = nsdef.next

                par = par.parent

            self.register_its_params(xpath, root, params=params)

            self.apply_its_rule(rule, xpath)

    def apply_local_its_rules(self, params={}):

        for rules in self._localrules:

            def reg_ns(xpath, node):

                if node.parent is not None:

                    reg_ns(xpath, node.parent)

                nsdef = node.nsDefs()

                while nsdef is not None:

                    if nsdef.name is not None:

                        xpath.xpathRegisterNs(nsdef.name, nsdef.content)

                    nsdef = nsdef.next

            xpath = self._doc.xpathNewContext()

            reg_ns(xpath, rules)

            self.register_its_params(xpath, rules, params=params)

            for rule in xml_child_iter(rules):

                if rule.type != 'element':

                    continue

                if rule.nsDefs() is not None:

                    rule_xpath = self._doc.xpathNewContext()

                    reg_ns(rule_xpath, rule)

                    self.register_its_params(rule_xpath, rules, params=params)

                else:

                    rule_xpath = xpath

                self.apply_its_rule(rule, rule_xpath)

    def _append_credits(self, parent, node, trdata):

        if xml_is_ns_name(node, NS_ITST, 'for-each'):

            select = node.nsProp('select', None)

            if select == 'years':

                for year in trdata[2].split(','):

                    for child in xml_child_iter(node):

                        self._append_credits(parent, child, trdata + (year.strip(),))

        elif xml_is_ns_name(node, NS_ITST, 'value-of'):

            select = node.nsProp('select', None)

            val = None

            if select == 'name':

                val = trdata[0]

            elif select == 'email':

                val = trdata[1]

            elif select == 'years':

                val = trdata[2]

            elif select == 'year' and len(trdata) == 4:

                val = trdata[3]

            if val is not None:

                if not PY3:

                    val = val.encode('utf-8')

                parent.addContent(val)

        else:

            newnode = node.copyNode(2)

            parent.addChild(newnode)

            for child in xml_child_iter(node):

                self._append_credits(newnode, child, trdata)

    def merge_credits(self, translations, language, node):

        if self._itst_credits is None:

            return

        # Dear Python, please implement pgettext.

        # http://bugs.python.org/issue2504

        # Sincerely, Shaun

        trans = translations.ugettext('_\x04translator-credits')

        if trans is None or trans == 'translator-credits':

            return

        regex = re.compile('(.*) \<(.*)\>, (.*)')

        for credit in trans.split('\n'):

            match = regex.match(credit)

            if not match:

                continue

            trdata = match.groups()