# This file is part of Rubber and thus covered by the GPL

# (c) Emmanuel Beffara, 2004--2006

"""

Indexing support with package 'index'.

This module handles the processing of the document's indices using a tool like

makeindex or xindy. It stores an MD5 sum of the source (.idx) file between two

runs, in order to detect modifications.

The following directives are provided to specify options for makeindex:

  tool <tool> =

    Choose which indexing tool should be used. Currently this can be either

    "makeindex" (by default) or "xindy".

  language <lang> =

    Choose the language used for sorting the index (xindy only).

  modules <mod> <mod> ... =

      Specify which modules xindy should use for the index.

  order <ordering> =

    Modify the ordering to be used (makeindex only, supported by xindy with

    warnings). The argument must be a space separated list of:

    - standard = use default ordering (no options, this is the default)

    - german = use German ordering (option "-g")

    - letter = use letter instead of word ordering (option "-l")

  path <directory> =

    Add the specified directory to the search path for styles.

  style <name> =

    Use the specified style file.

They all accept an optional argument first, enclosed in parentheses as in

"index.path (foo,bar) here/", to specify which index they apply to. Without

this argument, they apply to all indices declared at the point where they

occur.

"""

import os

from os.path import *

import re, string

import subprocess

import xml.dom.minidom

from subprocess import Popen, PIPE

from msg import _, msg

from plugins import TexModule

from util import md5_file

class Xindy:

    """

    Xindy command wrapper

    """

    def __init__(self, doc, idxfile, target, transcript="",

                 opts=None, modules=None,

                 index_lang="", style=""):

        self.doc = doc

        self.idxfile = idxfile

        self.target = target

        self.transcript = transcript

        self.opts = opts or []

        self.modules = modules or []

        self.index_lang = index_lang

        self.path_var = "XINDY_SEARCHPATH"

        mapfile = os.path.join(os.path.dirname(__file__), "xindylang.xml")

        self.languages = self.map_languages(mapfile)

        self._re_hyperindex = re.compile(r"hyperindexformat{\\(.*?)}}{",

                                         re.M|re.DOTALL)

        self.invalid_index_ranges = []

    def map_languages(self, mapfile):

        languages = {}

        dom_document = xml.dom.minidom.parse(mapfile)

        for dom_fontspec in dom_document.getElementsByTagName('map'):

            lang = dom_fontspec.getAttribute('lang')

            xindylang = dom_fontspec.getAttribute('xindylang')

            if xindylang:

                languages[lang] = xindylang

        dom_document.unlink()

        return languages

    def command(self):

        cmd = []

        if self.doc.program == "xelatex":

            # If raw index is in UTF-8 the texindy command cannot be used

            cmd.extend(["xindy", "-M", "texindy", "-C", self.doc.encoding])

            # To behave even more like texindy

            cmd.extend(["-q", "-M", "page-ranges"])

        else:

            # Call texindy to handle LICR encoded raw index

            # Equivalent to xindy arguments (beware of module order):

            #   "xindy", "-M", "tex/inputenc/latin",

            #            "-M", "texindy", "-C", "latin",

            #            "-I", "latex"

            cmd.extend(["texindy"])

        # Specific output files?

        if self.target:

            cmd.extend(["-o", self.target])

        if self.transcript:

            cmd.extend(["-t", self.transcript])

        # Find out which language to use

        if self.index_lang:

            lang = self.index_lang

        elif self.doc.lang:

            lang = self.languages.get(self.doc.lang)

            if not(lang):

                msg.warn(_("xindy: lang '%s' not found" % \

                           self.doc.lang), pkg="index")

            else:

                msg.log(_("xindy: lang '%s' mapped to '%s'" % \

                           (self.doc.lang, lang)), pkg="index")

        else:

            lang = None

        if lang:

            cmd.extend(["-L", lang])

        for mod in self.modules:

            cmd.extend(["-M", mod])

        if self.opts:

            cmd.extend(self.opts)

        cmd.append(self.idxfile)

        return cmd

    def _find_index_encoding(self, logname):

        # Texindy produces latin-* indexes. Try to find out which one from

        # the modules loaded by the script (language dependent)

        re_lang = re.compile("loading module \"lang/.*/(latin[^.-]*)")

        logfile = open(logname)

        encoding = ""

        for line in logfile:

            m = re_lang.search(line)

            if m:

                encoding = m.group(1)

                break

        logfile.close()

        return encoding

    def _index_is_unicode(self):

        f = file(self.target, "r")

        is_unicode = True 

        for line in f:

            try:

                line.decode("utf8")

            except:

                is_unicode = False

                break

        f.close()

        return is_unicode

    def _sanitize_idxfile(self):

        #

        # Remove the 'hyperindexformat' of the new hyperref that makes a mess

        # with Xindy. If not, the following error is raised by Xindy:

        # "WARNING: unknown cross-reference-class `hyperindexformat'! (ignored)"

        #

        f = file(self.idxfile, "r")

        data = f.read()

        f.close()

        data, nsub = self._re_hyperindex.subn(r"\1}{", data)

        if not(nsub):

            return

        msg.debug("Remove %d unsupported 'hyperindexformat' calls" % nsub)

        f = file(self.idxfile, "w")

        f.write(data)

        f.close()

    def _fix_invalid_ranges(self):

        if not(self.invalid_index_ranges): return

        f = open(self.idxfile)

        lines = f.readlines()

        f.close()

        # Track the lines with the wrong index ranges

        for i, line in enumerate(lines):

            for entry in self.invalid_index_ranges:

                if entry.index_key in line:

                    entry.add_line(i, line)

        # Summary of the lines to remove in order to fix the ranges

        skip_lines = []

        for entry in self.invalid_index_ranges:

            skip_lines.extend(entry.skip_lines)

            entry.reinit()

        if not(skip_lines): return

        # Remove the lines starting from the end to always have valid line num

        msg.debug("xindy: lines to remove from %s to fix ranges: %s" %\

                  (self.idxfile, skip_lines))

        skip_lines.sort()

        skip_lines.reverse()

        for line_num in skip_lines:

            del lines[line_num]

        f = open(self.idxfile, "w")

        f.writelines(lines)

        f.close()

    def _detect_invalid_ranges(self, data):

        # Look for warnings like this:

        #

        # WARNING: Found a :close-range in the index that wasn't opened before!

        #          Location-reference is 76 in keyword (Statute of Anne (1710))

        #          I'll continue and ignore this.

        #

        # Do it only once on the first run to find wrong indexes.

        if (self.invalid_index_ranges): return

        blocks = re.split("(WARNING:|ERROR:)", data, re.M)

        check_next_block = False

        for block in blocks:

            if "WARNING" in block:

                check_next_block = True

            elif check_next_block:

                m = re.search("Found.*?-range .*"\

                              "Location-reference is \d+ in keyword \((.*)\)",

                              block, re.M|re.DOTALL)

                if m: self.invalid_index_ranges.append(Indexentry(m.group(1)))

                check_next_block = False

    def run(self):

        self._sanitize_idxfile()

        self._fix_invalid_ranges()

        cmd = self.command()

        msg.debug(" ".join(cmd))

        # Collect the script output, and errors

        logname = join(dirname(self.target), "xindy.log")

        logfile = open(logname, "w")

        p = Popen(cmd, stdout=logfile, stderr=PIPE)

        errdata = p.communicate()[1]

        rc = p.wait()

        if msg.stdout:

            msg.stdout.write(errdata)

        else:

            msg.warn(_(errdata.strip()))

        logfile.close()

        if (rc != 0):

            msg.error(_("could not make index %s") % self.target)

            return 1

        self._detect_invalid_ranges(errdata)

        # Now convert the built index to UTF-8 if required

        if cmd[0] == "texindy" and self.doc.encoding == "utf8":

            if not(self._index_is_unicode()):

                encoding = self._find_index_encoding(logname)

                tmpindex = join(dirname(self.target), "new.ind")

                cmd = ["iconv", "-f", encoding, "-t", "utf8",

                       "-o", tmpindex, self.target]

                msg.debug(" ".join(cmd))

                rc = subprocess.call(cmd)

                if rc == 0: os.rename(tmpindex, self.target)

        return rc

class Indexentry:

    """

    Index entry wrapper from idxfile. Its role is to detect range anomalies

    """

    _re_entry = re.compile("\indexentry{(.*)\|([\(\)]?).*}{(\d+)}", re.DOTALL)

    def __init__(self, index_key):

        self.index_key = index_key

        self.skip_lines = []

        self.last_range_page = 0

        self.last_range_line = -1

        self.last_range_open = False

    def reinit(self):

        self.__init__(self.index_key)

    def add_line(self, line_num, indexentry):

        m = self._re_entry.search(indexentry)

        if not(m):

            return

        index_key = m.group(1).split("!")[-1]

        if index_key != self.index_key:

            return

        range_state = m.group(2)

        page = int(m.group(3))

        #print "Found %s at %d" % (index_key, page)

        if range_state == "(":

            # If a starting range overlap the previous range remove

            # this intermediate useless range close/open

            if page <= self.last_range_page:

                self.skip_lines += [self.last_range_line, line_num]

            self.last_range_page = page

            self.last_range_line = line_num

            self.last_range_open = True

        elif range_state == ")":

            self.last_range_page = page

            self.last_range_line = line_num

            self.last_range_open = False

        elif range_state == "":

            # If a single indexentry is within a range, skip it

            if self.last_range_open == True:

                self.skip_lines += [line_num]

class Makeindex:

    """

    Makeindex command wrapper

    """

    def __init__(self, doc, idxfile, target, transcript="",

                 opts=None, modules=None,

                 index_lang="", style=""):

        self.doc = doc

        self.idxfile = idxfile

        self.target = target

        self.transcript = transcript

        self.opts = opts or []

        self.path_var = "INDEXSTYLE"

        self.style = style

    def command(self):

        cmd = ["makeindex", "-o", self.target] + self.opts

        if self.transcript:

            cmd.extend(["-t", self.transcript])

        if self.style:

            cmd.extend(["-s", self.style])

        cmd.append(self.idxfile)

        return cmd

    def _index_is_unicode(self):

        f = file(self.target, "r")

        is_unicode = True 

        for line in f:

            try:

                line.decode("utf8")

            except:

                is_unicode = False

                break

        f.close()

        return is_unicode

    def run(self):

        cmd = self.command()

        msg.debug(" ".join(cmd))

        # Makeindex outputs everything to stderr, even progress messages

        rc = subprocess.call(cmd, stderr=msg.stdout)

        if (rc != 0):

            msg.error(_("could not make index %s") % self.target)

            return 1

        # Beware with UTF-8 encoding, makeindex with headings can be messy

        # because it puts in the headings the first 8bits char of the words

        # under the heading which can be an invalid character in UTF-8

        if (self.style and self.doc.encoding == "utf8"):

            if not(self._index_is_unicode()):

                # Retry without style to avoid headings

                msg.warn(_("makeindex on UTF8 failed. Retry..."))

                self.style = ""

                return self.run()

        return rc

class Index(TexModule):

    """

    This class represents a single index.

    """

    def __init__ (self, doc, source, target, transcript):

        """

        Initialize the index, by specifying the source file (generated by

        LaTeX), the target file (the output of makeindex) and the transcript

        (e.g. .ilg) file.  Transcript is used by glosstex.py.

        """

        self.paranoid = True

        self.doc = doc

        self.pbase = doc.src_base

        self.source = doc.src_base + "." + source

        self.target = doc.src_base + "." + target

        self.transcript = doc.src_base + "." + transcript

        # In paranoid mode, can output only in current working dir

        if self.paranoid and (os.path.dirname(self.target) == os.getcwd()):

            self.target = os.path.basename(self.target)

            self.transcript = os.path.basename(self.transcript)

        if os.path.exists(self.source):

            self.md5 = md5_file(self.source)

        else:

            self.md5 = None

        self.tool = "makeindex"

        self.tool_obj = None

        self.lang = None   # only for xindy

        self.modules = []  # only for xindy

        self.opts = []

        self.path = []

        self.style = None  # only for makeindex

    def do_language (self, lang):

        self.lang = lang

    def do_modules (self, *args):

        self.modules.extend(args)

    def do_order (self, *args):

        for opt in args:

            if opt == "standard": self.opts = []

            elif opt == "german": self.opts.append("-g")

            elif opt == "letter": self.opts.append("-l")

            else: msg.warn(

                _("unknown option '%s' for 'makeidx.order'") % opt)

    def do_path (self, path):

        self.path.append(self.doc.abspath(path))

    def do_style (self, style):

        self.style = style

    def do_tool (self, tool):

        if tool not in ("makeindex", "xindy"):

            msg.error(_("unknown indexing tool '%s'") % tool)

        self.tool = tool

    def post_compile (self):

        """

        Run the indexer tool

        """

        if not os.path.exists(self.source):

            msg.log(_("strange, there is no %s") % self.source, pkg="index")

            return 0

        if not self.run_needed():

            return 0

        msg.progress(_("processing index %s") % self.source)

        if not(self.tool_obj):

            if self.tool == "makeindex":

                index_cls = Makeindex

            elif self.tool == "xindy":

                index_cls = Xindy

            self.tool_obj = index_cls(self.doc,

                              self.source,

                              self.target,

                              transcript=self.transcript,

                              opts=self.opts,

                              modules=self.modules,

                              index_lang=self.lang,

                              style=self.style)

        rc = self.tool_obj.run()

        if rc != 0:

            return rc

        self.doc.must_compile = 1

        return 0

    def run_needed (self):

        """

        Check if makeindex has to be run. This is the case either if the

        target file does not exist or if the source file has changed.

        """

        if os.path.getsize(self.source) == 0:

            msg.log(_("the index file %s is empty") % self.source, pkg="index")

            return 0

        new = md5_file(self.source)

        if not os.path.exists(self.target):

            self.md5 = new

            return 1

        if not self.md5:

            self.md5 = new

            msg.log(_("the index file %s is new") % self.source, pkg="index")

            return 1

        if self.md5 == new:

            msg.log(_("the index %s did not change") % self.source, pkg="index")

            return 0

        self.md5 = new

        msg.log(_("the index %s has changed") % self.source, pkg="index")

        return 1

    def clean (self):

        """

        Remove all generated files related to the index.

        """

        for file in self.source, self.target, self.transcript:

            if exists(file):

                msg.log(_("removing %s") % file, pkg="index")

                os.unlink(file)

re_newindex = re.compile(" *{(?P<idx>[^{}]*)} *{(?P<ind>[^{}]*)}")

re_optarg = re.compile("\((?P<list>[^()]*)\) *")

class Module (TexModule):

    def __init__ (self, doc, dict):

        """

        Initialize the module with no index.

        """

        self.doc = doc

        self.indices = {}

        self.defaults = []

        self.commands = {}

        doc.parser.add_hook("makeindex", self.makeindex)

        doc.parser.add_hook("newindex", self.newindex)

    def register (self, name, idx, ind, ilg):

        """

        Register a new index.

        """

        index = self.indices[name] = Index(self.doc, idx, ind, ilg)

        for cmd in self.defaults:

            index.command(*cmd)

        if self.commands.has_key(name):

            for cmd in self.commands[name]:

                index.command(*cmd)

    def makeindex (self, dict):

        """

        Register the standard index.

        """

        self.register("default", "idx", "ind", "ilg")

    def newindex (self, dict):

        """

        Register a new index.

        """

        m = re_newindex.match(dict["line"])

        if not m:

            return

        index = dict["arg"]

        d = m.groupdict()

        self.register(index, d["idx"], d["ind"], "ilg")

        msg.log(_("index %s registered") % index, pkg="index")

    def command (self, cmd, args):

        indices = self.indices

        names = None

        if len(args) > 0:

            m = re_optarg.match(args[0])

            if m:

                names = m.group("list").split(",")

                args = args[1:]

        if names is None:

            self.defaults.append([cmd, args])

            names = indices.keys()

        for index in names:

            if indices.has_key(index):

                indices[index].command(cmd, args[1:])

            elif self.commands.has_key(index):

                self.commands[index].append([cmd, args])

            else:

                self.commands[index] = [[cmd, args]]

    def post_compile (self):

        for index in self.indices.values():

            if index.post_compile():

                return 1

        return 0

    def clean (self):

        for index in self.indices.values():

            index.clean()

        return 0