#!/usr/bin/env python3

#

#  Syntax: mkdoc.py [-I<path> ..] [.. a list of header files ..]

#

#  Extract documentation from C++ header files to use it in Python bindings

#

import os

import sys

import platform

import re

import textwrap

from clang import cindex

from clang.cindex import CursorKind

from collections import OrderedDict

from glob import glob

from threading import Thread, Semaphore

from multiprocessing import cpu_count

RECURSE_LIST = [

    CursorKind.TRANSLATION_UNIT,

    CursorKind.NAMESPACE,

    CursorKind.CLASS_DECL,

    CursorKind.STRUCT_DECL,

    CursorKind.ENUM_DECL,

    CursorKind.CLASS_TEMPLATE

]

PRINT_LIST = [

    CursorKind.CLASS_DECL,

    CursorKind.STRUCT_DECL,

    CursorKind.ENUM_DECL,

    CursorKind.ENUM_CONSTANT_DECL,

    CursorKind.CLASS_TEMPLATE,

    CursorKind.FUNCTION_DECL,

    CursorKind.FUNCTION_TEMPLATE,

    CursorKind.CONVERSION_FUNCTION,

    CursorKind.CXX_METHOD,

    CursorKind.CONSTRUCTOR,

    CursorKind.FIELD_DECL

]

PREFIX_BLACKLIST = [

    CursorKind.TRANSLATION_UNIT

]

CPP_OPERATORS = {

    '<=': 'le', '>=': 'ge', '==': 'eq', '!=': 'ne', '[]': 'array',

    '+=': 'iadd', '-=': 'isub', '*=': 'imul', '/=': 'idiv', '%=':

    'imod', '&=': 'iand', '|=': 'ior', '^=': 'ixor', '<<=': 'ilshift',

    '>>=': 'irshift', '++': 'inc', '--': 'dec', '<<': 'lshift', '>>':

    'rshift', '&&': 'land', '||': 'lor', '!': 'lnot', '~': 'bnot',

    '&': 'band', '|': 'bor', '+': 'add', '-': 'sub', '*': 'mul', '/':

    'div', '%': 'mod', '<': 'lt', '>': 'gt', '=': 'assign', '()': 'call'

}

CPP_OPERATORS = OrderedDict(

    sorted(CPP_OPERATORS.items(), key=lambda t: -len(t[0])))

job_count = cpu_count()

job_semaphore = Semaphore(job_count)

class NoFilenamesError(ValueError):

    pass

def d(s):

    return s if isinstance(s, str) else s.decode('utf8')

def sanitize_name(name):

    name = re.sub(r'type-parameter-0-([0-9]+)', r'T\1', name)

    for k, v in CPP_OPERATORS.items():

        name = name.replace('operator%s' % k, 'operator_%s' % v)

    name = re.sub('<.*>', '', name)

    name = ''.join([ch if ch.isalnum() else '_' for ch in name])

    name = re.sub('_$', '', re.sub('_+', '_', name))

    return '__doc_' + name

def process_comment(comment):

    result = ''

    # Remove C++ comment syntax

    leading_spaces = float('inf')

    for s in comment.expandtabs(tabsize=4).splitlines():

        s = s.strip()

        if s.startswith('/*'):

            s = s[2:].lstrip('*')

        elif s.endswith('*/'):

            s = s[:-2].rstrip('*')

        elif s.startswith('///'):

            s = s[3:]

        if s.startswith('*'):

            s = s[1:]

        if len(s) > 0:

            leading_spaces = min(leading_spaces, len(s) - len(s.lstrip()))

        result += s + '\n'

    if leading_spaces != float('inf'):

        result2 = ""

        for s in result.splitlines():

            result2 += s[leading_spaces:] + '\n'

        result = result2

    # Doxygen tags

    cpp_group = '([\w:]+)'

    param_group = '([\[\w:\]]+)'

    s = result

    s = re.sub(r'\\c\s+%s' % cpp_group, r'``\1``', s)

    s = re.sub(r'\\a\s+%s' % cpp_group, r'*\1*', s)

    s = re.sub(r'\\e\s+%s' % cpp_group, r'*\1*', s)

    s = re.sub(r'\\em\s+%s' % cpp_group, r'*\1*', s)

    s = re.sub(r'\\b\s+%s' % cpp_group, r'**\1**', s)

    s = re.sub(r'\\ingroup\s+%s' % cpp_group, r'', s)

    s = re.sub(r'\\param%s?\s+%s' % (param_group, cpp_group),

               r'\n\n$Parameter ``\2``:\n\n', s)

    s = re.sub(r'\\tparam%s?\s+%s' % (param_group, cpp_group),

               r'\n\n$Template parameter ``\2``:\n\n', s)

    for in_, out_ in {

        'return': 'Returns',

        'author': 'Author',

        'authors': 'Authors',

        'copyright': 'Copyright',

        'date': 'Date',

        'remark': 'Remark',

        'sa': 'See also',

        'see': 'See also',

        'extends': 'Extends',

        'throw': 'Throws',

        'throws': 'Throws'

    }.items():

        s = re.sub(r'\\%s\s*' % in_, r'\n\n$%s:\n\n' % out_, s)

    s = re.sub(r'\\details\s*', r'\n\n', s)

    s = re.sub(r'\\brief\s*', r'', s)

    s = re.sub(r'\\short\s*', r'', s)

    s = re.sub(r'\\ref\s*', r'', s)

    s = re.sub(r'\\code\s?(.*?)\s?\\endcode',

               r"```\n\1\n```\n", s, flags=re.DOTALL)

    # HTML/TeX tags

    s = re.sub(r'<tt>(.*?)</tt>', r'``\1``', s, flags=re.DOTALL)

    s = re.sub(r'
(.*?)
', r"```\n\1\n```\n", s, flags=re.DOTALL)

    s = re.sub(r'(.*?)', r'*\1*', s, flags=re.DOTALL)

    s = re.sub(r'(.*?)', r'**\1**', s, flags=re.DOTALL)

    s = re.sub(r'\\f\$(.*?)\\f\$', r'$\1$', s, flags=re.DOTALL)

    s = re.sub(r'
', r'\n\n* ', s)

    s = re.sub(r'</?ul>', r'', s)

    s = re.sub(r'', r'\n\n', s)

    s = s.replace('``true``', '``True``')

    s = s.replace('``false``', '``False``')

    # Re-flow text

    wrapper = textwrap.TextWrapper()

    wrapper.expand_tabs = True

    wrapper.replace_whitespace = True

    wrapper.drop_whitespace = True

    wrapper.width = 70

    wrapper.initial_indent = wrapper.subsequent_indent = ''

    result = ''

    in_code_segment = False

    for x in re.split(r'(```)', s):

        if x == '```':

            if not in_code_segment:

                result += '```\n'

            else:

                result += '\n```\n\n'

            in_code_segment = not in_code_segment

        elif in_code_segment:

            result += x.strip()

        else:

            for y in re.split(r'(?: *\n *){2,}', x):

                wrapped = wrapper.fill(re.sub(r'\s+', ' ', y).strip())

                if len(wrapped) > 0 and wrapped[0] == '$':

                    result += wrapped[1:] + '\n'

                    wrapper.initial_indent = \

                        wrapper.subsequent_indent = ' ' * 4

                else:

                    if len(wrapped) > 0:

                        result += wrapped + '\n\n'

                    wrapper.initial_indent = wrapper.subsequent_indent = ''

    return result.rstrip().lstrip('\n')

def extract(filename, node, prefix, output):

    if not (node.location.file is None or

            os.path.samefile(d(node.location.file.name), filename)):

        return 0

    if node.kind in RECURSE_LIST:

        sub_prefix = prefix

        if node.kind not in PREFIX_BLACKLIST:

            if len(sub_prefix) > 0:

                sub_prefix += '_'

            sub_prefix += d(node.spelling)

        for i in node.get_children():

            extract(filename, i, sub_prefix, output)

    if node.kind in PRINT_LIST:

        comment = d(node.raw_comment) if node.raw_comment is not None else ''

        comment = process_comment(comment)

        sub_prefix = prefix

        if len(sub_prefix) > 0:

            sub_prefix += '_'

        if len(node.spelling) > 0:

            name = sanitize_name(sub_prefix + d(node.spelling))

            output.append((name, filename, comment))

class ExtractionThread(Thread):

    def __init__(self, filename, parameters, output):

        Thread.__init__(self)

        self.filename = filename

        self.parameters = parameters

        self.output = output

        job_semaphore.acquire()

    def run(self):

        print('Processing "%s" ..' % self.filename, file=sys.stderr)

        try:

            index = cindex.Index(

                cindex.conf.lib.clang_createIndex(False, True))

            tu = index.parse(self.filename, self.parameters)

            extract(self.filename, tu.cursor, '', self.output)

        finally:

            job_semaphore.release()

def read_args(args):

    parameters = []

    filenames = []

    if "-x" not in args:

        parameters.extend(['-x', 'c++'])

    if not any(it.startswith("-std=") for it in args):

        parameters.append('-std=c++11')

    if platform.system() == 'Darwin':

        dev_path = '/Applications/Xcode.app/Contents/Developer/'

        lib_dir = dev_path + 'Toolchains/XcodeDefault.xctoolchain/usr/lib/'

        sdk_dir = dev_path + 'Platforms/MacOSX.platform/Developer/SDKs'

        libclang = lib_dir + 'libclang.dylib'

        if os.path.exists(libclang):

            cindex.Config.set_library_path(os.path.dirname(libclang))

        if os.path.exists(sdk_dir):

            sysroot_dir = os.path.join(sdk_dir, next(os.walk(sdk_dir))[1][0])

            parameters.append('-isysroot')

            parameters.append(sysroot_dir)

    elif platform.system() == 'Linux':

        # clang doesn't find its own base includes by default on Linux,

        # but different distros install them in different paths.

        # Try to autodetect, preferring the highest numbered version.

        def clang_folder_version(d):

            return [int(ver) for ver in re.findall(r'(?

        clang_include_dir = max((

            path

            for libdir in ['lib64', 'lib', 'lib32']

            for path in glob('/usr/%s/clang/*/include' % libdir)

            if os.path.isdir(path)

        ), default=None, key=clang_folder_version)

        if clang_include_dir:

            parameters.extend(['-isystem', clang_include_dir])

    for item in args:

        if item.startswith('-'):

            parameters.append(item)

        else:

            filenames.append(item)

    if len(filenames) == 0:

        raise NoFilenamesError("args parameter did not contain any filenames")

    return parameters, filenames

def extract_all(args):

    parameters, filenames = read_args(args)

    output = []

    for filename in filenames:

        thr = ExtractionThread(filename, parameters, output)

        thr.start()

    print('Waiting for jobs to finish ..', file=sys.stderr)

    for i in range(job_count):

        job_semaphore.acquire()

    return output

def write_header(comments, out_file=sys.stdout):

    print('''/*

  This file contains docstrings for the Python bindings.

  Do not edit! These were automatically extracted by mkdoc.py

 */

#define __EXPAND(x)                                      x

#define __COUNT(_1, _2, _3, _4, _5, _6, _7, COUNT, ...)  COUNT

#define __VA_SIZE(...)                                   __EXPAND(__COUNT(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1))

#define __CAT1(a, b)                                     a ## b

#define __CAT2(a, b)                                     __CAT1(a, b)

#define __DOC1(n1)                                       __doc_##n1

#define __DOC2(n1, n2)                                   __doc_##n1##_##n2

#define __DOC3(n1, n2, n3)                               __doc_##n1##_##n2##_##n3

#define __DOC4(n1, n2, n3, n4)                           __doc_##n1##_##n2##_##n3##_##n4

#define __DOC5(n1, n2, n3, n4, n5)                       __doc_##n1##_##n2##_##n3##_##n4##_##n5

#define __DOC6(n1, n2, n3, n4, n5, n6)                   __doc_##n1##_##n2##_##n3##_##n4##_##n5##_##n6

#define __DOC7(n1, n2, n3, n4, n5, n6, n7)               __doc_##n1##_##n2##_##n3##_##n4##_##n5##_##n6##_##n7

#define DOC(...)                                         __EXPAND(__EXPAND(__CAT2(__DOC, __VA_SIZE(__VA_ARGS__)))(__VA_ARGS__))

#if defined(__GNUG__)

#pragma GCC diagnostic push

#pragma GCC diagnostic ignored "-Wunused-variable"

#endif

''', file=out_file)

    name_ctr = 1

    name_prev = None

    for name, _, comment in list(sorted(comments, key=lambda x: (x[0], x[1]))):

        if name == name_prev:

            name_ctr += 1

            name = name + "_%i" % name_ctr

        else:

            name_prev = name

            name_ctr = 1

        print('\nstatic const char *%s =%sR"doc(%s)doc";' %

              (name, '\n' if '\n' in comment else ' ', comment), file=out_file)

    print('''

#if defined(__GNUG__)

#pragma GCC diagnostic pop

#endif

''', file=out_file)

def mkdoc(args):

    args = list(args)

    out_path = None

    for idx, arg in enumerate(args):

        if arg.startswith("-o"):

            args.remove(arg)

            try:

                out_path = arg[2:] or args.pop(idx)

            except IndexError:

                print("-o flag requires an argument")

                exit(-1)

            break

    comments = extract_all(args)

    if out_path:

        try:

            with open(out_path, 'w') as out_file:

                write_header(comments, out_file)

        except:

            # In the event of an error, don't leave a partially-written

            # output file.

            try:

                os.unlink(out_path)

            except:

                pass

            raise

    else:

        write_header(comments)

if __name__ == '__main__':

    try:

        mkdoc(sys.argv[1:])

    except NoFilenamesError:

        print('Syntax: %s [.. a list of header files ..]' % sys.argv[0])

        exit(-1)