# -*- python -*-
#
# gtk-doc - GTK DocBook documentation generator.
# Copyright (C) 2001 Damon Chaplin
# 2007-2016 Stefan Sauer
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# Support both Python 2 and 3
from __future__ import print_function
from collections import OrderedDict
import logging
import os
import re
import subprocess
import sys
import six
import codecs
from . import config
def open_text(filename, mode='r', encoding='utf-8'):
"""An open() which removes some differences between Python 2 and 3 and
has saner defaults.
Unlike the builtin open by default utf-8 is use and not the locale
encoding (which is ANSI on Windows for example, not very helpful)
For Python 2, files are opened in text mode like with Python 3.
"""
if mode not in ('r', 'w'):
raise ValueError("mode %r not supported, must be 'r' or 'w'" % mode)
if six.PY3:
return open(filename, mode, encoding=encoding)
else:
# We can't use io.open() here as its write method is too strict and
# only allows unicode instances and not everything in the codebase
# forces unicode at the moment. codecs.open() on the other hand
# happily takes ASCII str and decodes it.
return codecs.open(filename, mode, encoding=encoding)
def setup_logging():
"""Check GTKDOC_TRACE environment variable.
Set python log level to the value of the environment variable (DEBUG, INFO,
WARNING, ERROR and CRITICAL) or INFO if the environment variable is empty.
"""
log_level = os.environ.get('GTKDOC_TRACE', 'WARNING')
if log_level == '':
log_level = 'WARNING'
logging.basicConfig(stream=sys.stdout,
level=logging.getLevelName(log_level.upper()),
format='%(asctime)s:%(filename)s:%(funcName)s:%(lineno)d:%(levelname)s:%(message)s')
# When redirecting the output on python2 or if run with a non utf-8 locale
# we get UnicodeEncodeError:
encoding = sys.stdout.encoding
if 'PYTHONIOENCODING' not in os.environ and (not encoding or encoding != 'UTF-8'):
sys.stdout.flush()
if six.PY3:
sys.stdout = open(sys.stdout.fileno(), mode='w', encoding='utf8', buffering=1)
else:
import codecs
sys.stdout = codecs.getwriter('utf8')(sys.stdout)
def UpdateFileIfChanged(old_file, new_file, make_backup):
"""Compares the old version of the file with the new version and if the
file has changed it moves the new version into the old versions place. This
is used so we only change files if needed, so we can do proper dependency
tracking.
Args:
old_file (str): The pathname of the old file.
new_file (str): The pathname of the new version of the file.
make_backup (bool): True if a backup of the old file should be kept.
It will have the .bak suffix added to the file name.
Returns:
bool: It returns False if the file hasn't changed, and True if it has.
"""
logging.debug("Comparing %s with %s...", old_file, new_file)
if os.path.exists(old_file):
old_contents = new_contents = None
with open(old_file, 'rb') as f:
old_contents = f.read()
with open(new_file, 'rb') as f:
new_contents = f.read()
if old_contents == new_contents:
os.unlink(new_file)
logging.debug("-> content is the same.")
return False
if make_backup:
backupname = old_file + '.bak'
if os.path.exists(backupname):
os.unlink(backupname)
os.rename(old_file, backupname)
else:
os.unlink(old_file)
logging.debug("-> content differs.")
else:
logging.debug("-> %s created.", old_file)
os.rename(new_file, old_file)
return True
def GetModuleDocDir(module_name):
"""Get the docdir for the given module via pkg-config
Args:
module_name (string): The module, e.g. 'glib-2.0'
Returns:
str: the doc directory or None
"""
path = None
try:
path = subprocess.check_output([config.pkg_config, '--variable=prefix', module_name], universal_newlines=True)
except subprocess.CalledProcessError:
return None
return os.path.join(path.strip(), 'share/gtk-doc/html')
def LogWarning(filename, line, message):
"""Log a warning in gcc style format
Args:
file (str): The file the error comes from
line (int): line number in the file
message (str): the error message to print
"""
filename = filename or "unknown"
# TODO: write to stderr
print("%s:%d: warning: %s" % (filename, line, message))
def CreateValidSGMLID(xml_id):
"""Creates a valid SGML 'id' from the given string.
According to http://www.w3.org/TR/html4/types.html#type-id "ID and NAME
tokens must begin with a letter ([A-Za-z]) and may be followed by any number
of letters, digits ([0-9]), hyphens ("-"), underscores ("_"), colons (":"),
and periods (".")."
When creating SGML IDS, we append ":CAPS" to all all-caps identifiers to
prevent name clashes (SGML ids are case-insensitive). (It basically never is
the case that mixed-case identifiers would collide.)
Args:
id (str): The text to be converted into a valid SGML id.
Returns:
str: The converted id.
"""
# Special case, '_' would end up as '' so we use 'gettext-macro' instead.
if xml_id == '_':
return "gettext-macro"
xml_id = re.sub(r'[,;]', '', xml_id)
xml_id = re.sub(r'[_ ]', '-', xml_id)
xml_id = re.sub(r'^-+', '', xml_id)
xml_id = xml_id.replace('::', '-')
xml_id = xml_id.replace(':', '--')
# Append ":CAPS" to all all-caps identifiers
# FIXME: there are some inconsistencies here, we have index files containing e.g. TRUE--CAPS
if xml_id.isupper() and not xml_id.endswith('-CAPS'):
xml_id += ':CAPS'
return xml_id
# Parsing helpers (move to mkdb ?)
class ParseError(Exception):
pass
def PreprocessStructOrEnum(declaration):
"""Trim a type declaration for display.
Removes private sections and comments from the declaration.
Args:
declaration (str): the type declaration (struct or enum)
Returns:
str: the trimmed declaration
"""
# Remove private symbols
# Assume end of declaration if line begins with '}'
declaration = re.sub(r'\n?[ \t]*/\*\s*<\s*(private|protected)\s*>\s*\*/.*?(?:/\*\s*<\s*public\s*>\s*\*/|(?=^\}))',
'', declaration, flags=re.MULTILINE | re.DOTALL)
# Remove all other comments
declaration = re.sub(r'\n\s*/\*.*?\*/\s*\n', r'\n', declaration, flags=re.MULTILINE | re.DOTALL)
declaration = re.sub(r'/\*([^*]+|\*(?!/))*\*/', r' ', declaration)
declaration = re.sub(r'\n\s*//.*?\n', r'\n', declaration, flags=re.MULTILINE | re.DOTALL)
declaration = re.sub(r'//.*', '', declaration)
return declaration
# TODO: output_function_params is always passed as 0
# TODO: we always pass both functions
def ParseStructDeclaration(declaration, is_object, output_function_params, typefunc=None, namefunc=None):
""" Parse a struct declaration.
Takes a structure declaration and breaks it into individual type declarations.
Args:
declaration (str): the declaration to parse
is_object (bool): true if this is an object structure
output_function_params (bool): true if full type is wanted for function pointer members
typefunc (func): function to apply to type
namefunc (func): function to apply to name
Returns:
dict: map of (symbol, decl) pairs describing the public declaration
"""
# For forward struct declarations just return an empty array.
if re.search(r'(?:struct|union)\s+\S+\s*;', declaration, flags=re.MULTILINE | re.DOTALL):
return {}
# Remove all private parts of the declaration
# For objects, assume private
if is_object:
declaration = re.sub(r'''((?:struct|union)\s+\w*\s*\{)
.*?
(?:/\*\s*<\s*public\s*>\s*\*/|(?=\}))''',
r'\1', declaration, flags=re.MULTILINE | re.DOTALL | re.VERBOSE)
# Remove g_iface, parent_instance and parent_class if they are first member
declaration = re.sub(r'(\{)\s*(\w)+\s+(g_iface|parent_instance|parent_class)\s*;', r'\1', declaration)
declaration = PreprocessStructOrEnum(declaration)
if declaration.strip() == '':
return {}
# Prime match after "struct/union {" declaration
match = re.search(r'(?:struct|union)\s+\w*\s*\{', declaration, flags=re.MULTILINE | re.DOTALL)
if not match:
raise ParseError('Declaration "%s" does not begin with "struct/union [NAME] {"' % declaration)
logging.debug('public fields in struct/union: %s', declaration)
result = OrderedDict()
# Treat lines in sequence, allowing singly nested anonymous structs and unions.
for m in re.finditer(r'\s*([^{;]+(\{[^\}]*\}[^{;]+)?);', declaration[match.end():], flags=re.MULTILINE | re.DOTALL):
line = m.group(1)
logging.debug('checking "%s"', line)
if re.search(r'^\s*\}\s*\w*\s*$', line):
break
# FIXME: Just ignore nested structs and unions for now
if '{' in line:
continue
# ignore preprocessor directives
line = re.sub(r'^#.*?\n\s*', '', line, flags=re.MULTILINE | re.DOTALL)
if re.search(r'^\s*\}\s*\w*\s*$', line):
break
func_match = re.search(r'''^
(const\s+|G_CONST_RETURN\s+|unsigned\s+|signed\s+|long\s+|short\s+)*(struct\s+|enum\s+)? # mod1
(\w+)\s* # type
(\**(?:\s*restrict)?)\s* # ptr1
(const\s+)? # mod2
(\**\s*) # ptr2
(const\s+)? # mod3
\(\s*\*\s*(\w+)\s*\)\s* # name
\(([^)]*)\)\s* # func_params
$''', line, flags=re.VERBOSE)
vars_match = re.search(r'''^
((?:const\s+|volatile\s+|unsigned\s+|signed\s+|short\s+|long\s+)?)(struct\s+|enum\s+)? # mod1
(\w+)\s* # type
(\** \s* const\s+)? # mod2
(.*) # variables
$''', line, flags=re.VERBOSE)
# Try to match structure members which are functions
if func_match:
mod1 = func_match.group(1) or ''
if func_match.group(2):
mod1 += func_match.group(2)
func_type = func_match.group(3)
ptr1 = func_match.group(4)
mod2 = func_match.group(5) or ''
ptr2 = func_match.group(6)
mod3 = func_match.group(7) or ''
name = func_match.group(8)
func_params = func_match.group(9)
ptype = func_type
if typefunc:
ptype = typefunc(func_type, '<type>%s</type>' % func_type)
pname = name
if namefunc:
pname = namefunc(name)
if output_function_params:
result[name] = '%s%s%s%s%s%s (*%s) (%s)' % (
mod1, ptype, ptr1, mod2, ptr2, mod3, pname, func_params)
else:
result[name] = '%s ()' % pname
# Try to match normal struct fields of comma-separated variables/
elif vars_match:
mod1 = vars_match.group(1) or ''
if vars_match.group(2):
mod1 += vars_match.group(2)
vtype = vars_match.group(3)
ptype = vtype
if typefunc:
ptype = typefunc(vtype, '<type>%s</type>' % vtype)
mod2 = vars_match.group(4) or ''
if mod2:
mod2 = ' ' + mod2
var_list = vars_match.group(5)
logging.debug('"%s" "%s" "%s" "%s"', mod1, vtype, mod2, var_list)
mod1 = mod1.replace(' ', ' ')
mod2 = mod2.replace(' ', ' ')
for n in var_list.split(','):
# Each variable can have any number of '*' before the identifier,
# and be followed by any number of pairs of brackets or a bit field specifier.
# e.g. *foo, ***bar, *baz[12][23], foo : 25.
m = re.search(
r'^\s* (\**(?:\s*restrict\b)?) \s* (\w+) \s* (?: ((?:\[[^\]]*\]\s*)+) | (:\s*\d+)?) \s* $',
n, flags=re.VERBOSE)
if m:
ptrs = m.group(1)
name = m.group(2)
array = m.group(3) or ''
bits = m.group(4)
if bits:
bits = ' ' + bits
else:
bits = ''
if ptrs and not ptrs.endswith('*'):
ptrs += ' '
array = array.replace(' ', ' ')
bits = bits.replace(' ', ' ')
pname = name
if namefunc:
pname = namefunc(name)
result[name] = '%s%s%s %s%s%s%s;' % (mod1, ptype, mod2, ptrs, pname, array, bits)
logging.debug('Matched line: %s%s%s %s%s%s%s', mod1, ptype, mod2, ptrs, pname, array, bits)
else:
logging.warning('Cannot parse struct field: "%s"', n)
else:
logging.warning('Cannot parse structure field: "%s"', line)
return result
def ParseEnumDeclaration(declaration):
"""Parse an enum declaration.
This function takes a enumeration declaration and breaks it into individual
enum member declarations.
Args:
declaration (str): the declaration to parse
Returns:
str: list of strings describing the public declaration
"""
# For forward struct declarations just return an empty array.
if re.search(r'enum\s+\S+\s*;', declaration, flags=re.MULTILINE | re.DOTALL):
return ()
declaration = PreprocessStructOrEnum(declaration)
if declaration.strip() == '':
return ()
result = []
# Remove parenthesized expressions (in macros like GTK_BLAH = BLAH(1,3))
# to avoid getting confused by commas they might contain. This doesn't
# handle nested parentheses correctly.
declaration = re.sub(r'\([^)\n]+\)', '', declaration)
# Remove apostrophed characters (e.g. '}' or ',') values to avoid getting
# confused with end of enumeration.
# See https://bugzilla.gnome.org/show_bug.cgi?id=741305
declaration = re.sub(r'\'.\'', '', declaration)
# Remove comma from comma - possible whitespace - closing brace sequence
# since it is legal in GNU C and C99 to have a trailing comma but doesn't
# result in an actual enum member
declaration = re.sub(r',(\s*})', r'\1', declaration)
# Prime match after "typedef enum {" declaration
match = re.search(r'(typedef\s+)?enum\s*(\S+\s*)?\{', declaration, flags=re.MULTILINE | re.DOTALL)
if not match:
raise ParseError('Enum declaration "%s" does not begin with "typedef enum {" or "enum [NAME] {"' % declaration)
logging.debug("public fields in enum: %s'", declaration)
# Treat lines in sequence.
for m in re.finditer(r'\s*([^,\}]+)([,\}])', declaration[match.end():], flags=re.MULTILINE | re.DOTALL):
line = m.group(1)
terminator = m.group(2)
# ignore preprocessor directives
line = re.sub(r'^#.*?\n\s*', '', line, flags=re.MULTILINE | re.DOTALL)
m1 = re.search(r'^(\w+)\s*(=.*)?$', line, flags=re.MULTILINE | re.DOTALL)
# Special case for GIOCondition, where the values are specified by
# macros which expand to include the equal sign like '=1'.
m2 = re.search(r'^(\w+)\s*GLIB_SYSDEF_POLL', line, flags=re.MULTILINE | re.DOTALL)
if m1:
result.append(m1.group(1))
elif m2:
result.append(m2.group(1))
elif line.strip().startswith('#'):
# Special case include of <gdk/gdkcursors.h>, just ignore it
# Special case for #ifdef/#else/#endif, just ignore it
break
else:
logging.warning('Cannot parse enumeration member: %s', line)
if terminator == '}':
break
return result
def ParseFunctionDeclaration(declaration, typefunc, namefunc):
"""Parse a function declaration.
This function takes a function declaration and breaks it into individual
parameter declarations.
Args:
declaration (str): the declaration to parse
typefunc (func): function to apply to type
namefunc (func): function to apply to name
Returns:
dict: map of (symbol, decl) pairs describing the prototype
"""
result = OrderedDict()
param_num = 0
while declaration:
logging.debug('decl=[%s]', declaration)
# skip whitespace and commas
declaration, n = re.subn(r'^[\s,]+', '', declaration)
if n:
continue
declaration, n = re.subn(r'^void\s*[,\n]', '', declaration)
if n:
if param_num != 0:
logging.warning('void used as parameter %d in function %s', param_num, declaration)
result['void'] = namefunc('<type>void</type>')
param_num += 1
continue
declaration, n = re.subn(r'^\s*[_a-zA-Z0-9]*\.\.\.\s*[,\n]', '', declaration)
if n:
result['...'] = namefunc('...')
param_num += 1
continue
# allow alphanumerics, '_', '[' & ']' in param names, try to match a standard parameter
# $1 $2 $3 $4 $5
regex = r'^\s*((?:(?:G_CONST_RETURN|G_GNUC_[A-Z_]+\s+|unsigned long|unsigned short|signed long|signed short|unsigned|signed|long|short|volatile|const)\s+)*)((?:struct\b|enum\b)?\s*\w+)\s*((?:(?:const\b|restrict\b|G_GNUC_[A-Z_]+\b)?\s*\*?\s*(?:const\b|restrict\b|G_GNUC_[A-Z_]+\b)?\s*)*)(\w+)?\s*((?:\[\S*\])*)\s*(?:G_GNUC_[A-Z_]+)?\s*[,\n]'
m = re.match(regex, declaration)
if m:
declaration = re.sub(regex, '', declaration)
pre = m.group(1) or ''
type = m.group(2)
ptr = m.group(3) or ''
name = m.group(4) or ''
array = m.group(5) or ''
pre = re.sub(r'\s+', ' ', pre)
type = re.sub(r'\s+', ' ', type)
ptr = re.sub(r'\s+', ' ', ptr)
ptr = re.sub(r'\s+$', '', ptr)
if ptr and not ptr.endswith('*'):
ptr += ' '
logging.debug('"%s" "%s" "%s" "%s" "%s"', pre, type, ptr, name, array)
m = re.search(r'^((un)?signed .*)\s?', pre)
if name == '' and m:
name = type
type = m.group(1)
pre = ''
if name == '':
name = 'Param' + str(param_num + 1)
logging.debug('"%s" "%s" "%s" "%s" "%s"', pre, type, ptr, name, array)
xref = typefunc(type, '<type>%s</type>' % type)
result[name] = namefunc('%s%s %s%s%s' % (pre, xref, ptr, name, array))
param_num += 1
continue
# Try to match parameters which are functions
# $1 $2 $3 $4 $5 $6 $7 $8
regex = r'^(const\s+|G_CONST_RETURN\s+|G_GNUC_[A-Z_]+\s+|signed\s+|unsigned\s+)*(struct\s+)?(\w+)\s*(\**)\s*(?:restrict\b)?\s*(const\s+)?\(\s*(\*[\s\*]*)\s*(\w+)\s*\)\s*\(([^)]*)\)\s*[,\n]'
m = re.match(regex, declaration)
if m:
declaration = re.sub(regex, '', declaration)
mod1 = m.group(1) or ''
if m.group(2):
mod1 += m.group(2)
type = m.group(3)
ptr1 = m.group(4)
mod2 = m.group(5) or ''
func_ptr = m.group(6)
name = m.group(7)
func_params = m.group(8) or ''
if ptr1 and not ptr1.endswith('*'):
ptr1 += ' '
func_ptr = re.sub(r'\s+', ' ', func_ptr)
logging.debug('"%s" "%s" "%s" "%s" "%s"', mod1, type, mod2, func_ptr, name)
xref = typefunc(type, '<type>%s</type>' % type)
result[name] = namefunc('%s%s%s%s (%s%s) (%s)' % (mod1, xref, ptr1, mod2, func_ptr, name, func_params))
param_num += 1
continue
logging.warning('Cannnot parse args for function in "%s"', declaration)
break
return result
def ParseMacroDeclaration(declaration, namefunc):
"""Parse a macro declaration.
This function takes a macro declaration and breaks it into individual
parameter declarations.
Args:
declaration (str): the declaration to parse
namefunc (func): function to apply to name
Returns:
dict: map of (symbol, decl) pairs describing the macro
"""
result = OrderedDict()
logging.debug('decl=[%s]', declaration)
m = re.search(r'^\s*#\s*define\s+\w+\(([^\)]*)\)', declaration)
if m:
params = m.group(1)
params = re.sub(r'\n', '', params)
logging.debug('params=[%s]', params)
for param in params.split(','):
param = param.strip()
# Allow varargs variations
if param.endswith('...'):
param = '...'
if param != '':
result[param] = namefunc(param)
return result