/******************************************************************************
*
*
*
* Copyright (C) 1997-2015 by Dimitri van Heesch.
*
* Permission to use, copy, modify, and distribute this software and its
* documentation under the terms of the GNU General Public License is hereby
* granted. No representations are made about the suitability of this software
* for any purpose. It is provided "as is" without express or implied warranty.
* See the GNU General Public License for more details.
*
* Documents produced by Doxygen are derivative works derived from the
* input used in their production; they are not affected by this license.
*
*/
/* This code is based on the work done by the MoxyPyDoxy team
* (Linda Leong, Mike Rivera, Kim Truong, and Gabriel Estrada)
* in Spring 2005 as part of CS 179E: Compiler Design Project
* at the University of California, Riverside; the course was
* taught by Peter H. Froehlich <phf@acm.org>.
*/
%option never-interactive
%option prefix="pycodeYY"
%{
#include <stdio.h>
#include <qvaluestack.h>
#include "pycode.h"
#include "message.h"
#include "scanner.h"
#include "entry.h"
#include "doxygen.h"
#include "outputlist.h"
#include "util.h"
#include "membername.h"
#include "searchindex.h"
#include "config.h"
#include "groupdef.h"
#include "classlist.h"
#include "filedef.h"
#include "namespacedef.h"
#include "tooltip.h"
// Toggle for some debugging info
//#define DBG_CTX(x) fprintf x
#define DBG_CTX(x) do { } while(0)
#define YY_NO_INPUT 1
#define YY_NO_UNISTD_H 1
static ClassSDict g_codeClassSDict(17);
static QCString g_curClassName;
static QStrList g_curClassBases;
static CodeOutputInterface * g_code;
static const char * g_inputString; //!< the code fragment as text
static int g_inputPosition; //!< read offset during parsing
static const char * g_currentFontClass;
static bool g_needsTermination;
static Definition *g_searchCtx;
static bool g_collectXRefs;
static int g_inputLines; //!< number of line in the code fragment
static int g_yyLineNr; //!< current line number
static FileDef * g_sourceFileDef;
static Definition * g_currentDefinition;
static MemberDef * g_currentMemberDef;
static bool g_includeCodeFragment;
static QCString g_realScope;
//static bool g_insideBody;
static int g_bodyCurlyCount;
static bool g_searchingForBody;
static QCString g_classScope;
static int g_paramParens;
//static int g_anchorCount;
static bool g_exampleBlock;
static QCString g_exampleName;
static QCString g_type;
static QCString g_name;
static bool g_doubleStringIsDoc;
static bool g_doubleQuote;
static bool g_noSuiteFound;
static int g_stringContext;
static QValueStack<uint> g_indents; //!< Tracks indentation levels for scoping in python
static QCString g_docBlock; //!< contents of all lines of a documentation block
static bool g_endComment;
static void endFontClass();
static void adjustScopesAndSuites(unsigned indentLength);
/*! Represents a stack of variable to class mappings as found in the
* code. Each scope is enclosed in pushScope() and popScope() calls.
* Variables are added by calling addVariables() and one can search
* for variable using findVariable().
*/
class PyVariableContext
{
public:
static const ClassDef *dummyContext;
class Scope : public SDict<ClassDef>
{
public:
Scope() : SDict<ClassDef>(17) {}
};
PyVariableContext()
{
m_scopes.setAutoDelete(TRUE);
}
virtual ~PyVariableContext()
{
}
void pushScope()
{
m_scopes.append(new Scope);
}
void popScope()
{
if (m_scopes.count()>0)
{
m_scopes.remove(m_scopes.count()-1);
}
}
void clear()
{
m_scopes.clear();
m_globalScope.clear();
}
void clearExceptGlobal()
{
m_scopes.clear();
}
void addVariable(const QCString &type,const QCString &name);
ClassDef *findVariable(const QCString &name);
private:
Scope m_globalScope;
QList<Scope> m_scopes;
};
void PyVariableContext::addVariable(const QCString &type,const QCString &name)
{
//printf("PyVariableContext::addVariable(%s,%s)\n",type.data(),name.data());
QCString ltype = type.simplifyWhiteSpace();
QCString lname = name.simplifyWhiteSpace();
Scope *scope = m_scopes.count()==0 ? &m_globalScope : m_scopes.getLast();
ClassDef *varType;
if (
(varType=g_codeClassSDict[ltype]) || // look for class definitions inside the code block
(varType=getResolvedClass(g_currentDefinition,g_sourceFileDef,ltype)) // look for global class definitions
)
{
scope->append(lname,varType); // add it to a list
}
else
{
if (m_scopes.count()>0) // for local variables add a dummy entry so the name
// is hidden to avoid FALSE links to global variables with the same name
// TODO: make this work for namespaces as well!
{
scope->append(lname,dummyContext);
}
}
}
ClassDef *PyVariableContext::findVariable(const QCString &name)
{
if (name.isEmpty()) return 0;
ClassDef *result = 0;
QListIterator<Scope> sli(m_scopes);
Scope *scope;
// search from inner to outer scope
for (sli.toLast();(scope=sli.current());--sli)
{
result = scope->find(name);
if (result)
{
return result;
}
}
// nothing found -> also try the global scope
result=m_globalScope.find(name);
return result;
}
static PyVariableContext g_theVarContext;
const ClassDef *PyVariableContext::dummyContext = (ClassDef*)0x8;
class PyCallContext
{
public:
struct Ctx
{
Ctx() : name(g_name), type(g_type), cd(0) {}
QCString name;
QCString type;
ClassDef *cd;
};
PyCallContext()
{
m_classList.append(new Ctx);
m_classList.setAutoDelete(TRUE);
}
virtual ~PyCallContext() {}
void setClass(ClassDef *cd)
{
Ctx *ctx = m_classList.getLast();
if (ctx)
{
ctx->cd=cd;
}
}
void pushScope()
{
m_classList.append(new Ctx);
}
void popScope()
{
if (m_classList.count()>1)
{
Ctx *ctx = m_classList.getLast();
if (ctx)
{
g_name = ctx->name;
g_type = ctx->type;
}
m_classList.removeLast();
}
else
{
}
}
void clear()
{
m_classList.clear();
m_classList.append(new Ctx);
}
ClassDef *getClass() const
{
Ctx *ctx = m_classList.getLast();
if (ctx)
return ctx->cd;
else
return 0;
}
private:
QList<Ctx> m_classList;
};
static PyCallContext g_theCallContext;
/*! counts the number of lines in the input */
static int countLines()
{
const char *p=g_inputString;
char c;
int count=1;
while ((c=*p))
{
p++ ;
if (c=='\n') count++;
}
if (p>g_inputString && *(p-1)!='\n')
{ // last line does not end with a \n, so we add an extra
// line and explicitly terminate the line after parsing.
count++,
g_needsTermination=TRUE;
}
return count;
}
static void setCurrentDoc(const QCString &anchor)
{
if (Doxygen::searchIndex)
{
if (g_searchCtx)
{
Doxygen::searchIndex->setCurrentDoc(g_searchCtx,g_searchCtx->anchor(),FALSE);
}
else
{
Doxygen::searchIndex->setCurrentDoc(g_sourceFileDef,anchor,TRUE);
}
}
}
static void addToSearchIndex(const char *text)
{
if (Doxygen::searchIndex)
{
Doxygen::searchIndex->addWord(text,FALSE);
}
}
static ClassDef *stripClassName(const char *s,Definition *d=g_currentDefinition)
{
int pos=0;
QCString type = s;
QCString className;
QCString templSpec;
while (extractClassNameFromType(type,pos,className,templSpec)!=-1)
{
QCString clName=className+templSpec;
ClassDef *cd=0;
if (!g_classScope.isEmpty())
{
cd=getResolvedClass(d,g_sourceFileDef,g_classScope+"::"+clName);
}
if (cd==0)
{
cd=getResolvedClass(d,g_sourceFileDef,clName);
}
if (cd)
{
return cd;
}
}
return 0;
}
/*! start a new line of code, inserting a line number if g_sourceFileDef
* is TRUE. If a definition starts at the current line, then the line
* number is linked to the documentation of that definition.
*/
static void startCodeLine()
{
//if (g_currentFontClass) { g_code->endFontClass(); }
if (g_sourceFileDef)
{
//QCString lineNumber,lineAnchor;
//lineNumber.sprintf("%05d",g_yyLineNr);
//lineAnchor.sprintf("l%05d",g_yyLineNr);
Definition *d = g_sourceFileDef->getSourceDefinition(g_yyLineNr);
//printf("startCodeLine %d d=%p\n",g_yyLineNr,d);
//g_code->startLineNumber();
if (!g_includeCodeFragment && d && d->isLinkableInProject())
{
g_currentDefinition = d;
g_currentMemberDef = g_sourceFileDef->getSourceMember(g_yyLineNr);
//g_insideBody = FALSE;
g_endComment = FALSE;
g_searchingForBody = TRUE;
g_realScope = d->name().copy();
g_classScope = d->name().copy();
//printf("Real scope: `%s'\n",g_realScope.data());
g_bodyCurlyCount = 0;
QCString lineAnchor;
lineAnchor.sprintf("l%05d",g_yyLineNr);
if (g_currentMemberDef)
{
g_code->writeLineNumber(g_currentMemberDef->getReference(),
g_currentMemberDef->getOutputFileBase(),
g_currentMemberDef->anchor(),g_yyLineNr);
setCurrentDoc(lineAnchor);
}
else
{
g_code->writeLineNumber(d->getReference(),
d->getOutputFileBase(),
0,g_yyLineNr);
setCurrentDoc(lineAnchor);
}
}
else
{
//g_code->codify(lineNumber);
g_code->writeLineNumber(0,0,0,g_yyLineNr);
}
//g_code->endLineNumber();
}
g_code->startCodeLine(g_sourceFileDef);
if (g_currentFontClass)
{
g_code->startFontClass(g_currentFontClass);
}
}
static void codify(const char* text)
{
g_code->codify(text);
}
static void endCodeLine()
{
endFontClass();
g_code->endCodeLine();
}
static void nextCodeLine()
{
const char *fc = g_currentFontClass;
endCodeLine();
if (g_yyLineNr<g_inputLines)
{
g_currentFontClass = fc;
startCodeLine();
}
}
/*! writes a link to a fragment \a text that may span multiple lines, inserting
* line numbers for each line. If \a text contains newlines, the link will be
* split into multiple links with the same destination, one for each line.
*/
static void writeMultiLineCodeLink(CodeOutputInterface &ol,
Definition *d,
const char *text)
{
static bool sourceTooltips = Config_getBool(SOURCE_TOOLTIPS);
TooltipManager::instance()->addTooltip(d);
QCString ref = d->getReference();
QCString file = d->getOutputFileBase();
QCString anchor = d->anchor();
QCString tooltip;
if (!sourceTooltips) // fall back to simple "title" tooltips
{
tooltip = d->briefDescriptionAsTooltip();
}
bool done=FALSE;
char *p=(char *)text;
while (!done)
{
char *sp=p;
char c;
while ((c=*p++) && c!='\n') { }
if (c=='\n')
{
g_yyLineNr++;
*(p-1)='\0';
//printf("writeCodeLink(%s,%s,%s,%s)\n",ref,file,anchor,sp);
ol.writeCodeLink(ref,file,anchor,sp,tooltip);
nextCodeLine();
}
else
{
//printf("writeCodeLink(%s,%s,%s,%s)\n",ref,file,anchor,sp);
ol.writeCodeLink(ref,file,anchor,sp,tooltip);
done=TRUE;
}
}
}
static void startFontClass(const char *s)
{
// if font class is already set don't stop and start it.
// strcmp does not like null pointers as input.
if (!g_currentFontClass || !s || strcmp(g_currentFontClass,s))
{
endFontClass();
g_code->startFontClass(s);
g_currentFontClass=s;
}
}
static void endFontClass()
{
if (g_currentFontClass)
{
g_code->endFontClass();
g_currentFontClass=0;
}
}
static void codifyLines(char *text)
{
//printf("codifyLines(%d,\"%s\")\n",g_yyLineNr,text);
char *p=text,*sp=p;
char c;
bool done=FALSE;
const char * tmp_currentFontClass = g_currentFontClass;
while (!done)
{
sp=p;
while ((c=*p++) && c!='\n') { }
if (c=='\n')
{
g_yyLineNr++;
*(p-1)='\0';
g_code->codify(sp);
endCodeLine();
if (g_yyLineNr<g_inputLines)
{
startCodeLine();
}
if (tmp_currentFontClass)
{
startFontClass(tmp_currentFontClass);
}
}
else
{
g_code->codify(sp);
done=TRUE;
}
}
}
static void codifyLines(const QCString &str)
{
char *tmp= (char *)malloc(str.length()+1);
qstrcpy(tmp, str);
codifyLines(tmp);
free(tmp);
}
static bool getLinkInScope(const QCString &c, // scope
const QCString &m, // member
const char *memberText, // exact text
CodeOutputInterface &ol,
const char *text
)
{
MemberDef *md;
ClassDef *cd;
FileDef *fd;
NamespaceDef *nd;
GroupDef *gd;
//printf("Trying `%s'::`%s'\n",c.data(),m.data());
if (getDefs(c,m,"()",md,cd,fd,nd,gd,FALSE,g_sourceFileDef) &&
md->isLinkable())
{
//Definition *d=0;
//if (cd) d=cd; else if (nd) d=nd; else if (fd) d=fd; else d=gd;
Definition *d = md->getOuterScope()==Doxygen::globalScope ?
md->getBodyDef() : md->getOuterScope();
//printf("Found! d=%s\n",d?d->name().data():"<none>");
if (md->getGroupDef()) d = md->getGroupDef();
if (d && d->isLinkable())
{
g_theCallContext.setClass(stripClassName(md->typeString(),md->getOuterScope()));
//printf("g_currentDefinition=%p g_currentMemberDef=%p\n",
// g_currentDefinition,g_currentMemberDef);
if (g_currentDefinition && g_currentMemberDef &&
md!=g_currentMemberDef && g_collectXRefs)
{
addDocCrossReference(g_currentMemberDef,md);
}
//printf("d->getReference()=`%s' d->getOutputBase()=`%s' name=`%s' member name=`%s'\n",d->getReference().data(),d->getOutputFileBase().data(),d->name().data(),md->name().data());
writeMultiLineCodeLink(ol,md, text ? text : memberText);
addToSearchIndex(text ? text : memberText);
return TRUE;
}
}
return FALSE;
}
static bool getLink(const char *className,
const char *memberName,
CodeOutputInterface &ol,
const char *text=0)
{
QCString m=removeRedundantWhiteSpace(memberName);
QCString c=className;
if (!getLinkInScope(c,m,memberName,ol,text))
{
if (!g_curClassName.isEmpty())
{
if (!c.isEmpty()) c.prepend("::");
c.prepend(g_curClassName);
return getLinkInScope(c,m,memberName,ol,text);
}
return FALSE;
}
return TRUE;
}
/*
For a given string in the source code,
finds its class or global id and links to it.
*/
static void generateClassOrGlobalLink(CodeOutputInterface &ol,char *clName,
bool typeOnly=FALSE)
{
QCString className=clName;
// Don't do anything for empty text
if (className.isEmpty()) return;
DBG_CTX((stderr,"generateClassOrGlobalLink(className=%s)\n",className.data()));
ClassDef *cd=0,*lcd=0; /** Class def that we may find */
MemberDef *md=0; /** Member def that we may find */
//bool isLocal=FALSE;
if ((lcd=g_theVarContext.findVariable(className))==0) // not a local variable
{
Definition *d = g_currentDefinition;
QCString scope = substitute(className,".","::");
cd = getResolvedClass(d,g_sourceFileDef,substitute(className,".","::"),&md);
DBG_CTX((stderr,"d=%s g_sourceFileDef=%s\n",
d?d->displayName().data():"<null>",
g_currentDefinition?g_currentDefinition->displayName().data():"<null>"));
DBG_CTX((stderr,"is found as a type %s\n",cd?cd->name().data():"<null>"));
if (cd==0 && md==0) // also see if it is variable or enum or enum value
{
NamespaceDef *nd = getResolvedNamespace(scope);
if (nd)
{
writeMultiLineCodeLink(ol,nd,clName);
addToSearchIndex(className);
return;
}
else if (getLink(g_classScope,clName,ol,clName))
{
return;
}
}
}
else
{
if (lcd!=PyVariableContext::dummyContext)
{
g_theCallContext.setClass(lcd);
}
//isLocal=TRUE;
DBG_CTX((stderr,"is a local variable cd=%p!\n",cd));
}
if (cd && cd->isLinkable()) // is it a linkable class
{
writeMultiLineCodeLink(ol,cd,clName);
addToSearchIndex(className);
if (md)
{
Definition *d = md->getOuterScope()==Doxygen::globalScope ?
md->getBodyDef() : md->getOuterScope();
if (md->getGroupDef()) d = md->getGroupDef();
if (d && d->isLinkable() && md->isLinkable() &&
g_currentMemberDef && g_collectXRefs)
{
addDocCrossReference(g_currentMemberDef,md);
}
}
}
else // not a class, maybe a global member
{
int scopeEnd = className.findRev(".");
if (scopeEnd!=-1 && !typeOnly) // name with explicit scope
{
QCString scope = substitute(className.left(scopeEnd),".","::");
QCString locName = className.right(className.length()-scopeEnd-1);
ClassDef *mcd = getClass(scope);
DBG_CTX((stderr,"scope=%s locName=%s mcd=%p\n",scope.data(),locName.data(),mcd));
if (mcd)
{
MemberDef *md = mcd->getMemberByName(locName);
if (md)
{
g_theCallContext.setClass(stripClassName(md->typeString(),md->getOuterScope()));
writeMultiLineCodeLink(ol,md,clName);
addToSearchIndex(className);
Definition *d = md->getOuterScope()==Doxygen::globalScope ?
md->getBodyDef() : md->getOuterScope();
if (md->getGroupDef()) d = md->getGroupDef();
if (d && d->isLinkable() && md->isLinkable() &&
g_currentMemberDef && g_collectXRefs)
{
addDocCrossReference(g_currentMemberDef,md);
}
return;
}
}
else // check namespace as well
{
NamespaceDef *mnd = getResolvedNamespace(scope);
if (mnd)
{
MemberDef *md=mnd->getMemberByName(locName);
if (md)
{
//printf("name=%s scope=%s\n",locName.data(),scope.data());
g_theCallContext.setClass(stripClassName(md->typeString(),md->getOuterScope()));
writeMultiLineCodeLink(ol,md,clName);
addToSearchIndex(className);
Definition *d = md->getOuterScope()==Doxygen::globalScope ?
md->getBodyDef() : md->getOuterScope();
if (md->getGroupDef()) d = md->getGroupDef();
if (d && d->isLinkable() && md->isLinkable() &&
g_currentMemberDef && g_collectXRefs)
{
addDocCrossReference(g_currentMemberDef,md);
}
return;
}
}
}
}
// nothing found, just write out the word
codifyLines(clName);
addToSearchIndex(clName);
}
}
/*
As of June 1, this function seems to work
for file members, but scopes are not
being correctly tracked for classes
so it doesn't work for classes yet.
*/
static void generateFunctionLink(CodeOutputInterface &ol,char *funcName)
{
//CodeClassDef *ccd=0;
ClassDef *ccd=0;
QCString locScope=g_classScope.copy();
QCString locFunc=removeRedundantWhiteSpace(funcName);
DBG_CTX((stdout,"*** locScope=%s locFunc=%s\n",locScope.data(),locFunc.data()));
int i=locFunc.findRev("::");
if (i>0)
{
locScope=locFunc.left(i);
locFunc=locFunc.right(locFunc.length()-i-2).stripWhiteSpace();
}
//printf("generateFunctionLink(%s) classScope=`%s'\n",locFunc.data(),locScope.data());
if (!locScope.isEmpty() && (ccd=g_codeClassSDict[locScope]))
{
//printf("using classScope %s\n",g_classScope.data());
if (ccd->baseClasses())
{
BaseClassListIterator bcli(*ccd->baseClasses());
for ( ; bcli.current() ; ++bcli)
{
if (getLink(bcli.current()->classDef->name(),locFunc,ol,funcName))
{
return;
}
}
}
}
if (!getLink(locScope,locFunc,ol,funcName))
{
generateClassOrGlobalLink(ol,funcName);
}
return;
}
static bool findMemberLink(CodeOutputInterface &ol,Definition *sym,const char *symName)
{
//printf("sym %s outerScope=%s equal=%d\n",
// sym->name().data(),sym->getOuterScope()->name().data(),
// sym->getOuterScope()==g_currentDefinition);
if (sym->getOuterScope() &&
sym->getOuterScope()->definitionType()==Definition::TypeClass &&
g_currentDefinition->definitionType()==Definition::TypeClass)
{
ClassDef *cd = (ClassDef*)sym->getOuterScope();
ClassDef *thisCd = (ClassDef *)g_currentDefinition;
if (sym->definitionType()==Definition::TypeMember)
{
if (g_currentMemberDef && g_collectXRefs)
{
addDocCrossReference(g_currentMemberDef,(MemberDef*)sym);
}
}
DBG_CTX((stderr,"cd=%s thisCd=%s\n",cd?cd->name().data():"<none>",thisCd?thisCd->name().data():"<none>"));
// TODO: find the nearest base class in case cd is a base class of
// thisCd
if (cd==thisCd || (thisCd && thisCd->isBaseClass(cd,TRUE)))
{
writeMultiLineCodeLink(ol,sym,symName);
return TRUE;
}
}
return FALSE;
}
static void findMemberLink(CodeOutputInterface &ol,char *symName)
{
//printf("Member reference: %s scope=%s member=%s\n",
// yytext,
// g_currentDefinition?g_currentDefinition->name().data():"<none>",
// g_currentMemberDef?g_currentMemberDef->name().data():"<none>"
// );
if (g_currentDefinition)
{
DefinitionIntf *di = Doxygen::symbolMap->find(symName);
if (di)
{
if (di->definitionType()==DefinitionIntf::TypeSymbolList) // multiple symbols
{
DefinitionListIterator dli(*(DefinitionList*)di);
Definition *sym;
for (dli.toFirst();(sym=dli.current());++dli)
{
if (findMemberLink(ol,sym,symName)) return;
}
}
else // single symbol
{
if (findMemberLink(ol,(Definition*)di,symName)) return;
}
}
}
//printf("sym %s not found\n",&yytext[5]);
codify(symName);
}
#undef YY_INPUT
#define YY_INPUT(buf,result,max_size) result=yyread(buf,max_size);
static int yyread(char *buf,int max_size)
{
int c=0;
while( c < max_size && g_inputString[g_inputPosition] )
{
*buf = g_inputString[g_inputPosition++] ;
c++; buf++;
}
return c;
}
%}
BB [ \t]+
B [ \t]*
NEWLINE \n
DIGIT [0-9]
LETTER [A-Za-z\x80-\xFF]
NONEMPTY [A-Za-z0-9_\x80-\xFF]
EXPCHAR [#(){}\[\],:.%/\\=`*~|&<>!;+-]
NONEMPTYEXP [^ \t\n:]
PARAMNONEMPTY [^ \t\n():]
IDENTIFIER ({LETTER}|"_")({LETTER}|{DIGIT}|"_")*
BORDER ([^A-Za-z0-9])
POUNDCOMMENT "##"
TRISINGLEQUOTE "'''"
TRIDOUBLEQUOTE "\"\"\""
LONGSTRINGCHAR [^\\"']
ESCAPESEQ ("\\")(.)
LONGSTRINGITEM ({LONGSTRINGCHAR}|{ESCAPESEQ})
SMALLQUOTE ("\"\""|"\""|"'"|"''")
LONGSTRINGBLOCK ({LONGSTRINGITEM}+|{SMALLQUOTE})
SHORTSTRING ("'"{SHORTSTRINGITEM}*"'"|'"'{SHORTSTRINGITEM}*'"')
SHORTSTRINGITEM ({SHORTSTRINGCHAR}|{ESCAPESEQ})
SHORTSTRINGCHAR [^\\\n"]
STRINGLITERAL {STRINGPREFIX}?( {SHORTSTRING} | {LONGSTRING})
STRINGPREFIX ("r"|"u"|"ur"|"R"|"U"|"UR"|"Ur"|"uR")
KEYWORD ("lambda"|"import"|"class"|"assert"|"as"|"from"|"global"|"def"|"True"|"False")
FLOWKW ("or"|"and"|"is"|"not"|"print"|"for"|"in"|"if"|"try"|"except"|"yield"|"raise"|"break"|"continue"|"pass"|"if"|"return"|"while"|"elif"|"else"|"finally")
QUOTES ("\""[^"]*"\"")
SINGLEQUOTES ("'"[^']*"'")
LONGINTEGER {INTEGER}("l"|"L")
INTEGER ({DECIMALINTEGER}|{OCTINTEGER}|{HEXINTEGER})
DECIMALINTEGER ({NONZERODIGIT}{DIGIT}*|"0")
OCTINTEGER "0"{OCTDIGIT}+
HEXINTEGER "0"("x"|"X"){HEXDIGIT}+
NONZERODIGIT [1-9]
OCTDIGIT [0-7]
HEXDIGIT ({DIGIT}|[a-f]|[A-F])
FLOATNUMBER ({POINTFLOAT}|{EXPONENTFLOAT})
POINTFLOAT ({INTPART}?{FRACTION}|{INTPART}".")
EXPONENTFLOAT ({INTPART}|{POINTFLOAT}){EXPONENT}
INTPART {DIGIT}+
FRACTION "."{DIGIT}+
EXPONENT ("e"|"E")("+"|"-")?{DIGIT}+
IMAGNUMBER ({FLOATNUMBER}|{INTPART})("j"|"J")
ATOM ({IDENTIFIER}|{LITERAL}|{ENCLOSURE})
ENCLOSURE ({PARENTH_FORM}|{LIST_DISPLAY}|{DICT_DISPLAY}|{STRING_CONVERSION})
LITERAL ({STRINGLITERAL}|{INTEGER}|{LONGINTEGER}|{FLOATNUMBER}|{IMAGNUMBER})
PARENTH_FORM "("{EXPRESSION_LIST}?")"
TEST ({AND_TEST}("or"{AND_TEST})*|{LAMBDA_FORM})
TESTLIST {TEST}( ","{TEST})*","?
LIST_DISPLAY "["{LISTMAKER}?"]"
LISTMAKER {EXPRESSION}({LIST_FOR}|(","{EXPRESSION})*","?)
LIST_ITER ({LIST_FOR}|{LIST_IF})
LIST_FOR "for"{EXPRESSION_LIST}"in"{TESTLIST}{LIST_ITER}?
LIST_IF "if"{TEST}{LIST_ITER}?
DICT_DISPLAY "\{"{KEY_DATUM_LIST}?"\}"
KEY_DATUM_LIST {KEY_DATUM}(","{KEY_DATUM})*","?
KEY_DATUM {EXPRESSION}":"{EXPRESSION}
STRING_CONVERSION "`"{EXPRESSION_LIST}"`"
PRIMARY ({ATOM}|{ATTRIBUTEREF}|{SUBSCRIPTION}|{SLICING}|{CALL})
ATTRIBUTEREF {PRIMARY}"."{IDENTIFIER}
SUBSCRIPTION {PRIMARY}"["{EXPRESSION_LIST}"]"
SLICING ({SIMPLE_SLICING}|{EXTENDED_SLICING})
SIMPLE_SLICING {PRIMARY}"["{SHORT_SLICE}"]"
EXTENDED_SLICING {PRIMARY}"["{SLICE_LIST}"]"
SLICE_LIST {SLICE_ITEM}(","{SLICE_ITEM})*","?
SLICE_ITEM ({EXPRESSION}|{PROPER_SLICE}|{ELLIPSIS})
PROPER_SLICE ({SHORT_SLICE}|{LONG_SLICE})
SHORT_SLICE {LOWER_BOUND}?":"{UPPER_BOUND}?
LONG_SLICE {SHORT_SLICE}":"{STRIDE}?
LOWER_BOUND {EXPRESSION}
UPPER_BOUND {EXPRESSION}
STRIDE {EXPRESSION}
ELLIPSIS "..."
CALL {PRIMARY}"("({ARGUMENT_LIST}","?)?")"
ARGUMENT_LIST ({POSITIONAL_ARGUMENTS}(","{KEYWORD_ARGUMENTS})?(",""*"{EXPRESSION})?(",""**"{EXPRESSION})?|{KEYWORD_ARGUMENTS}(",""*"{EXPRESSION})?(",""**"{EXPRESSION})?|"*"{EXPRESSION}(",""**"{EXPRESSION})?|"**"{EXPRESSION})
POSITIONAL_ARGUMENTS {EXPRESSION}(","{EXPRESSION})*
KEYWORD_ARGUMENTS {KEYWORD_ITEM}(","{KEYWORD_ITEM})*
KEYWORD_ITEM {IDENTIFIER}"="{EXPRESSION}
POWER {PRIMARY}("**"{U_EXPR})?
U_EXPR ({POWER}|"-"{U_EXPR}|"+"{U_EXPR}|"\~"{U_EXPR})
M_EXPR ({U_EXPR}|{M_EXPR}"*"{U_EXPR}|{M_EXPR}"//"{U_EXPR}|{M_EXPR}"/"{U_EXPR}|{M_EXPR}"\%"{U_EXPR})
A_EXPR ({M_EXPR}|{A_EXPR}"+"{M_EXPR}|{A_EXPR}"-"{M_EXPR}
SHIFT_EXPR ({A_EXPR}|{SHIFT_EXPR}("<<"|">>"){A_EXPR})
AND_EXPR ({SHIFT_EXPR}|{AND_EXPR}"\;SPMamp;"{SHIFT_EXPR}
XOR_EXPR ({AND_EXPR}|{XOR_EXPR}"\textasciicircum"{AND_EXPR})
OR_EXPR ({XOR_EXPR}|{OR_EXPR}"|"{ XOR_EXPR})
COMPARISON {OR_EXPR}({COMP_OPERATOR}{OR_EXPR})*
COMP_OPERATOR ("<"|">"|"=="|">="|"<="|"<>"|"!="|"is""not"?|"not"?"in")
EXPRESSION ({OR_TEST}|{LAMBDA_FORM})
OR_TEST ({AND_TEST}|{OR_TEST}"or"{AND_TEST})
AND_TEST ({NOT_TEST}|{AND_TEST}"and"{NOT_TEST})
NOT_TEST ({COMPARISON}|"not"{NOT_TEST})
LAMBDA_FORM "lambda"{PARAMETER_LIST}?":"{EXPRESSION}
EXPRESSION_LIST {EXPRESSION}(","{EXPRESSION})*","?
SIMPLE_STMT ({EXPRESSION_STMT}|{ASSERT_STMT}|{ASSIGNMENT_STMT}|{AUGMENTED_ASSIGNMENT_STMT}|{PASS_STMT}|{DEL_STMT}|{PRINT_STMT}|{RETURN_STMT}|{YIELD_STMT}|{RAISE_STMT}|{BREAK_STMT}|{CONTINUE_STMT}|{IMPORT_STMT}|{GLOBAL_STMT}|{EXEC_STMT})
EXPRESSION_STMT {EXPRESSION_LIST}
ASSERT_STMT "assert"{EXPRESSION}(","{EXPRESSION})?
ASSIGNMENT_STMT ({TARGET_LIST}"=")+{EXPRESSION_LIST}
TARGET_LIST {TARGET}(","{TARGET})*","?
TARGET ({IDENTIFIER}|"("{TARGET_LIST}")"|"["{TARGET_LIST}"]"|{ATTRIBUTEREF}|{SUBSCRIPTION}|{SLICING})
%option noyywrap
%option stack
%x Body
%x FunctionDec
%x FunctionParams
%x ClassDec
%x ClassInheritance
%x Suite
%x SuiteCaptureIndent
%x SuiteStart
%x SuiteMaintain
%x SuiteContinuing
%x LongString
%x SingleQuoteString
%x DoubleQuoteString
%x TripleString
%x DocBlock
%%
<Body,Suite>{
"def"{BB} {
startFontClass("keyword");
codify(yytext);
endFontClass();
BEGIN( FunctionDec );
}
"class"{BB} {
startFontClass("keyword");
codify(yytext);
endFontClass();
BEGIN( ClassDec );
}
"None" {
startFontClass("keywordtype");
codify(yytext);
endFontClass();
}
"self."{IDENTIFIER}/"."({IDENTIFIER}".")*{IDENTIFIER}"(" {
codify("self.");
findMemberLink(*g_code,&yytext[5]);
}
"self."{IDENTIFIER}/"(" {
codify("self.");
findMemberLink(*g_code,&yytext[5]);
}
"self."{IDENTIFIER}/"."({IDENTIFIER}".")*{IDENTIFIER} {
codify("self.");
findMemberLink(*g_code,&yytext[5]);
}
"self."{IDENTIFIER} {
codify("self.");
findMemberLink(*g_code,&yytext[5]);
}
"cls."{IDENTIFIER}/"."({IDENTIFIER}".")*{IDENTIFIER}"(" {
codify("cls.");
findMemberLink(*g_code,&yytext[4]);
}
"cls."{IDENTIFIER}/"(" {
codify("cls.");
findMemberLink(*g_code,&yytext[4]);
}
"cls."{IDENTIFIER}/"."({IDENTIFIER}".")*{IDENTIFIER} {
codify("cls.");
findMemberLink(*g_code,&yytext[4]);
}
"cls."{IDENTIFIER} {
codify("cls.");
findMemberLink(*g_code,&yytext[4]);
}
}
<ClassDec>{IDENTIFIER} {
generateClassOrGlobalLink(*g_code,yytext);
// codify(yytext);
g_curClassName = yytext;
g_curClassBases.clear();
BEGIN( ClassInheritance );
}
<ClassInheritance>{
({BB}|[(,)]) {
codify(yytext);
}
({IDENTIFIER}".")*{IDENTIFIER} {
// The parser
// is assuming
// that ALL identifiers
// in this state
// are base classes;
// it doesn't check to see
// that the first parenthesis
// has been seen.
// This is bad - it should
// probably be more strict
// about what to accept.
g_curClassBases.inSort(yytext);
generateClassOrGlobalLink(*g_code,yytext);
// codify(yytext);
}
":" {
codify(yytext);
// Assume this will
// be a one-line suite;
// found counter-example
// in SuiteStart.
// Push a class scope
ClassDef *classDefToAdd = new ClassDef("<code>",1,1,g_curClassName,ClassDef::Class,0,0,FALSE);
g_codeClassSDict.append(g_curClassName,classDefToAdd);
char *s=g_curClassBases.first();
while (s)
{
ClassDef *baseDefToAdd;
baseDefToAdd=g_codeClassSDict[s];
// Try to find class in global
// scope
if (baseDefToAdd==0)
{
baseDefToAdd=getResolvedClass(g_currentDefinition,g_sourceFileDef,s);
}
if (baseDefToAdd && baseDefToAdd!=classDefToAdd)
{
classDefToAdd->insertBaseClass(baseDefToAdd,s,Public,Normal);
}
s=g_curClassBases.next();
}
// Reset class-parsing variables.
g_curClassName.resize(0);
g_curClassBases.clear();
g_noSuiteFound = TRUE;
BEGIN( SuiteStart );
}
}
<FunctionDec>{
{IDENTIFIER} {
generateFunctionLink(*g_code,yytext);
}
{B}"(" {
codify(yytext);
BEGIN( FunctionParams );
}
}
<FunctionParams>{
({BB}|",") {
// Parses delimiters
codify(yytext);
}
({IDENTIFIER}|{PARAMNONEMPTY}+) {
codify(yytext);
}
")" {
codify(yytext);
}
"\n" {
codifyLines(yytext);
}
":" {
codify(yytext);
// Assume this will
// be a one-line suite;
// found counter-example
// in SuiteStart.
g_noSuiteFound = TRUE;
BEGIN( SuiteStart );
}
}
<Body,Suite>{
{KEYWORD} {
// Position-sensitive rules!
// Must come AFTER keyword-triggered rules
// Must come BEFORE identifier NONEMPTY-like rules
// to syntax highlight.
startFontClass("keyword");
codify(yytext);
endFontClass();
}
{FLOWKW} {
startFontClass("keywordflow");
codify(yytext);
endFontClass();
}
({IDENTIFIER}".")*{IDENTIFIER}/"(" {
generateClassOrGlobalLink(*g_code,yytext);
}
({IDENTIFIER}".")+{IDENTIFIER} {
generateClassOrGlobalLink(*g_code,yytext,TRUE);
}
{IDENTIFIER} { codify(yytext); }
}
<SuiteStart>{
{BB} {
codify(yytext);
}
"pass" {
startFontClass("keyword");
codifyLines(yytext);
endFontClass();
BEGIN(Body);
}
{KEYWORD} {
startFontClass("keyword");
codifyLines(yytext);
endFontClass();
// No indentation necessary
g_noSuiteFound = FALSE;
}
{FLOWKW} {
startFontClass("keywordflow");
codifyLines(yytext);
endFontClass();
// No indentation necessary
g_noSuiteFound = FALSE;
}
{IDENTIFIER} {
codify(yytext);
}
{POUNDCOMMENT} {
if (YY_START==SingleQuoteString ||
YY_START==DoubleQuoteString ||
YY_START==TripleString
)
{
REJECT;
}
yy_push_state(YY_START);
BEGIN(DocBlock);
g_docBlock=yytext;
}
{NEWLINE} {
codifyLines(yytext);
if ( g_noSuiteFound )
{
// printf("New suite to capture! [%d]\n", g_yyLineNr);
BEGIN ( SuiteCaptureIndent );
}
}
}
<SuiteCaptureIndent>{
"\n"|({BB}"\n") {
// Blankline - ignore, keep looking for indentation.
codifyLines(yytext);
}
{BB} {
// This state lasts momentarily,
// to check the indentation
// level that is about to be
// used.
codifyLines(yytext);
g_indents.push(yyleng);
// printf("Captured indent of %d [line %d]\n", yyleng, g_yyLineNr);
BEGIN( Suite );
}
}
<SuiteMaintain>{
{BB}/({NONEMPTY}|{EXPCHAR}) {
// This implements poor
// indendation-tracking;
// should be improved.
// (translate tabs to space, etc)
codifyLines(yytext);
adjustScopesAndSuites((int)yyleng);
}
"\n"|({BB}"\n") {
// If this ever succeeds,
// it means that this is
// a blank line, and
// can be ignored.
codifyLines(yytext);
}
""/({NONEMPTY}|{EXPCHAR}) {
// Default rule; matches
// the empty string, assuming
// real text starts here.
// Just go straight to Body.
adjustScopesAndSuites(0);
}
}
<Suite>{NEWLINE} {
codifyLines(yytext);
BEGIN( SuiteMaintain );
}
<Body>{IDENTIFIER} {
codify(yytext);
}
<Body>{NEWLINE} {
codifyLines(yytext);
}
<SingleQuoteString>{ // Single quoted string like 'That\'s a """nice""" string!'
\\{B}\n { // line continuation
codifyLines(yytext);
}
\\. { // espaced char
codify(yytext);
}
{STRINGPREFIX}?{TRIDOUBLEQUOTE} { // tripple double quotes
codify(yytext);
}
"'" { // end of the string
codify(yytext);
endFontClass();
BEGIN(g_stringContext);
}
[^"'\n\\]+ { // normal chars
codify(yytext);
}
. { // normal char
codify(yytext);
}
}
<DoubleQuoteString>{ // Double quoted string like "That's \"a '''nice'''\" string!"
\\{B}\n { // line continuation
codifyLines(yytext);
}
\\. { // espaced char
codify(yytext);
}
{STRINGPREFIX}?{TRISINGLEQUOTE} { // tripple single quotes
codify(yytext);
}
"\"" { // end of the string
codify(yytext);
endFontClass();
BEGIN(g_stringContext);
}
[^"'\n\\]+ { // normal chars
codify(yytext);
}
. { // normal char
codify(yytext);
}
}
<TripleString>{
{TRIDOUBLEQUOTE} |
{TRISINGLEQUOTE} {
codify(yytext);
if (g_doubleQuote==(yytext[0]=='"'))
{
endFontClass();
BEGIN(g_stringContext);
}
}
{LONGSTRINGBLOCK} {
codifyLines(yytext);
}
\n {
codifyLines(yytext);
}
. {
codify(yytext);
}
}
/*
<*>({NONEMPTY}|{EXPCHAR}|{BB}) { // This should go one character at a time.
codify(yytext);
// printf("[pycode] '%s' [ state %d ] [line %d] no match\n",
// yytext, YY_START, g_yyLineNr);
//endFontClass();
BEGIN(Body);
}
*/
<*>{STRINGPREFIX}?{TRISINGLEQUOTE} |
<*>{STRINGPREFIX}?{TRIDOUBLEQUOTE} {
startFontClass("stringliteral");
g_stringContext=YY_START;
g_doubleQuote=yytext[yyleng-1]=='"';
codify(yytext);
BEGIN(TripleString);
}
<*>{STRINGPREFIX}?"'" { // single quoted string
startFontClass("stringliteral");
g_stringContext=YY_START;
codify(yytext);
BEGIN(SingleQuoteString);
}
<*>{STRINGPREFIX}?"\"" { // double quoted string
startFontClass("stringliteral");
g_stringContext=YY_START;
codify(yytext);
BEGIN(DoubleQuoteString);
}
<DocBlock>.* { // contents of current comment line
g_docBlock+=yytext;
}
<DocBlock>"\n"{B}("#") { // comment block (next line is also comment line)
g_docBlock+=yytext;
}
<DocBlock>{NEWLINE} { // comment block ends at the end of this line
// remove special comment (default config)
if (Config_getBool(STRIP_CODE_COMMENTS))
{
g_yyLineNr+=((QCString)g_docBlock).contains('\n');
g_endComment=TRUE;
}
else // do not remove comment
{
startFontClass("comment");
codifyLines(g_docBlock);
endFontClass();
}
unput(*yytext);
yy_pop_state();
}
<*>{POUNDCOMMENT}.* {
if (YY_START==SingleQuoteString ||
YY_START==DoubleQuoteString ||
YY_START==TripleString
)
{
REJECT;
}
yy_push_state(YY_START);
BEGIN(DocBlock);
g_docBlock=yytext;
}
<*>"#".* { // normal comment
if (YY_START==SingleQuoteString ||
YY_START==DoubleQuoteString ||
YY_START==TripleString
)
{
REJECT;
}
startFontClass("comment");
codifyLines(yytext);
endFontClass();
}
<*>{NEWLINE} {
if (g_endComment)
{
g_endComment=FALSE;
}
else
{
codifyLines(yytext);
}
//printf("[pycode] %d NEWLINE [line %d] no match\n",
// YY_START, g_yyLineNr);
//endFontClass();
BEGIN(Body);
}
<*>[ \t]+ {
codify(yytext);
BEGIN(Body);
}
<*>. {
codify(yytext);
// printf("[pycode] '%s' [ state %d ] [line %d] no match\n",
// yytext, YY_START, g_yyLineNr);
//endFontClass();
BEGIN(Body);
}
<*><<EOF>> {
if (YY_START == DocBlock) {
if (!Config_getBool(STRIP_CODE_COMMENTS))
{
startFontClass("comment");
codifyLines(g_docBlock);
endFontClass();
}
}
yyterminate();
}
%%
/*@ ----------------------------------------------------------------------------
*/
void resetPythonCodeParserState()
{
g_currentDefinition = 0;
g_currentMemberDef = 0;
g_doubleStringIsDoc = FALSE;
g_paramParens = 0;
g_indents.clear();
BEGIN( Body );
}
/*!
Examines current stack of white-space indentations;
re-syncs the parser with the correct scope.
*/
static void adjustScopesAndSuites(unsigned indentLength)
{
// States to pop
if (!g_indents.isEmpty() && indentLength < g_indents.top())
{
while (!g_indents.isEmpty() && indentLength < g_indents.top())
{
// printf("Exited scope indent of [%d]\n", g_indents.top());
g_indents.pop(); // Pop the old suite's indentation
g_currentMemberDef=0;
if (g_currentDefinition)
g_currentDefinition=g_currentDefinition->getOuterScope();
}
}
// Are there any remaining indentation levels for suites?
if (!g_indents.isEmpty())
{
BEGIN( Suite );
}
else
{
BEGIN( Body );
}
}
void parsePythonCode(CodeOutputInterface &od,const char * /*className*/,
const QCString &s,bool exBlock, const char *exName,
FileDef *fd,int startLine,int endLine,bool inlineFragment,
MemberDef *,bool,Definition *searchCtx,bool collectXRefs)
{
//printf("***parseCode()\n");
//--------------------------------------
if (s.isEmpty()) return;
printlex(yy_flex_debug, TRUE, __FILE__, fd ? fd->fileName().data(): NULL);
TooltipManager::instance()->clearTooltips();
g_code = &od;
g_inputString = s;
g_inputPosition = 0;
g_currentFontClass = 0;
g_needsTermination = FALSE;
g_searchCtx=searchCtx;
g_collectXRefs=collectXRefs;
if (startLine!=-1)
g_yyLineNr = startLine;
else
g_yyLineNr = 1;
if (endLine!=-1)
g_inputLines = endLine+1;
else
g_inputLines = g_yyLineNr + countLines() - 1;
g_exampleBlock = exBlock;
g_exampleName = exName;
g_sourceFileDef = fd;
bool cleanupSourceDef = FALSE;
if (exBlock && fd==0)
{
// create a dummy filedef for the example
g_sourceFileDef = new FileDef("",(exName?exName:"generated"));
cleanupSourceDef = TRUE;
}
if (g_sourceFileDef)
{
setCurrentDoc("l00001");
}
g_includeCodeFragment = inlineFragment;
// Starts line 1 on the output
startCodeLine();
pycodeYYrestart( pycodeYYin );
pycodeYYlex();
if (!g_indents.isEmpty())
{
// printf("Exited pysourceparser in inconsistent state!\n");
}
if (g_needsTermination)
{
endCodeLine();
}
if (fd)
{
TooltipManager::instance()->writeTooltips(*g_code);
}
if (cleanupSourceDef)
{
// delete the temporary file definition used for this example
delete g_sourceFileDef;
g_sourceFileDef=0;
}
printlex(yy_flex_debug, FALSE, __FILE__, fd ? fd->fileName().data(): NULL);
return;
}
#if !defined(YY_FLEX_SUBMINOR_VERSION)
extern "C" { // some bogus code to keep the compiler happy
void pycodeYYdummy() { yy_flex_realloc(0,0); }
}
#elif YY_FLEX_MAJOR_VERSION<=2 && YY_FLEX_MINOR_VERSION<=5 && YY_FLEX_SUBMINOR_VERSION<33
#error "You seem to be using a version of flex newer than 2.5.4. These are currently incompatible with 2.5.4, and do NOT work with doxygen! Please use version 2.5.4 or expect things to be parsed wrongly! A bug report has been submitted (#732132)."
#endif