/****************************************************************************** * * * * Copyright (C) 1997-2015 by Dimitri van Heesch. * * Permission to use, copy, modify, and distribute this software and its * documentation under the terms of the GNU General Public License is hereby * granted. No representations are made about the suitability of this software * for any purpose. It is provided "as is" without express or implied warranty. * See the GNU General Public License for more details. * * Documents produced by Doxygen are derivative works derived from the * input used in their production; they are not affected by this license. * */ /* This code is based on the work done by the MoxyPyDoxy team * (Linda Leong, Mike Rivera, Kim Truong, and Gabriel Estrada) * in Spring 2005 as part of CS 179E: Compiler Design Project * at the University of California, Riverside; the course was * taught by Peter H. Froehlich . */ %option never-interactive %option prefix="pycodeYY" %{ #include #include #include "pycode.h" #include "message.h" #include "scanner.h" #include "entry.h" #include "doxygen.h" #include "outputlist.h" #include "util.h" #include "membername.h" #include "searchindex.h" #include "config.h" #include "groupdef.h" #include "classlist.h" #include "filedef.h" #include "namespacedef.h" #include "tooltip.h" // Toggle for some debugging info //#define DBG_CTX(x) fprintf x #define DBG_CTX(x) do { } while(0) #define YY_NO_INPUT 1 #define YY_NO_UNISTD_H 1 static ClassSDict g_codeClassSDict(17); static QCString g_curClassName; static QStrList g_curClassBases; static CodeOutputInterface * g_code; static const char * g_inputString; //!< the code fragment as text static int g_inputPosition; //!< read offset during parsing static const char * g_currentFontClass; static bool g_needsTermination; static Definition *g_searchCtx; static bool g_collectXRefs; static int g_inputLines; //!< number of line in the code fragment static int g_yyLineNr; //!< current line number static FileDef * g_sourceFileDef; static Definition * g_currentDefinition; static MemberDef * g_currentMemberDef; static bool g_includeCodeFragment; static QCString g_realScope; //static bool g_insideBody; static int g_bodyCurlyCount; static bool g_searchingForBody; static QCString g_classScope; static int g_paramParens; //static int g_anchorCount; static bool g_exampleBlock; static QCString g_exampleName; static QCString g_type; static QCString g_name; static bool g_doubleStringIsDoc; static bool g_doubleQuote; static bool g_noSuiteFound; static int g_stringContext; static QValueStack g_indents; //!< Tracks indentation levels for scoping in python static QCString g_docBlock; //!< contents of all lines of a documentation block static bool g_endComment; static void endFontClass(); static void adjustScopesAndSuites(unsigned indentLength); /*! Represents a stack of variable to class mappings as found in the * code. Each scope is enclosed in pushScope() and popScope() calls. * Variables are added by calling addVariables() and one can search * for variable using findVariable(). */ class PyVariableContext { public: static const ClassDef *dummyContext; class Scope : public SDict { public: Scope() : SDict(17) {} }; PyVariableContext() { m_scopes.setAutoDelete(TRUE); } virtual ~PyVariableContext() { } void pushScope() { m_scopes.append(new Scope); } void popScope() { if (m_scopes.count()>0) { m_scopes.remove(m_scopes.count()-1); } } void clear() { m_scopes.clear(); m_globalScope.clear(); } void clearExceptGlobal() { m_scopes.clear(); } void addVariable(const QCString &type,const QCString &name); ClassDef *findVariable(const QCString &name); private: Scope m_globalScope; QList m_scopes; }; void PyVariableContext::addVariable(const QCString &type,const QCString &name) { //printf("PyVariableContext::addVariable(%s,%s)\n",type.data(),name.data()); QCString ltype = type.simplifyWhiteSpace(); QCString lname = name.simplifyWhiteSpace(); Scope *scope = m_scopes.count()==0 ? &m_globalScope : m_scopes.getLast(); ClassDef *varType; if ( (varType=g_codeClassSDict[ltype]) || // look for class definitions inside the code block (varType=getResolvedClass(g_currentDefinition,g_sourceFileDef,ltype)) // look for global class definitions ) { scope->append(lname,varType); // add it to a list } else { if (m_scopes.count()>0) // for local variables add a dummy entry so the name // is hidden to avoid FALSE links to global variables with the same name // TODO: make this work for namespaces as well! { scope->append(lname,dummyContext); } } } ClassDef *PyVariableContext::findVariable(const QCString &name) { if (name.isEmpty()) return 0; ClassDef *result = 0; QListIterator sli(m_scopes); Scope *scope; // search from inner to outer scope for (sli.toLast();(scope=sli.current());--sli) { result = scope->find(name); if (result) { return result; } } // nothing found -> also try the global scope result=m_globalScope.find(name); return result; } static PyVariableContext g_theVarContext; const ClassDef *PyVariableContext::dummyContext = (ClassDef*)0x8; class PyCallContext { public: struct Ctx { Ctx() : name(g_name), type(g_type), cd(0) {} QCString name; QCString type; ClassDef *cd; }; PyCallContext() { m_classList.append(new Ctx); m_classList.setAutoDelete(TRUE); } virtual ~PyCallContext() {} void setClass(ClassDef *cd) { Ctx *ctx = m_classList.getLast(); if (ctx) { ctx->cd=cd; } } void pushScope() { m_classList.append(new Ctx); } void popScope() { if (m_classList.count()>1) { Ctx *ctx = m_classList.getLast(); if (ctx) { g_name = ctx->name; g_type = ctx->type; } m_classList.removeLast(); } else { } } void clear() { m_classList.clear(); m_classList.append(new Ctx); } ClassDef *getClass() const { Ctx *ctx = m_classList.getLast(); if (ctx) return ctx->cd; else return 0; } private: QList m_classList; }; static PyCallContext g_theCallContext; /*! counts the number of lines in the input */ static int countLines() { const char *p=g_inputString; char c; int count=1; while ((c=*p)) { p++ ; if (c=='\n') count++; } if (p>g_inputString && *(p-1)!='\n') { // last line does not end with a \n, so we add an extra // line and explicitly terminate the line after parsing. count++, g_needsTermination=TRUE; } return count; } static void setCurrentDoc(const QCString &anchor) { if (Doxygen::searchIndex) { if (g_searchCtx) { Doxygen::searchIndex->setCurrentDoc(g_searchCtx,g_searchCtx->anchor(),FALSE); } else { Doxygen::searchIndex->setCurrentDoc(g_sourceFileDef,anchor,TRUE); } } } static void addToSearchIndex(const char *text) { if (Doxygen::searchIndex) { Doxygen::searchIndex->addWord(text,FALSE); } } static ClassDef *stripClassName(const char *s,Definition *d=g_currentDefinition) { int pos=0; QCString type = s; QCString className; QCString templSpec; while (extractClassNameFromType(type,pos,className,templSpec)!=-1) { QCString clName=className+templSpec; ClassDef *cd=0; if (!g_classScope.isEmpty()) { cd=getResolvedClass(d,g_sourceFileDef,g_classScope+"::"+clName); } if (cd==0) { cd=getResolvedClass(d,g_sourceFileDef,clName); } if (cd) { return cd; } } return 0; } /*! start a new line of code, inserting a line number if g_sourceFileDef * is TRUE. If a definition starts at the current line, then the line * number is linked to the documentation of that definition. */ static void startCodeLine() { //if (g_currentFontClass) { g_code->endFontClass(); } if (g_sourceFileDef) { //QCString lineNumber,lineAnchor; //lineNumber.sprintf("%05d",g_yyLineNr); //lineAnchor.sprintf("l%05d",g_yyLineNr); Definition *d = g_sourceFileDef->getSourceDefinition(g_yyLineNr); //printf("startCodeLine %d d=%p\n",g_yyLineNr,d); //g_code->startLineNumber(); if (!g_includeCodeFragment && d && d->isLinkableInProject()) { g_currentDefinition = d; g_currentMemberDef = g_sourceFileDef->getSourceMember(g_yyLineNr); //g_insideBody = FALSE; g_endComment = FALSE; g_searchingForBody = TRUE; g_realScope = d->name().copy(); g_classScope = d->name().copy(); //printf("Real scope: `%s'\n",g_realScope.data()); g_bodyCurlyCount = 0; QCString lineAnchor; lineAnchor.sprintf("l%05d",g_yyLineNr); if (g_currentMemberDef) { g_code->writeLineNumber(g_currentMemberDef->getReference(), g_currentMemberDef->getOutputFileBase(), g_currentMemberDef->anchor(),g_yyLineNr); setCurrentDoc(lineAnchor); } else { g_code->writeLineNumber(d->getReference(), d->getOutputFileBase(), 0,g_yyLineNr); setCurrentDoc(lineAnchor); } } else { //g_code->codify(lineNumber); g_code->writeLineNumber(0,0,0,g_yyLineNr); } //g_code->endLineNumber(); } g_code->startCodeLine(g_sourceFileDef); if (g_currentFontClass) { g_code->startFontClass(g_currentFontClass); } } static void codify(const char* text) { g_code->codify(text); } static void endCodeLine() { endFontClass(); g_code->endCodeLine(); } static void nextCodeLine() { const char *fc = g_currentFontClass; endCodeLine(); if (g_yyLineNraddTooltip(d); QCString ref = d->getReference(); QCString file = d->getOutputFileBase(); QCString anchor = d->anchor(); QCString tooltip; if (!sourceTooltips) // fall back to simple "title" tooltips { tooltip = d->briefDescriptionAsTooltip(); } bool done=FALSE; char *p=(char *)text; while (!done) { char *sp=p; char c; while ((c=*p++) && c!='\n') { } if (c=='\n') { g_yyLineNr++; *(p-1)='\0'; //printf("writeCodeLink(%s,%s,%s,%s)\n",ref,file,anchor,sp); ol.writeCodeLink(ref,file,anchor,sp,tooltip); nextCodeLine(); } else { //printf("writeCodeLink(%s,%s,%s,%s)\n",ref,file,anchor,sp); ol.writeCodeLink(ref,file,anchor,sp,tooltip); done=TRUE; } } } static void startFontClass(const char *s) { // if font class is already set don't stop and start it. // strcmp does not like null pointers as input. if (!g_currentFontClass || !s || strcmp(g_currentFontClass,s)) { endFontClass(); g_code->startFontClass(s); g_currentFontClass=s; } } static void endFontClass() { if (g_currentFontClass) { g_code->endFontClass(); g_currentFontClass=0; } } static void codifyLines(char *text) { //printf("codifyLines(%d,\"%s\")\n",g_yyLineNr,text); char *p=text,*sp=p; char c; bool done=FALSE; const char * tmp_currentFontClass = g_currentFontClass; while (!done) { sp=p; while ((c=*p++) && c!='\n') { } if (c=='\n') { g_yyLineNr++; *(p-1)='\0'; g_code->codify(sp); endCodeLine(); if (g_yyLineNrcodify(sp); done=TRUE; } } } static void codifyLines(const QCString &str) { char *tmp= (char *)malloc(str.length()+1); qstrcpy(tmp, str); codifyLines(tmp); free(tmp); } static bool getLinkInScope(const QCString &c, // scope const QCString &m, // member const char *memberText, // exact text CodeOutputInterface &ol, const char *text ) { MemberDef *md; ClassDef *cd; FileDef *fd; NamespaceDef *nd; GroupDef *gd; //printf("Trying `%s'::`%s'\n",c.data(),m.data()); if (getDefs(c,m,"()",md,cd,fd,nd,gd,FALSE,g_sourceFileDef) && md->isLinkable()) { //Definition *d=0; //if (cd) d=cd; else if (nd) d=nd; else if (fd) d=fd; else d=gd; Definition *d = md->getOuterScope()==Doxygen::globalScope ? md->getBodyDef() : md->getOuterScope(); //printf("Found! d=%s\n",d?d->name().data():""); if (md->getGroupDef()) d = md->getGroupDef(); if (d && d->isLinkable()) { g_theCallContext.setClass(stripClassName(md->typeString(),md->getOuterScope())); //printf("g_currentDefinition=%p g_currentMemberDef=%p\n", // g_currentDefinition,g_currentMemberDef); if (g_currentDefinition && g_currentMemberDef && md!=g_currentMemberDef && g_collectXRefs) { addDocCrossReference(g_currentMemberDef,md); } //printf("d->getReference()=`%s' d->getOutputBase()=`%s' name=`%s' member name=`%s'\n",d->getReference().data(),d->getOutputFileBase().data(),d->name().data(),md->name().data()); writeMultiLineCodeLink(ol,md, text ? text : memberText); addToSearchIndex(text ? text : memberText); return TRUE; } } return FALSE; } static bool getLink(const char *className, const char *memberName, CodeOutputInterface &ol, const char *text=0) { QCString m=removeRedundantWhiteSpace(memberName); QCString c=className; if (!getLinkInScope(c,m,memberName,ol,text)) { if (!g_curClassName.isEmpty()) { if (!c.isEmpty()) c.prepend("::"); c.prepend(g_curClassName); return getLinkInScope(c,m,memberName,ol,text); } return FALSE; } return TRUE; } /* For a given string in the source code, finds its class or global id and links to it. */ static void generateClassOrGlobalLink(CodeOutputInterface &ol,char *clName, bool typeOnly=FALSE) { QCString className=clName; // Don't do anything for empty text if (className.isEmpty()) return; DBG_CTX((stderr,"generateClassOrGlobalLink(className=%s)\n",className.data())); ClassDef *cd=0,*lcd=0; /** Class def that we may find */ MemberDef *md=0; /** Member def that we may find */ //bool isLocal=FALSE; if ((lcd=g_theVarContext.findVariable(className))==0) // not a local variable { Definition *d = g_currentDefinition; QCString scope = substitute(className,".","::"); cd = getResolvedClass(d,g_sourceFileDef,substitute(className,".","::"),&md); DBG_CTX((stderr,"d=%s g_sourceFileDef=%s\n", d?d->displayName().data():"", g_currentDefinition?g_currentDefinition->displayName().data():"")); DBG_CTX((stderr,"is found as a type %s\n",cd?cd->name().data():"")); if (cd==0 && md==0) // also see if it is variable or enum or enum value { NamespaceDef *nd = getResolvedNamespace(scope); if (nd) { writeMultiLineCodeLink(ol,nd,clName); addToSearchIndex(className); return; } else if (getLink(g_classScope,clName,ol,clName)) { return; } } } else { if (lcd!=PyVariableContext::dummyContext) { g_theCallContext.setClass(lcd); } //isLocal=TRUE; DBG_CTX((stderr,"is a local variable cd=%p!\n",cd)); } if (cd && cd->isLinkable()) // is it a linkable class { writeMultiLineCodeLink(ol,cd,clName); addToSearchIndex(className); if (md) { Definition *d = md->getOuterScope()==Doxygen::globalScope ? md->getBodyDef() : md->getOuterScope(); if (md->getGroupDef()) d = md->getGroupDef(); if (d && d->isLinkable() && md->isLinkable() && g_currentMemberDef && g_collectXRefs) { addDocCrossReference(g_currentMemberDef,md); } } } else // not a class, maybe a global member { int scopeEnd = className.findRev("."); if (scopeEnd!=-1 && !typeOnly) // name with explicit scope { QCString scope = substitute(className.left(scopeEnd),".","::"); QCString locName = className.right(className.length()-scopeEnd-1); ClassDef *mcd = getClass(scope); DBG_CTX((stderr,"scope=%s locName=%s mcd=%p\n",scope.data(),locName.data(),mcd)); if (mcd) { MemberDef *md = mcd->getMemberByName(locName); if (md) { g_theCallContext.setClass(stripClassName(md->typeString(),md->getOuterScope())); writeMultiLineCodeLink(ol,md,clName); addToSearchIndex(className); Definition *d = md->getOuterScope()==Doxygen::globalScope ? md->getBodyDef() : md->getOuterScope(); if (md->getGroupDef()) d = md->getGroupDef(); if (d && d->isLinkable() && md->isLinkable() && g_currentMemberDef && g_collectXRefs) { addDocCrossReference(g_currentMemberDef,md); } return; } } else // check namespace as well { NamespaceDef *mnd = getResolvedNamespace(scope); if (mnd) { MemberDef *md=mnd->getMemberByName(locName); if (md) { //printf("name=%s scope=%s\n",locName.data(),scope.data()); g_theCallContext.setClass(stripClassName(md->typeString(),md->getOuterScope())); writeMultiLineCodeLink(ol,md,clName); addToSearchIndex(className); Definition *d = md->getOuterScope()==Doxygen::globalScope ? md->getBodyDef() : md->getOuterScope(); if (md->getGroupDef()) d = md->getGroupDef(); if (d && d->isLinkable() && md->isLinkable() && g_currentMemberDef && g_collectXRefs) { addDocCrossReference(g_currentMemberDef,md); } return; } } } } // nothing found, just write out the word codifyLines(clName); addToSearchIndex(clName); } } /* As of June 1, this function seems to work for file members, but scopes are not being correctly tracked for classes so it doesn't work for classes yet. */ static void generateFunctionLink(CodeOutputInterface &ol,char *funcName) { //CodeClassDef *ccd=0; ClassDef *ccd=0; QCString locScope=g_classScope.copy(); QCString locFunc=removeRedundantWhiteSpace(funcName); DBG_CTX((stdout,"*** locScope=%s locFunc=%s\n",locScope.data(),locFunc.data())); int i=locFunc.findRev("::"); if (i>0) { locScope=locFunc.left(i); locFunc=locFunc.right(locFunc.length()-i-2).stripWhiteSpace(); } //printf("generateFunctionLink(%s) classScope=`%s'\n",locFunc.data(),locScope.data()); if (!locScope.isEmpty() && (ccd=g_codeClassSDict[locScope])) { //printf("using classScope %s\n",g_classScope.data()); if (ccd->baseClasses()) { BaseClassListIterator bcli(*ccd->baseClasses()); for ( ; bcli.current() ; ++bcli) { if (getLink(bcli.current()->classDef->name(),locFunc,ol,funcName)) { return; } } } } if (!getLink(locScope,locFunc,ol,funcName)) { generateClassOrGlobalLink(ol,funcName); } return; } static bool findMemberLink(CodeOutputInterface &ol,Definition *sym,const char *symName) { //printf("sym %s outerScope=%s equal=%d\n", // sym->name().data(),sym->getOuterScope()->name().data(), // sym->getOuterScope()==g_currentDefinition); if (sym->getOuterScope() && sym->getOuterScope()->definitionType()==Definition::TypeClass && g_currentDefinition->definitionType()==Definition::TypeClass) { ClassDef *cd = (ClassDef*)sym->getOuterScope(); ClassDef *thisCd = (ClassDef *)g_currentDefinition; if (sym->definitionType()==Definition::TypeMember) { if (g_currentMemberDef && g_collectXRefs) { addDocCrossReference(g_currentMemberDef,(MemberDef*)sym); } } DBG_CTX((stderr,"cd=%s thisCd=%s\n",cd?cd->name().data():"",thisCd?thisCd->name().data():"")); // TODO: find the nearest base class in case cd is a base class of // thisCd if (cd==thisCd || (thisCd && thisCd->isBaseClass(cd,TRUE))) { writeMultiLineCodeLink(ol,sym,symName); return TRUE; } } return FALSE; } static void findMemberLink(CodeOutputInterface &ol,char *symName) { //printf("Member reference: %s scope=%s member=%s\n", // yytext, // g_currentDefinition?g_currentDefinition->name().data():"", // g_currentMemberDef?g_currentMemberDef->name().data():"" // ); if (g_currentDefinition) { DefinitionIntf *di = Doxygen::symbolMap->find(symName); if (di) { if (di->definitionType()==DefinitionIntf::TypeSymbolList) // multiple symbols { DefinitionListIterator dli(*(DefinitionList*)di); Definition *sym; for (dli.toFirst();(sym=dli.current());++dli) { if (findMemberLink(ol,sym,symName)) return; } } else // single symbol { if (findMemberLink(ol,(Definition*)di,symName)) return; } } } //printf("sym %s not found\n",&yytext[5]); codify(symName); } #undef YY_INPUT #define YY_INPUT(buf,result,max_size) result=yyread(buf,max_size); static int yyread(char *buf,int max_size) { int c=0; while( c < max_size && g_inputString[g_inputPosition] ) { *buf = g_inputString[g_inputPosition++] ; c++; buf++; } return c; } %} BB [ \t]+ B [ \t]* NEWLINE \n DIGIT [0-9] LETTER [A-Za-z\x80-\xFF] NONEMPTY [A-Za-z0-9_\x80-\xFF] EXPCHAR [#(){}\[\],:.%/\\=`*~|&<>!;+-] NONEMPTYEXP [^ \t\n:] PARAMNONEMPTY [^ \t\n():] IDENTIFIER ({LETTER}|"_")({LETTER}|{DIGIT}|"_")* BORDER ([^A-Za-z0-9]) POUNDCOMMENT "##" TRISINGLEQUOTE "'''" TRIDOUBLEQUOTE "\"\"\"" LONGSTRINGCHAR [^\\"'] ESCAPESEQ ("\\")(.) LONGSTRINGITEM ({LONGSTRINGCHAR}|{ESCAPESEQ}) SMALLQUOTE ("\"\""|"\""|"'"|"''") LONGSTRINGBLOCK ({LONGSTRINGITEM}+|{SMALLQUOTE}) SHORTSTRING ("'"{SHORTSTRINGITEM}*"'"|'"'{SHORTSTRINGITEM}*'"') SHORTSTRINGITEM ({SHORTSTRINGCHAR}|{ESCAPESEQ}) SHORTSTRINGCHAR [^\\\n"] STRINGLITERAL {STRINGPREFIX}?( {SHORTSTRING} | {LONGSTRING}) STRINGPREFIX ("r"|"u"|"ur"|"R"|"U"|"UR"|"Ur"|"uR") KEYWORD ("lambda"|"import"|"class"|"assert"|"as"|"from"|"global"|"def"|"True"|"False") FLOWKW ("or"|"and"|"is"|"not"|"print"|"for"|"in"|"if"|"try"|"except"|"yield"|"raise"|"break"|"continue"|"pass"|"if"|"return"|"while"|"elif"|"else"|"finally") QUOTES ("\""[^"]*"\"") SINGLEQUOTES ("'"[^']*"'") LONGINTEGER {INTEGER}("l"|"L") INTEGER ({DECIMALINTEGER}|{OCTINTEGER}|{HEXINTEGER}) DECIMALINTEGER ({NONZERODIGIT}{DIGIT}*|"0") OCTINTEGER "0"{OCTDIGIT}+ HEXINTEGER "0"("x"|"X"){HEXDIGIT}+ NONZERODIGIT [1-9] OCTDIGIT [0-7] HEXDIGIT ({DIGIT}|[a-f]|[A-F]) FLOATNUMBER ({POINTFLOAT}|{EXPONENTFLOAT}) POINTFLOAT ({INTPART}?{FRACTION}|{INTPART}".") EXPONENTFLOAT ({INTPART}|{POINTFLOAT}){EXPONENT} INTPART {DIGIT}+ FRACTION "."{DIGIT}+ EXPONENT ("e"|"E")("+"|"-")?{DIGIT}+ IMAGNUMBER ({FLOATNUMBER}|{INTPART})("j"|"J") ATOM ({IDENTIFIER}|{LITERAL}|{ENCLOSURE}) ENCLOSURE ({PARENTH_FORM}|{LIST_DISPLAY}|{DICT_DISPLAY}|{STRING_CONVERSION}) LITERAL ({STRINGLITERAL}|{INTEGER}|{LONGINTEGER}|{FLOATNUMBER}|{IMAGNUMBER}) PARENTH_FORM "("{EXPRESSION_LIST}?")" TEST ({AND_TEST}("or"{AND_TEST})*|{LAMBDA_FORM}) TESTLIST {TEST}( ","{TEST})*","? LIST_DISPLAY "["{LISTMAKER}?"]" LISTMAKER {EXPRESSION}({LIST_FOR}|(","{EXPRESSION})*","?) LIST_ITER ({LIST_FOR}|{LIST_IF}) LIST_FOR "for"{EXPRESSION_LIST}"in"{TESTLIST}{LIST_ITER}? LIST_IF "if"{TEST}{LIST_ITER}? DICT_DISPLAY "\{"{KEY_DATUM_LIST}?"\}" KEY_DATUM_LIST {KEY_DATUM}(","{KEY_DATUM})*","? KEY_DATUM {EXPRESSION}":"{EXPRESSION} STRING_CONVERSION "`"{EXPRESSION_LIST}"`" PRIMARY ({ATOM}|{ATTRIBUTEREF}|{SUBSCRIPTION}|{SLICING}|{CALL}) ATTRIBUTEREF {PRIMARY}"."{IDENTIFIER} SUBSCRIPTION {PRIMARY}"["{EXPRESSION_LIST}"]" SLICING ({SIMPLE_SLICING}|{EXTENDED_SLICING}) SIMPLE_SLICING {PRIMARY}"["{SHORT_SLICE}"]" EXTENDED_SLICING {PRIMARY}"["{SLICE_LIST}"]" SLICE_LIST {SLICE_ITEM}(","{SLICE_ITEM})*","? SLICE_ITEM ({EXPRESSION}|{PROPER_SLICE}|{ELLIPSIS}) PROPER_SLICE ({SHORT_SLICE}|{LONG_SLICE}) SHORT_SLICE {LOWER_BOUND}?":"{UPPER_BOUND}? LONG_SLICE {SHORT_SLICE}":"{STRIDE}? LOWER_BOUND {EXPRESSION} UPPER_BOUND {EXPRESSION} STRIDE {EXPRESSION} ELLIPSIS "..." CALL {PRIMARY}"("({ARGUMENT_LIST}","?)?")" ARGUMENT_LIST ({POSITIONAL_ARGUMENTS}(","{KEYWORD_ARGUMENTS})?(",""*"{EXPRESSION})?(",""**"{EXPRESSION})?|{KEYWORD_ARGUMENTS}(",""*"{EXPRESSION})?(",""**"{EXPRESSION})?|"*"{EXPRESSION}(",""**"{EXPRESSION})?|"**"{EXPRESSION}) POSITIONAL_ARGUMENTS {EXPRESSION}(","{EXPRESSION})* KEYWORD_ARGUMENTS {KEYWORD_ITEM}(","{KEYWORD_ITEM})* KEYWORD_ITEM {IDENTIFIER}"="{EXPRESSION} POWER {PRIMARY}("**"{U_EXPR})? U_EXPR ({POWER}|"-"{U_EXPR}|"+"{U_EXPR}|"\~"{U_EXPR}) M_EXPR ({U_EXPR}|{M_EXPR}"*"{U_EXPR}|{M_EXPR}"//"{U_EXPR}|{M_EXPR}"/"{U_EXPR}|{M_EXPR}"\%"{U_EXPR}) A_EXPR ({M_EXPR}|{A_EXPR}"+"{M_EXPR}|{A_EXPR}"-"{M_EXPR} SHIFT_EXPR ({A_EXPR}|{SHIFT_EXPR}("<<"|">>"){A_EXPR}) AND_EXPR ({SHIFT_EXPR}|{AND_EXPR}"\;SPMamp;"{SHIFT_EXPR} XOR_EXPR ({AND_EXPR}|{XOR_EXPR}"\textasciicircum"{AND_EXPR}) OR_EXPR ({XOR_EXPR}|{OR_EXPR}"|"{ XOR_EXPR}) COMPARISON {OR_EXPR}({COMP_OPERATOR}{OR_EXPR})* COMP_OPERATOR ("<"|">"|"=="|">="|"<="|"<>"|"!="|"is""not"?|"not"?"in") EXPRESSION ({OR_TEST}|{LAMBDA_FORM}) OR_TEST ({AND_TEST}|{OR_TEST}"or"{AND_TEST}) AND_TEST ({NOT_TEST}|{AND_TEST}"and"{NOT_TEST}) NOT_TEST ({COMPARISON}|"not"{NOT_TEST}) LAMBDA_FORM "lambda"{PARAMETER_LIST}?":"{EXPRESSION} EXPRESSION_LIST {EXPRESSION}(","{EXPRESSION})*","? SIMPLE_STMT ({EXPRESSION_STMT}|{ASSERT_STMT}|{ASSIGNMENT_STMT}|{AUGMENTED_ASSIGNMENT_STMT}|{PASS_STMT}|{DEL_STMT}|{PRINT_STMT}|{RETURN_STMT}|{YIELD_STMT}|{RAISE_STMT}|{BREAK_STMT}|{CONTINUE_STMT}|{IMPORT_STMT}|{GLOBAL_STMT}|{EXEC_STMT}) EXPRESSION_STMT {EXPRESSION_LIST} ASSERT_STMT "assert"{EXPRESSION}(","{EXPRESSION})? ASSIGNMENT_STMT ({TARGET_LIST}"=")+{EXPRESSION_LIST} TARGET_LIST {TARGET}(","{TARGET})*","? TARGET ({IDENTIFIER}|"("{TARGET_LIST}")"|"["{TARGET_LIST}"]"|{ATTRIBUTEREF}|{SUBSCRIPTION}|{SLICING}) %option noyywrap %option stack %x Body %x FunctionDec %x FunctionParams %x ClassDec %x ClassInheritance %x Suite %x SuiteCaptureIndent %x SuiteStart %x SuiteMaintain %x SuiteContinuing %x LongString %x SingleQuoteString %x DoubleQuoteString %x TripleString %x DocBlock %% { "def"{BB} { startFontClass("keyword"); codify(yytext); endFontClass(); BEGIN( FunctionDec ); } "class"{BB} { startFontClass("keyword"); codify(yytext); endFontClass(); BEGIN( ClassDec ); } "None" { startFontClass("keywordtype"); codify(yytext); endFontClass(); } "self."{IDENTIFIER}/"."({IDENTIFIER}".")*{IDENTIFIER}"(" { codify("self."); findMemberLink(*g_code,&yytext[5]); } "self."{IDENTIFIER}/"(" { codify("self."); findMemberLink(*g_code,&yytext[5]); } "self."{IDENTIFIER}/"."({IDENTIFIER}".")*{IDENTIFIER} { codify("self."); findMemberLink(*g_code,&yytext[5]); } "self."{IDENTIFIER} { codify("self."); findMemberLink(*g_code,&yytext[5]); } "cls."{IDENTIFIER}/"."({IDENTIFIER}".")*{IDENTIFIER}"(" { codify("cls."); findMemberLink(*g_code,&yytext[4]); } "cls."{IDENTIFIER}/"(" { codify("cls."); findMemberLink(*g_code,&yytext[4]); } "cls."{IDENTIFIER}/"."({IDENTIFIER}".")*{IDENTIFIER} { codify("cls."); findMemberLink(*g_code,&yytext[4]); } "cls."{IDENTIFIER} { codify("cls."); findMemberLink(*g_code,&yytext[4]); } } {IDENTIFIER} { generateClassOrGlobalLink(*g_code,yytext); // codify(yytext); g_curClassName = yytext; g_curClassBases.clear(); BEGIN( ClassInheritance ); } { ({BB}|[(,)]) { codify(yytext); } ({IDENTIFIER}".")*{IDENTIFIER} { // The parser // is assuming // that ALL identifiers // in this state // are base classes; // it doesn't check to see // that the first parenthesis // has been seen. // This is bad - it should // probably be more strict // about what to accept. g_curClassBases.inSort(yytext); generateClassOrGlobalLink(*g_code,yytext); // codify(yytext); } ":" { codify(yytext); // Assume this will // be a one-line suite; // found counter-example // in SuiteStart. // Push a class scope ClassDef *classDefToAdd = new ClassDef("",1,1,g_curClassName,ClassDef::Class,0,0,FALSE); g_codeClassSDict.append(g_curClassName,classDefToAdd); char *s=g_curClassBases.first(); while (s) { ClassDef *baseDefToAdd; baseDefToAdd=g_codeClassSDict[s]; // Try to find class in global // scope if (baseDefToAdd==0) { baseDefToAdd=getResolvedClass(g_currentDefinition,g_sourceFileDef,s); } if (baseDefToAdd && baseDefToAdd!=classDefToAdd) { classDefToAdd->insertBaseClass(baseDefToAdd,s,Public,Normal); } s=g_curClassBases.next(); } // Reset class-parsing variables. g_curClassName.resize(0); g_curClassBases.clear(); g_noSuiteFound = TRUE; BEGIN( SuiteStart ); } } { {IDENTIFIER} { generateFunctionLink(*g_code,yytext); } {B}"(" { codify(yytext); BEGIN( FunctionParams ); } } { ({BB}|",") { // Parses delimiters codify(yytext); } ({IDENTIFIER}|{PARAMNONEMPTY}+) { codify(yytext); } ")" { codify(yytext); } "\n" { codifyLines(yytext); } ":" { codify(yytext); // Assume this will // be a one-line suite; // found counter-example // in SuiteStart. g_noSuiteFound = TRUE; BEGIN( SuiteStart ); } } { {KEYWORD} { // Position-sensitive rules! // Must come AFTER keyword-triggered rules // Must come BEFORE identifier NONEMPTY-like rules // to syntax highlight. startFontClass("keyword"); codify(yytext); endFontClass(); } {FLOWKW} { startFontClass("keywordflow"); codify(yytext); endFontClass(); } ({IDENTIFIER}".")*{IDENTIFIER}/"(" { generateClassOrGlobalLink(*g_code,yytext); } ({IDENTIFIER}".")+{IDENTIFIER} { generateClassOrGlobalLink(*g_code,yytext,TRUE); } {IDENTIFIER} { codify(yytext); } } { {BB} { codify(yytext); } "pass" { startFontClass("keyword"); codifyLines(yytext); endFontClass(); BEGIN(Body); } {KEYWORD} { startFontClass("keyword"); codifyLines(yytext); endFontClass(); // No indentation necessary g_noSuiteFound = FALSE; } {FLOWKW} { startFontClass("keywordflow"); codifyLines(yytext); endFontClass(); // No indentation necessary g_noSuiteFound = FALSE; } {IDENTIFIER} { codify(yytext); } {POUNDCOMMENT} { if (YY_START==SingleQuoteString || YY_START==DoubleQuoteString || YY_START==TripleString ) { REJECT; } yy_push_state(YY_START); BEGIN(DocBlock); g_docBlock=yytext; } {NEWLINE} { codifyLines(yytext); if ( g_noSuiteFound ) { // printf("New suite to capture! [%d]\n", g_yyLineNr); BEGIN ( SuiteCaptureIndent ); } } } { "\n"|({BB}"\n") { // Blankline - ignore, keep looking for indentation. codifyLines(yytext); } {BB} { // This state lasts momentarily, // to check the indentation // level that is about to be // used. codifyLines(yytext); g_indents.push(yyleng); // printf("Captured indent of %d [line %d]\n", yyleng, g_yyLineNr); BEGIN( Suite ); } } { {BB}/({NONEMPTY}|{EXPCHAR}) { // This implements poor // indendation-tracking; // should be improved. // (translate tabs to space, etc) codifyLines(yytext); adjustScopesAndSuites((int)yyleng); } "\n"|({BB}"\n") { // If this ever succeeds, // it means that this is // a blank line, and // can be ignored. codifyLines(yytext); } ""/({NONEMPTY}|{EXPCHAR}) { // Default rule; matches // the empty string, assuming // real text starts here. // Just go straight to Body. adjustScopesAndSuites(0); } } {NEWLINE} { codifyLines(yytext); BEGIN( SuiteMaintain ); } {IDENTIFIER} { codify(yytext); } {NEWLINE} { codifyLines(yytext); } { // Single quoted string like 'That\'s a """nice""" string!' \\{B}\n { // line continuation codifyLines(yytext); } \\. { // espaced char codify(yytext); } {STRINGPREFIX}?{TRIDOUBLEQUOTE} { // tripple double quotes codify(yytext); } "'" { // end of the string codify(yytext); endFontClass(); BEGIN(g_stringContext); } [^"'\n\\]+ { // normal chars codify(yytext); } . { // normal char codify(yytext); } } { // Double quoted string like "That's \"a '''nice'''\" string!" \\{B}\n { // line continuation codifyLines(yytext); } \\. { // espaced char codify(yytext); } {STRINGPREFIX}?{TRISINGLEQUOTE} { // tripple single quotes codify(yytext); } "\"" { // end of the string codify(yytext); endFontClass(); BEGIN(g_stringContext); } [^"'\n\\]+ { // normal chars codify(yytext); } . { // normal char codify(yytext); } } { {TRIDOUBLEQUOTE} | {TRISINGLEQUOTE} { codify(yytext); if (g_doubleQuote==(yytext[0]=='"')) { endFontClass(); BEGIN(g_stringContext); } } {LONGSTRINGBLOCK} { codifyLines(yytext); } \n { codifyLines(yytext); } . { codify(yytext); } } /* <*>({NONEMPTY}|{EXPCHAR}|{BB}) { // This should go one character at a time. codify(yytext); // printf("[pycode] '%s' [ state %d ] [line %d] no match\n", // yytext, YY_START, g_yyLineNr); //endFontClass(); BEGIN(Body); } */ <*>{STRINGPREFIX}?{TRISINGLEQUOTE} | <*>{STRINGPREFIX}?{TRIDOUBLEQUOTE} { startFontClass("stringliteral"); g_stringContext=YY_START; g_doubleQuote=yytext[yyleng-1]=='"'; codify(yytext); BEGIN(TripleString); } <*>{STRINGPREFIX}?"'" { // single quoted string startFontClass("stringliteral"); g_stringContext=YY_START; codify(yytext); BEGIN(SingleQuoteString); } <*>{STRINGPREFIX}?"\"" { // double quoted string startFontClass("stringliteral"); g_stringContext=YY_START; codify(yytext); BEGIN(DoubleQuoteString); } .* { // contents of current comment line g_docBlock+=yytext; } "\n"{B}("#") { // comment block (next line is also comment line) g_docBlock+=yytext; } {NEWLINE} { // comment block ends at the end of this line // remove special comment (default config) if (Config_getBool(STRIP_CODE_COMMENTS)) { g_yyLineNr+=((QCString)g_docBlock).contains('\n'); g_endComment=TRUE; } else // do not remove comment { startFontClass("comment"); codifyLines(g_docBlock); endFontClass(); } unput(*yytext); yy_pop_state(); } <*>{POUNDCOMMENT}.* { if (YY_START==SingleQuoteString || YY_START==DoubleQuoteString || YY_START==TripleString ) { REJECT; } yy_push_state(YY_START); BEGIN(DocBlock); g_docBlock=yytext; } <*>"#".* { // normal comment if (YY_START==SingleQuoteString || YY_START==DoubleQuoteString || YY_START==TripleString ) { REJECT; } startFontClass("comment"); codifyLines(yytext); endFontClass(); } <*>{NEWLINE} { if (g_endComment) { g_endComment=FALSE; } else { codifyLines(yytext); } //printf("[pycode] %d NEWLINE [line %d] no match\n", // YY_START, g_yyLineNr); //endFontClass(); BEGIN(Body); } <*>[ \t]+ { codify(yytext); BEGIN(Body); } <*>. { codify(yytext); // printf("[pycode] '%s' [ state %d ] [line %d] no match\n", // yytext, YY_START, g_yyLineNr); //endFontClass(); BEGIN(Body); } <*><> { if (YY_START == DocBlock) { if (!Config_getBool(STRIP_CODE_COMMENTS)) { startFontClass("comment"); codifyLines(g_docBlock); endFontClass(); } } yyterminate(); } %% /*@ ---------------------------------------------------------------------------- */ void resetPythonCodeParserState() { g_currentDefinition = 0; g_currentMemberDef = 0; g_doubleStringIsDoc = FALSE; g_paramParens = 0; g_indents.clear(); BEGIN( Body ); } /*! Examines current stack of white-space indentations; re-syncs the parser with the correct scope. */ static void adjustScopesAndSuites(unsigned indentLength) { // States to pop if (!g_indents.isEmpty() && indentLength < g_indents.top()) { while (!g_indents.isEmpty() && indentLength < g_indents.top()) { // printf("Exited scope indent of [%d]\n", g_indents.top()); g_indents.pop(); // Pop the old suite's indentation g_currentMemberDef=0; if (g_currentDefinition) g_currentDefinition=g_currentDefinition->getOuterScope(); } } // Are there any remaining indentation levels for suites? if (!g_indents.isEmpty()) { BEGIN( Suite ); } else { BEGIN( Body ); } } void parsePythonCode(CodeOutputInterface &od,const char * /*className*/, const QCString &s,bool exBlock, const char *exName, FileDef *fd,int startLine,int endLine,bool inlineFragment, MemberDef *,bool,Definition *searchCtx,bool collectXRefs) { //printf("***parseCode()\n"); //-------------------------------------- if (s.isEmpty()) return; printlex(yy_flex_debug, TRUE, __FILE__, fd ? fd->fileName().data(): NULL); TooltipManager::instance()->clearTooltips(); g_code = &od; g_inputString = s; g_inputPosition = 0; g_currentFontClass = 0; g_needsTermination = FALSE; g_searchCtx=searchCtx; g_collectXRefs=collectXRefs; if (startLine!=-1) g_yyLineNr = startLine; else g_yyLineNr = 1; if (endLine!=-1) g_inputLines = endLine+1; else g_inputLines = g_yyLineNr + countLines() - 1; g_exampleBlock = exBlock; g_exampleName = exName; g_sourceFileDef = fd; bool cleanupSourceDef = FALSE; if (exBlock && fd==0) { // create a dummy filedef for the example g_sourceFileDef = new FileDef("",(exName?exName:"generated")); cleanupSourceDef = TRUE; } if (g_sourceFileDef) { setCurrentDoc("l00001"); } g_includeCodeFragment = inlineFragment; // Starts line 1 on the output startCodeLine(); pycodeYYrestart( pycodeYYin ); pycodeYYlex(); if (!g_indents.isEmpty()) { // printf("Exited pysourceparser in inconsistent state!\n"); } if (g_needsTermination) { endCodeLine(); } if (fd) { TooltipManager::instance()->writeTooltips(*g_code); } if (cleanupSourceDef) { // delete the temporary file definition used for this example delete g_sourceFileDef; g_sourceFileDef=0; } printlex(yy_flex_debug, FALSE, __FILE__, fd ? fd->fileName().data(): NULL); return; } #if !defined(YY_FLEX_SUBMINOR_VERSION) extern "C" { // some bogus code to keep the compiler happy void pycodeYYdummy() { yy_flex_realloc(0,0); } } #elif YY_FLEX_MAJOR_VERSION<=2 && YY_FLEX_MINOR_VERSION<=5 && YY_FLEX_SUBMINOR_VERSION<33 #error "You seem to be using a version of flex newer than 2.5.4. These are currently incompatible with 2.5.4, and do NOT work with doxygen! Please use version 2.5.4 or expect things to be parsed wrongly! A bug report has been submitted (#732132)." #endif