#!/usr/bin/python -u

#

# imports the API description and fills up a database with

# name relevance to modules, functions or web pages

#

# Operation needed:

# =================

#

# install mysqld, the python wrappers for mysql and libxml2, start mysqld

# Change the root passwd of mysql:

#    mysqladmin -u root password new_password

# Create the new database xmlsoft

#    mysqladmin -p create xmlsoft

# Create a database user 'veillard' and give him passord access

# change veillard and abcde with the right user name and passwd

#    mysql -p

#    password:

#    mysql> GRANT ALL PRIVILEGES ON xmlsoft TO veillard@localhost

#           IDENTIFIED BY 'abcde' WITH GRANT OPTION;

#

# As the user check the access:

#    mysql -p xmlsoft

#    Enter password:

#    Welcome to the MySQL monitor....

#    mysql> use xmlsoft

#    Database changed

#    mysql> quit

#    Bye

#

# Then run the script in the doc subdir, it will create the symbols and

# word tables and populate them with informations extracted from 

# the libxml2-api.xml API description, and make then accessible read-only

# by nobody@loaclhost the user expected to be Apache's one

#

# On the Apache configuration, make sure you have php support enabled

#

import MySQLdb

import libxml2

import sys

import string

import os

#

# We are not interested in parsing errors here

#

def callback(ctx, str):

    return

libxml2.registerErrorHandler(callback, None)

#

# The dictionary of tables required and the SQL command needed

# to create them

#

TABLES={

  "symbols" : """CREATE TABLE symbols (

           name varchar(255) BINARY NOT NULL,

	   module varchar(255) BINARY NOT NULL,

           type varchar(25) NOT NULL,

	   descr varchar(255),

	   UNIQUE KEY name (name),

	   KEY module (module))""",

  "words" : """CREATE TABLE words (

           name varchar(50) BINARY NOT NULL,

	   symbol varchar(255) BINARY NOT NULL,

           relevance int,

	   KEY name (name),

	   KEY symbol (symbol),

	   UNIQUE KEY ID (name, symbol))""",

  "wordsHTML" : """CREATE TABLE wordsHTML (

           name varchar(50) BINARY NOT NULL,

	   resource varchar(255) BINARY NOT NULL,

	   section varchar(255),

	   id varchar(50),

           relevance int,

	   KEY name (name),

	   KEY resource (resource),

	   UNIQUE KEY ref (name, resource))""",

  "wordsArchive" : """CREATE TABLE wordsArchive (

           name varchar(50) BINARY NOT NULL,

	   ID int(11) NOT NULL,

           relevance int,

	   KEY name (name),

	   UNIQUE KEY ref (name, ID))""",

  "pages" : """CREATE TABLE pages (

           resource varchar(255) BINARY NOT NULL,

	   title varchar(255) BINARY NOT NULL,

	   UNIQUE KEY name (resource))""",

  "archives" : """CREATE TABLE archives (

           ID int(11) NOT NULL auto_increment,

           resource varchar(255) BINARY NOT NULL,

	   title varchar(255) BINARY NOT NULL,

	   UNIQUE KEY id (ID,resource(255)),

	   INDEX (ID),

	   INDEX (resource))""",

  "Queries" : """CREATE TABLE Queries (

           ID int(11) NOT NULL auto_increment,

	   Value varchar(50) NOT NULL,

	   Count int(11) NOT NULL,

	   UNIQUE KEY id (ID,Value(35)),

	   INDEX (ID))""",

  "AllQueries" : """CREATE TABLE AllQueries (

           ID int(11) NOT NULL auto_increment,

	   Value varchar(50) NOT NULL,

	   Count int(11) NOT NULL,

	   UNIQUE KEY id (ID,Value(35)),

	   INDEX (ID))""",

}

#

# The XML API description file to parse

#

API="libxml2-api.xml"

DB=None

#########################################################################

#									#

#                  MySQL database interfaces				#

#									#

#########################################################################

def createTable(db, name):

    global TABLES

    if db == None:

        return -1

    if name == None:

        return -1

    c = db.cursor()

    ret = c.execute("DROP TABLE IF EXISTS %s" % (name))

    if ret == 1:

        print "Removed table %s" % (name)

    print "Creating table %s" % (name)

    try:

        ret = c.execute(TABLES[name])

    except:

        print "Failed to create table %s" % (name)

	return -1

    return ret

def checkTables(db, verbose = 1):

    global TABLES

    if db == None:

        return -1

    c = db.cursor()

    nbtables = c.execute("show tables")

    if verbose:

	print "Found %d tables" % (nbtables)

    tables = {}

    i = 0

    while i < nbtables:

        l = c.fetchone()

	name = l[0]

	tables[name] = {}

        i = i + 1

    for table in TABLES.keys():

        if not tables.has_key(table):

	    print "table %s missing" % (table)

	    createTable(db, table)

	try:

	    ret = c.execute("SELECT count(*) from %s" % table);

	    row = c.fetchone()

	    if verbose:

		print "Table %s contains %d records" % (table, row[0])

	except:

	    print "Troubles with table %s : repairing" % (table)

	    ret = c.execute("repair table %s" % table);

	    print "repairing returned %d" % (ret)

	    ret = c.execute("SELECT count(*) from %s" % table);

	    row = c.fetchone()

	    print "Table %s contains %d records" % (table, row[0])

    if verbose:

	print "checkTables finished"

    # make sure apache can access the tables read-only

    try:

	ret = c.execute("GRANT SELECT ON xmlsoft.* TO nobody@localhost")

	ret = c.execute("GRANT INSERT,SELECT,UPDATE  ON xmlsoft.Queries TO nobody@localhost")

    except:

        pass

    return 0

def openMySQL(db="xmlsoft", passwd=None, verbose = 1):

    global DB

    if passwd == None:

        try:

	    passwd = os.environ["MySQL_PASS"]

	except:

	    print "No password available, set environment MySQL_PASS"

	    sys.exit(1)

    DB = MySQLdb.connect(passwd=passwd, db=db)

    if DB == None:

        return -1

    ret = checkTables(DB, verbose)

    return ret

def updateWord(name, symbol, relevance):

    global DB

    if DB == None:

        openMySQL()

    if DB == None:

        return -1

    if name == None:

        return -1

    if symbol == None:

        return -1

    c = DB.cursor()

    try:

	ret = c.execute(

"""INSERT INTO words (name, symbol, relevance) VALUES ('%s','%s', %d)""" %

		(name, symbol, relevance))

    except:

        try:

	    ret = c.execute(

    """UPDATE words SET relevance = %d where name = '%s' and symbol = '%s'""" %

		    (relevance, name, symbol))

	except:

	    print "Update word (%s, %s, %s) failed command" % (name, symbol, relevance)

	    print "UPDATE words SET relevance = %d where name = '%s' and symbol = '%s'" % (relevance, name, symbol)

	    print sys.exc_type, sys.exc_value

	    return -1

    return ret

def updateSymbol(name, module, type, desc):

    global DB

    updateWord(name, name, 50)

    if DB == None:

        openMySQL()

    if DB == None:

        return -1

    if name == None:

        return -1

    if module == None:

        return -1

    if type == None:

        return -1

    try:

	desc = string.replace(desc, "'", " ")

	l = string.split(desc, ".")

	desc = l[0]

	desc = desc[0:99]

    except:

        desc = ""

    c = DB.cursor()

    try:

	ret = c.execute(

"""INSERT INTO symbols (name, module, type, descr) VALUES ('%s','%s', '%s', '%s')""" %

                    (name, module, type, desc))

    except:

        try:

	    ret = c.execute(

"""UPDATE symbols SET module='%s', type='%s', descr='%s' where name='%s'""" %

                    (module, type, desc, name))

        except:

	    print "Update symbol (%s, %s, %s) failed command" % (name, module, type)

	    print """UPDATE symbols SET module='%s', type='%s', descr='%s' where name='%s'""" % (module, type, desc, name)

	    print sys.exc_type, sys.exc_value

	    return -1

    return ret

def addFunction(name, module, desc = ""):

    return updateSymbol(name, module, 'function', desc)

def addMacro(name, module, desc = ""):

    return updateSymbol(name, module, 'macro', desc)

def addEnum(name, module, desc = ""):

    return updateSymbol(name, module, 'enum', desc)

def addStruct(name, module, desc = ""):

    return updateSymbol(name, module, 'struct', desc)

def addConst(name, module, desc = ""):

    return updateSymbol(name, module, 'const', desc)

def addType(name, module, desc = ""):

    return updateSymbol(name, module, 'type', desc)

def addFunctype(name, module, desc = ""):

    return updateSymbol(name, module, 'functype', desc)

def addPage(resource, title):

    global DB

    if DB == None:

        openMySQL()

    if DB == None:

        return -1

    if resource == None:

        return -1

    c = DB.cursor()

    try:

	ret = c.execute(

	    """INSERT INTO pages (resource, title) VALUES ('%s','%s')""" %

                    (resource, title))

    except:

        try:

	    ret = c.execute(

		"""UPDATE pages SET title='%s' WHERE resource='%s'""" %

                    (title, resource))

        except:

	    print "Update symbol (%s, %s, %s) failed command" % (name, module, type)

	    print """UPDATE pages SET title='%s' WHERE resource='%s'""" % (title, resource)

	    print sys.exc_type, sys.exc_value

	    return -1

    return ret

def updateWordHTML(name, resource, desc, id, relevance):

    global DB

    if DB == None:

        openMySQL()

    if DB == None:

        return -1

    if name == None:

        return -1

    if resource == None:

        return -1

    if id == None:

        id = ""

    if desc == None:

        desc = ""

    else:

	try:

	    desc = string.replace(desc, "'", " ")

	    desc = desc[0:99]

	except:

	    desc = ""

    c = DB.cursor()

    try:

	ret = c.execute(

"""INSERT INTO wordsHTML (name, resource, section, id, relevance) VALUES ('%s','%s', '%s', '%s', '%d')""" %

                    (name, resource, desc, id, relevance))

    except:

        try:

	    ret = c.execute(

"""UPDATE wordsHTML SET section='%s', id='%s', relevance='%d' where name='%s' and resource='%s'""" %

                    (desc, id, relevance, name, resource))

        except:

	    print "Update symbol (%s, %s, %d) failed command" % (name, resource, relevance)

	    print """UPDATE wordsHTML SET section='%s', id='%s', relevance='%d' where name='%s' and resource='%s'""" % (desc, id, relevance, name, resource)

	    print sys.exc_type, sys.exc_value

	    return -1

    return ret

def checkXMLMsgArchive(url):

    global DB

    if DB == None:

        openMySQL()

    if DB == None:

        return -1

    if url == None:

        return -1

    c = DB.cursor()

    try:

	ret = c.execute(

	    """SELECT ID FROM archives WHERE resource='%s'""" % (url))

	row = c.fetchone()

	if row == None:

	    return -1

    except:

	return -1

    return row[0]

def addXMLMsgArchive(url, title):

    global DB

    if DB == None:

        openMySQL()

    if DB == None:

        return -1

    if url == None:

        return -1

    if title == None:

        title = ""

    else:

	title = string.replace(title, "'", " ")

	title = title[0:99]

    c = DB.cursor()

    try:

        cmd = """INSERT INTO archives (resource, title) VALUES ('%s','%s')""" % (url, title)

        ret = c.execute(cmd)

	cmd = """SELECT ID FROM archives WHERE resource='%s'""" % (url)

        ret = c.execute(cmd)

	row = c.fetchone()

	if row == None:

	    print "addXMLMsgArchive failed to get the ID: %s" % (url)

	    return -1

    except:

        print "addXMLMsgArchive failed command: %s" % (cmd)

	return -1

    return((int)(row[0]))

def updateWordArchive(name, id, relevance):

    global DB

    if DB == None:

        openMySQL()

    if DB == None:

        return -1

    if name == None:

        return -1

    if id == None:

        return -1

    c = DB.cursor()

    try:

	ret = c.execute(

"""INSERT INTO wordsArchive (name, id, relevance) VALUES ('%s', '%d', '%d')""" %

                    (name, id, relevance))

    except:

        try:

	    ret = c.execute(

"""UPDATE wordsArchive SET relevance='%d' where name='%s' and ID='%d'""" %

                    (relevance, name, id))

        except:

	    print "Update word archive (%s, %d, %d) failed command" % (name, id, relevance)

	    print """UPDATE wordsArchive SET relevance='%d' where name='%s' and ID='%d'""" % (relevance, name, id)

	    print sys.exc_type, sys.exc_value

	    return -1

    return ret

#########################################################################

#									#

#                  Word dictionary and analysis routines		#

#									#

#########################################################################

#

# top 100 english word without the one len < 3 + own set

#

dropWords = {

    'the':0, 'this':0, 'can':0, 'man':0, 'had':0, 'him':0, 'only':0,

    'and':0, 'not':0, 'been':0, 'other':0, 'even':0, 'are':0, 'was':0,

    'new':0, 'most':0, 'but':0, 'when':0, 'some':0, 'made':0, 'from':0,

    'who':0, 'could':0, 'after':0, 'that':0, 'will':0, 'time':0, 'also':0,

    'have':0, 'more':0, 'these':0, 'did':0, 'was':0, 'two':0, 'many':0,

    'they':0, 'may':0, 'before':0, 'for':0, 'which':0, 'out':0, 'then':0,

    'must':0, 'one':0, 'through':0, 'with':0, 'you':0, 'said':0,

    'first':0, 'back':0, 'were':0, 'what':0, 'any':0, 'years':0, 'his':0,

    'her':0, 'where':0, 'all':0, 'its':0, 'now':0, 'much':0, 'she':0,

    'about':0, 'such':0, 'your':0, 'there':0, 'into':0, 'like':0, 'may':0,

    'would':0, 'than':0, 'our':0, 'well':0, 'their':0, 'them':0, 'over':0,

    'down':0,

    'net':0, 'www':0, 'bad':0, 'Okay':0, 'bin':0, 'cur':0,

}

wordsDict = {}

wordsDictHTML = {}

wordsDictArchive = {}

def cleanupWordsString(str):

    str = string.replace(str, ".", " ")

    str = string.replace(str, "!", " ")

    str = string.replace(str, "?", " ")

    str = string.replace(str, ",", " ")

    str = string.replace(str, "'", " ")

    str = string.replace(str, '"', " ")

    str = string.replace(str, ";", " ")

    str = string.replace(str, "(", " ")

    str = string.replace(str, ")", " ")

    str = string.replace(str, "{", " ")

    str = string.replace(str, "}", " ")

    str = string.replace(str, "<", " ")

    str = string.replace(str, ">", " ")

    str = string.replace(str, "=", " ")

    str = string.replace(str, "/", " ")

    str = string.replace(str, "*", " ")

    str = string.replace(str, ":", " ")

    str = string.replace(str, "#", " ")

    str = string.replace(str, "\\", " ")

    str = string.replace(str, "\n", " ")

    str = string.replace(str, "\r", " ")

    str = string.replace(str, "\xc2", " ")

    str = string.replace(str, "\xa0", " ")

    return str

def cleanupDescrString(str):

    str = string.replace(str, "'", " ")

    str = string.replace(str, "\n", " ")

    str = string.replace(str, "\r", " ")

    str = string.replace(str, "\xc2", " ")

    str = string.replace(str, "\xa0", " ")

    l = string.split(str)

    str = string.join(str)

    return str

def splitIdentifier(str):

    ret = []

    while str != "":

        cur = string.lower(str[0])

	str = str[1:]

	if ((cur < 'a') or (cur > 'z')):

	    continue

	while (str != "") and (str[0] >= 'A') and (str[0] <= 'Z'):

	    cur = cur + string.lower(str[0])

	    str = str[1:]

	while (str != "") and (str[0] >= 'a') and (str[0] <= 'z'):

	    cur = cur + str[0]

	    str = str[1:]

	while (str != "") and (str[0] >= '0') and (str[0] <= '9'):

	    str = str[1:]

	ret.append(cur)

    return ret

def addWord(word, module, symbol, relevance):

    global wordsDict

    if word == None or len(word) < 3:

        return -1

    if module == None or symbol == None:

        return -1

    if dropWords.has_key(word):

        return 0

    if ord(word[0]) > 0x80:

        return 0

    if wordsDict.has_key(word):

        d = wordsDict[word]

	if d == None:

	    return 0

	if len(d) > 500:

	    wordsDict[word] = None

	    return 0

	try:

	    relevance = relevance + d[(module, symbol)]

	except:

	    pass

    else:

        wordsDict[word] = {}

    wordsDict[word][(module, symbol)] = relevance

    return relevance

def addString(str, module, symbol, relevance):

    if str == None or len(str) < 3:

        return -1

    ret = 0

    str = cleanupWordsString(str)

    l = string.split(str)

    for word in l:

	if len(word) > 2:

	    ret = ret + addWord(word, module, symbol, 5)

    return ret

def addWordHTML(word, resource, id, section, relevance):

    global wordsDictHTML

    if word == None or len(word) < 3:

        return -1

    if resource == None or section == None:

        return -1

    if dropWords.has_key(word):

        return 0

    if ord(word[0]) > 0x80:

        return 0

    section = cleanupDescrString(section)

    if wordsDictHTML.has_key(word):

        d = wordsDictHTML[word]

	if d == None:

	    print "skipped %s" % (word)

	    return 0

	try:

	    (r,i,s) = d[resource]

	    if i != None:

	        id = i

	    if s != None:

	        section = s

	    relevance = relevance + r

	except:

	    pass

    else:

        wordsDictHTML[word] = {}

    d = wordsDictHTML[word];

    d[resource] = (relevance, id, section)

    return relevance

def addStringHTML(str, resource, id, section, relevance):

    if str == None or len(str) < 3:

        return -1

    ret = 0

    str = cleanupWordsString(str)

    l = string.split(str)

    for word in l:

	if len(word) > 2:

	    try:

		r = addWordHTML(word, resource, id, section, relevance)

		if r < 0:

		    print "addWordHTML failed: %s %s" % (word, resource)

		ret = ret + r

	    except:

		print "addWordHTML failed: %s %s %d" % (word, resource, relevance)

		print sys.exc_type, sys.exc_value

    return ret

def addWordArchive(word, id, relevance):

    global wordsDictArchive

    if word == None or len(word) < 3:

        return -1

    if id == None or id == -1:

        return -1

    if dropWords.has_key(word):

        return 0

    if ord(word[0]) > 0x80:

        return 0

    if wordsDictArchive.has_key(word):

        d = wordsDictArchive[word]

	if d == None:

	    print "skipped %s" % (word)

	    return 0

	try:

	    r = d[id]

	    relevance = relevance + r

	except:

	    pass

    else:

        wordsDictArchive[word] = {}

    d = wordsDictArchive[word];

    d[id] = relevance

    return relevance

def addStringArchive(str, id, relevance):

    if str == None or len(str) < 3:

        return -1

    ret = 0

    str = cleanupWordsString(str)

    l = string.split(str)

    for word in l:

        i = len(word)

	if i > 2:

	    try:

		r = addWordArchive(word, id, relevance)

		if r < 0:

		    print "addWordArchive failed: %s %s" % (word, id)

		else:

		    ret = ret + r

	    except:

		print "addWordArchive failed: %s %s %d" % (word, id, relevance)

		print sys.exc_type, sys.exc_value

    return ret

#########################################################################

#									#

#                  XML API description analysis				#

#									#

#########################################################################

def loadAPI(filename):

    doc = libxml2.parseFile(filename)

    print "loaded %s" % (filename)

    return doc

def foundExport(file, symbol):

    if file == None:

        return 0

    if symbol == None:

        return 0

    addFunction(symbol, file)

    l = splitIdentifier(symbol)

    for word in l:

	addWord(word, file, symbol, 10)

    return 1

def analyzeAPIFile(top):

    count = 0

    name = top.prop("name")

    cur = top.children

    while cur != None:

        if cur.type == 'text':

	    cur = cur.next

	    continue

	if cur.name == "exports":

	    count = count + foundExport(name, cur.prop("symbol"))

	else:

	    print "unexpected element %s in API doc <file name='%s'>" % (name)

        cur = cur.next

    return count

def analyzeAPIFiles(top):

    count = 0

    cur = top.children

    while cur != None:

        if cur.type == 'text':

	    cur = cur.next

	    continue

	if cur.name == "file":

	    count = count + analyzeAPIFile(cur)

	else:

	    print "unexpected element %s in API doc <files>" % (cur.name)

        cur = cur.next

    return count

def analyzeAPIEnum(top):

    file = top.prop("file")

    if file == None:

        return 0

    symbol = top.prop("name")

    if symbol == None:

        return 0

    addEnum(symbol, file)

    l = splitIdentifier(symbol)

    for word in l:

	addWord(word, file, symbol, 10)

    return 1

def analyzeAPIConst(top):

    file = top.prop("file")

    if file == None:

        return 0

    symbol = top.prop("name")

    if symbol == None:

        return 0

    addConst(symbol, file)

    l = splitIdentifier(symbol)

    for word in l:

	addWord(word, file, symbol, 10)

    return 1

def analyzeAPIType(top):

    file = top.prop("file")

    if file == None:

        return 0

    symbol = top.prop("name")

    if symbol == None:

        return 0

    addType(symbol, file)

    l = splitIdentifier(symbol)

    for word in l:

	addWord(word, file, symbol, 10)

    return 1

def analyzeAPIFunctype(top):

    file = top.prop("file")

    if file == None:

        return 0

    symbol = top.prop("name")

    if symbol == None:

        return 0

    addFunctype(symbol, file)

    l = splitIdentifier(symbol)

    for word in l:

	addWord(word, file, symbol, 10)

    return 1

def analyzeAPIStruct(top):

    file = top.prop("file")

    if file == None:

        return 0

    symbol = top.prop("name")

    if symbol == None:

        return 0

    addStruct(symbol, file)

    l = splitIdentifier(symbol)

    for word in l:

	addWord(word, file, symbol, 10)

    info = top.prop("info")

    if info != None:

	info = string.replace(info, "'", " ")

	info = string.strip(info)

	l = string.split(info)

	for word in l:

	    if len(word) > 2:

		addWord(word, file, symbol, 5)

    return 1

def analyzeAPIMacro(top):

    file = top.prop("file")

    if file == None:

        return 0

    symbol = top.prop("name")

    if symbol == None:

        return 0

    symbol = string.replace(symbol, "'", " ")

    symbol = string.strip(symbol)

    info = None

    cur = top.children

    while cur != None:

        if cur.type == 'text':

	    cur = cur.next

	    continue

	if cur.name == "info":

	    info = cur.content

	    break

        cur = cur.next

    l = splitIdentifier(symbol)

    for word in l:

	addWord(word, file, symbol, 10)

    if info == None:

	addMacro(symbol, file)

        print "Macro %s description has no <info>" % (symbol)

        return 0

    info = string.replace(info, "'", " ")

    info = string.strip(info)

    addMacro(symbol, file, info)

    l = string.split(info)

    for word in l:

	if len(word) > 2:

	    addWord(word, file, symbol, 5)

    return 1

def analyzeAPIFunction(top):

    file = top.prop("file")

    if file == None:

        return 0

    symbol = top.prop("name")

    if symbol == None:

        return 0

    symbol = string.replace(symbol, "'", " ")

    symbol = string.strip(symbol)

    info = None

    cur = top.children

    while cur != None:

        if cur.type == 'text':

	    cur = cur.next

	    continue

	if cur.name == "info":

	    info = cur.content

	elif cur.name == "return":

	    rinfo = cur.prop("info")

	    if rinfo != None:

		rinfo = string.replace(rinfo, "'", " ")

		rinfo = string.strip(rinfo)

	        addString(rinfo, file, symbol, 7)

	elif cur.name == "arg":

	    ainfo = cur.prop("info")

	    if ainfo != None:

		ainfo = string.replace(ainfo, "'", " ")

		ainfo = string.strip(ainfo)

	        addString(ainfo, file, symbol, 5)

	    name = cur.prop("name")

	    if name != None:

		name = string.replace(name, "'", " ")

		name = string.strip(name)

	        addWord(name, file, symbol, 7)

        cur = cur.next

    if info == None:

        print "Function %s description has no <info>" % (symbol)

	addFunction(symbol, file, "")

    else:

        info = string.replace(info, "'", " ")