Blame check-xsddata-test-suite.py

Packit 21b7a2
#!/usr/bin/python
Packit 21b7a2
import sys
Packit 21b7a2
import time
Packit 21b7a2
import os
Packit 21b7a2
import string
Packit 21b7a2
import StringIO
Packit 21b7a2
sys.path.insert(0, "python")
Packit 21b7a2
import libxml2
Packit 21b7a2
Packit 21b7a2
# Memory debug specific
Packit 21b7a2
libxml2.debugMemory(1)
Packit 21b7a2
debug = 0
Packit 21b7a2
verbose = 0
Packit 21b7a2
quiet = 1
Packit 21b7a2
Packit 21b7a2
#
Packit 21b7a2
# the testsuite description
Packit 21b7a2
#
Packit 21b7a2
CONF=os.path.join(os.path.dirname(__file__), "test/xsdtest/xsdtestsuite.xml")
Packit 21b7a2
LOG="check-xsddata-test-suite.log"
Packit 21b7a2
Packit 21b7a2
log = open(LOG, "w")
Packit 21b7a2
nb_schemas_tests = 0
Packit 21b7a2
nb_schemas_success = 0
Packit 21b7a2
nb_schemas_failed = 0
Packit 21b7a2
nb_instances_tests = 0
Packit 21b7a2
nb_instances_success = 0
Packit 21b7a2
nb_instances_failed = 0
Packit 21b7a2
Packit 21b7a2
libxml2.lineNumbersDefault(1)
Packit 21b7a2
#
Packit 21b7a2
# Error and warnng callbacks
Packit 21b7a2
#
Packit 21b7a2
def callback(ctx, str):
Packit 21b7a2
    global log
Packit 21b7a2
    log.write("%s%s" % (ctx, str))
Packit 21b7a2
Packit 21b7a2
libxml2.registerErrorHandler(callback, "")
Packit 21b7a2
Packit 21b7a2
#
Packit 21b7a2
# Resolver callback
Packit 21b7a2
#
Packit 21b7a2
resources = {}
Packit 21b7a2
def resolver(URL, ID, ctxt):
Packit 21b7a2
    global resources
Packit 21b7a2
Packit 21b7a2
    if resources.has_key(URL):
Packit 21b7a2
        return(StringIO.StringIO(resources[URL]))
Packit 21b7a2
    log.write("Resolver failure: asked %s\n" % (URL))
Packit 21b7a2
    log.write("resources: %s\n" % (resources))
Packit 21b7a2
    return None
Packit 21b7a2
Packit 21b7a2
#
Packit 21b7a2
# handle a valid instance
Packit 21b7a2
#
Packit 21b7a2
def handle_valid(node, schema):
Packit 21b7a2
    global log
Packit 21b7a2
    global nb_instances_success
Packit 21b7a2
    global nb_instances_failed
Packit 21b7a2
Packit 21b7a2
    instance = node.prop("dtd")
Packit 21b7a2
    if instance == None:
Packit 21b7a2
        instance = ""
Packit 21b7a2
    child = node.children
Packit 21b7a2
    while child != None:
Packit 21b7a2
        if child.type != 'text':
Packit 21b7a2
	    instance = instance + child.serialize()
Packit 21b7a2
	child = child.next
Packit 21b7a2
Packit 21b7a2
    mem = libxml2.debugMemory(1);
Packit 21b7a2
    try:
Packit 21b7a2
	doc = libxml2.parseDoc(instance)
Packit 21b7a2
    except:
Packit 21b7a2
        doc = None
Packit 21b7a2
Packit 21b7a2
    if doc == None:
Packit 21b7a2
        log.write("\nFailed to parse correct instance:\n-----\n")
Packit 21b7a2
	log.write(instance)
Packit 21b7a2
        log.write("\n-----\n")
Packit 21b7a2
	nb_instances_failed = nb_instances_failed + 1
Packit 21b7a2
	return
Packit 21b7a2
Packit 21b7a2
    if debug:
Packit 21b7a2
        print "instance line %d" % (node.lineNo())
Packit 21b7a2
       
Packit 21b7a2
    try:
Packit 21b7a2
        ctxt = schema.relaxNGNewValidCtxt()
Packit 21b7a2
	ret = doc.relaxNGValidateDoc(ctxt)
Packit 21b7a2
	del ctxt
Packit 21b7a2
    except:
Packit 21b7a2
        ret = -1
Packit 21b7a2
Packit 21b7a2
    doc.freeDoc()
Packit 21b7a2
    if mem != libxml2.debugMemory(1):
Packit 21b7a2
	print "validating instance %d line %d leaks" % (
Packit 21b7a2
		  nb_instances_tests, node.lineNo())
Packit 21b7a2
Packit 21b7a2
    if ret != 0:
Packit 21b7a2
        log.write("\nFailed to validate correct instance:\n-----\n")
Packit 21b7a2
	log.write(instance)
Packit 21b7a2
        log.write("\n-----\n")
Packit 21b7a2
	nb_instances_failed = nb_instances_failed + 1
Packit 21b7a2
    else:
Packit 21b7a2
	nb_instances_success = nb_instances_success + 1
Packit 21b7a2
Packit 21b7a2
#
Packit 21b7a2
# handle an invalid instance
Packit 21b7a2
#
Packit 21b7a2
def handle_invalid(node, schema):
Packit 21b7a2
    global log
Packit 21b7a2
    global nb_instances_success
Packit 21b7a2
    global nb_instances_failed
Packit 21b7a2
Packit 21b7a2
    instance = node.prop("dtd")
Packit 21b7a2
    if instance == None:
Packit 21b7a2
        instance = ""
Packit 21b7a2
    child = node.children
Packit 21b7a2
    while child != None:
Packit 21b7a2
        if child.type != 'text':
Packit 21b7a2
	    instance = instance + child.serialize()
Packit 21b7a2
	child = child.next
Packit 21b7a2
Packit 21b7a2
#    mem = libxml2.debugMemory(1);
Packit 21b7a2
Packit 21b7a2
    try:
Packit 21b7a2
	doc = libxml2.parseDoc(instance)
Packit 21b7a2
    except:
Packit 21b7a2
        doc = None
Packit 21b7a2
Packit 21b7a2
    if doc == None:
Packit 21b7a2
        log.write("\nStrange: failed to parse incorrect instance:\n-----\n")
Packit 21b7a2
	log.write(instance)
Packit 21b7a2
        log.write("\n-----\n")
Packit 21b7a2
	return
Packit 21b7a2
Packit 21b7a2
    if debug:
Packit 21b7a2
        print "instance line %d" % (node.lineNo())
Packit 21b7a2
       
Packit 21b7a2
    try:
Packit 21b7a2
        ctxt = schema.relaxNGNewValidCtxt()
Packit 21b7a2
	ret = doc.relaxNGValidateDoc(ctxt)
Packit 21b7a2
	del ctxt
Packit 21b7a2
Packit 21b7a2
    except:
Packit 21b7a2
        ret = -1
Packit 21b7a2
Packit 21b7a2
    doc.freeDoc()
Packit 21b7a2
#    if mem != libxml2.debugMemory(1):
Packit 21b7a2
#	print "validating instance %d line %d leaks" % (
Packit 21b7a2
#		  nb_instances_tests, node.lineNo())
Packit 21b7a2
    
Packit 21b7a2
    if ret == 0:
Packit 21b7a2
        log.write("\nFailed to detect validation problem in instance:\n-----\n")
Packit 21b7a2
	log.write(instance)
Packit 21b7a2
        log.write("\n-----\n")
Packit 21b7a2
	nb_instances_failed = nb_instances_failed + 1
Packit 21b7a2
    else:
Packit 21b7a2
	nb_instances_success = nb_instances_success + 1
Packit 21b7a2
Packit 21b7a2
#
Packit 21b7a2
# handle an incorrect test
Packit 21b7a2
#
Packit 21b7a2
def handle_correct(node):
Packit 21b7a2
    global log
Packit 21b7a2
    global nb_schemas_success
Packit 21b7a2
    global nb_schemas_failed
Packit 21b7a2
Packit 21b7a2
    schema = ""
Packit 21b7a2
    child = node.children
Packit 21b7a2
    while child != None:
Packit 21b7a2
        if child.type != 'text':
Packit 21b7a2
	    schema = schema + child.serialize()
Packit 21b7a2
	child = child.next
Packit 21b7a2
Packit 21b7a2
    try:
Packit 21b7a2
	rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema))
Packit 21b7a2
	rngs = rngp.relaxNGParse()
Packit 21b7a2
    except:
Packit 21b7a2
        rngs = None
Packit 21b7a2
    if rngs == None:
Packit 21b7a2
        log.write("\nFailed to compile correct schema:\n-----\n")
Packit 21b7a2
	log.write(schema)
Packit 21b7a2
        log.write("\n-----\n")
Packit 21b7a2
	nb_schemas_failed = nb_schemas_failed + 1
Packit 21b7a2
    else:
Packit 21b7a2
	nb_schemas_success = nb_schemas_success + 1
Packit 21b7a2
    return rngs
Packit 21b7a2
        
Packit 21b7a2
def handle_incorrect(node):
Packit 21b7a2
    global log
Packit 21b7a2
    global nb_schemas_success
Packit 21b7a2
    global nb_schemas_failed
Packit 21b7a2
Packit 21b7a2
    schema = ""
Packit 21b7a2
    child = node.children
Packit 21b7a2
    while child != None:
Packit 21b7a2
        if child.type != 'text':
Packit 21b7a2
	    schema = schema + child.serialize()
Packit 21b7a2
	child = child.next
Packit 21b7a2
Packit 21b7a2
    try:
Packit 21b7a2
	rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema))
Packit 21b7a2
	rngs = rngp.relaxNGParse()
Packit 21b7a2
    except:
Packit 21b7a2
        rngs = None
Packit 21b7a2
    if rngs != None:
Packit 21b7a2
        log.write("\nFailed to detect schema error in:\n-----\n")
Packit 21b7a2
	log.write(schema)
Packit 21b7a2
        log.write("\n-----\n")
Packit 21b7a2
	nb_schemas_failed = nb_schemas_failed + 1
Packit 21b7a2
    else:
Packit 21b7a2
#	log.write("\nSuccess detecting schema error in:\n-----\n")
Packit 21b7a2
#	log.write(schema)
Packit 21b7a2
#	log.write("\n-----\n")
Packit 21b7a2
	nb_schemas_success = nb_schemas_success + 1
Packit 21b7a2
    return None
Packit 21b7a2
Packit 21b7a2
#
Packit 21b7a2
# resource handling: keep a dictionary of URL->string mappings
Packit 21b7a2
#
Packit 21b7a2
def handle_resource(node, dir):
Packit 21b7a2
    global resources
Packit 21b7a2
Packit 21b7a2
    try:
Packit 21b7a2
	name = node.prop('name')
Packit 21b7a2
    except:
Packit 21b7a2
        name = None
Packit 21b7a2
Packit 21b7a2
    if name == None or name == '':
Packit 21b7a2
        log.write("resource has no name")
Packit 21b7a2
	return;
Packit 21b7a2
        
Packit 21b7a2
    if dir != None:
Packit 21b7a2
#        name = libxml2.buildURI(name, dir)
Packit 21b7a2
        name = dir + '/' + name
Packit 21b7a2
Packit 21b7a2
    res = ""
Packit 21b7a2
    child = node.children
Packit 21b7a2
    while child != None:
Packit 21b7a2
        if child.type != 'text':
Packit 21b7a2
	    res = res + child.serialize()
Packit 21b7a2
	child = child.next
Packit 21b7a2
    resources[name] = res
Packit 21b7a2
Packit 21b7a2
#
Packit 21b7a2
# dir handling: pseudo directory resources
Packit 21b7a2
#
Packit 21b7a2
def handle_dir(node, dir):
Packit 21b7a2
    try:
Packit 21b7a2
	name = node.prop('name')
Packit 21b7a2
    except:
Packit 21b7a2
        name = None
Packit 21b7a2
Packit 21b7a2
    if name == None or name == '':
Packit 21b7a2
        log.write("resource has no name")
Packit 21b7a2
	return;
Packit 21b7a2
        
Packit 21b7a2
    if dir != None:
Packit 21b7a2
#        name = libxml2.buildURI(name, dir)
Packit 21b7a2
        name = dir + '/' + name
Packit 21b7a2
Packit 21b7a2
    dirs = node.xpathEval('dir')
Packit 21b7a2
    for dir in dirs:
Packit 21b7a2
        handle_dir(dir, name)
Packit 21b7a2
    res = node.xpathEval('resource')
Packit 21b7a2
    for r in res:
Packit 21b7a2
        handle_resource(r, name)
Packit 21b7a2
Packit 21b7a2
#
Packit 21b7a2
# handle a testCase element
Packit 21b7a2
#
Packit 21b7a2
def handle_testCase(node):
Packit 21b7a2
    global nb_schemas_tests
Packit 21b7a2
    global nb_instances_tests
Packit 21b7a2
    global resources
Packit 21b7a2
Packit 21b7a2
    sections = node.xpathEval('string(section)')
Packit 21b7a2
    log.write("\n    ======== test %d line %d section %s ==========\n" % (
Packit 21b7a2
Packit 21b7a2
              nb_schemas_tests, node.lineNo(), sections))
Packit 21b7a2
    resources = {}
Packit 21b7a2
    if debug:
Packit 21b7a2
        print "test %d line %d" % (nb_schemas_tests, node.lineNo())
Packit 21b7a2
Packit 21b7a2
    dirs = node.xpathEval('dir')
Packit 21b7a2
    for dir in dirs:
Packit 21b7a2
        handle_dir(dir, None)
Packit 21b7a2
    res = node.xpathEval('resource')
Packit 21b7a2
    for r in res:
Packit 21b7a2
        handle_resource(r, None)
Packit 21b7a2
Packit 21b7a2
    tsts = node.xpathEval('incorrect')
Packit 21b7a2
    if tsts != []:
Packit 21b7a2
        if len(tsts) != 1:
Packit 21b7a2
	    print "warning test line %d has more than one <incorrect> example" %(node.lineNo())
Packit 21b7a2
	schema = handle_incorrect(tsts[0])
Packit 21b7a2
    else:
Packit 21b7a2
        tsts = node.xpathEval('correct')
Packit 21b7a2
	if tsts != []:
Packit 21b7a2
	    if len(tsts) != 1:
Packit 21b7a2
		print "warning test line %d has more than one <correct> example"% (node.lineNo())
Packit 21b7a2
	    schema = handle_correct(tsts[0])
Packit 21b7a2
	else:
Packit 21b7a2
	    print "warning <testCase> line %d has no <correct> nor <incorrect> child" % (node.lineNo())
Packit 21b7a2
Packit 21b7a2
    nb_schemas_tests = nb_schemas_tests + 1;
Packit 21b7a2
    
Packit 21b7a2
    valids = node.xpathEval('valid')
Packit 21b7a2
    invalids = node.xpathEval('invalid')
Packit 21b7a2
    nb_instances_tests = nb_instances_tests + len(valids) + len(invalids)
Packit 21b7a2
    if schema != None:
Packit 21b7a2
        for valid in valids:
Packit 21b7a2
	    handle_valid(valid, schema)
Packit 21b7a2
        for invalid in invalids:
Packit 21b7a2
	    handle_invalid(invalid, schema)
Packit 21b7a2
Packit 21b7a2
Packit 21b7a2
#
Packit 21b7a2
# handle a testSuite element
Packit 21b7a2
#
Packit 21b7a2
def handle_testSuite(node, level = 0):
Packit 21b7a2
    global nb_schemas_tests, nb_schemas_success, nb_schemas_failed
Packit 21b7a2
    global nb_instances_tests, nb_instances_success, nb_instances_failed
Packit 21b7a2
    if verbose and level >= 0:
Packit 21b7a2
	old_schemas_tests = nb_schemas_tests
Packit 21b7a2
	old_schemas_success = nb_schemas_success
Packit 21b7a2
	old_schemas_failed = nb_schemas_failed
Packit 21b7a2
	old_instances_tests = nb_instances_tests
Packit 21b7a2
	old_instances_success = nb_instances_success
Packit 21b7a2
	old_instances_failed = nb_instances_failed
Packit 21b7a2
Packit 21b7a2
    docs = node.xpathEval('documentation')
Packit 21b7a2
    authors = node.xpathEval('author')
Packit 21b7a2
    if docs != []:
Packit 21b7a2
        msg = ""
Packit 21b7a2
        for doc in docs:
Packit 21b7a2
	    msg = msg + doc.content + " "
Packit 21b7a2
	if authors != []:
Packit 21b7a2
	    msg = msg + "written by "
Packit 21b7a2
	    for author in authors:
Packit 21b7a2
	        msg = msg + author.content + " "
Packit 21b7a2
	if quiet == 0:
Packit 21b7a2
	    print msg
Packit 21b7a2
    sections = node.xpathEval('section')
Packit 21b7a2
    if verbose and sections != [] and level <= 0:
Packit 21b7a2
        msg = ""
Packit 21b7a2
        for section in sections:
Packit 21b7a2
	    msg = msg + section.content + " "
Packit 21b7a2
	if quiet == 0:
Packit 21b7a2
	    print "Tests for section %s" % (msg)
Packit 21b7a2
    for test in node.xpathEval('testCase'):
Packit 21b7a2
        handle_testCase(test)
Packit 21b7a2
    for test in node.xpathEval('testSuite'):
Packit 21b7a2
        handle_testSuite(test, level + 1)
Packit 21b7a2
	        
Packit 21b7a2
Packit 21b7a2
    if verbose and level >= 0 :
Packit 21b7a2
        if sections != []:
Packit 21b7a2
	    msg = ""
Packit 21b7a2
	    for section in sections:
Packit 21b7a2
		msg = msg + section.content + " "
Packit 21b7a2
	    print "Result of tests for section %s" % (msg)
Packit 21b7a2
	elif docs != []:
Packit 21b7a2
	    msg = ""
Packit 21b7a2
	    for doc in docs:
Packit 21b7a2
	        msg = msg + doc.content + " "
Packit 21b7a2
	    print "Result of tests for %s" % (msg)
Packit 21b7a2
Packit 21b7a2
        if nb_schemas_tests != old_schemas_tests:
Packit 21b7a2
	    print "found %d test schemas: %d success %d failures" % (
Packit 21b7a2
		  nb_schemas_tests - old_schemas_tests,
Packit 21b7a2
		  nb_schemas_success - old_schemas_success,
Packit 21b7a2
		  nb_schemas_failed - old_schemas_failed)
Packit 21b7a2
	if nb_instances_tests != old_instances_tests:
Packit 21b7a2
	    print "found %d test instances: %d success %d failures" % (
Packit 21b7a2
		  nb_instances_tests - old_instances_tests,
Packit 21b7a2
		  nb_instances_success - old_instances_success,
Packit 21b7a2
		  nb_instances_failed - old_instances_failed)
Packit 21b7a2
#
Packit 21b7a2
# Parse the conf file
Packit 21b7a2
#
Packit 21b7a2
libxml2.substituteEntitiesDefault(1);
Packit 21b7a2
testsuite = libxml2.parseFile(CONF)
Packit 21b7a2
Packit 21b7a2
#
Packit 21b7a2
# Error and warnng callbacks
Packit 21b7a2
#
Packit 21b7a2
def callback(ctx, str):
Packit 21b7a2
    global log
Packit 21b7a2
    log.write("%s%s" % (ctx, str))
Packit 21b7a2
Packit 21b7a2
libxml2.registerErrorHandler(callback, "")
Packit 21b7a2
Packit 21b7a2
libxml2.setEntityLoader(resolver)
Packit 21b7a2
root = testsuite.getRootElement()
Packit 21b7a2
if root.name != 'testSuite':
Packit 21b7a2
    print "%s doesn't start with a testSuite element, aborting" % (CONF)
Packit 21b7a2
    sys.exit(1)
Packit 21b7a2
if quiet == 0:
Packit 21b7a2
    print "Running Relax NG testsuite"
Packit 21b7a2
handle_testSuite(root)
Packit 21b7a2
Packit 21b7a2
if quiet == 0 or nb_schemas_failed != 0:
Packit 21b7a2
    print "\nTOTAL:\nfound %d test schemas: %d success %d failures" % (
Packit 21b7a2
      nb_schemas_tests, nb_schemas_success, nb_schemas_failed)
Packit 21b7a2
if quiet == 0 or nb_instances_failed != 0:
Packit 21b7a2
    print "found %d test instances: %d success %d failures" % (
Packit 21b7a2
      nb_instances_tests, nb_instances_success, nb_instances_failed)
Packit 21b7a2
Packit 21b7a2
testsuite.freeDoc()
Packit 21b7a2
Packit 21b7a2
# Memory debug specific
Packit 21b7a2
libxml2.relaxNGCleanupTypes()
Packit 21b7a2
libxml2.cleanupParser()
Packit 21b7a2
if libxml2.debugMemory(1) == 0:
Packit 21b7a2
    if quiet == 0:
Packit 21b7a2
	print "OK"
Packit 21b7a2
else:
Packit 21b7a2
    print "Memory leak %d bytes" % (libxml2.debugMemory(1))
Packit 21b7a2
    libxml2.dumpMemory()