Blame testing/runtests.py

Packit 1c1d7e
#!/usr/bin/python
Packit 1c1d7e
Packit 1c1d7e
from __future__ import print_function
Packit 1c1d7e
import argparse, glob, itertools, re, shutil, os, sys
Packit 1c1d7e
Packit 1c1d7e
config_reg = re.compile('.*\/\/\s*(?P<name>\S+):\s*(?P<value>.*)$')
Packit 1c1d7e
Packit 1c1d7e
class Tester:
Packit 1c1d7e
	def __init__(self,args,test):
Packit 1c1d7e
		self.args      = args
Packit 1c1d7e
		self.test      = test
Packit 1c1d7e
		self.update    = args.updateref
Packit 1c1d7e
		self.config    = self.get_config()
Packit 1c1d7e
		self.test_name = '[%s]: %s' % (self.test,self.config['objective'][0])
Packit 1c1d7e
		self.test_id   = self.test.split('_')[0]
Packit 1c1d7e
		if self.update:
Packit 1c1d7e
			self.test_out = self.args.inputdir+'/'+self.test_id
Packit 1c1d7e
		else:
Packit 1c1d7e
			self.test_out = self.args.outputdir+'/test_output_'+self.test_id
Packit 1c1d7e
		self.prepare_test()
Packit 1c1d7e
Packit 1c1d7e
	def compare_ok(self,got_file,expected_file,name):
Packit 1c1d7e
		if not os.path.isfile(got_file):
Packit 1c1d7e
			return (True,'%s absent' % got_file)
Packit 1c1d7e
		elif not os.path.isfile(expected_file):
Packit 1c1d7e
			return (True,'%s absent' % expected_file)
Packit 1c1d7e
		else:
Packit 1c1d7e
			diff = os.popen('diff -b -w -u %s %s' % (got_file,expected_file)).read()
Packit 1c1d7e
			if diff and not diff.startswith("No differences"):
Packit 1c1d7e
				return (True,'Difference between generated output and reference:\n%s' % diff)
Packit 1c1d7e
		return (False,'')
Packit 1c1d7e
Packit 1c1d7e
	def get_config(self):
Packit 1c1d7e
		config = {}
Packit 1c1d7e
		with open(self.args.inputdir+'/'+self.test,'r') as f:
Packit 1c1d7e
			for line in f.readlines():
Packit 1c1d7e
				m = config_reg.match(line)
Packit 1c1d7e
				if m:
Packit 1c1d7e
					key   = m.group('name')
Packit 1c1d7e
					value = m.group('value')
Packit 1c1d7e
					if (key=='config'):
Packit 1c1d7e
						value = value.replace('$INPUTDIR',self.args.inputdir)
Packit 1c1d7e
					#print('key=%s value=%s' % (key,value))
Packit 1c1d7e
					config.setdefault(key, []).append(value)
Packit 1c1d7e
		return config
Packit 1c1d7e
Packit 1c1d7e
	def prepare_test(self):
Packit 1c1d7e
		# prepare test environment
Packit 1c1d7e
		shutil.rmtree(self.test_out,ignore_errors=True)
Packit 1c1d7e
		os.mkdir(self.test_out)
Packit 1c1d7e
		shutil.copy(self.args.inputdir+'/Doxyfile',self.test_out)
Packit 1c1d7e
		with open(self.test_out+'/Doxyfile','a') as f:
Packit 1c1d7e
			print('INPUT=%s/%s' % (self.args.inputdir,self.test), file=f)
Packit 1c1d7e
			print('STRIP_FROM_PATH=%s' % self.args.inputdir, file=f)
Packit 1c1d7e
			print('XML_OUTPUT=%s/out' % self.test_out, file=f)
Packit 1c1d7e
			print('EXAMPLE_PATH=%s' % self.args.inputdir, file=f)
Packit 1c1d7e
			if 'config' in self.config:
Packit 1c1d7e
				for option in self.config['config']:
Packit 1c1d7e
					print(option, file=f)
Packit 1c1d7e
Packit 1c1d7e
		if 'check' not in self.config or not self.config['check']:
Packit 1c1d7e
			print('Test doesn\'t specify any files to check')
Packit 1c1d7e
			sys.exit(1)
Packit 1c1d7e
Packit 1c1d7e
		# run doxygen
Packit 1c1d7e
		if (sys.platform == 'win32'):
Packit 1c1d7e
			redir=' > nul:'             
Packit 1c1d7e
		else:
Packit 1c1d7e
			redir=' 2> /dev/null'             
Packit 1c1d7e
		if os.system('%s %s/Doxyfile %s' % (self.args.doxygen,self.test_out, redir))!=0:
Packit 1c1d7e
			print('Error: failed to run %s on %s/Doxyfile' % (self.args.doxygen,self.test_out));
Packit 1c1d7e
			sys.exit(1)
Packit 1c1d7e
Packit 1c1d7e
	# update the reference data for this test
Packit 1c1d7e
	def update_test(self,testmgr):
Packit 1c1d7e
		print('Updating reference for %s' % self.test_name)
Packit 1c1d7e
Packit 1c1d7e
		if 'check' in self.config:
Packit 1c1d7e
			for check in self.config['check']:
Packit 1c1d7e
				check_file='%s/out/%s' % (self.test_out,check)
Packit 1c1d7e
				# check if the file we need to check is actually generated
Packit 1c1d7e
				if not os.path.isfile(check_file):
Packit 1c1d7e
					print('Non-existing file %s after \'check:\' statement' % check_file)
Packit 1c1d7e
					return
Packit 1c1d7e
				# convert output to canonical form
Packit 1c1d7e
				data = os.popen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file)).read()
Packit 1c1d7e
				if data:
Packit 1c1d7e
					# strip version
Packit 1c1d7e
					data = re.sub(r'xsd" version="[0-9.-]+"','xsd" version=""',data).rstrip('\n')
Packit 1c1d7e
				else:
Packit 1c1d7e
					print('Failed to run %s on the doxygen output file %s' % (self.args.xmllint,self.test_out))
Packit 1c1d7e
					return
Packit 1c1d7e
				out_file='%s/%s' % (self.test_out,check)
Packit 1c1d7e
				with open(out_file,'w') as f:
Packit 1c1d7e
					print(data,file=f)
Packit 1c1d7e
		shutil.rmtree(self.test_out+'/out',ignore_errors=True)
Packit 1c1d7e
		os.remove(self.test_out+'/Doxyfile')
Packit 1c1d7e
Packit 1c1d7e
	# check the relevant files of a doxygen run with the reference material
Packit 1c1d7e
	def perform_test(self,testmgr):
Packit 1c1d7e
		# look for files to check against the reference
Packit 1c1d7e
		if 'check' in self.config:
Packit 1c1d7e
			for check in self.config['check']:
Packit 1c1d7e
				check_file='%s/out/%s' % (self.test_out,check)
Packit 1c1d7e
				# check if the file we need to check is actually generated
Packit 1c1d7e
				if not os.path.isfile(check_file):
Packit 1c1d7e
					testmgr.ok(False,self.test_name,msg='Non-existing file %s after \'check:\' statement' % check_file)
Packit 1c1d7e
					return
Packit 1c1d7e
				# convert output to canonical form
Packit 1c1d7e
				data = os.popen('%s --format --noblanks --nowarning %s' % (self.args.xmllint,check_file)).read()
Packit 1c1d7e
				if data:
Packit 1c1d7e
					# strip version
Packit 1c1d7e
					data = re.sub(r'xsd" version="[0-9.-]+"','xsd" version=""',data).rstrip('\n')
Packit 1c1d7e
				else:
Packit 1c1d7e
					testmgr.ok(False,self.test_name,msg='Failed to run %s on the doxygen output file %s' % (self.args.xmllint,self.test_out))
Packit 1c1d7e
					return
Packit 1c1d7e
				out_file='%s/%s' % (self.test_out,check)
Packit 1c1d7e
				with open(out_file,'w') as f:
Packit 1c1d7e
					print(data,file=f)
Packit 1c1d7e
				ref_file='%s/%s/%s' % (self.args.inputdir,self.test_id,check)
Packit 1c1d7e
				(failed,msg) = self.compare_ok(out_file,ref_file,self.test_name)
Packit 1c1d7e
				if failed:
Packit 1c1d7e
					testmgr.ok(False,self.test_name,msg)
Packit 1c1d7e
					return
Packit 1c1d7e
		shutil.rmtree(self.test_out,ignore_errors=True)
Packit 1c1d7e
		testmgr.ok(True,self.test_name)
Packit 1c1d7e
Packit 1c1d7e
	def run(self,testmgr):
Packit 1c1d7e
		if self.update:
Packit 1c1d7e
			self.update_test(testmgr)
Packit 1c1d7e
		else:
Packit 1c1d7e
			self.perform_test(testmgr)
Packit 1c1d7e
Packit 1c1d7e
class TestManager:
Packit 1c1d7e
	def __init__(self,args,tests):
Packit 1c1d7e
		self.args  = args
Packit 1c1d7e
		self.tests = tests
Packit 1c1d7e
		self.num_tests = len(tests)
Packit 1c1d7e
		self.count=1
Packit 1c1d7e
		self.passed=0
Packit 1c1d7e
		print('1..%d' % self.num_tests)
Packit 1c1d7e
Packit 1c1d7e
	def ok(self,result,test_name,msg='Ok'):
Packit 1c1d7e
		if result:
Packit 1c1d7e
			print('ok %s - %s' % (self.count,test_name))
Packit 1c1d7e
			self.passed = self.passed + 1
Packit 1c1d7e
		else:
Packit 1c1d7e
			print('not ok %s - %s' % (self.count,test_name))
Packit 1c1d7e
			print('-------------------------------------')
Packit 1c1d7e
			print(msg)
Packit 1c1d7e
			print('-------------------------------------')
Packit 1c1d7e
		self.count = self.count + 1
Packit 1c1d7e
Packit 1c1d7e
	def result(self):
Packit 1c1d7e
		if self.passed==self.num_tests:
Packit 1c1d7e
			print('All tests passed!')
Packit 1c1d7e
		else:
Packit 1c1d7e
			print('%d out of %s tests failed' % (self.num_tests-self.passed,self.num_tests))
Packit 1c1d7e
		return 0 if self.passed==self.num_tests else 1
Packit 1c1d7e
Packit 1c1d7e
	def perform_tests(self):
Packit 1c1d7e
		for test in self.tests:
Packit 1c1d7e
			tester = Tester(self.args,test)
Packit 1c1d7e
			tester.run(self)
Packit 1c1d7e
		return 0 if self.args.updateref else self.result()
Packit 1c1d7e
Packit 1c1d7e
def main():
Packit 1c1d7e
	# argument handling
Packit 1c1d7e
	parser = argparse.ArgumentParser(description='run doxygen tests')
Packit 1c1d7e
	parser.add_argument('--updateref',help='update the reference data for a test',action="store_true")
Packit 1c1d7e
	parser.add_argument('--doxygen',nargs='?',default='doxygen',help='path/name of the doxygen executable')
Packit 1c1d7e
	parser.add_argument('--xmllint',nargs='?',default='xmllint',help='path/name of the xmllint executable')
Packit 1c1d7e
	parser.add_argument('--id',nargs='+',dest='ids',action='append',type=int,help='id of the test to perform')
Packit 1c1d7e
	parser.add_argument('--all',help='perform all tests',action="store_true")
Packit 1c1d7e
	parser.add_argument('--inputdir',nargs='?',default='.',help='input directory containing the tests')
Packit 1c1d7e
	parser.add_argument('--outputdir',nargs='?',default='.',help='output directory to write the doxygen output to')
Packit 1c1d7e
	args = parser.parse_args()
Packit 1c1d7e
Packit 1c1d7e
    # sanity check
Packit 1c1d7e
	if (not args.updateref is None) and (args.ids is None) and (args.all is None):
Packit 1c1d7e
		parser.error('--updateref requires either --id or --all')
Packit 1c1d7e
Packit 1c1d7e
	starting_directory = os.getcwd()
Packit 1c1d7e
	os.chdir(args.inputdir)
Packit 1c1d7e
    # find the tests to run
Packit 1c1d7e
	if args.ids: # test ids are given by user
Packit 1c1d7e
		tests = []
Packit 1c1d7e
		for id in list(itertools.chain.from_iterable(args.ids)):
Packit 1c1d7e
			tests.append(glob.glob('%s_*'%id))
Packit 1c1d7e
			tests.append(glob.glob('0%s_*'%id))
Packit 1c1d7e
			tests.append(glob.glob('00%s_*'%id))
Packit 1c1d7e
		tests = list(itertools.chain.from_iterable(tests))
Packit 1c1d7e
	else: # find all tests
Packit 1c1d7e
		tests = glob.glob('[0-9][0-9][0-9]_*')
Packit 1c1d7e
	os.chdir(starting_directory)
Packit 1c1d7e
Packit 1c1d7e
	# create test manager to run the tests
Packit 1c1d7e
	testManager = TestManager(args,tests)
Packit 1c1d7e
	sys.exit(testManager.perform_tests())
Packit 1c1d7e
Packit 1c1d7e
if __name__ == '__main__':
Packit 1c1d7e
	main()