# This file is part of cloud-init. See LICENSE file for license information.
"""Verify test results."""
import os
import unittest
from tests.cloud_tests import (config, LOG, util, testcases)
def verify_data(data_dir, platform, os_name, tests):
"""Verify test data is correct.
@param data_dir: top level directory for all tests
@param platform: The platform name we for this test data (e.g. lxd)
@param os_name: The operating system under test (xenial, artful, etc.).
@param tests: list of test names
@return_value: {<test_name>: {passed: True/False, failures: []}}
"""
base_dir = os.sep.join((data_dir, platform, os_name))
runner = unittest.TextTestRunner(verbosity=util.current_verbosity())
res = {}
for test_name in tests:
LOG.debug('verifying test data for %s', test_name)
# get cloudconfig for test
test_conf = config.load_test_config(test_name)
test_module = config.name_to_module(test_name)
cloud_conf = test_conf['cloud_config']
# load script outputs
data = {'platform': platform, 'os_name': os_name}
test_dir = os.path.join(base_dir, test_name)
for script_name in os.listdir(test_dir):
with open(os.path.join(test_dir, script_name), 'rb') as fp:
data[script_name] = fp.read()
# get test suite and launch tests
suite = testcases.get_suite(test_module, data, cloud_conf)
suite_results = runner.run(suite)
res[test_name] = {
'passed': suite_results.wasSuccessful(),
'failures': [{'module': type(test_class).__base__.__module__,
'class': type(test_class).__base__.__name__,
'function': str(test_class).split()[0],
'error': trace.splitlines()[-1],
'traceback': trace, }
for test_class, trace in suite_results.failures]
}
for failure in res[test_name]['failures']:
LOG.warning('test case: %s failed %s.%s with: %s',
test_name, failure['class'], failure['function'],
failure['error'])
return res
def format_test_failures(test_result):
"""Return a human-readable printable format of test failures."""
if not test_result['failures']:
return ''
failure_hdr = ' test failures:'
failure_fmt = ' * {module}.{class}.{function}\n '
output = []
for failure in test_result['failures']:
if not output:
output = [failure_hdr]
msg = failure_fmt.format(**failure)
if failure.get('error'):
msg += failure['error']
else:
msg += failure.get('traceback', '')
output.append(msg)
return '\n'.join(output)
def format_results(res):
"""Return human-readable results as a string"""
platform_hdr = 'Platform: {platform}'
distro_hdr = ' Distro: {distro}'
distro_summary_fmt = (
' test modules passed:{passed} tests failed:{failed}')
output = ['']
counts = {}
for platform, platform_data in res.items():
output.append(platform_hdr.format(platform=platform))
counts[platform] = {}
for distro, distro_data in platform_data.items():
distro_failure_output = []
output.append(distro_hdr.format(distro=distro))
counts[platform][distro] = {'passed': 0, 'failed': 0}
for _, test_result in distro_data.items():
if test_result['passed']:
counts[platform][distro]['passed'] += 1
else:
counts[platform][distro]['failed'] += len(
test_result['failures'])
failure_output = format_test_failures(test_result)
if failure_output:
distro_failure_output.append(failure_output)
output.append(
distro_summary_fmt.format(**counts[platform][distro]))
if distro_failure_output:
output.extend(distro_failure_output)
return '\n'.join(output)
def verify(args):
"""Verify test data.
@param args: directory of test data
@return_value: 0 for success, or number of failed tests
"""
failed = 0
res = {}
# find test data
tests = util.list_test_data(args.data_dir)
for platform in tests.keys():
res[platform] = {}
for os_name in tests[platform].keys():
test_name = "platform='{}', os='{}'".format(platform, os_name)
LOG.info('test: %s verifying test data', test_name)
# run test
res[platform][os_name] = verify_data(
args.data_dir, platform, os_name,
tests[platform][os_name])
# handle results
fail_list = [k for k, v in res[platform][os_name].items()
if not v.get('passed')]
if len(fail_list) == 0:
LOG.info('test: %s passed all tests', test_name)
else:
LOG.warning('test: %s failed %s tests', test_name,
len(fail_list))
failed += len(fail_list)
# dump results
LOG.debug('\n---- Verify summarized results:\n%s', format_results(res))
if args.result:
util.merge_results({'verify': res}, args.result)
return failed
# vi: ts=4 expandtab