Blame tests/run_tests.py

Packit 2ba279
#!/usr/bin/python3
Packit 2ba279
Packit 2ba279
from __future__ import print_function
Packit 2ba279
Packit 2ba279
import argparse
Packit 2ba279
import datetime
Packit 2ba279
import os
Packit 2ba279
import re
Packit 2ba279
import six
Packit 2ba279
import subprocess
Packit 2ba279
import sys
Packit 2ba279
import unittest
Packit 2ba279
import yaml
Packit 2ba279
Packit 2ba279
from distutils.spawn import find_executable
Packit 2ba279
Packit 2ba279
LIBDIRS = 'src/utils/.libs:src/plugins/.libs:src/plugins/fs/.libs:src/lib/.libs'
Packit 2ba279
GIDIR = 'src/lib'
Packit 2ba279
Packit 2ba279
SKIP_CONFIG = 'skip.yml'
Packit 2ba279
Packit 2ba279
Packit 2ba279
def _get_tests_from_suite(suite, tests):
Packit 2ba279
    """ Extract tests from the test suite """
Packit 2ba279
    # 'tests' we get from 'unittest.defaultTestLoader.discover' are "wrapped"
Packit 2ba279
    # in multiple 'unittest.suite.TestSuite' classes/lists so we need to "unpack"
Packit 2ba279
    # the indivudual test cases
Packit 2ba279
    for test in suite:
Packit 2ba279
        if isinstance(test, unittest.suite.TestSuite):
Packit 2ba279
            _get_tests_from_suite(test, tests)
Packit 2ba279
Packit 2ba279
        if isinstance(test, unittest.TestCase):
Packit 2ba279
            tests.append(test)
Packit 2ba279
Packit 2ba279
    return tests
Packit 2ba279
Packit 2ba279
Packit 2ba279
def _get_test_tags(test):
Packit 2ba279
    """ Get test tags for single test case """
Packit 2ba279
Packit 2ba279
    tags = []
Packit 2ba279
Packit 2ba279
    # test failed to load, usually some ImportError or something really broken
Packit 2ba279
    # in the test file, just return empty list and let it fail
Packit 2ba279
    # with python2 the loader will raise an exception directly without returning
Packit 2ba279
    # a "fake" FailedTest test case
Packit 2ba279
    if six.PY3 and isinstance(test, unittest.loader._FailedTest):
Packit 2ba279
        return tags
Packit 2ba279
Packit 2ba279
    test_fn = getattr(test, test._testMethodName)
Packit 2ba279
Packit 2ba279
    # it is possible to either tag a test funcion or the class so we need to
Packit 2ba279
    # check both for the tag
Packit 2ba279
    if getattr(test_fn, "slow", False) or getattr(test_fn.__self__, "slow", False):
Packit 2ba279
        tags.append(TestTags.SLOW)
Packit 2ba279
    if getattr(test_fn, "unstable", False) or getattr(test_fn.__self__, "unstable", False):
Packit 2ba279
        tags.append(TestTags.UNSTABLE)
Packit 2ba279
    if getattr(test_fn, "unsafe", False) or getattr(test_fn.__self__, "unsafe", False):
Packit 2ba279
        tags.append(TestTags.UNSAFE)
Packit 2ba279
    if getattr(test_fn, "core", False) or getattr(test_fn.__self__, "core", False):
Packit 2ba279
        tags.append(TestTags.CORE)
Packit 2ba279
    if getattr(test_fn, "nostorage", False) or getattr(test_fn.__self__, "nostorage", False):
Packit 2ba279
        tags.append(TestTags.NOSTORAGE)
Packit 2ba279
    if getattr(test_fn, "extradeps", False) or getattr(test_fn.__self__, "extradeps", False):
Packit 2ba279
        tags.append(TestTags.EXTRADEPS)
Packit 2ba279
    if getattr(test_fn, "regression", False) or getattr(test_fn.__self__, "regression", False):
Packit 2ba279
        tags.append(TestTags.REGRESSION)
Packit 2ba279
    if getattr(test_fn, "sourceonly", False) or getattr(test_fn.__self__, "sourceonly", False):
Packit 2ba279
        tags.append(TestTags.SOURCEONLY)
Packit 2ba279
Packit 2ba279
    return tags
Packit 2ba279
Packit 2ba279
Packit 2ba279
def parse_args():
Packit 2ba279
    """ Parse cmdline arguments """
Packit 2ba279
Packit 2ba279
    argparser = argparse.ArgumentParser(description='libblockdev test suite')
Packit 2ba279
    argparser.add_argument('testname', nargs='*', help='name of test class or '
Packit 2ba279
                           'method (e. g. "CryptoTestFormat", '
Packit 2ba279
                           '"GenericResize.test_ext2_generic_resize")')
Packit 2ba279
    argparser.add_argument('-f', '--fast', dest='fast', help='skip slow tests',
Packit 2ba279
                           action='store_true')
Packit 2ba279
    argparser.add_argument('-l', '--lucky', dest='lucky',
Packit 2ba279
                           help='run also potentially dangerous/failing tests',
Packit 2ba279
                           action='store_true')
Packit 2ba279
    argparser.add_argument('-j', '--jenkins', dest='jenkins',
Packit 2ba279
                           help='run also tests that should run only in a CI environment',
Packit 2ba279
                           action='store_true')
Packit 2ba279
    argparser.add_argument('-c', '--core', dest='core',
Packit 2ba279
                           help='run tests that cover basic functionality of the library and regression tests',
Packit 2ba279
                           action='store_true')
Packit 2ba279
    argparser.add_argument('-s', '--stop', dest='stop',
Packit 2ba279
                           help='stop executing after first failed test',
Packit 2ba279
                           action='store_true')
Packit 2ba279
    argparser.add_argument('-i', '--installed', dest='installed',
Packit 2ba279
                           help='run tests against installed version of libblockdev',
Packit 2ba279
                           action='store_true')
Packit 2ba279
    args = argparser.parse_args()
Packit 2ba279
Packit 2ba279
    if args.fast:
Packit 2ba279
        os.environ['SKIP_SLOW'] = ''
Packit 2ba279
    if args.lucky:
Packit 2ba279
        os.environ['FEELINGLUCKY'] = ''
Packit 2ba279
    if args.jenkins:
Packit 2ba279
        os.environ['JENKINS_HOME'] = ''
Packit 2ba279
Packit 2ba279
    # read the environmental variables for backwards compatibility
Packit 2ba279
    if 'JENKINS_HOME' in os.environ:
Packit 2ba279
        args.jenkins = True
Packit 2ba279
    if 'SKIP_SLOW' in os.environ:
Packit 2ba279
        args.fast = True
Packit 2ba279
    if 'FEELINGLUCKY' in os.environ:
Packit 2ba279
        args.lucky = True
Packit 2ba279
Packit 2ba279
    return args
Packit 2ba279
Packit 2ba279
def _split_test_id(test_id):
Packit 2ba279
    # test.id() looks like 'crypto_test.CryptoTestResize.test_luks2_resize'
Packit 2ba279
    # and we want to print 'test_luks2_resize (crypto_test.CryptoTestResize)'
Packit 2ba279
    test_desc = test.id().split(".")
Packit 2ba279
    test_name = test_desc[-1]
Packit 2ba279
    test_module = ".".join(test_desc[:-1])
Packit 2ba279
Packit 2ba279
    return test_name, test_module
Packit 2ba279
Packit 2ba279
Packit 2ba279
def _print_skip_message(test, skip_tag):
Packit 2ba279
    test_id = test.id()
Packit 2ba279
    test_module, test_name = _split_test_id(test_id)
Packit 2ba279
Packit 2ba279
    if skip_tag == TestTags.SLOW:
Packit 2ba279
        reason = "skipping slow tests"
Packit 2ba279
    elif skip_tag == TestTags.UNSTABLE:
Packit 2ba279
        reason = "skipping unstable tests"
Packit 2ba279
    elif skip_tag == TestTags.UNSAFE:
Packit 2ba279
        reason = "skipping test that modifies system configuration"
Packit 2ba279
    elif skip_tag == TestTags.EXTRADEPS:
Packit 2ba279
        reason = "skipping test that requires special configuration"
Packit 2ba279
    elif skip_tag == TestTags.CORE:
Packit 2ba279
        reason = "skipping non-core test"
Packit 2ba279
    elif skip_tag == TestTags.SOURCEONLY:
Packit 2ba279
        reason = "skipping test that can run only against library compiled from source"
Packit 2ba279
    else:
Packit 2ba279
        reason = "unknown reason"  # just to be sure there is some default value
Packit 2ba279
Packit 2ba279
    if test._testMethodDoc:
Packit 2ba279
        print("%s (%s)\n%s ... skipped '%s'" % (test_name, test_module, test._testMethodDoc, reason),
Packit 2ba279
              file=sys.stderr)
Packit 2ba279
    else:
Packit 2ba279
        print("%s (%s) ... skipped '%s'" % (test_name, test_module, reason),
Packit 2ba279
              file=sys.stderr)
Packit 2ba279
Packit 2ba279
Packit 2ba279
def _should_skip(distro=None, version=None, arch=None, reason=None):
Packit 2ba279
    # all these can be lists or a single value, so covert everything to list
Packit 2ba279
    if distro is not None and type(distro) is not list:
Packit 2ba279
        distro = [distro]
Packit 2ba279
    if version is not None and type(version) is not list:
Packit 2ba279
        version = [version]
Packit 2ba279
    if arch is not None and type(arch) is not list:
Packit 2ba279
        arch = [arch]
Packit 2ba279
Packit 2ba279
    # DISTRO, VERSION and ARCH variables are set in main, we don't need to
Packit 2ba279
    # call hostnamectl etc. for every test run
Packit 2ba279
    if (distro is None or DISTRO in distro) and (version is None or VERSION in version) and \
Packit 2ba279
       (arch is None or ARCH in arch):
Packit 2ba279
        return True
Packit 2ba279
Packit 2ba279
    return False
Packit 2ba279
Packit 2ba279
Packit 2ba279
def _parse_skip_config(config):
Packit 2ba279
    with open(config) as f:
Packit 2ba279
        data = f.read()
Packit 2ba279
    parsed = yaml.load(data, Loader=yaml.SafeLoader)
Packit 2ba279
Packit 2ba279
    skipped_tests = dict()
Packit 2ba279
Packit 2ba279
    for entry in parsed:
Packit 2ba279
        for skip in entry["skip_on"]:
Packit 2ba279
            if _should_skip(**skip):
Packit 2ba279
                skipped_tests[entry["test"]] = skip["reason"]
Packit 2ba279
Packit 2ba279
    return skipped_tests
Packit 2ba279
Packit 2ba279
Packit 2ba279
if __name__ == '__main__':
Packit 2ba279
Packit 2ba279
    testdir = os.path.abspath(os.path.dirname(__file__))
Packit 2ba279
    projdir = os.path.abspath(os.path.normpath(os.path.join(testdir, '..')))
Packit 2ba279
Packit 2ba279
    args = parse_args()
Packit 2ba279
    if args.installed:
Packit 2ba279
        os.environ['LIBBLOCKDEV_TESTS_SKIP_OVERRIDE'] = ''
Packit 2ba279
        os.environ['LIBBLOCKDEV_CONFIG_DIR'] = '/etc/libblockdev/conf.d/'
Packit 2ba279
    else:
Packit 2ba279
        if 'LD_LIBRARY_PATH' not in os.environ and 'GI_TYPELIB_PATH' not in os.environ:
Packit 2ba279
            os.environ['LD_LIBRARY_PATH'] = LIBDIRS
Packit 2ba279
            os.environ['GI_TYPELIB_PATH'] = GIDIR
Packit 2ba279
            os.environ['LIBBLOCKDEV_CONFIG_DIR'] = os.path.join(testdir, 'default_config')
Packit 2ba279
Packit 2ba279
            try:
Packit 2ba279
                pyver = 'python3' if six.PY3 else 'python'
Packit 2ba279
                os.execv(sys.executable, [pyver] + sys.argv)
Packit 2ba279
            except OSError as e:
Packit 2ba279
                print('Failed re-exec with a new LD_LIBRARY_PATH and GI_TYPELIB_PATH: %s' % str(e))
Packit 2ba279
                sys.exit(1)
Packit 2ba279
Packit 2ba279
        sys.path.append(testdir)
Packit 2ba279
        sys.path.append(projdir)
Packit 2ba279
        sys.path.append(os.path.join(projdir, 'src/python'))
Packit 2ba279
Packit 2ba279
    start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
Packit 2ba279
Packit 2ba279
    loader = unittest.defaultTestLoader
Packit 2ba279
    suite = unittest.TestSuite()
Packit 2ba279
Packit 2ba279
    if args.testname:
Packit 2ba279
        test_cases = loader.loadTestsFromNames(args.testname)
Packit 2ba279
    else:
Packit 2ba279
        test_cases = loader.discover(start_dir=testdir, pattern='*_test*.py')
Packit 2ba279
Packit 2ba279
    # extract list of test classes so we can check/run them manually one by one
Packit 2ba279
    tests = []
Packit 2ba279
    tests = _get_tests_from_suite(test_cases, tests)
Packit 2ba279
Packit 2ba279
    # get distro and arch here so we don't have to do this for every test
Packit 2ba279
    from utils import get_version
Packit 2ba279
    DISTRO, VERSION = get_version()
Packit 2ba279
    ARCH = os.uname()[-1]
Packit 2ba279
Packit 2ba279
    # get list of tests to skip from the config file
Packit 2ba279
    skipping = _parse_skip_config(os.path.join(testdir, SKIP_CONFIG))
Packit 2ba279
Packit 2ba279
    # for some reason overrides_hack will fail if we import this at the start
Packit 2ba279
    # of the file
Packit 2ba279
    from utils import TestTags
Packit 2ba279
Packit 2ba279
    for test in tests:
Packit 2ba279
        test_id = test.id()
Packit 2ba279
Packit 2ba279
        # get tags and (possibly) skip the test
Packit 2ba279
        tags = _get_test_tags(test)
Packit 2ba279
Packit 2ba279
        if TestTags.SLOW in tags and args.fast:
Packit 2ba279
            _print_skip_message(test, TestTags.SLOW)
Packit 2ba279
            continue
Packit 2ba279
        if TestTags.UNSTABLE in tags and not args.lucky:
Packit 2ba279
            _print_skip_message(test, TestTags.UNSTABLE)
Packit 2ba279
            continue
Packit 2ba279
        if TestTags.UNSAFE in tags or TestTags.EXTRADEPS in tags and not args.jenkins:
Packit 2ba279
            _print_skip_message(test, TestTags.UNSAFE)
Packit 2ba279
            continue
Packit 2ba279
        if TestTags.EXTRADEPS in tags and not args.jenkins:
Packit 2ba279
            _print_skip_message(test, TestTags.EXTRADEPS)
Packit 2ba279
            continue
Packit 2ba279
        if TestTags.SOURCEONLY in tags and args.installed:
Packit 2ba279
            _print_skip_message(test, TestTags.SOURCEONLY)
Packit 2ba279
            continue
Packit 2ba279
Packit 2ba279
        if args.core and TestTags.CORE not in tags and TestTags.REGRESSION not in tags:
Packit 2ba279
            _print_skip_message(test, TestTags.CORE)
Packit 2ba279
            continue
Packit 2ba279
Packit 2ba279
        # check if the test is in the list of tests to skip
Packit 2ba279
        skip_id = next((test_pattern for test_pattern in skipping.keys() if re.search(test_pattern, test_id)), None)
Packit 2ba279
        if skip_id:
Packit 2ba279
            test_name, test_module = _split_test_id(test_id)
Packit 2ba279
            reason = "not supported on this distribution in this version and arch: %s" % skipping[skip_id]
Packit 2ba279
            print("%s (%s)\n%s ... skipped '%s'" % (test_name, test_module,
Packit 2ba279
                                                    test._testMethodDoc, reason),
Packit 2ba279
                  file=sys.stderr)
Packit 2ba279
            continue
Packit 2ba279
Packit 2ba279
        # finally add the test to the suite
Packit 2ba279
        suite.addTest(test)
Packit 2ba279
Packit 2ba279
    result = unittest.TextTestRunner(verbosity=2, failfast=args.stop).run(suite)
Packit 2ba279
Packit 2ba279
    # dump cropped journal to log file
Packit 2ba279
    if find_executable('journalctl'):
Packit 2ba279
        with open('journaldump.log', 'w') as outfile:
Packit 2ba279
            subprocess.call(['journalctl', '-S', start_time], stdout=outfile)
Packit 2ba279
Packit 2ba279
    if result.wasSuccessful():
Packit 2ba279
        sys.exit(0)
Packit 2ba279
    else:
Packit 2ba279
        sys.exit(1)