/util/testlib.py

#!python
# Copyright (c) 2000-2008 ActiveState Software Inc.
# License: MIT License (http://www.opensource.org/licenses/mit-license.php)

"""
    test suite harness

    Usage:

        test --list [<tags>...]  # list available tests modules
        test [<tags>...]         # run test modules

    Options:
        -v, --verbose   more verbose output
        -q, --quiet     don't print anything except if a test fails
        -d, --debug     log debug information        
        -h, --help      print this text and exit
        -l, --list      Just list the available test modules. You can also
                        specify tags to play with module filtering.
        -n, --no-default-tags   Ignore default tags
        -L <directive>  Specify a logging level via
                            <logname>:<levelname>
                        For example:
                            codeintel.db:DEBUG
                        This option can be used multiple times.

    By default this will run all tests in all available "test_*" modules.
    Tags can be specified to control which tests are run. For example:
    
        test python         # run tests with the 'python' tag
        test python cpln    # run tests with both 'python' and 'cpln' tags
        test -- -python     # exclude tests with the 'python' tag
                            # (the '--' is necessary to end the option list)
    
    The full name and base name of a test module are implicit tags for that
    module, e.g. module "test_xdebug.py" has tags "test_xdebug" and "xdebug".
    A TestCase's class name (with and without "TestCase") is an implicit
    tag for an test_* methods. A "test_foo" method also has "test_foo"
    and "foo" implicit tags.

    Tags can be added explicitly added:
    - to modules via a __tags__ global list; and
    - to individual test_* methods via a "tags" attribute list (you can
      use the testlib.tag() decorator for this).
"""
#TODO:
# - Document how tests are found (note the special "test_cases()" and
#   "test_suite_class" hooks).
# - See the optparse "TODO" below.
# - Make the quiet option actually quiet.

__version_info__ = (0, 6, 6)
__version__ = '.'.join(map(str, __version_info__))


import os
from os.path import join, basename, dirname, abspath, splitext, \
                    isfile, isdir, normpath, exists
import sys
import getopt
import glob
import time
import types
import tempfile
import unittest
from pprint import pprint
import imp
import optparse
import logging
import textwrap
import traceback



#---- globals and exceptions

log = logging.getLogger("test")



#---- exports generally useful to test cases

class TestError(Exception):
    pass

class TestSkipped(Exception):
    """Raise this to indicate that a test is being skipped.

    ConsoleTestRunner knows to interpret these at NOT failures.
    """
    pass

class TestFailed(Exception):
    pass

def tag(*tags):
    """Decorator to add tags to test_* functions.
    
    Example:
        class MyTestCase(unittest.TestCase):
            @testlib.tag("knownfailure")
            def test_foo(self):
                #...
    """
    def decorate(f):
        if not hasattr(f, "tags"):
            f.tags = []
        f.tags += tags
        return f
    return decorate


#---- timedtest decorator
# Use this to assert that a test completes in a given amount of time.
# This is from http://www.artima.com/forums/flat.jsp?forum=122&thread=129497
# Including here, becase it might be useful.
# NOTE: Untested and I suspect some breakage.

TOLERANCE = 0.05

class DurationError(AssertionError): pass

def timedtest(max_time, tolerance=TOLERANCE):
    """ timedtest decorator
    decorates the test method with a timer
    when the time spent by the test exceeds
    max_time in seconds, an Assertion error is thrown.
    """
    def _timedtest(function):
        def wrapper(*args, **kw):
            start_time = time.time()
            try:
                function(*args, **kw)
            finally:
                total_time = time.time() - start_time
                if total_time > max_time + tolerance:
                    raise DurationError(('Test was too long (%.2f s)'
                                           % total_time))
        return wrapper

    return _timedtest



#---- module api

class Test(object):
    def __init__(self, ns, testmod, testcase, testfn_name,
                 testsuite_class=None):
        self.ns = ns
        self.testmod = testmod
        self.testcase = testcase
        self.testfn_name = testfn_name
        self.testsuite_class = testsuite_class
        # Give each testcase some extra testlib attributes for useful
        # introspection on TestCase instances later on.
        self.testcase._testlib_shortname_ = self.shortname()
        self.testcase._testlib_explicit_tags_ = self.explicit_tags()
        self.testcase._testlib_implicit_tags_ = self.implicit_tags()
    def __str__(self):
        return self.shortname()
    def __repr__(self):
        return "<Test %s>" % self.shortname()
    def shortname(self):
        bits = [self._normname(self.testmod.__name__),
                self._normname(self.testcase.__class__.__name__),
                self._normname(self.testfn_name)]
        if self.ns:
            bits.insert(0, self.ns)
        return '/'.join(bits)
    def _flatten_tags(self, tags):
        """Split tags with '/' in them into multiple tags.
        
        '/' is the reserved tag separator and allowing tags with
        embedded '/' results in one being unable to select those via
        filtering. As long as tag order is stable then presentation of
        these subsplit tags should be fine.
        """
        flattened = []
        for t in tags:
            flattened += t.replace(os.sep, '/').split('/')
        return flattened
    def explicit_tags(self):
        tags = []
        if hasattr(self.testmod, "__tags__"):
            tags += self.testmod.__tags__
        if hasattr(self.testcase, "__tags__"):
            tags += self.testcase.__tags__
        testfn = getattr(self.testcase, self.testfn_name)
        if hasattr(testfn, "tags"):
            tags += testfn.tags
        return self._flatten_tags(tags)
    def implicit_tags(self):
        tags = [
            self.testmod.__name__.lower(),
            self._normname(self.testmod.__name__),
            self.testcase.__class__.__name__.lower(),
            self._normname(self.testcase.__class__.__name__),
            self.testfn_name,
            self._normname(self.testfn_name),
        ]
        if self.ns:
            tags.insert(0, self.ns)
        return self._flatten_tags(tags)
    def tags(self):
        return self.explicit_tags() + self.implicit_tags()
    def doc(self):
        testfn = getattr(self.testcase, self.testfn_name)
        return testfn.__doc__ or ""
    def _normname(self, name):
        if name.startswith("test_"):
            return name[5:].lower()
        elif name.startswith("test"):
            return name[4:].lower()
        elif name.endswith("TestCase"):
            return name[:-8].lower()
        else:
            return name


def testmod_paths_from_testdir(testdir):
    """Generate test module paths in the given dir."""
    for path in glob.glob(join(testdir, "test_*.py")):
        yield path

    for path in glob.glob(join(testdir, "test_*")):
        if not isdir(path): continue
        if not isfile(join(path, "__init__.py")): continue
        yield path

def testmods_from_testdir(testdir):
    """Generate test modules in the given test dir.
    
    Modules are imported with 'testdir' first on sys.path.
    """
    testdir = normpath(testdir)
    for testmod_path in testmod_paths_from_testdir(testdir):
        testmod_name = splitext(basename(testmod_path))[0]
        log.debug("import test module '%s'", testmod_path)
        try:
            iinfo = imp.find_module(testmod_name, [dirname(testmod_path)])
            testabsdir = abspath(testdir)
            sys.path.insert(0, testabsdir)
            old_dir = os.getcwd()
            os.chdir(testdir)
            try:
                testmod = imp.load_module(testmod_name, *iinfo)
            finally:
                os.chdir(old_dir)
                sys.path.remove(testabsdir)
        except TestSkipped:
            _, ex, _ = sys.exc_info()
            log.warn("'%s' module skipped: %s", testmod_name, ex)
        except Exception:
            _, ex, _ = sys.exc_info()
            log.warn("could not import test module '%s': %s (skipping, "
                     "run with '-d' for full traceback)",
                     testmod_path, ex)
            if log.isEnabledFor(logging.DEBUG):
                traceback.print_exc()
        else:
            yield testmod

def testcases_from_testmod(testmod):
    """Gather tests from a 'test_*' module.
    
    Returns a list of TestCase-subclass instances. One instance for each
    found test function.
    
    In general the normal unittest TestLoader.loadTests*() semantics are
    used for loading tests with some differences:
    - TestCase subclasses beginning with '_' are skipped (presumed to be
      internal).
    - If the module has a top-level "test_cases", it is called for a list of
      TestCase subclasses from which to load tests (can be a generator). This
      allows for run-time setup of test cases.
    - If the module has a top-level "test_suite_class", it is used to group
      all test cases from that module into an instance of that TestSuite
      subclass. This allows for overriding of test running behaviour.
    """
    class TestListLoader(unittest.TestLoader):
        suiteClass = list

    loader = TestListLoader()
    if hasattr(testmod, "test_cases"):
        try:
            for testcase_class in testmod.test_cases():
                if testcase_class.__name__.startswith("_"):
                    log.debug("skip private TestCase class '%s'",
                              testcase_class.__name__)
                    continue
                for testcase in loader.loadTestsFromTestCase(testcase_class):
                    yield testcase
        except Exception:
            _, ex, _ = sys.exc_info()
            testmod_path = testmod.__file__
            if testmod_path.endswith(".pyc"):
                testmod_path = testmod_path[:-1]
            log.warn("error running test_cases() in '%s': %s (skipping, "
                     "run with '-d' for full traceback)",
                     testmod_path, ex)
            if log.isEnabledFor(logging.DEBUG):
                traceback.print_exc()
    else:
        class_names_skipped = []
        for testcases in loader.loadTestsFromModule(testmod):
            for testcase in testcases:
                class_name = testcase.__class__.__name__
                if class_name in class_names_skipped:
                    pass
                elif class_name.startswith("_"):
                    log.debug("skip private TestCase class '%s'", class_name)
                    class_names_skipped.append(class_name)
                else:
                    yield testcase


def tests_from_manifest(testdir_from_ns):
    """Return a list of `testlib.Test` instances for each test found in
    the manifest.
    
    There will be a test for
    (a) each "test*" function of
    (b) each TestCase-subclass in
    (c) each "test_*" Python module in
    (d) each test dir in the manifest.
    
    If a "test_*" module has a top-level "test_suite_class", it will later
    be used to group all test cases from that module into an instance of that
    TestSuite subclass. This allows for overriding of test running behaviour.
    """
    for ns, testdir in testdir_from_ns.items():
        for testmod in testmods_from_testdir(testdir):
            if hasattr(testmod, "test_suite_class"):
                testsuite_class = testmod.test_suite_class
                if not issubclass(testsuite_class, unittest.TestSuite):
                    testmod_path = testmod.__file__
                    if testmod_path.endswith(".pyc"):
                        testmod_path = testmod_path[:-1]
                    log.warn("'test_suite_class' of '%s' module is not a "
                             "subclass of 'unittest.TestSuite': ignoring",
                             testmod_path)
            else:
                testsuite_class = None
            for testcase in testcases_from_testmod(testmod):
                try:
                    yield Test(ns, testmod, testcase,
                               testcase._testMethodName,
                               testsuite_class)
                except AttributeError:
                    # Python 2.4 and older:
                    yield Test(ns, testmod, testcase,
                               testcase._TestCase__testMethodName,
                               testsuite_class)

def tests_from_manifest_and_tags(testdir_from_ns, tags):
    include_tags = [tag.lower() for tag in tags if not tag.startswith('-')]
    exclude_tags = [tag[1:].lower() for tag in tags if tag.startswith('-')]

    for test in tests_from_manifest(testdir_from_ns):
        test_tags = [t.lower() for t in test.tags()]

        matching_exclude_tags = [t for t in exclude_tags if t in test_tags]
        if matching_exclude_tags:
            #log.debug("test '%s' matches exclude tag(s) '%s': skipping",
            #          test.shortname(), "', '".join(matching_exclude_tags))
            continue

        if not include_tags:
            yield test
        else:
            for tag in include_tags:
                if tag not in test_tags:
                    #log.debug("test '%s' does not match tag '%s': skipping",
                    #          test.shortname(), tag)
                    break
            else:
                #log.debug("test '%s' matches tags: %s", test.shortname(),
                #          ' '.join(tags))
                yield test
                
def test(testdir_from_ns, tags=[], setup_func=None):
    log.debug("test(testdir_from_ns=%r, tags=%r, ...)",
              testdir_from_ns, tags)
    if setup_func is not None:
        setup_func()
    tests = list(tests_from_manifest_and_tags(testdir_from_ns, tags))
    if not tests:
        return None
    
    # Groups test cases into a test suite class given by their test module's
    # "test_suite_class" hook, if any.
    suite = unittest.TestSuite()
    suite_for_testmod = None
    testmod = None
    for test in tests:
        if test.testmod != testmod:
            if suite_for_testmod is not None:
                suite.addTest(suite_for_testmod)
            suite_for_testmod = (test.testsuite_class or unittest.TestSuite)()
            testmod = test.testmod
        suite_for_testmod.addTest(test.testcase)
    if suite_for_testmod is not None:
        suite.addTest(suite_for_testmod)
    
    runner = ConsoleTestRunner(sys.stdout)
    result = runner.run(suite)
    return result

def list_tests(testdir_from_ns, tags):
    # Say I have two test_* modules:
    #   test_python.py:
    #       __tags__ = ["guido"]
    #       class BasicTestCase(unittest.TestCase):
    #           def test_def(self):
    #           def test_class(self):
    #       class ComplexTestCase(unittest.TestCase):
    #           def test_foo(self):
    #           def test_bar(self):
    #   test_perl/__init__.py:
    #       __tags__ = ["larry", "wall"]
    #       class BasicTestCase(unittest.TestCase):
    #           def test_sub(self):
    #           def test_package(self):
    #       class EclecticTestCase(unittest.TestCase):
    #           def test_foo(self):
    #           def test_bar(self):
    # The short-form list output for this should look like:
    #   python/basic/def [guido]
    #   python/basic/class [guido]
    #   python/complex/foo [guido]
    #   python/complex/bar [guido]
    #   perl/basic/sub [larry, wall]
    #   perl/basic/package [larry, wall]
    #   perl/eclectic/foo [larry, wall]
    #   perl/eclectic/bar [larry, wall]
    log.debug("list_tests(testdir_from_ns=%r, tags=%r)",
              testdir_from_ns, tags)

    tests = list(tests_from_manifest_and_tags(testdir_from_ns, tags))
    if not tests:
        return

    WIDTH = 78
    if log.isEnabledFor(logging.INFO): # long-form
        for i, t in enumerate(tests):
            if i:
                print()
            testfile = t.testmod.__file__
            if testfile.endswith(".pyc"):
                testfile = testfile[:-1]
            print("%s:" % t.shortname())
            print("  from: %s#%s.%s" % (testfile,
                t.testcase.__class__.__name__, t.testfn_name))
            wrapped = textwrap.fill(' '.join(t.tags()), WIDTH-10)
            print("  tags: %s" % _indent(wrapped, 8, True))
            if t.doc():
                print(_indent(t.doc(), width=2))
    else:
        for t in tests:
            line = t.shortname() + ' '
            if t.explicit_tags():
                line += '[%s]' % ' '.join(t.explicit_tags())
            print(line)


#---- text test runner that can handle TestSkipped reasonably

class ConsoleTestResult(unittest.TestResult):
    """A test result class that can print formatted text results to a stream.

    Used by ConsoleTestRunner.
    """
    separator1 = '=' * 70
    separator2 = '-' * 70

    def __init__(self, stream):
        unittest.TestResult.__init__(self)
        self.skips = []
        self.stream = stream

    def getDescription(self, test):
        try:
            if test._testlib_explicit_tags_:
                return "%s [%s]" % (test._testlib_shortname_,
                                    ', '.join(test._testlib_explicit_tags_))
            else:
                return test._testlib_shortname_
        except:
            return "<Error getting description for %s>" % (test,)

    def startTest(self, test):
        unittest.TestResult.startTest(self, test)
        self.stream.write(self.getDescription(test))
        self.stream.write(" ... ")

    def addSuccess(self, test):
        unittest.TestResult.addSuccess(self, test)
        self.stream.write("ok\n")

    def addSkip(self, test, err):
        why = str(err[1])
        self.skips.append((test, why))
        self.stream.write("skipped (%s)\n" % why)

    def addError(self, test, err):
        if isinstance(err[1], TestSkipped):
            self.addSkip(test, err)
        else:
            unittest.TestResult.addError(self, test, err)
            self.stream.write("ERROR\n")

    def addFailure(self, test, err):
        unittest.TestResult.addFailure(self, test, err)
        self.stream.write("FAIL\n")

    def printSummary(self):
        self.stream.write('\n')
        self.printErrorList('ERROR', self.errors)
        self.printErrorList('FAIL', self.failures)

    def printErrorList(self, flavour, errors):
        for test, err in errors:
            self.stream.write(self.separator1 + '\n')
            self.stream.write("%s: %s\n"
                              % (flavour, self.getDescription(test)))
            self.stream.write(self.separator2 + '\n')
            self.stream.write("%s\n" % err)


class ConsoleTestRunner(object):
    """A test runner class that displays results on the console.

    It prints out the names of tests as they are run, errors as they
    occur, and a summary of the results at the end of the test run.
    
    Differences with unittest.TextTestRunner:
    - adds support for *skipped* tests (those that raise TestSkipped)
    - no verbosity option (only have equiv of verbosity=2)
    - test "short desc" is it 3-level tag name (e.g. 'foo/bar/baz' where
      that identifies: 'test_foo.py::BarTestCase.test_baz'.
    """
    def __init__(self, stream=sys.stderr):
        self.stream = stream

    def run(self, test_or_suite, test_result_class=ConsoleTestResult):
        """Run the given test case or test suite."""
        result = test_result_class(self.stream)
        start_time = time.time()
        test_or_suite.run(result)
        time_taken = time.time() - start_time

        result.printSummary()
        self.stream.write(result.separator2 + '\n')
        self.stream.write("Ran %d test%s in %.3fs\n\n"
            % (result.testsRun, result.testsRun != 1 and "s" or "",
               time_taken))
        details = []
        num_skips = len(result.skips)
        if num_skips:
            details.append("%d skip%s"
                % (num_skips, (num_skips != 1 and "s" or "")))
        if not result.wasSuccessful():
            num_failures = len(result.failures)
            if num_failures:
                details.append("%d failure%s"
                    % (num_failures, (num_failures != 1 and "s" or "")))
            num_errors = len(result.errors)
            if num_errors:
                details.append("%d error%s"
                    % (num_errors, (num_errors != 1 and "s" or "")))
            self.stream.write("FAILED (%s)\n" % ', '.join(details))
        elif details:
            self.stream.write("OK (%s)\n" % ', '.join(details))
        else:
            self.stream.write("OK\n")
        return result



#---- internal support stuff

# Recipe: indent (0.2.1)
def _indent(s, width=4, skip_first_line=False):
    """_indent(s, [width=4]) -> 's' indented by 'width' spaces

    The optional "skip_first_line" argument is a boolean (default False)
    indicating if the first line should NOT be indented.
    """
    lines = s.splitlines(1)
    indentstr = ' '*width
    if skip_first_line:
        return indentstr.join(lines)
    else:
        return indentstr + indentstr.join(lines)





#---- mainline

#TODO: pass in add_help_option=False and add it ourself here.
## Optparse's handling of the doc passed in for -h|--help handling is
## abysmal. Hence we'll stick with getopt.
#def _parse_opts(args):
#    """_parse_opts(args) -> (options, tags)"""
#    usage = "usage: %prog [OPTIONS...] [TAGS...]"
#    parser = optparse.OptionParser(prog="test", usage=usage,
#                                   description=__doc__)
#    parser.add_option("-v", "--verbose", dest="log_level",
#                      action="store_const", const=logging.DEBUG,
#                      help="more verbose output")
#    parser.add_option("-q", "--quiet", dest="log_level",
#                      action="store_const", const=logging.WARNING,
#                      help="quieter output")
#    parser.add_option("-l", "--list", dest="action",
#                      action="store_const", const="list",
#                      help="list available tests")
#    parser.set_defaults(log_level=logging.INFO, action="test")
#    opts, raw_tags = parser.parse_args()
#
#    # Trim '.py' from user-supplied tags. They might have gotten there
#    # via shell expansion.
#    ...
#
#    return opts, raw_tags

def _parse_opts(args, default_tags):
    """_parse_opts(args) -> (log_level, action, tags)"""
    opts, raw_tags = getopt.getopt(args, "hvqdlL:n",
        ["help", "verbose", "quiet", "debug", "list", "no-default-tags"])
    log_level = logging.WARN
    action = "test"
    no_default_tags = False
    for opt, optarg in opts:
        if opt in ("-h", "--help"):
            action = "help"
        elif opt in ("-v", "--verbose"):
            log_level = logging.INFO
        elif opt in ("-q", "--quiet"):
            log_level = logging.ERROR
        elif opt in ("-d", "--debug"):
            log_level = logging.DEBUG
        elif opt in ("-l", "--list"):
            action = "list"
        elif opt in ("-n", "--no-default-tags"):
            no_default_tags = True
        elif opt == "-L":
            # Optarg is of the form '<logname>:<levelname>', e.g.
            # "codeintel:DEBUG", "codeintel.db:INFO".
            # if :<levelname> is not given, default to DEBUG
            lname, llevelname = (optarg.split(':', 1) + ["DEBUG"])[:2]
            llevel = getattr(logging, llevelname)
            logging.getLogger(lname).setLevel(llevel)

    # Clean up the given tags.
    if no_default_tags:
        tags = []
    else:
        tags = default_tags
    for raw_tag in raw_tags:
        if splitext(raw_tag)[1] in (".py", ".pyc", ".pyo", ".pyw") \
           and exists(raw_tag):
            # Trim '.py' from user-supplied tags if it looks to be from
            # shell expansion.
            tags.append(splitext(raw_tag)[0])
        elif '/' in raw_tag:
            # Split one '/' to allow the shortname from the test listing
            # to be used as a filter.
            tags += raw_tag.split('/')
        else:
            tags.append(raw_tag)

    return log_level, action, tags


def harness(testdir_from_ns={None: os.curdir}, argv=sys.argv,
            setup_func=None, default_tags=None):
    """Convenience mainline for a test harness "test.py" script.

        "testdir_from_ns" (optional) is basically a set of directories in
            which to look for test cases. It is a dict with:
                <namespace>: <testdir>
            where <namespace> is a (short) string that becomes part of the
            included test names and an implicit tag for filtering those
            tests. By default the current dir is use with an empty namespace:
                {None: os.curdir}
        "setup_func" (optional) is a callable that will be called once
            before any tests are run to prepare for the test suite. It
            is not called if no tests will be run.
        "default_tags" (optional)
    
    Typically, if you have a number of test_*.py modules you can create
    a test harness, "test.py", for them that looks like this:

        #!/usr/bin/env python
        if __name__ == "__main__":
            retval = testlib.harness()
            sys.exit(retval)
    """
    if not logging.root.handlers:
        logging.basicConfig()
    try:
        log_level, action, tags = _parse_opts(argv[1:], default_tags or [])
    except getopt.error:
        _, ex, _ = sys.exc_info()
        log.error(str(ex) + " (did you need a '--' before a '-TAG' argument?)")
        return 1
    log.setLevel(log_level)

    if action == "help":
        print(__doc__)
        return 0
    if action == "list":
        return list_tests(testdir_from_ns, tags)
    elif action == "test":
        result = test(testdir_from_ns, tags, setup_func=setup_func)
        if result is None:
            return None
        return len(result.errors) + len(result.failures)
    else:
        raise TestError("unexpected action/mode: '%s'" % action)