TestSuite.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

# TestSuite.py
# 2009 Oct 13 . ccr

"""Exercise PythonTidy over the Python Test Suite.

"""

from __future__ import division
import os
import subprocess
import re
import tarfile
import PythonTidy

ZERO = 0
SPACE = ' '
NULL = ''
NUL = '\x00'
NA = -1

PATH_ARCHIVE = '/var/cache/apt/archives/Python-2.5.2.tar.bz2'
PATTERN_MEMBER = 'Python-2.5.2/Lib/test/'
PATH_TARGET = '/home/crhode/lab'
PATH_TESTS = os.path.join(PATH_TARGET, PATTERN_MEMBER[:-1])
PATH_TEST_PACKAGE = os.path.split(PATH_TESTS)[ZERO]
PATH_REGRTEST_STDOUT = '/tmp/regrtest.txt'
PAT_PASS_FAIL = re.compile(r'^\d+\s+tests?\s+((?:failed)|(?:skipped)):\s+(.*)$', re.MULTILINE)
PAT_STATE = re.compile(r'^\s*(?:#\s*([^:]*):)?\s*(.*)\s*$')

SENTINEL = {
    'run': None,
    'u_skip': 'User skipped',        
    'a_skip': 'Failed after tidying',
    'b_skip': 'Failed before tidying',
    'r_fail': 'Regression test failed', 
    'r_skip': 'Regression test skipped',
    }

STATE = dict(
    (value, key)
    for (key, value) in SENTINEL.iteritems()
    )

class Archive(object):

    def __init__(self, path):
        self.name = path
        self.tar = tarfile.open(self.name)
        return

    def get_members(self):
        return self.tar.getmembers()

    def extract(self, pattern, path):
        targets = [
            member
            for member in self.get_members()
            if member.name.startswith(pattern)
            ]
        return self.tar.extractall(path, targets)


class Test(object):

    def __init__(self, name, state='run'):
        self.name = name
        self.set_state(state)
        return

    def __str__(self):
        sentinel = SENTINEL[self.state]
        if sentinel is None:
            result = '  %s' % self.name
        else:
            result = '# %s:  %s' % (sentinel, self.name)
        return result

    def set_state(self, state):
        self.state = state
        return self

    def is_run(self):
        return self.state in ['run']


def get_test(line):
    match = PAT_STATE.match(line)
    (sentinel, name) = match.group(1,2)
    if sentinel is None:
        if line.startswith('#'):
            result = Test(name=name, state='u_skip')
        else:
            result = Test(name=name, state='run')
    else:
        sentinel = sentinel.capitalize()
        result = Test(name=name, state=STATE[sentinel])
    return result


class Repertoire(list):

    def __init__(self):
        list.__init__(self)
        self.file_name = os.path.join(PATH_TESTS, 'repertoire.dat')
        return

    def get_from_dir(self):
        for test in os.listdir(PATH_TESTS):
            if test.startswith('test_') and test.endswith('.py'):
                test = Test(test[:-3])
                self.append(test)
        return self

    def load(self):
        del self[:]
        flob = open(self.file_name, 'r')
        for line in flob:
            test = get_test(line[:-1])
            self.append(test)
        flob.close()
        return self

    def save(self):
        flob = open(self.file_name, 'w')
        for test in self:
            flob.write('%s\n' % str(test))
        flob.close()
        return self

    def spike(self):

        """On my system these tests fail before tidying:

        """
        
        for test in self:
            if test.name in [
                'test_imageop',
                'test_pyclbr',
                'test_sys',
                ]:
                test.set_state('b_skip')
        return self

    def omit(self):

        """These tests fail after tidying:
        
        """

        for test in self:
            if test.name in [
                'test_grammar',

                                # *test_grammar* exposes bug 6978 in
                                # the *compiler* module.  Tuples are
                                # immutable and hashable and thus
                                # suitable as dict indices.  Whereas a
                                # singleton tuple literal (x,) is
                                # valid as an index, the *compiler*
                                # module parses it as x when it
                                # appears.

                'test_dis',

                                # *test_dis* compares "disassembled"
                                # Python byte code to what is
                                # expected.  While byte code for a
                                # tidied script should be functionally
                                # equivalent to the untidied version,
                                # it will not necessarily be
                                # identical.
                
                'test_trace',

                                # *test_trace* compares the line
                                # numbers in a functional trace of a
                                # running script with those expected.
                                # A statement in a tidied script will
                                # generally have a line number
                                # slightly different from the same
                                # statement in the untidied version.

                'test_doctest',

                                # *test_doctest* is an extensive suite
                                # of tests of the *doctest* module,
                                # which itself is used to document
                                # test code within docstrings and at
                                # need to compare instant results
                                # against those expected.  One of the
                                # tests in *test_doctest* appears to
                                # require line numbers consistent with
                                # expectations, but tidied scripts
                                # generally violate such conditions as
                                # explained above.
                
                ]:
                test.set_state('a_skip')
        return self

    def log_results(self):
        flob = open(PATH_REGRTEST_STDOUT, 'r')
        _buffer = flob.read()
        flob.close()
        _buffer = _buffer.replace('\n    ', SPACE)
        result = PAT_PASS_FAIL.split(_buffer)
        while result:
            sentinel = result.pop(ZERO)
            if sentinel in ['failed', 'skipped']:
                if result:
                    tests = result.pop(ZERO)
                    tests = tests.split()
                    for test in tests:
                        self.log_result(sentinel, test)
        return self

    def log_result(self, sentinel, target):
        for test in self:
            if test.name == target:
                if sentinel in ['failed']:
                    test.set_state('r_fail')
                elif sentinel in ['skipped']:
                    test.set_state('r_skip')
                break
        return self

    def tidy_all(self):
        for test in self:
            if test.is_run():
                self.tidy(test.name)
        return self

    def tidy(self, name):
        test_name = os.path.join(PATH_TESTS, '%s.py' % name)
        back_name = os.path.join(PATH_TESTS, '%s.bak' % name)
        if os.path.exists(back_name):
            pass
        else:
            os.rename(test_name, back_name)
            print 'Converting', name
            PythonTidy.tidy_up(file_in=back_name, file_out=test_name)
        return self

    def untidy_all(self):
        for test in self:
            self.untidy(test.name)
        return self
        
    def untidy(self, name):
        test_name = os.path.join(PATH_TESTS, '%s.py' % name)
        back_name = os.path.join(PATH_TESTS, '%s.bak' % name)
        if os.path.exists(back_name):
            os.rename(back_name, test_name)
        return self


class Run(subprocess.Popen):

    def __init__(self, repertoire):
        results = open(PATH_REGRTEST_STDOUT, 'w')
        env = os.environ
        env.setdefault('PYTHONPATH', PATH_TEST_PACKAGE)
        subprocess.Popen.__init__(
            self,
            '%s/regrtest.py -f %s' % (PATH_TESTS, repertoire.file_name),
            stdout=results,
            env=env,
            shell=True,
            )
        results.close()
        return
        

def main_line():
    Archive(PATH_ARCHIVE).extract(PATTERN_MEMBER, PATH_TARGET)
    repertoire = Repertoire()
    repertoire.get_from_dir()
    repertoire.spike()
    repertoire.save()
    repertoire.load()
    repertoire.omit()
    repertoire.untidy_all()
    repertoire.tidy_all()
    process = Run(repertoire)
    retcd = process.wait()
    repertoire.log_results()
    repertoire.save()
    return


if __name__ == "__main__":
    main_line()


# Fin