# -*- test-case-name: twisted.test.test_trial -*- # # Twisted, the Framework of Your Internet # Copyright (C) 2001-2003 Matthew W. Lefkowitz # # This library is free software; you can redistribute it and/or # modify it under the terms of version 2.1 of the GNU Lesser General Public # License as published by the Free Software Foundation. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # import sys, time, pdb, string, types, inspect, traceback from twisted.python import reflect, failure # test results, passed as resultType to Reporter.reportResults() SKIP, EXPECTED_FAILURE, FAILURE, ERROR, UNEXPECTED_SUCCESS, SUCCESS = \ "skip", "expected failure", "failure", "error", "unexpected success", \ "success" class Reporter: """I report results from a run of a test suite. In all lists below, 'Results' are either a twisted.python.failure.Failure object, an exc_info tuple, or a string. @ivar errors: Tests which have encountered an error. @type errors: List of (testClass, method, Results) tuples. @ivar failures: Tests which have failed. @type failures: List of (testClass, method, Results) tuples. @ivar skips: Tests which have been skipped. @type skips: List of (testClass, method, Results) tuples. @ivar expectedFailures: Tests which failed but are marked as 'todo' @type expectedFailures: List of (testClass, method, Results) tuples. @ivar unexpectedSuccesses: Tests which passed but are marked as 'todo' @type unexpectedSuccesses: List of (testClass, method, Results) tuples. @ivar imports: Import errors encountered while assembling the test suite. @type imports: List of (moduleName, exception) tuples. @ivar numTests: The number of tests I have reports for. @type numTests: int @ivar expectedTests: The number of tests I expect to run. @type expectedTests: int @ivar debugger: Run the debugger when encountering a failing test. @type debugger: bool """ def __init__(self): self.errors = [] self.failures = [] self.skips = [] self.expectedFailures = [] self.unexpectedSuccesses = [] self.imports = [] self.numTests = 0 self.expectedTests = 0 self.debugger = False def start(self, expectedTests): self.expectedTests = expectedTests self.startTime = time.time() def reportImportError(self, name, exc): self.imports.append((name, exc)) def reportStart(self, testClass, method): pass def reportResults(self, testClass, method, resultType, results=None): tup = (testClass, method, results) self.numTests += 1 if resultType in (FAILURE, ERROR, EXPECTED_FAILURE): if self.debugger: if isinstance(results, failure.Failure): print "Failure, not Exception -- can't postmortem." pdb.set_trace() else: pdb.post_mortem(results[2]) if resultType == SKIP: self.skips.append(tup) elif resultType == FAILURE: self.failures.append(tup) elif resultType == EXPECTED_FAILURE: self.expectedFailures.append(tup) elif resultType == ERROR: self.errors.append(tup) elif resultType == UNEXPECTED_SUCCESS: self.unexpectedSuccesses.append(tup) elif resultType == SUCCESS: pass # SUCCESS COUNTS FOR NOTHING! else: raise ValueError, "bad value for resultType: %s" % resultType def getRunningTime(self): if hasattr(self, 'stopTime'): return self.stopTime - self.startTime else: return time.time() - self.startTime def allPassed(self): return not (self.errors or self.failures) def stop(self): self.stopTime = time.time() class MinimalReporter(Reporter): def __init__(self, fp): Reporter.__init__(self) self.fp = fp def stop(self): Reporter.stop(self) t = (self.getRunningTime(), self.expectedTests, self.numTests, len(self.imports), len(self.errors), len(self.failures), len(self.skips)) self.fp.write(' '.join(map(str,t))+'\n') class TextReporter(Reporter): SEPARATOR = '-' * 79 DOUBLE_SEPARATOR = '=' * 79 def __init__(self, stream=sys.stdout, tbformat='plain'): self.stream = stream self.tbformat = tbformat Reporter.__init__(self) def reportResults(self, testClass, method, resultType, results=None): letters = {SKIP: 'S', EXPECTED_FAILURE: 'T', FAILURE: 'F', ERROR: 'E', UNEXPECTED_SUCCESS: '!', SUCCESS: '.'} self.write(letters.get(resultType, '?')) Reporter.reportResults(self, testClass, method, resultType, results) def _formatError(self, flavor, (testClass, method, error)): if isinstance(error, failure.Failure): tb = error.getBriefTraceback() elif isinstance(error, types.TupleType): d = {'plain': traceback, 'emacs': util} tb = ''.join(d[self.tbformat].format_exception(*error)) else: tb = "%s\n" % error ret = ("%s\n%s: %s (%s)\n%s\n%s" % (self.DOUBLE_SEPARATOR, flavor, method.__name__, reflect.qual(testClass), self.SEPARATOR, tb)) return ret def _formatImportError(self, name, error): if isinstance(error, failure.Failure): what = error.getBriefTraceback() elif type(error) == types.TupleType: what = error.args[0] else: what = "%s\n" % error ret = "Could not import %s: %s\n" % (name, what) return ret def write(self, format, *args): if args: self.stream.write(format % args) else: self.stream.write(format) self.stream.flush() def writeln(self, format=None, *args): if format is not None: self.stream.write(format % args) self.stream.write('\n') self.stream.flush() def _statusReport(self): summaries = [] if self.failures: summaries.append('failures=%d' % len(self.failures)) if self.errors: summaries.append('errors=%d' % len(self.errors)) if self.skips: summaries.append('skips=%d' % len(self.skips)) if self.expectedFailures: summaries.append('expectedFailures=%d' % \ len(self.expectedFailures)) if self.unexpectedSuccesses: summaries.append('unexpectedSuccesses=%d' % \ len(self.unexpectedSuccesses)) summary = (summaries and ' ('+', '.join(summaries)+')') or '' if self.failures or self.errors: # maybe include self.unexpectedSuccesses here # do *not* include self.expectedFailures.. that's the whole point status = 'FAILED' else: status = 'OK' return '%s%s' % (status, summary) def stop(self): Reporter.stop(self) self.writeln() for error in self.skips: self.write(self._formatError('SKIPPED', error)) for error in self.expectedFailures: self.write(self._formatError('EXPECTED FAILURE', error)) for error in self.unexpectedSuccesses: self.write(self._formatError('UNEXPECTED SUCCESS', error)) for error in self.failures: self.write(self._formatError('FAILURE', error)) for error in self.errors: self.write(self._formatError('ERROR', error)) self.writeln(self.SEPARATOR) self.writeln('Ran %d tests in %.3fs', self.numTests, self.getRunningTime()) if self.imports: self.writeln() for name, error in self.imports: self.write(self._formatImportError(name, error)) self.writeln() self.writeln(self._statusReport()) class TimingTextReporter(TextReporter): def reportStart(self, testClass, method): self.testStartedAt = time.time() self.write('%s (%s) ... ', method.__name__, reflect.qual(testClass)) def reportResults(self, testClass, method, resultType, results=None): stopped = time.time() t = stopped-self.testStartedAt words = {SKIP: '[SKIPPED]', EXPECTED_FAILURE: '[TODO]', FAILURE: '[FAIL]', ERROR: '[ERROR]', UNEXPECTED_SUCCESS: '[SUCCESS!?!]', SUCCESS: '[OK]'} self.writeln(words.get(resultType, "[??]")+" "+"(%.02f secs)" % t) Reporter.reportResults(self, testClass, method, resultType, results) class VerboseTextReporter(TextReporter): def reportStart(self, testClass, method): self.write('%s (%s) ... ', method.__name__, reflect.qual(testClass)) def reportResults(self, testClass, method, resultType, results=None): words = {SKIP: '[SKIPPED]', EXPECTED_FAILURE: '[TODO]', FAILURE: '[FAIL]', ERROR: '[ERROR]', UNEXPECTED_SUCCESS: '[SUCCESS!?!]', SUCCESS: '[OK]'} self.writeln(words.get(resultType, "[??]")) Reporter.reportResults(self, testClass, method, resultType, results) class TreeReporter(TextReporter): columns = 79 BLACK = 30 RED = 31 GREEN = 32 YELLOW = 33 BLUE = 34 MAGENTA = 35 CYAN = 36 WHITE = 37 def __init__(self, stream=sys.stdout, tbformat='plain'): TextReporter.__init__(self, stream, tbformat) self.lastModule = None self.lastClass = None def reportStart(self, testClass, method): if testClass.__module__ != self.lastModule: self.writeln(testClass.__module__) self.lastModule = testClass.__module__ if testClass != self.lastClass: self.writeln(' %s' % testClass.__name__) self.lastClass = testClass docstr = inspect.getdoc(method) if docstr: # inspect trims whitespace on the left; the lstrip here is # for those odd folks who start docstrings with a blank line. what = docstr.lstrip().split('\n', 1)[0] else: what = method.__name__ self.currentLine = ' %s ... ' % (what,) self.write(self.currentLine) def color(self, text, color): return '%s%s;1m%s%s0m' % ('\x1b[', color, text, '\x1b[') def endLine(self, message, color): spaces = ' ' * (self.columns - len(self.currentLine) - len(message)) self.write(spaces) self.writeln(self.color(message, color)) def reportResults(self, testClass, method, resultType, results=None): words = {SKIP: ('[SKIPPED]', self.BLUE), EXPECTED_FAILURE: ('[TODO]', self.BLUE), FAILURE: ('[FAIL]', self.RED), ERROR: ('[ERROR]', self.RED), UNEXPECTED_SUCCESS: ('[SUCCESS!?!]', self.RED), SUCCESS: ('[OK]', self.GREEN)} text = words.get(resultType, ('[??]', self.BLUE)) self.endLine(text[0], text[1]) Reporter.reportResults(self, testClass, method, resultType, results) import util