How to fix verbose unittest output with subtests? - python-unittest

Question

When using unittest with subtests and verbose=2, the summary of the failed tests at the top are missing 'FAIL\n'.
for example, this test (in Python 3.7):
import unittest
class TestThing(unittest.TestCase):
def test_thing1(self):
for i in range(10):
with self.subTest(i=i):
self.assertLess(i, 10)
def test_thing2(self):
for i in range(10):
with self.subTest(i=i):
self.assertLess(i, 9)
def test_thing3(self):
for i in range(10):
with self.subTest(i=i):
self.assertLess(i, 10)
if __name__ == '__main__':
unittest.main(verbosity=2)
Results in this output:
>python .\blah.py
test_thing1 (__main__.TestThing) ... ok
test_thing2 (__main__.TestThing) ... test_thing3 (__main__.TestThing) ... ok
======================================================================
FAIL: test_thing2 (__main__.TestThing) (i=9)
----------------------------------------------------------------------
Traceback (most recent call last):
File ".\blah.py", line 16, in test_thing2
self.assertLess(i, 9)
AssertionError: 9 not less than 9
----------------------------------------------------------------------
Ran 3 tests in 0.003s
FAILED (failures=1)
If I remove the subtest, then the output becomes:
test_thing1 (__main__.TestThing) ... ok
test_thing2 (__main__.TestThing) ... FAIL
test_thing3 (__main__.TestThing) ... ok
======================================================================
FAIL: test_thing2 (__main__.TestThing)
----------------------------------------------------------------------
Traceback (most recent call last):
File ".\blah.py", line 16, in test_thing2
self.assertLess(i, 9)
AssertionError: 9 not less than 9
----------------------------------------------------------------------
Ran 3 tests in 0.002s
FAILED (failures=1)
I've tried using my own resultclass but it appears that the addFailure doesn't get called for subtests (like it does for regular tests).
class MyTextTestResult(unittest.TextTestResult):
def addFailure(self, test, err):
print('adding a failure')
super().addFailure(test, err)
if self.showAll:
self.stream.writeln("FAIL")
elif self.dots:
self.stream.write('F')
self.stream.flush()
def addSuccess(self, test):
print('adding a success')
super().addSuccess(test)
if self.showAll:
self.stream.writeln("ok")
elif self.dots:
self.stream.write('.')
self.stream.flush()
Does anyone know how to fix the output for subtest results (using verbose=2)?

Answer 1

Looks like if I change the addSubTest in TextTestResult, that does it.
class MyTextTestResult(unittest.TextTestResult):
def addSubTest(self, test, subtest, err):
super().addSubTest(test, subtest, err)
if err is not None:
if not self.stream.getvalue().strip().endswith('FAIL'):
self.stream.writeln("FAIL")
if __name__ == '__main__':
# unittest.main(verbosity=2)
runner = unittest.TextTestRunner(None, resultclass=MyTextTestResult, verbosity=2)
suite = unittest.TestSuite()
suite.addTest(TestThing('test_thing1'))
suite.addTest(TestThing('test_thing2'))
suite.addTest(TestThing('test_thing3'))
runner.run(suite)
test_thing1 (__main__.TestThing) ... ok
test_thing2 (__main__.TestThing) ... FAIL
test_thing3 (__main__.TestThing) ... ok
======================================================================
FAIL: test_thing2 (__main__.TestThing) (i=9)
----------------------------------------------------------------------
Traceback (most recent call last):
File ".\blah.py", line 22, in test_thing2
self.assertLess(i, 9)
AssertionError: 9 not less than 9
----------------------------------------------------------------------
Ran 3 tests in 0.004s
FAILED (failures=2)

Answer 2

It seems that the TextTestRunner doesn't print anything (including the newline) for tests unless all subtests succeed.
Instead of not printing anything, the following TestResult will cause the TextTestRunner to print "some subtests weren't successful" (including a newline):
class MyTestResult(unittest.TextTestResult):
def startTest(self, *args):
self.current_test_has_subtests = False
super().startTest(*args)
def addSubTest(self, *args):
super().addSubTest(*args)
self.current_test_has_subtests = True
def stopTest(self, test):
super().stopTest(test)
if self.current_test_has_subtests and not self.wasSuccessful():
print("some subtests weren't successful")
To use this, change the testRunner in the unittest.main call. For example:
unittest.main(
verbosity=2,
testRunner=unittest.TextTestRunner(verbosity=2, resultclass=MyTestRunner)
)