diff --git a/NEWS b/NEWS index 6cfeb0ec..ca38f65f 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,11 @@ Changes and improvements to testtools_, grouped by release. NEXT ~~~~ +Improvements +------------ +* Add support for ``TestCase.subTest``, reporting each subtest with its own + parameters. (Cyril Roelandt, #317) + 2.9.1 ~~~~~ diff --git a/tests/test_testcase.py b/tests/test_testcase.py index e03be132..1901c4cc 100644 --- a/tests/test_testcase.py +++ b/tests/test_testcase.py @@ -51,6 +51,10 @@ LogEvent, TestResult, ) +from testtools.testresult.real import ( + ExtendedToStreamDecorator, + StreamSummary, +) from .helpers import ( AsText, @@ -2353,6 +2357,23 @@ def _run_case(self, case): case.run(result) return result + def _outcomes(self, result, case): + """The (event name, subtest label) pairs from a run. + + Labels are relative to ``case``'s own id, so that tests can assert on + the subtest description without depending on the enclosing qualname. + """ + prefix = case.id() + outcomes = [] + for event in result._events: + if event[0] in ("startTest", "stopTest"): + continue + reported = event[1].id() + if reported.startswith(prefix): + reported = reported[len(prefix) :].lstrip() + outcomes.append((event[0], reported)) + return outcomes + def test_passing_subtests(self): class Case(TestCase): def test_it(self): @@ -2360,9 +2381,12 @@ def test_it(self): with self.subTest(i=i): self.assertEqual(i % 2, 0) - result = self._run_case(Case("test_it")) - self.assertIn("addSuccess", [e[0] for e in result._events]) - self.assertEqual([], result.failures) + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addSuccess", "")], + self._outcomes(result, case), + ) def test_single_failure(self): class Case(TestCase): @@ -2371,11 +2395,12 @@ def test_it(self): with self.subTest(i=i): self.assertEqual(i % 2, 0) - result = self._run_case(Case("test_it")) - self.assertNotIn("addSuccess", [e[0] for e in result._events]) - self.assertEqual(1, len(result.failures)) - subtest = result.failures[0][0] - self.assertIn("(i=1)", str(subtest)) + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addFailure", "(i=1)")], + self._outcomes(result, case), + ) def test_multiple_failures(self): class Case(TestCase): @@ -2384,16 +2409,14 @@ def test_it(self): with self.subTest(i=i): self.assertEqual(i % 2, 0) - result = self._run_case(Case("test_it")) - self.assertEqual(2, len(result.failures)) - descriptions = [str(f[0]) for f in result.failures] - self.assertTrue( - any("(i=1)" in d for d in descriptions), - f"Expected a failure for (i=1), got {descriptions}", - ) - self.assertTrue( - any("(i=3)" in d for d in descriptions), - f"Expected a failure for (i=3), got {descriptions}", + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [ + ("addFailure", "(i=1)"), + ("addFailure", "(i=3)"), + ], + self._outcomes(result, case), ) def test_failure_continues_loop(self): @@ -2418,11 +2441,12 @@ def test_it(self): with self.subTest(msg="my label", x=42): self.fail("boom") - result = self._run_case(Case("test_it")) - self.assertEqual(1, len(result.failures)) - description = str(result.failures[0][0]) - self.assertIn("[my label]", description) - self.assertIn("(x=42)", description) + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addFailure", "[my label] (x=42)")], + self._outcomes(result, case), + ) def test_multiple_params(self): class Case(TestCase): @@ -2430,11 +2454,12 @@ def test_it(self): with self.subTest(a=1, b="two"): self.fail("boom") - result = self._run_case(Case("test_it")) - self.assertEqual(1, len(result.failures)) - description = str(result.failures[0][0]) - self.assertIn("a=1", description) - self.assertIn("b='two'", description) + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addFailure", "(a=1, b='two')")], + self._outcomes(result, case), + ) def test_no_params(self): class Case(TestCase): @@ -2442,10 +2467,12 @@ def test_it(self): with self.subTest(): self.fail("boom") - result = self._run_case(Case("test_it")) - self.assertEqual(1, len(result.failures)) - description = str(result.failures[0][0]) - self.assertIn("()", description) + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addFailure", "()")], + self._outcomes(result, case), + ) def test_nested_subtests(self): class Case(TestCase): @@ -2456,16 +2483,16 @@ def test_it(self): with self.subTest(b=b): self.assertEqual(a, b) - result = self._run_case(Case("test_it")) - self.assertEqual(4, len(result.failures)) - descriptions = [str(f[0]) for f in result.failures] - self.assertTrue( - any("a=1" in d and "b=3" in d for d in descriptions), - f"Expected a failure with both a=1 and b=3, got {descriptions}", - ) - self.assertTrue( - any("a=2" in d and "b=4" in d for d in descriptions), - f"Expected a failure with both a=2 and b=4, got {descriptions}", + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [ + ("addFailure", "(a=1, b=3)"), + ("addFailure", "(a=1, b=4)"), + ("addFailure", "(a=2, b=3)"), + ("addFailure", "(a=2, b=4)"), + ], + self._outcomes(result, case), ) def test_skip_inside_subtest(self): @@ -2482,9 +2509,186 @@ def test_it(self): self.reached.append(i) case = Case("test_it") - self._run_case(case) + result = self._run_case(case) self.assertEqual([0, 2], case.reached) - self.assertEqual(1, len(case._subtest_skips)) + # The skipped subtest is reported on its own; the parent is not + # additionally reported as a success. + self.assertEqual( + [("addSkip", "(i=1)")], + self._outcomes(result, case), + ) + + def test_skip_uses_custom_skip_exception(self): + class CustomSkip(Exception): + pass + + class Case(TestCase): + skipException = CustomSkip + + def test_it(self): + with self.subTest(i=1): + self.skipTest("skip this one") + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addSkip", "(i=1)")], + self._outcomes(result, case), + ) + + def test_error_is_reported_as_error(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + raise ValueError("boom") + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addError", "(i=1)")], + self._outcomes(result, case), + ) + + def test_expected_failure_inside_subtest(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + self.expectFailure("known bug", self.assertEqual, 1, 2) + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addExpectedFailure", "(i=1)")], + self._outcomes(result, case), + ) + + def test_details_are_reported_with_the_subtest(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + self.addDetail("clue", text_content("important diagnostic")) + self.fail("boom") + + case = Case("test_it") + result = self._run_case(case) + [(_, _, details)] = [e for e in result._events if e[0] == "addFailure"] + self.assertEqual("important diagnostic", details["clue"].as_text()) + + def test_details_do_not_leak_to_the_parent(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + self.addDetail("clue", text_content("subtest only")) + self.fail("boom") + self.assertNotIn("clue", self.getDetails()) + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addFailure", "(i=1)")], + self._outcomes(result, case), + ) + + def test_multiple_exceptions_are_unpacked(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + raise MultipleExceptions( + (ValueError, ValueError("one"), None), + (KeyError, KeyError("two"), None), + ) + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [ + ("addError", "(i=1)"), + ("addError", "(i=1)"), + ], + self._outcomes(result, case), + ) + + def test_failfast_stops_at_first_failing_subtest(self): + class Case(TestCase): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.iterations = [] + + def test_it(self): + for i in range(4): + with self.subTest(i=i): + self.iterations.append(i) + self.fail("boom") + + case = Case("test_it") + result = ExtendedTestResult() + result.failfast = True + case.run(result) + self.assertEqual([0], case.iterations) + self.assertEqual( + [("addFailure", "(i=0)")], + self._outcomes(result, case), + ) + + def test_keyboard_interrupt_propagates(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + raise KeyboardInterrupt + + self.assertRaises(KeyboardInterrupt, Case("test_it").run, ExtendedTestResult()) + + def test_subtest_is_not_runnable(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + self.fail("boom") + + case = Case("test_it") + result = self._run_case(case) + [(_, subtest, _)] = [e for e in result._events if e[0] == "addFailure"] + self.assertRaises(NotImplementedError, subtest.runTest) + + def test_stream_result_counts_one_test(self): + # A subtest outcome is attached to the parent test rather than + # opening a stream record of its own, as unittest does. + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + self.fail("boom") + + summary = StreamSummary() + decorator = ExtendedToStreamDecorator(summary) + decorator.startTestRun() + Case("test_it").run(decorator) + decorator.stopTestRun() + self.assertEqual(1, summary.testsRun) + self.assertEqual(False, summary.wasSuccessful()) + + def test_stream_result_counts_one_test_for_skip(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + self.skipTest("skip this one") + + summary = StreamSummary() + decorator = ExtendedToStreamDecorator(summary) + decorator.startTestRun() + Case("test_it").run(decorator) + decorator.stopTestRun() + self.assertEqual(1, summary.testsRun) + self.assertEqual(True, summary.wasSuccessful()) + + def test_str_matches_unittest_format(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + self.fail("boom") + + case = Case("test_it") + result = self._run_case(case) + [(_, subtest, _)] = [e for e in result._events if e[0] == "addFailure"] + self.assertEqual(f"{case} (i=1)", str(subtest)) def test_suite(): diff --git a/testtools/runtest.py b/testtools/runtest.py index 1d62a010..6012f305 100644 --- a/testtools/runtest.py +++ b/testtools/runtest.py @@ -125,6 +125,8 @@ def _run_prepared_result(self, result: TestResult) -> TestResult: self.case.__testtools_tb_locals__ = getattr( # type: ignore[attr-defined] result, "tb_locals", False ) + # Let subTest report each subtest against this result as it runs. + self.case._subtest_result = result self._run_core() if self._exceptions: # One or more caught exceptions, now trigger the test's @@ -138,6 +140,7 @@ def _run_prepared_result(self, result: TestResult) -> TestResult: self.last_resort(self.case, self.result, e) raise e finally: + self.case._subtest_result = None result.stopTest(self.case) return result @@ -184,16 +187,11 @@ def _run_core(self) -> None: if getattr(self.case, "force_failure", None): self._run_user(_raise_force_fail_error) failed = True - for subtest, err in getattr(self.case, "_subtest_failures", ()): - add_subtest = getattr(self.result, "addSubTest", None) - if add_subtest is not None: - add_subtest(self.case, subtest, err) - else: - self.result.addFailure(self.case, err) + if getattr(self.case, "_subtest_failed", False): failed = True - for subtest, reason in getattr(self.case, "_subtest_skips", ()): - self.result.addSkip(subtest, reason=reason) - if not failed: + if not failed and not getattr( + self.case, "_subtest_reported", False + ): self.result.addSuccess( self.case, details=self.case.getDetails() ) diff --git a/testtools/testcase.py b/testtools/testcase.py index c1603e9d..9df9ab2a 100644 --- a/testtools/testcase.py +++ b/testtools/testcase.py @@ -100,14 +100,21 @@ class _ExpectedFailure(Exception): """ +class _SubTestAborted(Exception): + """Raised to abort a test whose result asked us to stop.""" + + _subtest_msg_sentinel = object() -class SubTest(unittest.TestCase): +class _SubTest(unittest.TestCase): """Describes a single subTest iteration for failure reporting. Carries the message and parameters passed to ``subTest`` so that result objects can label each failure with its subTest context. + + Like ``unittest.case._SubTest``, this is a description rather than a + runnable test: it has no test method and must not be run. """ def __init__( @@ -119,6 +126,9 @@ def __init__( self._msg = msg self._params = params + def runTest(self) -> NoReturn: + raise NotImplementedError("_SubTest describes a subtest; it cannot be run") + def _subDescription(self) -> str: parts: list[str] = [] if self._msg is not _subtest_msg_sentinel: @@ -135,7 +145,7 @@ def shortDescription(self) -> str | None: return self.test_case.shortDescription() def __str__(self) -> str: - return self.id() + return f"{self.test_case} {self._subDescription()}" # TypeVar for decorators @@ -387,10 +397,12 @@ def __init__(self, *args: object, **kwargs: object) -> None: # data from exceptions. self.__exception_handlers: list[Callable[[OptExcInfo], None]] = [] # Passed to RunTest to map exceptions to result actions + # Handlers accept an optional details override, used to report a + # subtest against its own _SubTest rather than the parent test. self.exception_handlers: list[ tuple[ type[BaseException], - Callable[[TestCase, TestResult, BaseException], None], + Callable[..., None], ] ] = [ (self.skipException, self._report_skip), @@ -417,9 +429,12 @@ def _reset(self) -> None: # force_failure is set by expectThat() on mismatch; must be # cleared so re-runs of the same test can succeed. self.force_failure: bool | None = None - self._subtest_failures: list[tuple[SubTest, ExcInfo]] = [] - self._subtest_skips: list[tuple[SubTest, str]] = [] self._subtest_params: dict[str, Any] = {} + # Set by RunTest for the duration of the run, so that subTest can + # report each subtest as it completes. + self._subtest_result: TestResult | None = None + self._subtest_failed: bool = False + self._subtest_reported: bool = False def __eq__(self, other: object) -> bool: eq = getattr(unittest.TestCase, "__eq__", None) @@ -886,29 +901,63 @@ def onException(self, exc_info: OptExcInfo, tb_label: str = "traceback") -> None handler(exc_info) @staticmethod - def _report_error(self: "TestCase", result: TestResult, err: BaseException) -> None: - result.addError(self, details=self.getDetails()) + def _report_error( + self: "TestCase", + result: TestResult, + err: BaseException, + details: DetailsDict | None = None, + ) -> None: + result.addError(self, details=TestCase._report_details(self, details)) @staticmethod def _report_expected_failure( - self: "TestCase", result: TestResult, err: BaseException + self: "TestCase", + result: TestResult, + err: BaseException, + details: DetailsDict | None = None, ) -> None: - result.addExpectedFailure(self, details=self.getDetails()) + result.addExpectedFailure(self, details=TestCase._report_details(self, details)) @staticmethod def _report_failure( - self: "TestCase", result: TestResult, err: BaseException + self: "TestCase", + result: TestResult, + err: BaseException, + details: DetailsDict | None = None, ) -> None: - result.addFailure(self, details=self.getDetails()) + result.addFailure(self, details=TestCase._report_details(self, details)) @staticmethod - def _report_skip(self: "TestCase", result: TestResult, err: BaseException) -> None: + def _report_skip( + self: "TestCase", + result: TestResult, + err: BaseException, + details: DetailsDict | None = None, + ) -> None: if err.args: reason = err.args[0] else: reason = "no reason given." - self._add_reason(reason) - result.addSkip(self, details=self.getDetails()) + if details is None: + self._add_reason(reason) + result.addSkip(self, details=self.getDetails()) + else: + details = dict(details) + details["reason"] = content.text_content(reason) + result.addSkip(self, details=details) + + @staticmethod + def _report_details( + self: unittest.TestCase, details: DetailsDict | None + ) -> DetailsDict: + """Details to report an outcome with. + + Subtests supply their own details; everything else reports the + details accumulated on the test itself. + """ + if details is None: + return cast("TestCase", self).getDetails() + return dict(details) def _report_traceback( self, exc_info: OptExcInfo, tb_label: str = "traceback" @@ -933,27 +982,85 @@ def _report_traceback( def subTest( self, msg: object = _subtest_msg_sentinel, **params: Any ) -> Iterator[None]: - """Return a context manager for a subTest.""" + """Return a context manager that reports a subtest independently. + + Exceptions raised inside the block are reported against a + ``_SubTest`` describing ``msg`` and ``params``, and do not abort the + rest of the test method. + """ merged_params = {**self._subtest_params, **params} - subtest = SubTest(self, msg, merged_params) + subtest = _SubTest(self, msg, merged_params) old_params, self._subtest_params = self._subtest_params, merged_params + # Details added inside the block belong to this subtest, not to the + # parent test, so report and then discard whatever it accumulated. + outer_details = dict(self.getDetails()) try: yield - except SkipTest as e: - reason = str(e) - self._subtest_skips.append((subtest, reason)) - except Exception: - # Inside except block, exc_info() is guaranteed to have non-None values - exc_info = sys.exc_info() - self._subtest_failures.append((subtest, exc_info)) # type: ignore[arg-type] + except BaseException: + # Inside except block, exc_info() is guaranteed to have non-None + # values. + exc_info = cast(ExcInfo, sys.exc_info()) + if not self._report_subtest(subtest, exc_info, outer_details): + raise finally: self._subtest_params = old_params + def _report_subtest( + self, subtest: "_SubTest", exc_info: ExcInfo, outer_details: DetailsDict + ) -> bool: + """Report the outcome of a failed subtest. + + :return: True if the exception was reported, False if it should + propagate (there is nothing to report to, or it is not an + exception this test knows how to handle). + """ + result = self._subtest_result + if result is None: + return False + if exc_info[0] is MultipleExceptions: + for sub_exc_info in exc_info[1].args: + self._report_subtest(subtest, sub_exc_info, outer_details) + return True + for exc_class, handler in self.exception_handlers: + if isinstance(exc_info[1], exc_class): + break + else: + return False + details = self.getDetails() + try: + self.onException(exc_info, tb_label="traceback") + subtest_details = { + name: detail + for name, detail in details.items() + if name not in outer_details + } + handler(subtest, result, exc_info[1], subtest_details) + # A reported subtest outcome stands on its own; the parent must + # not also be reported as a success. + self._subtest_reported = True + if not isinstance(exc_info[1], self.skipException): + self._subtest_failed = True + # Match unittest: stop at the first failing subtest when the + # result asks us to. + if getattr(result, "shouldStop", False): + raise _SubTestAborted from None + finally: + # Leave the parent's details as they were, so a subtest failure + # is not re-reported as part of the parent test. + details.clear() + details.update(outer_details) + return True + @staticmethod def _report_unexpected_success( - self: "TestCase", result: TestResult, err: BaseException + self: "TestCase", + result: TestResult, + err: BaseException, + details: DetailsDict | None = None, ) -> None: - result.addUnexpectedSuccess(self, details=self.getDetails()) + result.addUnexpectedSuccess( + self, details=TestCase._report_details(self, details) + ) def run(self, result: TestResult | None = None) -> TestResult: # type: ignore[override] self._reset() @@ -1061,7 +1168,11 @@ def _run_test_method(self, result: TestResult) -> object: :param result: A testtools.TestResult to report activity to. :return: None. """ - return self._get_test_method()() + try: + return self._get_test_method()() + except _SubTestAborted: + # A subtest already reported the failure that stopped the run. + return None def useFixture(self, fixture: "FixtureT") -> "FixtureT": """Use fixture in a test case. diff --git a/testtools/testresult/doubles.py b/testtools/testresult/doubles.py index 7ce1ee26..d024ac3f 100644 --- a/testtools/testresult/doubles.py +++ b/testtools/testresult/doubles.py @@ -69,6 +69,12 @@ ] | tuple[Literal["addUnexpectedSuccess"], unittest.TestCase] | tuple[Literal["addUnexpectedSuccess"], unittest.TestCase, dict[str, object]] + | tuple[ + Literal["addSubTest"], + unittest.TestCase, + unittest.TestCase, + tuple[type, Exception, object] | object | None, + ] | tuple[Literal["addDuration"], unittest.TestCase, float] | tuple[Literal["progress"], int, int] | tuple[Literal["tags"], Iterable[str], Iterable[str]] @@ -124,6 +130,15 @@ def addUnexpectedSuccess(self, test: unittest.TestCase) -> None: if self.failfast: self.stop() + def addSubTest( + self, + test: unittest.TestCase, + subtest: unittest.TestCase, + err: OptExcInfo | None, + ) -> None: + super().addSubTest(test, subtest, err) # type: ignore[arg-type] + self._events.append(("addSubTest", test, subtest, err)) + def addDuration(self, test: unittest.TestCase, duration: float) -> None: self._events.append(("addDuration", test, duration)) self.collectedDurations.append((test, duration)) diff --git a/testtools/testresult/real.py b/testtools/testresult/real.py index bb887cac..780d7d24 100644 --- a/testtools/testresult/real.py +++ b/testtools/testresult/real.py @@ -2301,14 +2301,30 @@ def _convert( status: str, reason: str | None = None, ) -> None: + # Imported here to avoid a circular import at module load. + from testtools.testcase import _SubTest + if not self._started: self.startTestRun() - test_id = test.id() now = self._now() if err is not None: if details is None: details = {} details["traceback"] = TracebackContent(err, test) + if isinstance(test, _SubTest): + # A subtest outcome belongs to the test that contains it: attach + # its details to the parent so that the run counts one test, as + # unittest does. A skip or expected failure still needs a status + # of its own, since the parent reports nothing in that case. + test_id = test.test_case.id() + label = test._subDescription() + details = { + f"{name} {label}": detail for name, detail in (details or {}).items() + } + if status not in ("skip", "xfail"): + status = None # type: ignore[assignment] + else: + test_id = test.id() if details is not None: for name, content in details.items(): mime_type = repr(content.content_type) @@ -2342,12 +2358,13 @@ def _convert( test_id=test_id, timestamp=now, ) - self.status( - test_id=test_id, - test_status=status, - test_tags=self.current_tags, - timestamp=now, - ) + if status is not None: + self.status( + test_id=test_id, + test_status=status, + test_tags=self.current_tags, + timestamp=now, + ) def addExpectedFailure( self, diff --git a/testtools/twistedsupport/_runtest.py b/testtools/twistedsupport/_runtest.py index fca42e26..e6e48589 100644 --- a/testtools/twistedsupport/_runtest.py +++ b/testtools/twistedsupport/_runtest.py @@ -455,7 +455,10 @@ def _run_core(self): successful = False self._log_user_exception(UncleanReactorError(junk)) - if successful: + if getattr(self.case, "_subtest_failed", False): + successful = False + + if successful and not getattr(self.case, "_subtest_reported", False): self.result.addSuccess(self.case, details=self.case.getDetails()) def _run_user(self, function, *args):