diff --git a/NEWS b/NEWS index 6cfeb0ec..ca38f65f 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,11 @@ Changes and improvements to testtools_, grouped by release. NEXT ~~~~ +Improvements +------------ +* Add support for ``TestCase.subTest``, reporting each subtest with its own + parameters. (Cyril Roelandt, #317) + 2.9.1 ~~~~~ diff --git a/tests/test_testcase.py b/tests/test_testcase.py index e7f0bcdc..1901c4cc 100644 --- a/tests/test_testcase.py +++ b/tests/test_testcase.py @@ -51,6 +51,10 @@ LogEvent, TestResult, ) +from testtools.testresult.real import ( + ExtendedToStreamDecorator, + StreamSummary, +) from .helpers import ( AsText, @@ -2343,6 +2347,350 @@ def test_other_attribute(self): self.assertRaises(AttributeError, getattr, orig, "thing") +class TestSubTest(TestCase): + """Tests for testtools.TestCase.subTest support.""" + + run_test_with = FullStackRunTest + + def _run_case(self, case): + result = ExtendedTestResult() + case.run(result) + return result + + def _outcomes(self, result, case): + """The (event name, subtest label) pairs from a run. + + Labels are relative to ``case``'s own id, so that tests can assert on + the subtest description without depending on the enclosing qualname. + """ + prefix = case.id() + outcomes = [] + for event in result._events: + if event[0] in ("startTest", "stopTest"): + continue + reported = event[1].id() + if reported.startswith(prefix): + reported = reported[len(prefix) :].lstrip() + outcomes.append((event[0], reported)) + return outcomes + + def test_passing_subtests(self): + class Case(TestCase): + def test_it(self): + for i in (0, 2, 4): + with self.subTest(i=i): + self.assertEqual(i % 2, 0) + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addSuccess", "")], + self._outcomes(result, case), + ) + + def test_single_failure(self): + class Case(TestCase): + def test_it(self): + for i in (0, 1, 2): + with self.subTest(i=i): + self.assertEqual(i % 2, 0) + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addFailure", "(i=1)")], + self._outcomes(result, case), + ) + + def test_multiple_failures(self): + class Case(TestCase): + def test_it(self): + for i in range(4): + with self.subTest(i=i): + self.assertEqual(i % 2, 0) + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [ + ("addFailure", "(i=1)"), + ("addFailure", "(i=3)"), + ], + self._outcomes(result, case), + ) + + def test_failure_continues_loop(self): + class Case(TestCase): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.iterations = [] + + def test_it(self): + for i in range(4): + with self.subTest(i=i): + self.iterations.append(i) + self.assertEqual(i % 2, 0) + + case = Case("test_it") + self._run_case(case) + self.assertEqual([0, 1, 2, 3], case.iterations) + + def test_with_msg(self): + class Case(TestCase): + def test_it(self): + with self.subTest(msg="my label", x=42): + self.fail("boom") + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addFailure", "[my label] (x=42)")], + self._outcomes(result, case), + ) + + def test_multiple_params(self): + class Case(TestCase): + def test_it(self): + with self.subTest(a=1, b="two"): + self.fail("boom") + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addFailure", "(a=1, b='two')")], + self._outcomes(result, case), + ) + + def test_no_params(self): + class Case(TestCase): + def test_it(self): + with self.subTest(): + self.fail("boom") + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addFailure", "()")], + self._outcomes(result, case), + ) + + def test_nested_subtests(self): + class Case(TestCase): + def test_it(self): + for a in (1, 2): + with self.subTest(a=a): + for b in (3, 4): + with self.subTest(b=b): + self.assertEqual(a, b) + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [ + ("addFailure", "(a=1, b=3)"), + ("addFailure", "(a=1, b=4)"), + ("addFailure", "(a=2, b=3)"), + ("addFailure", "(a=2, b=4)"), + ], + self._outcomes(result, case), + ) + + def test_skip_inside_subtest(self): + class Case(TestCase): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.reached = [] + + def test_it(self): + for i in range(3): + with self.subTest(i=i): + if i == 1: + self.skipTest("skip this one") + self.reached.append(i) + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual([0, 2], case.reached) + # The skipped subtest is reported on its own; the parent is not + # additionally reported as a success. + self.assertEqual( + [("addSkip", "(i=1)")], + self._outcomes(result, case), + ) + + def test_skip_uses_custom_skip_exception(self): + class CustomSkip(Exception): + pass + + class Case(TestCase): + skipException = CustomSkip + + def test_it(self): + with self.subTest(i=1): + self.skipTest("skip this one") + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addSkip", "(i=1)")], + self._outcomes(result, case), + ) + + def test_error_is_reported_as_error(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + raise ValueError("boom") + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addError", "(i=1)")], + self._outcomes(result, case), + ) + + def test_expected_failure_inside_subtest(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + self.expectFailure("known bug", self.assertEqual, 1, 2) + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addExpectedFailure", "(i=1)")], + self._outcomes(result, case), + ) + + def test_details_are_reported_with_the_subtest(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + self.addDetail("clue", text_content("important diagnostic")) + self.fail("boom") + + case = Case("test_it") + result = self._run_case(case) + [(_, _, details)] = [e for e in result._events if e[0] == "addFailure"] + self.assertEqual("important diagnostic", details["clue"].as_text()) + + def test_details_do_not_leak_to_the_parent(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + self.addDetail("clue", text_content("subtest only")) + self.fail("boom") + self.assertNotIn("clue", self.getDetails()) + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [("addFailure", "(i=1)")], + self._outcomes(result, case), + ) + + def test_multiple_exceptions_are_unpacked(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + raise MultipleExceptions( + (ValueError, ValueError("one"), None), + (KeyError, KeyError("two"), None), + ) + + case = Case("test_it") + result = self._run_case(case) + self.assertEqual( + [ + ("addError", "(i=1)"), + ("addError", "(i=1)"), + ], + self._outcomes(result, case), + ) + + def test_failfast_stops_at_first_failing_subtest(self): + class Case(TestCase): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.iterations = [] + + def test_it(self): + for i in range(4): + with self.subTest(i=i): + self.iterations.append(i) + self.fail("boom") + + case = Case("test_it") + result = ExtendedTestResult() + result.failfast = True + case.run(result) + self.assertEqual([0], case.iterations) + self.assertEqual( + [("addFailure", "(i=0)")], + self._outcomes(result, case), + ) + + def test_keyboard_interrupt_propagates(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + raise KeyboardInterrupt + + self.assertRaises(KeyboardInterrupt, Case("test_it").run, ExtendedTestResult()) + + def test_subtest_is_not_runnable(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + self.fail("boom") + + case = Case("test_it") + result = self._run_case(case) + [(_, subtest, _)] = [e for e in result._events if e[0] == "addFailure"] + self.assertRaises(NotImplementedError, subtest.runTest) + + def test_stream_result_counts_one_test(self): + # A subtest outcome is attached to the parent test rather than + # opening a stream record of its own, as unittest does. + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + self.fail("boom") + + summary = StreamSummary() + decorator = ExtendedToStreamDecorator(summary) + decorator.startTestRun() + Case("test_it").run(decorator) + decorator.stopTestRun() + self.assertEqual(1, summary.testsRun) + self.assertEqual(False, summary.wasSuccessful()) + + def test_stream_result_counts_one_test_for_skip(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + self.skipTest("skip this one") + + summary = StreamSummary() + decorator = ExtendedToStreamDecorator(summary) + decorator.startTestRun() + Case("test_it").run(decorator) + decorator.stopTestRun() + self.assertEqual(1, summary.testsRun) + self.assertEqual(True, summary.wasSuccessful()) + + def test_str_matches_unittest_format(self): + class Case(TestCase): + def test_it(self): + with self.subTest(i=1): + self.fail("boom") + + case = Case("test_it") + result = self._run_case(case) + [(_, subtest, _)] = [e for e in result._events if e[0] == "addFailure"] + self.assertEqual(f"{case} (i=1)", str(subtest)) + + def test_suite(): from unittest import TestLoader diff --git a/testtools/runtest.py b/testtools/runtest.py index f4ef9648..6012f305 100644 --- a/testtools/runtest.py +++ b/testtools/runtest.py @@ -125,6 +125,8 @@ def _run_prepared_result(self, result: TestResult) -> TestResult: self.case.__testtools_tb_locals__ = getattr( # type: ignore[attr-defined] result, "tb_locals", False ) + # Let subTest report each subtest against this result as it runs. + self.case._subtest_result = result self._run_core() if self._exceptions: # One or more caught exceptions, now trigger the test's @@ -138,6 +140,7 @@ def _run_prepared_result(self, result: TestResult) -> TestResult: self.last_resort(self.case, self.result, e) raise e finally: + self.case._subtest_result = None result.stopTest(self.case) return result @@ -184,7 +187,11 @@ def _run_core(self) -> None: if getattr(self.case, "force_failure", None): self._run_user(_raise_force_fail_error) failed = True - if not failed: + if getattr(self.case, "_subtest_failed", False): + failed = True + if not failed and not getattr( + self.case, "_subtest_reported", False + ): self.result.addSuccess( self.case, details=self.case.getDetails() ) diff --git a/testtools/testcase.py b/testtools/testcase.py index bdd640a9..9df9ab2a 100644 --- a/testtools/testcase.py +++ b/testtools/testcase.py @@ -15,6 +15,7 @@ "unique_text_generator", ] +import contextlib import copy import datetime import functools @@ -99,6 +100,54 @@ class _ExpectedFailure(Exception): """ +class _SubTestAborted(Exception): + """Raised to abort a test whose result asked us to stop.""" + + +_subtest_msg_sentinel = object() + + +class _SubTest(unittest.TestCase): + """Describes a single subTest iteration for failure reporting. + + Carries the message and parameters passed to ``subTest`` so that + result objects can label each failure with its subTest context. + + Like ``unittest.case._SubTest``, this is a description rather than a + runnable test: it has no test method and must not be run. + """ + + def __init__( + self, test_case: "TestCase", msg: object, params: dict[str, Any] + ) -> None: + super().__init__() + self.test_case = test_case + self.failureException = test_case.failureException + self._msg = msg + self._params = params + + def runTest(self) -> NoReturn: + raise NotImplementedError("_SubTest describes a subtest; it cannot be run") + + def _subDescription(self) -> str: + parts: list[str] = [] + if self._msg is not _subtest_msg_sentinel: + parts.append(f"[{self._msg}]") + if self._params: + params_desc = ", ".join(f"{k}={v!r}" for k, v in self._params.items()) + parts.append(f"({params_desc})") + return " ".join(parts) or "()" + + def id(self) -> str: + return f"{self.test_case.id()} {self._subDescription()}" + + def shortDescription(self) -> str | None: + return self.test_case.shortDescription() + + def __str__(self) -> str: + return f"{self.test_case} {self._subDescription()}" + + # TypeVar for decorators _P = ParamSpec("_P") _R = TypeVar("_R") @@ -348,10 +397,12 @@ def __init__(self, *args: object, **kwargs: object) -> None: # data from exceptions. self.__exception_handlers: list[Callable[[OptExcInfo], None]] = [] # Passed to RunTest to map exceptions to result actions + # Handlers accept an optional details override, used to report a + # subtest against its own _SubTest rather than the parent test. self.exception_handlers: list[ tuple[ type[BaseException], - Callable[[TestCase, TestResult, BaseException], None], + Callable[..., None], ] ] = [ (self.skipException, self._report_skip), @@ -378,6 +429,12 @@ def _reset(self) -> None: # force_failure is set by expectThat() on mismatch; must be # cleared so re-runs of the same test can succeed. self.force_failure: bool | None = None + self._subtest_params: dict[str, Any] = {} + # Set by RunTest for the duration of the run, so that subTest can + # report each subtest as it completes. + self._subtest_result: TestResult | None = None + self._subtest_failed: bool = False + self._subtest_reported: bool = False def __eq__(self, other: object) -> bool: eq = getattr(unittest.TestCase, "__eq__", None) @@ -844,29 +901,63 @@ def onException(self, exc_info: OptExcInfo, tb_label: str = "traceback") -> None handler(exc_info) @staticmethod - def _report_error(self: "TestCase", result: TestResult, err: BaseException) -> None: - result.addError(self, details=self.getDetails()) + def _report_error( + self: "TestCase", + result: TestResult, + err: BaseException, + details: DetailsDict | None = None, + ) -> None: + result.addError(self, details=TestCase._report_details(self, details)) @staticmethod def _report_expected_failure( - self: "TestCase", result: TestResult, err: BaseException + self: "TestCase", + result: TestResult, + err: BaseException, + details: DetailsDict | None = None, ) -> None: - result.addExpectedFailure(self, details=self.getDetails()) + result.addExpectedFailure(self, details=TestCase._report_details(self, details)) @staticmethod def _report_failure( - self: "TestCase", result: TestResult, err: BaseException + self: "TestCase", + result: TestResult, + err: BaseException, + details: DetailsDict | None = None, ) -> None: - result.addFailure(self, details=self.getDetails()) + result.addFailure(self, details=TestCase._report_details(self, details)) @staticmethod - def _report_skip(self: "TestCase", result: TestResult, err: BaseException) -> None: + def _report_skip( + self: "TestCase", + result: TestResult, + err: BaseException, + details: DetailsDict | None = None, + ) -> None: if err.args: reason = err.args[0] else: reason = "no reason given." - self._add_reason(reason) - result.addSkip(self, details=self.getDetails()) + if details is None: + self._add_reason(reason) + result.addSkip(self, details=self.getDetails()) + else: + details = dict(details) + details["reason"] = content.text_content(reason) + result.addSkip(self, details=details) + + @staticmethod + def _report_details( + self: unittest.TestCase, details: DetailsDict | None + ) -> DetailsDict: + """Details to report an outcome with. + + Subtests supply their own details; everything else reports the + details accumulated on the test itself. + """ + if details is None: + return cast("TestCase", self).getDetails() + return dict(details) def _report_traceback( self, exc_info: OptExcInfo, tb_label: str = "traceback" @@ -887,11 +978,89 @@ def _report_traceback( ), ) + @contextlib.contextmanager + def subTest( + self, msg: object = _subtest_msg_sentinel, **params: Any + ) -> Iterator[None]: + """Return a context manager that reports a subtest independently. + + Exceptions raised inside the block are reported against a + ``_SubTest`` describing ``msg`` and ``params``, and do not abort the + rest of the test method. + """ + merged_params = {**self._subtest_params, **params} + subtest = _SubTest(self, msg, merged_params) + old_params, self._subtest_params = self._subtest_params, merged_params + # Details added inside the block belong to this subtest, not to the + # parent test, so report and then discard whatever it accumulated. + outer_details = dict(self.getDetails()) + try: + yield + except BaseException: + # Inside except block, exc_info() is guaranteed to have non-None + # values. + exc_info = cast(ExcInfo, sys.exc_info()) + if not self._report_subtest(subtest, exc_info, outer_details): + raise + finally: + self._subtest_params = old_params + + def _report_subtest( + self, subtest: "_SubTest", exc_info: ExcInfo, outer_details: DetailsDict + ) -> bool: + """Report the outcome of a failed subtest. + + :return: True if the exception was reported, False if it should + propagate (there is nothing to report to, or it is not an + exception this test knows how to handle). + """ + result = self._subtest_result + if result is None: + return False + if exc_info[0] is MultipleExceptions: + for sub_exc_info in exc_info[1].args: + self._report_subtest(subtest, sub_exc_info, outer_details) + return True + for exc_class, handler in self.exception_handlers: + if isinstance(exc_info[1], exc_class): + break + else: + return False + details = self.getDetails() + try: + self.onException(exc_info, tb_label="traceback") + subtest_details = { + name: detail + for name, detail in details.items() + if name not in outer_details + } + handler(subtest, result, exc_info[1], subtest_details) + # A reported subtest outcome stands on its own; the parent must + # not also be reported as a success. + self._subtest_reported = True + if not isinstance(exc_info[1], self.skipException): + self._subtest_failed = True + # Match unittest: stop at the first failing subtest when the + # result asks us to. + if getattr(result, "shouldStop", False): + raise _SubTestAborted from None + finally: + # Leave the parent's details as they were, so a subtest failure + # is not re-reported as part of the parent test. + details.clear() + details.update(outer_details) + return True + @staticmethod def _report_unexpected_success( - self: "TestCase", result: TestResult, err: BaseException + self: "TestCase", + result: TestResult, + err: BaseException, + details: DetailsDict | None = None, ) -> None: - result.addUnexpectedSuccess(self, details=self.getDetails()) + result.addUnexpectedSuccess( + self, details=TestCase._report_details(self, details) + ) def run(self, result: TestResult | None = None) -> TestResult: # type: ignore[override] self._reset() @@ -999,7 +1168,11 @@ def _run_test_method(self, result: TestResult) -> object: :param result: A testtools.TestResult to report activity to. :return: None. """ - return self._get_test_method()() + try: + return self._get_test_method()() + except _SubTestAborted: + # A subtest already reported the failure that stopped the run. + return None def useFixture(self, fixture: "FixtureT") -> "FixtureT": """Use fixture in a test case. diff --git a/testtools/testresult/doubles.py b/testtools/testresult/doubles.py index 7ce1ee26..d024ac3f 100644 --- a/testtools/testresult/doubles.py +++ b/testtools/testresult/doubles.py @@ -69,6 +69,12 @@ ] | tuple[Literal["addUnexpectedSuccess"], unittest.TestCase] | tuple[Literal["addUnexpectedSuccess"], unittest.TestCase, dict[str, object]] + | tuple[ + Literal["addSubTest"], + unittest.TestCase, + unittest.TestCase, + tuple[type, Exception, object] | object | None, + ] | tuple[Literal["addDuration"], unittest.TestCase, float] | tuple[Literal["progress"], int, int] | tuple[Literal["tags"], Iterable[str], Iterable[str]] @@ -124,6 +130,15 @@ def addUnexpectedSuccess(self, test: unittest.TestCase) -> None: if self.failfast: self.stop() + def addSubTest( + self, + test: unittest.TestCase, + subtest: unittest.TestCase, + err: OptExcInfo | None, + ) -> None: + super().addSubTest(test, subtest, err) # type: ignore[arg-type] + self._events.append(("addSubTest", test, subtest, err)) + def addDuration(self, test: unittest.TestCase, duration: float) -> None: self._events.append(("addDuration", test, duration)) self.collectedDurations.append((test, duration)) diff --git a/testtools/testresult/real.py b/testtools/testresult/real.py index bb887cac..780d7d24 100644 --- a/testtools/testresult/real.py +++ b/testtools/testresult/real.py @@ -2301,14 +2301,30 @@ def _convert( status: str, reason: str | None = None, ) -> None: + # Imported here to avoid a circular import at module load. + from testtools.testcase import _SubTest + if not self._started: self.startTestRun() - test_id = test.id() now = self._now() if err is not None: if details is None: details = {} details["traceback"] = TracebackContent(err, test) + if isinstance(test, _SubTest): + # A subtest outcome belongs to the test that contains it: attach + # its details to the parent so that the run counts one test, as + # unittest does. A skip or expected failure still needs a status + # of its own, since the parent reports nothing in that case. + test_id = test.test_case.id() + label = test._subDescription() + details = { + f"{name} {label}": detail for name, detail in (details or {}).items() + } + if status not in ("skip", "xfail"): + status = None # type: ignore[assignment] + else: + test_id = test.id() if details is not None: for name, content in details.items(): mime_type = repr(content.content_type) @@ -2342,12 +2358,13 @@ def _convert( test_id=test_id, timestamp=now, ) - self.status( - test_id=test_id, - test_status=status, - test_tags=self.current_tags, - timestamp=now, - ) + if status is not None: + self.status( + test_id=test_id, + test_status=status, + test_tags=self.current_tags, + timestamp=now, + ) def addExpectedFailure( self, diff --git a/testtools/twistedsupport/_runtest.py b/testtools/twistedsupport/_runtest.py index fca42e26..e6e48589 100644 --- a/testtools/twistedsupport/_runtest.py +++ b/testtools/twistedsupport/_runtest.py @@ -455,7 +455,10 @@ def _run_core(self): successful = False self._log_user_exception(UncleanReactorError(junk)) - if successful: + if getattr(self.case, "_subtest_failed", False): + successful = False + + if successful and not getattr(self.case, "_subtest_reported", False): self.result.addSuccess(self.case, details=self.case.getDetails()) def _run_user(self, function, *args):