From 42e3238fea470160a23636ca89e2cb6af4b4aea0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 17:11:41 +0000 Subject: [PATCH 1/3] test(mutation): gate nightly mutation runs on untriaged survivors `mutmut run` (3.7.0, pinned) exits 0 however many mutants survive, so mutation.yml was green whatever it found, and the two accepted survivors lived in a pyproject.toml comment nothing checked. Running the audit locally showed the cost: a third survivor in the shared TLV walker had gone untriaged and unkilled. - scripts/check_mutmut_survivors.py fails the job on any undetected mutant not accepted in scripts/mutmut_survivors.txt, on a stale entry, and on a scoped file with no checked mutant at all; unknown statuses fail closed. Verified against a real run: passes with the record, fails on both survivors without it. - tests/test_tlv.py kills the third survivor (a caller's min_option_length discarded when the length counts its header). - tests/conftest.py clears every package-level functools cache before each test, and loads the Hypothesis profiles for every selection. mutmut 3 forks each mutant from a parent that already ran the suite, so a warm cache could hide a mutant; and mutmut's test selection never imported test_fuzz.py, where the profiles were registered. - fuzz.yml/mutation.yml key concurrency on the event, so the cron can't cancel the manual runs CONTRIBUTING.md's new release checklist needs. - tests/test_workflows.py guards each of these, plus every fuzz harness being run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CEQCZGyuvpMyZwUbDucF3G --- .github/workflows/fuzz.yml | 6 +- .github/workflows/mutation.yml | 32 ++++-- .gitignore | 4 +- CHANGELOG.md | 8 ++ CONTRIBUTING.md | 41 +++++++ pyproject.toml | 34 +++--- scripts/check_mutmut_survivors.py | 149 ++++++++++++++++++++++++++ scripts/mutmut_survivors.txt | 27 +++++ tests/conftest.py | 63 +++++++++++ tests/test_dns.py | 14 +++ tests/test_fuzz.py | 16 +-- tests/test_mutmut_gate.py | 172 ++++++++++++++++++++++++++++++ tests/test_tlv.py | 28 +++++ tests/test_workflows.py | 60 ++++++++++- 14 files changed, 611 insertions(+), 43 deletions(-) create mode 100644 scripts/check_mutmut_survivors.py create mode 100644 scripts/mutmut_survivors.txt create mode 100644 tests/test_mutmut_gate.py diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index d7afe47..2b77d56 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -12,8 +12,12 @@ on: - cron: "0 3 * * *" workflow_dispatch: +# Keyed on the triggering event too, so the nightly cron can never +# cancel a manually dispatched run on the same ref (the release +# checklist in CONTRIBUTING.md relies on one) -- runs of the same kind +# still supersede each other. concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true permissions: diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index 9a06211..4662ecc 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -11,8 +11,12 @@ on: - cron: "0 4 * * *" workflow_dispatch: +# Keyed on the triggering event too, so the nightly cron can never +# cancel a manually dispatched run on the same ref (the release +# checklist in CONTRIBUTING.md relies on one) -- runs of the same kind +# still supersede each other. concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true permissions: @@ -65,20 +69,32 @@ jobs: 'netprotocols._tlv.*' \ 'netprotocols.layer7.dns.x__read_name*' \ 'netprotocols.layer7.dns.x__labels*' + # `mutmut run` exits 0 however many mutants survive, so it can + # never turn this job red on its own. This step is what does: it + # fails on any undetected mutant not in the triage record + # (scripts/mutmut_survivors.txt), on a stale entry there, and on + # a scoped file with no checked mutant at all. + - name: Fail on survivors outside the triage record + run: | + uv run --frozen mutmut results --all true > mutmut-results.txt + uv run --frozen python scripts/check_mutmut_survivors.py \ + mutmut-results.txt # A failed scheduled run already notifies repo watchers by - # default (GitHub's standard behavior) — that red run *is* the - # report; nothing here files an issue on top of it. This just - # makes the surviving mutants recoverable without needing repo - # write access to re-run mutmut locally. Covers both the - # sqlite-file cache and the mutated-copies directory since either - # may be what a given mutmut version writes; whichever doesn't - # exist is silently skipped. + # default (GitHub's standard behavior), and with the step above a + # red run means an untriaged survivor or a broken run; nothing + # here files an issue on top of it. This makes the results + # recoverable without needing repo write access to re-run mutmut + # locally. Covers the status listing plus both the sqlite-file + # cache and the mutated-copies directory, since either of the + # latter may be what a given mutmut version writes; whatever + # doesn't exist is silently skipped. - name: Upload mutmut results on failure if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: mutmut-nightly-failure-${{ github.run_id }} path: | + mutmut-results.txt .mutmut-cache mutants if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 4984b86..6f99111 100644 --- a/.gitignore +++ b/.gitignore @@ -156,6 +156,8 @@ benchmark.out # lockfile-pinned dependency. scripts/pyodide/node_modules/ -# mutmut's mutation-testing workspace/cache +# mutmut's mutation-testing workspace/cache, and the status listing +# scripts/check_mutmut_survivors.py reads mutants/ .mutmut-cache +mutmut-results.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 35d3f6b..8cc3be9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Development +- **The nightly mutation run can now fail on a surviving mutant.** `mutmut run` (3.7.0) exits 0 however many mutants survive, so `mutation.yml` was green whatever it found. A new final step, `scripts/check_mutmut_survivors.py`, fails the job on any undetected mutant not accepted in the new triage record `scripts/mutmut_survivors.txt`, on a stale entry there, and on a scoped file with no checked mutant at all; unknown mutmut statuses fail closed. The two accepted survivors and their reasoning moved into that record from a `pyproject.toml` comment. +- **Closed a mutation-testing gap the gate exposed.** A third survivor — `_tlv.walk_kind_length_value` discarding a caller's `min_option_length` when the length byte counts its own header — was neither triaged nor killed, because no call site passes a floor above the structural 2. Two direct tests in `tests/test_tlv.py` now pin it. +- **Mutation runs can no longer report false survivors from warm caches.** mutmut 3 runs the clean suite in-process and then forks once per mutant, so a `functools` cache warmed by the original code (DNS's section parsers) could answer in the child and hide a mutant. `tests/conftest.py` now clears every package-level cache before each test, discovering them rather than listing them. +- **Hypothesis profiles load for every test selection.** They were registered only when `tests/test_fuzz.py` was imported, which mutmut's test selection never does, so property tests under mutation ran randomized and under the default deadline. They now live in `tests/conftest.py`. +- **`fuzz.yml` and `mutation.yml` key their concurrency groups on the triggering event**, so the nightly cron can no longer cancel a manually dispatched run. CONTRIBUTING.md gains *Mutation testing* and *Releasing* sections, the latter making those manual runs a release prerequisite. +- **New guards in `tests/test_workflows.py`**: every harness under `tests/fuzz/` must be run by `fuzz.yml`, the survivor gate must follow `mutmut run` and read the full status listing, and both nightly concurrency groups must be event-keyed. +- **Corrected `pyproject.toml`'s mutmut comment**, which said mutants were generated for the DNS dataclasses; mutmut 3.7.0 never mutates decorated classes or functions. + - **Tightened three DNS corpus assertions in `tests/test_dns.py` from substring/`endswith` checks to exact equality, closing 3 open CodeQL `py/incomplete-url-substring-sanitization` alerts (#166).** `test_names_decompress_to_real_domains` asserted `"example.com" in names` / `"accounts.youtube.com" in names` against a `set` decoded from a fixed pcap corpus, and `test_answers_parse_to_real_records` asserted `cname.rdata_text.endswith("google.com")` against the same corpus — not security-relevant trust checks, but exactly the substring/suffix pattern the rule flags regardless of what the value's actually validating. Both now compare against the known exact decoded values instead (`names == {"example.com", "accounts.youtube.com"}`, `rdata_text == "www3.l.google.com"`), which removes the pattern CodeQL keys on and is strictly more precise than the checks it replaces. - **Consolidated six open Dependabot updates and grouped `github/codeql-action` so its sub-actions can no longer be bumped apart (#198, #199, #204, #205, #206, #207).** Dependabot tracks `github/codeql-action/init`, `/analyze`, and `/upload-sarif` as three separate dependencies and opened one PR each; `analyze` refuses a config file written by a different `init` release ("Loaded a configuration file for version '4.37.9', but running version '4.38.1'"), so #204 and #206 each failed CodeQL alone and neither could ever merge. All three now move to v4.38.1 together, alongside `astral-sh/setup-uv` 10.0.1 → 10.2.0, `hypothesis` 6.167.1 → 6.168.0, `ruff` 0.16.6 → 0.16.7, and `mutmut` 3.7.0 → 3.8.0 (lockfile byte-identical to Dependabot's). `dependabot.yml` gains a `codeql-action` group for the `github-actions` ecosystem, and `tests/test_workflows.py` gains `TestCodeQLActionVersionLockstep`, which fails the ordinary test job — naming the mismatched steps — whenever `github/codeql-action/*` steps across all workflows are pinned to more than one commit. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 629ad1b..d7cece8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -122,6 +122,47 @@ only if the derivation is byte-faithful to the wire and the provenance is recorded in the MANIFEST; that script's synthesis fallback (splicing tag shims over a real untagged capture) is the worked example. +## Mutation testing + +`.github/workflows/mutation.yml` runs a scoped +[mutmut](https://github.com/boxed/mutmut) audit every night; the scope +and its reasoning live in `pyproject.toml`'s `[tool.mutmut]` table. +`mutmut run` itself exits 0 however many mutants survive, so the job's +final step, +[`scripts/check_mutmut_survivors.py`](scripts/check_mutmut_survivors.py), +is what fails it — on any undetected mutant not accepted in +[`scripts/mutmut_survivors.txt`](scripts/mutmut_survivors.txt). To +reproduce locally: + +```bash +uv run mutmut run 'netprotocols.checksum.*' 'netprotocols._tlv.*' \ + 'netprotocols.layer7.dns.x__read_name*' \ + 'netprotocols.layer7.dns.x__labels*' +uv run mutmut results --all true > mutmut-results.txt +python3 scripts/check_mutmut_survivors.py mutmut-results.txt +``` + +A survivor a test *could* kill gets that test. Only a true no-op, or one +out of any test's reach, goes in the triage record, under a comment +saying why. Two properties of the setup are easy to trip over: + +- mutmut 3.7.0 never mutates a decorated function (other than a lone + `@staticmethod`/`@classmethod`) or anything inside a decorated class, + so logic meant to be audited belongs in undecorated module-level + functions. +- `tests/conftest.py` clears every package-level `functools` cache + before each test, so a result cached by the original code can't hide + a mutant. + +## Releasing + +`release.yml` gates publishing on the CI ladder only; the nightly fuzz +and mutation jobs never run on a pull request. Before pushing a release +tag, dispatch both by hand (Actions → the workflow → *Run workflow*, +on `master` at the commit to be tagged) and tag only once both are +green. Their concurrency groups are keyed on the triggering event, so a +nightly run can't cancel a manual one. + ## Changelog This project keeps a [Keep a Changelog](https://keepachangelog.com/) diff --git a/pyproject.toml b/pyproject.toml index eaa3ecf..a77ad83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,25 +131,17 @@ pytest_add_cli_args_test_selection = [ # uv run mutmut run 'netprotocols.checksum.*' 'netprotocols._tlv.*' \ # 'netprotocols.layer7.dns.x__read_name*' \ # 'netprotocols.layer7.dns.x__labels*' -# Mutants mutmut generates elsewhere in dns.py (record/RDATA parsing, -# the DNS/DNSOverTCP dataclasses, ...) are out of scope for this audit -# and were not triaged. +# Mutants mutmut generates elsewhere in dns.py (the record/RDATA +# parsing helpers, ...) are out of scope for this audit and were not +# triaged. mutmut 3.7.0 generates none at all for a decorated function +# other than a lone @staticmethod/@classmethod, or for anything inside +# a decorated class -- so the @lru_cache section parsers and the +# @dataclass DNS/DNSOverTCP classes are never mutated, whatever the +# filters say. Code meant to be audited has to live in undecorated +# module-level functions. # -# Triage note — accepted-equivalent survivors (two, both true no-op -# mutations rather than gaps in coverage): -# -# 1. `_pseudo_header`'s IPv6 arm packs its 32-bit length field with -# `struct.pack("!IBBBB", ...)`; a mutant swapping "I" for the -# signed "i" only diverges from the original for a payload -# >= 2**31 bytes (2 GiB), which no test in this suite can construct -# or run in reasonable time/memory. The same mutation on the IPv4 -# arm's 16-bit length field *is* killed — see -# test_ipv4_pseudo_header_length_field_is_unsigned in -# test_checksum.py, which exercises the exact signed/unsigned -# boundary at a testable (40 KB) scale. -# -# 2. `_labels`' final label decode, `.decode("ascii", "replace")`, has -# a mutant that upper-cases the codec name to "ASCII". Python's -# codec lookup normalizes case, so "ascii" and "ASCII" name the -# exact same codec — every input decodes identically either way, -# with no observable difference for any test to catch. +# Triage record: scripts/mutmut_survivors.txt lists each accepted +# survivor with the reason it is equivalent, and +# scripts/check_mutmut_survivors.py (run by .github/workflows/ +# mutation.yml after `mutmut run`) fails the run on any other +# undetected mutant -- `mutmut run` itself exits 0 regardless. diff --git a/scripts/check_mutmut_survivors.py b/scripts/check_mutmut_survivors.py new file mode 100644 index 0000000..40ee148 --- /dev/null +++ b/scripts/check_mutmut_survivors.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Fail a mutmut run that left an unaccepted mutant undetected. + +``mutmut run`` (3.7.0, pinned in uv.lock) exits 0 however many mutants +survive: it only exits non-zero when the stats or clean run itself +fails. Without this check the nightly mutation workflow is green no +matter what it finds. This script reads the full status listing and +fails on any of: + +- a mutant that went undetected (survived, had no covering tests, or + ended in a status this script does not recognize, so a mutmut + upgrade that adds one fails closed) and is not in the allowlist; +- an allowlist entry that is no longer undetected, so the list cannot + quietly accumulate entries for mutants a later test kills, or whose + number shifted because the function's source changed (mutant numbers + are positional, so an edit to an allowlisted function usually + renumbers its mutants and the entry has to be re-triaged anyway); +- a file in pyproject.toml's ``[tool.mutmut] only_mutate`` with no + checked mutant at all, so a run that silently checked nothing cannot + pass. + +Usage, from the repository root after a ``mutmut run``:: + + uv run --frozen mutmut results --all true > mutmut-results.txt + python3 scripts/check_mutmut_survivors.py mutmut-results.txt + +The allowlist (``scripts/mutmut_survivors.txt`` by default) holds one +mutant name per line; ``#`` starts a comment. Every entry needs a +comment saying why that mutant is equivalent or out of reach: the file +is the triage record, not a mute button. +""" + +from __future__ import annotations + +import argparse +import re +import sys +import tomllib +from collections.abc import Iterable +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_ALLOWLIST = ROOT / "scripts" / "mutmut_survivors.txt" +DEFAULT_PYPROJECT = ROOT / "pyproject.toml" + +#: A test failed or hung, or the mutant crashed the interpreter or was +#: rejected by the type checker: in every case the change was noticed. +DETECTED = frozenset({"killed", "timeout", "segfault", "caught by type check"}) +#: Outside this run's filters, or skipped by mutmut itself. +UNCHECKED = frozenset({"not checked", "skipped"}) + +_LINE = re.compile(r"^\s*(?P\S+): (?P.+?)\s*$") + + +def parse_results(lines: Iterable[str]) -> dict[str, str]: + """Map each mutant name to its status in ``mutmut results`` output.""" + statuses: dict[str, str] = {} + for line in lines: + match = _LINE.match(line) + if match: + statuses[match["name"]] = match["status"] + return statuses + + +def parse_allowlist(lines: Iterable[str]) -> set[str]: + """The mutant names in an allowlist, ignoring comments and blanks.""" + names: set[str] = set() + for line in lines: + entry = line.split("#", 1)[0].strip() + if entry: + names.add(entry) + return names + + +def mutated_modules(pyproject: str) -> list[str]: + """Dotted module names of the files ``only_mutate`` scopes to.""" + paths = tomllib.loads(pyproject)["tool"]["mutmut"]["only_mutate"] + modules = [] + for path in paths: + relative = Path(path).relative_to("src").with_suffix("") + modules.append(".".join(relative.parts)) + return modules + + +def check( + statuses: dict[str, str], allowlist: set[str], modules: list[str] +) -> list[str]: + """Every reason the run should fail; empty when it passes.""" + if not statuses: + return ["no mutant statuses found -- was `mutmut results` empty?"] + + undetected = { + name + for name, status in statuses.items() + if status not in DETECTED and status not in UNCHECKED + } + problems = [ + f"undetected mutant: {name} ({statuses[name]})" + for name in sorted(undetected - allowlist) + ] + problems += [ + f"allowlisted mutant is no longer undetected: {name} " + f"({statuses.get(name, 'absent from this run')}) -- remove or " + f"re-triage its entry" + for name in sorted(allowlist - undetected) + ] + for module in modules: + checked = any( + name.startswith(f"{module}.") and status not in UNCHECKED + for name, status in statuses.items() + ) + if not checked: + problems.append( + f"no mutant of {module} was checked -- the run's filters " + f"no longer reach a file only_mutate lists" + ) + return problems + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "results", type=Path, help="`mutmut results --all true` output" + ) + parser.add_argument("--allowlist", type=Path, default=DEFAULT_ALLOWLIST) + parser.add_argument("--pyproject", type=Path, default=DEFAULT_PYPROJECT) + args = parser.parse_args(argv) + + statuses = parse_results( + args.results.read_text(encoding="utf-8").splitlines() + ) + allowlist = parse_allowlist( + args.allowlist.read_text(encoding="utf-8").splitlines() + ) + modules = mutated_modules(args.pyproject.read_text(encoding="utf-8")) + problems = check(statuses, allowlist, modules) + + checked = sum(status not in UNCHECKED for status in statuses.values()) + print( + f"{checked} mutants checked, {len(allowlist)} accepted survivors " + f"allowlisted" + ) + for problem in problems: + print(f"FAIL: {problem}", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/mutmut_survivors.txt b/scripts/mutmut_survivors.txt new file mode 100644 index 0000000..82eab90 --- /dev/null +++ b/scripts/mutmut_survivors.txt @@ -0,0 +1,27 @@ +# Accepted mutmut survivors: the triage record for the scoped audit +# configured in pyproject.toml's [tool.mutmut] table. +# +# scripts/check_mutmut_survivors.py fails the nightly mutation run on +# any undetected mutant not listed here, and on any entry listed here +# that is no longer undetected. Every entry must say why that mutant is +# a true no-op or out of any test's reach; one a test *could* kill gets +# that test instead of an entry. Mutant numbers are positional, so +# editing an allowlisted function usually renumbers its mutants -- the +# gate then reports the entry as stale and it has to be re-triaged. + +# checksum._pseudo_header's IPv6 arm packs its 32-bit length field with +# struct.pack("!IBBBB", ...); this mutant swaps "I" for the signed "i", +# which only diverges for a payload >= 2**31 bytes (2 GiB), which no +# test in this suite can construct or run in reasonable time/memory. +# The same mutation on the IPv4 arm's 16-bit length field *is* killed +# -- see test_ipv4_pseudo_header_length_field_is_unsigned in +# tests/test_checksum.py, which exercises the exact signed/unsigned +# boundary at a testable (40 KB) scale. +netprotocols.checksum.x__pseudo_header__mutmut_33 + +# dns._labels' final label decode, .decode("ascii", "replace"): this +# mutant upper-cases the codec name to "ASCII". Python's codec lookup +# normalizes case, so both name the exact same codec -- every input +# decodes identically either way, with no observable difference for +# any test to catch. +netprotocols.layer7.dns.x__labels__mutmut_122 diff --git a/tests/conftest.py b/tests/conftest.py index c22ec30..1b9c796 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,16 +4,79 @@ https://wiki.wireshark.org/SampleCaptures (arp-storm.pcap, among others), locally captured frames, and the real-capture corpus under ``tests/fixtures/`` (see its MANIFEST.md). + +Two pieces of suite-wide setup also live here, because conftest.py is +the one module pytest loads for *every* selection of test files — +including the narrow selection mutmut runs (pyproject.toml's +``pytest_add_cli_args_test_selection``), which never imports +test_fuzz.py: + +- The Hypothesis profiles (see test_fuzz.py's module docstring), so a + property test anywhere runs under the same derandomized, deadline-free + settings whether or not test_fuzz.py was collected. +- Clearing every module-level ``functools`` cache in the package before + each test. mutmut 3 runs the clean suite in-process and then forks + once per mutant, so without this a cache warmed by the original code + answers in the child and hides the mutant (a false survivor), and a + test whose input was already cached is never associated with the + cached function at all. """ +import importlib +import os +import pkgutil +from collections.abc import Callable, Iterator from pathlib import Path import pytest +from hypothesis import settings +import netprotocols from netprotocols.pcap import read_pcap FIXTURES = Path(__file__).parent / "fixtures" +settings.register_profile( + "netprotocols", max_examples=200, deadline=None, derandomize=True +) +settings.register_profile( + "nightly", max_examples=10_000, deadline=None, derandomize=False +) +settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "netprotocols")) + + +def _package_caches() -> tuple[Callable[[], None], ...]: + """The ``cache_clear`` of every ``functools`` cache defined at module + level anywhere in the package. + + Discovered rather than listed, so a cache added to any module later + is cleared without anyone remembering to register it here. The + ``__module__`` check skips a cached function re-exported from some + other module, so each cache is collected once. + """ + clears: list[Callable[[], None]] = [] + for info in pkgutil.walk_packages( + netprotocols.__path__, f"{netprotocols.__name__}." + ): + module = importlib.import_module(info.name) + for value in vars(module).values(): + clear = getattr(value, "cache_clear", None) + owner = getattr(value, "__module__", None) + if callable(clear) and owner == module.__name__: + clears.append(clear) + return tuple(clears) + + +PACKAGE_CACHES = _package_caches() + + +@pytest.fixture(autouse=True) +def _clear_package_caches() -> Iterator[None]: + """Start every test with every package-level cache empty.""" + for clear in PACKAGE_CACHES: + clear() + yield + def pcap_frames(path: Path) -> list[bytes]: """Every frame's raw bytes from one pcap file, in order. diff --git a/tests/test_dns.py b/tests/test_dns.py index 8a7ed12..bbc1465 100644 --- a/tests/test_dns.py +++ b/tests/test_dns.py @@ -733,6 +733,20 @@ def test_full_tcp_dns_walk_with_records(self): assert b"".join(bytes(layer) for layer in layers) == frame +class TestCachesStartEmpty: + """conftest.py clears every package-level cache before each test, so + a mutant can't hide behind a result the original code cached (see + its module docstring). The other tests in this file warm both + caches, so finding them empty here proves the discovery reaches + them and the clearing actually runs.""" + + @pytest.mark.parametrize("name", ["_parse_questions", "_parse_records"]) + def test_cache_is_empty_when_a_test_starts(self, name: str) -> None: + from netprotocols.layer7 import dns as dns_module + + assert getattr(dns_module, name).cache_info().currsize == 0 + + class TestSectionParsingIsShared: """#85: the three section accessors share one parse, and nothing round-trips through `bytes(self)` to get at the message.""" diff --git a/tests/test_fuzz.py b/tests/test_fuzz.py index 2180940..2c0c5bc 100644 --- a/tests/test_fuzz.py +++ b/tests/test_fuzz.py @@ -19,15 +19,17 @@ Select a profile with ``HYPOTHESIS_PROFILE`` (read once, at import time) — e.g. ``HYPOTHESIS_PROFILE=nightly pytest tests/test_fuzz.py`` -reproduces a nightly run locally. +reproduces a nightly run locally. The profiles are registered and +loaded in conftest.py rather than here, so every property test in the +suite gets them, including under a test selection that never imports +this file (mutmut's, for one). """ import contextlib -import os import sys import pytest -from hypothesis import given, settings +from hypothesis import given from hypothesis import strategies as st from conftest import corpus_frames @@ -59,14 +61,6 @@ from strategies import ROUND_TRIP_STRATEGIES from test_corpus import walk -settings.register_profile( - "netprotocols", max_examples=200, deadline=None, derandomize=True -) -settings.register_profile( - "nightly", max_examples=10_000, deadline=None, derandomize=False -) -settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "netprotocols")) - ALL_PROTOCOLS = ( Ethernet, VLAN, diff --git a/tests/test_mutmut_gate.py b/tests/test_mutmut_gate.py new file mode 100644 index 0000000..0e98f68 --- /dev/null +++ b/tests/test_mutmut_gate.py @@ -0,0 +1,172 @@ +"""Tests for scripts/check_mutmut_survivors.py, the nightly mutation gate. + +`mutmut run` exits 0 however many mutants survive, so this script is the +only thing that can fail the mutation workflow on a finding. A gate that +silently passes is worse than none -- it reads as coverage -- so each +way it is meant to fail gets its own test, alongside the committed +triage record and pyproject.toml it actually runs against. +""" + +import importlib.util +from pathlib import Path +from types import ModuleType + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "check_mutmut_survivors.py" + + +def _load_gate() -> ModuleType: + spec = importlib.util.spec_from_file_location("mutmut_gate", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +gate = _load_gate() + +MODULES = ["netprotocols.checksum", "netprotocols._tlv"] + + +def statuses() -> dict[str, str]: + """A passing status listing: one killed mutant in each of MODULES, + for a test to add to or override.""" + return { + "netprotocols.checksum.x_f__mutmut_1": "killed", + "netprotocols._tlv.x_g__mutmut_1": "killed", + } + + +class TestParsing: + def test_results_lines_map_names_to_statuses(self) -> None: + lines = [ + " netprotocols.checksum.x_f__mutmut_1: killed", + " netprotocols.checksum.x_f__mutmut_2: check was interrupted" + " by user", + "", + "some unrelated banner line", + ] + assert gate.parse_results(lines) == { + "netprotocols.checksum.x_f__mutmut_1": "killed", + "netprotocols.checksum.x_f__mutmut_2": ( + "check was interrupted by user" + ), + } + + def test_allowlist_ignores_comments_and_blanks(self) -> None: + lines = ["# why", "", " a.x_f__mutmut_3 # inline note", "b"] + assert gate.parse_allowlist(lines) == {"a.x_f__mutmut_3", "b"} + + def test_only_mutate_paths_become_dotted_modules(self) -> None: + pyproject = ( + '[tool.mutmut]\nonly_mutate = ["src/netprotocols/_tlv.py",' + ' "src/netprotocols/layer7/dns.py"]\n' + ) + assert gate.mutated_modules(pyproject) == [ + "netprotocols._tlv", + "netprotocols.layer7.dns", + ] + + +class TestCheck: + def test_a_clean_run_passes(self) -> None: + results = statuses() + results["netprotocols._tlv.x_g__mutmut_2"] = "timeout" + results["netprotocols.layer7.dns.x_h__mutmut_1"] = "not checked" + assert gate.check(results, set(), MODULES) == [] + + @pytest.mark.parametrize( + "status", ["survived", "no tests", "suspicious", "a status from v4"] + ) + def test_an_undetected_mutant_fails(self, status: str) -> None: + """Unknown statuses fail closed: a mutmut upgrade that adds one + must be triaged here, not silently counted as a kill.""" + results = statuses() + results["netprotocols._tlv.x_g__mutmut_2"] = status + (problem,) = gate.check(results, set(), MODULES) + assert "undetected mutant: netprotocols._tlv.x_g__mutmut_2" in problem + + def test_an_allowlisted_survivor_passes(self) -> None: + results = statuses() + results["netprotocols._tlv.x_g__mutmut_2"] = "survived" + allowlist = {"netprotocols._tlv.x_g__mutmut_2"} + assert gate.check(results, allowlist, MODULES) == [] + + @pytest.mark.parametrize("status", ["killed", None]) + def test_a_stale_allowlist_entry_fails(self, status: str | None) -> None: + """An entry whose mutant is now killed, or no longer exists + (renumbered by an edit), must be removed or re-triaged.""" + results = statuses() + name = "netprotocols._tlv.x_g__mutmut_9" + if status is not None: + results[name] = status + (problem,) = gate.check(results, {name}, MODULES) + assert "no longer undetected" in problem + assert name in problem + + def test_a_scoped_module_with_nothing_checked_fails(self) -> None: + results = statuses() + results["netprotocols._tlv.x_g__mutmut_1"] = "not checked" + (problem,) = gate.check(results, set(), MODULES) + assert "no mutant of netprotocols._tlv was checked" in problem + + def test_module_prefix_does_not_match_a_sibling(self) -> None: + """netprotocols.checksum must not be satisfied by a mutant of a + hypothetical netprotocols.checksum_extra module.""" + results = { + "netprotocols.checksum_extra.x_f__mutmut_1": "killed", + "netprotocols._tlv.x_g__mutmut_1": "killed", + } + (problem,) = gate.check(results, set(), MODULES) + assert "no mutant of netprotocols.checksum was checked" in problem + + def test_empty_results_fail(self) -> None: + assert gate.check({}, set(), MODULES) + + +class TestMain: + def test_exit_status_reflects_the_check(self, tmp_path: Path) -> None: + results = tmp_path / "results.txt" + allowlist = tmp_path / "allow.txt" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text( + '[tool.mutmut]\nonly_mutate = ["src/netprotocols/_tlv.py"]\n' + ) + allowlist.write_text("") + results.write_text(" netprotocols._tlv.x_g__mutmut_1: killed\n") + args = [ + str(results), + "--allowlist", + str(allowlist), + "--pyproject", + str(pyproject), + ] + assert gate.main(args) == 0 + results.write_text(" netprotocols._tlv.x_g__mutmut_1: survived\n") + assert gate.main(args) == 1 + + +class TestCommittedTriageRecord: + def test_every_entry_is_justified(self) -> None: + """Each allowlisted name must sit under a comment saying why it + is accepted -- the file is a triage record, not a mute list.""" + lines = gate.DEFAULT_ALLOWLIST.read_text().splitlines() + for index, line in enumerate(lines): + entry = line.split("#", 1)[0].strip() + if entry: + assert index and lines[index - 1].startswith("#"), ( + f"{entry} has no justifying comment directly above it" + ) + + def test_entries_name_modules_the_audit_scopes(self) -> None: + modules = gate.mutated_modules(gate.DEFAULT_PYPROJECT.read_text()) + entries = gate.parse_allowlist( + gate.DEFAULT_ALLOWLIST.read_text().splitlines() + ) + for entry in entries: + assert any(entry.startswith(f"{m}.") for m in modules), ( + f"{entry} is outside every file pyproject.toml's " + f"only_mutate scopes, so it can never be reported" + ) diff --git a/tests/test_tlv.py b/tests/test_tlv.py index eb33548..c33464c 100644 --- a/tests/test_tlv.py +++ b/tests/test_tlv.py @@ -124,6 +124,34 @@ def test_length_includes_header_enforces_a_floor_of_two_regardless( assert err.expected == ">=2" assert err.actual == declared_length + def test_a_floor_stronger_than_two_wins_when_length_includes_header( + self, + ): + """The structural floor of 2 is a minimum, not a replacement: a + caller's stricter min_option_length still applies on top of it. + No current call site passes more than 2, so only a direct test + pins this (it kills the mutant that drops the caller's floor).""" + with pytest.raises(InvalidFieldError) as excinfo: + walk(b"\x05\x02", min_option_length=3) + err = excinfo.value + assert err.expected == ">=3" + assert err.actual == 2 + + def test_a_floor_applies_when_length_excludes_header(self): + """Without the structural floor, a caller's min_option_length is + the only floor there is, so it must still be enforced.""" + with pytest.raises(InvalidFieldError) as excinfo: + walk( + b"\x05\x00", + lone_byte_kinds=frozenset(), + terminator_kind=None, + min_option_length=1, + length_includes_header=False, + ) + err = excinfo.value + assert err.expected == ">=1" + assert err.actual == 0 + class TestLengthIncludesHeaderTrue: """TCP/IPv4 shape: the length byte counts the kind and length bytes diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 7697659..8491a54 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -474,6 +474,11 @@ def test_publish_attests_build_provenance( ) +FUZZ_HARNESSES = sorted( + (WORKFLOWS.parent.parent / "tests" / "fuzz").glob("*.py") +) + + class TestFuzzWorkflow: def test_has_a_concurrency_group(self) -> None: """Without one, an overlapping nightly run could waste a runner @@ -482,6 +487,37 @@ def test_has_a_concurrency_group(self) -> None: "fuzz.yml no longer declares a top-level concurrency group" ) + def test_harness_glob_is_not_vacuous(self) -> None: + assert FUZZ_HARNESSES, "tests/fuzz/ holds no harness at all" + + @pytest.mark.parametrize( + "harness", FUZZ_HARNESSES, ids=lambda path: path.name + ) + def test_every_harness_is_run(self, harness: Path) -> None: + """A harness under tests/fuzz/ that no job invokes is dead code + that looks like coverage. Globbed, not listed, so a new harness + fails here until fuzz.yml runs it.""" + atheris = job_blocks(FUZZ).get("atheris") + assert atheris is not None, "fuzz.yml has no atheris job" + assert f"tests/fuzz/{harness.name}" in atheris, ( + f"fuzz.yml's atheris job never runs tests/fuzz/{harness.name}" + ) + + +@pytest.mark.parametrize("workflow", [FUZZ, MUTATION], ids=lambda p: p.name) +def test_nightly_concurrency_is_keyed_on_the_event(workflow: Path) -> None: + """The release checklist relies on manually dispatched fuzz and + mutation runs; keyed on workflow and ref alone, the nightly cron + would cancel a dispatched run in progress on the same branch.""" + match = re.search( + r"^concurrency:\n group: (?P.+)$", workflow.read_text(), re.M + ) + assert match, f"{workflow.name} declares no concurrency group" + assert "github.event_name" in match["group"], ( + f"{workflow.name}'s concurrency group would let the nightly run " + f"cancel a manually dispatched one" + ) + class TestMutationWorkflow: # pyproject.toml's [tool.mutmut] only_mutate covers the whole of @@ -490,7 +526,7 @@ class TestMutationWorkflow: # way to scope below a whole file, so pyproject.toml's own comment # documents these positional filters as the way to reproduce that # narrower scope. If the workflow step doesn't pass them, it mutates - # (and reports on) all of dns.py's untriaged record/RDATA/dataclass + # (and reports on) all of dns.py's untriaged record/RDATA parsing # code every night, silently wider than what was ever audited. EXPECTED_FILTERS = ( "netprotocols.checksum.*", @@ -586,6 +622,28 @@ def test_run_applies_the_documented_dns_scope_filters(self) -> None: f"more or less of dns.py than was ever audited" ) + def test_survivors_are_gated_after_the_run(self) -> None: + """`mutmut run` exits 0 however many mutants survive, so without + the gate step after it the nightly job is green regardless of + what it finds.""" + block = job_blocks(MUTATION).get("mutation") + assert block is not None, "mutation.yml has no mutation job" + run_at = block.find("mutmut run") + gate_at = block.find("scripts/check_mutmut_survivors.py") + assert gate_at != -1, ( + "mutation.yml never runs scripts/check_mutmut_survivors.py, " + "so surviving mutants can no longer fail the job" + ) + assert run_at < gate_at, ( + "the survivor gate runs before `mutmut run`, so it checks " + "a stale result set" + ) + assert "mutmut results --all true" in block[run_at:gate_at], ( + "the gate needs the full status listing (--all true): the " + "default listing omits killed mutants, so a run that checked " + "nothing would look the same as one that killed everything" + ) + def test_run_rejects_an_unquoted_argument_on_a_continued_line( self, ) -> None: From 98ffc700ebce445ed5ebbb28df985e927b36f68c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 17:21:22 +0000 Subject: [PATCH 2/3] docs(mutation): describe mutmut 3.8.0, which master now pins #208 bumped mutmut 3.7.0 -> 3.8.0 after the survivor gate was written. Re-verified on 3.8.0: `mutmut run` still exits 0 with mutants surviving, the status vocabulary is unchanged, and the two triaged survivors keep their names, so the gate passes as before. One behaviour did change: 3.8.0 recurses into decorated classes, so @dataclass methods are mutated again. The earlier "correction" of pyproject.toml's comment would have made it false, so the wording now says only that decorated functions (the @lru_cache parsers) are never mutated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CEQCZGyuvpMyZwUbDucF3G --- CHANGELOG.md | 4 ++-- CONTRIBUTING.md | 9 +++++---- pyproject.toml | 15 +++++++-------- scripts/check_mutmut_survivors.py | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cc3be9..d5a6ced 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,13 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Development -- **The nightly mutation run can now fail on a surviving mutant.** `mutmut run` (3.7.0) exits 0 however many mutants survive, so `mutation.yml` was green whatever it found. A new final step, `scripts/check_mutmut_survivors.py`, fails the job on any undetected mutant not accepted in the new triage record `scripts/mutmut_survivors.txt`, on a stale entry there, and on a scoped file with no checked mutant at all; unknown mutmut statuses fail closed. The two accepted survivors and their reasoning moved into that record from a `pyproject.toml` comment. +- **The nightly mutation run can now fail on a surviving mutant.** `mutmut run` (3.8.0) exits 0 however many mutants survive, so `mutation.yml` was green whatever it found. A new final step, `scripts/check_mutmut_survivors.py`, fails the job on any undetected mutant not accepted in the new triage record `scripts/mutmut_survivors.txt`, on a stale entry there, and on a scoped file with no checked mutant at all; unknown mutmut statuses fail closed. The two accepted survivors and their reasoning moved into that record from a `pyproject.toml` comment. - **Closed a mutation-testing gap the gate exposed.** A third survivor — `_tlv.walk_kind_length_value` discarding a caller's `min_option_length` when the length byte counts its own header — was neither triaged nor killed, because no call site passes a floor above the structural 2. Two direct tests in `tests/test_tlv.py` now pin it. - **Mutation runs can no longer report false survivors from warm caches.** mutmut 3 runs the clean suite in-process and then forks once per mutant, so a `functools` cache warmed by the original code (DNS's section parsers) could answer in the child and hide a mutant. `tests/conftest.py` now clears every package-level cache before each test, discovering them rather than listing them. - **Hypothesis profiles load for every test selection.** They were registered only when `tests/test_fuzz.py` was imported, which mutmut's test selection never does, so property tests under mutation ran randomized and under the default deadline. They now live in `tests/conftest.py`. - **`fuzz.yml` and `mutation.yml` key their concurrency groups on the triggering event**, so the nightly cron can no longer cancel a manually dispatched run. CONTRIBUTING.md gains *Mutation testing* and *Releasing* sections, the latter making those manual runs a release prerequisite. - **New guards in `tests/test_workflows.py`**: every harness under `tests/fuzz/` must be run by `fuzz.yml`, the survivor gate must follow `mutmut run` and read the full status listing, and both nightly concurrency groups must be event-keyed. -- **Corrected `pyproject.toml`'s mutmut comment**, which said mutants were generated for the DNS dataclasses; mutmut 3.7.0 never mutates decorated classes or functions. +- **`pyproject.toml`'s mutmut comment now says that decorated functions are never mutated**, so the `@lru_cache` DNS section parsers sit outside the audit whatever the filters say. - **Tightened three DNS corpus assertions in `tests/test_dns.py` from substring/`endswith` checks to exact equality, closing 3 open CodeQL `py/incomplete-url-substring-sanitization` alerts (#166).** `test_names_decompress_to_real_domains` asserted `"example.com" in names` / `"accounts.youtube.com" in names` against a `set` decoded from a fixed pcap corpus, and `test_answers_parse_to_real_records` asserted `cname.rdata_text.endswith("google.com")` against the same corpus — not security-relevant trust checks, but exactly the substring/suffix pattern the rule flags regardless of what the value's actually validating. Both now compare against the known exact decoded values instead (`names == {"example.com", "accounts.youtube.com"}`, `rdata_text == "www3.l.google.com"`), which removes the pattern CodeQL keys on and is strictly more precise than the checks it replaces. - **Consolidated six open Dependabot updates and grouped `github/codeql-action` so its sub-actions can no longer be bumped apart (#198, #199, #204, #205, #206, #207).** Dependabot tracks `github/codeql-action/init`, `/analyze`, and `/upload-sarif` as three separate dependencies and opened one PR each; `analyze` refuses a config file written by a different `init` release ("Loaded a configuration file for version '4.37.9', but running version '4.38.1'"), so #204 and #206 each failed CodeQL alone and neither could ever merge. All three now move to v4.38.1 together, alongside `astral-sh/setup-uv` 10.0.1 → 10.2.0, `hypothesis` 6.167.1 → 6.168.0, `ruff` 0.16.6 → 0.16.7, and `mutmut` 3.7.0 → 3.8.0 (lockfile byte-identical to Dependabot's). `dependabot.yml` gains a `codeql-action` group for the `github-actions` ecosystem, and `tests/test_workflows.py` gains `TestCodeQLActionVersionLockstep`, which fails the ordinary test job — naming the mismatched steps — whenever `github/codeql-action/*` steps across all workflows are pinned to more than one commit. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d7cece8..6c54d20 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -146,10 +146,11 @@ A survivor a test *could* kill gets that test. Only a true no-op, or one out of any test's reach, goes in the triage record, under a comment saying why. Two properties of the setup are easy to trip over: -- mutmut 3.7.0 never mutates a decorated function (other than a lone - `@staticmethod`/`@classmethod`) or anything inside a decorated class, - so logic meant to be audited belongs in undecorated module-level - functions. +- mutmut never mutates a decorated function (other than a lone + `@staticmethod`/`@classmethod`) — an `@lru_cache`'d parser, say — so + logic meant to be audited belongs in undecorated functions. (Methods + of a decorated class such as a `@dataclass` are mutated, as of + mutmut 3.8.0.) - `tests/conftest.py` clears every package-level `functools` cache before each test, so a result cached by the original code can't hide a mutant. diff --git a/pyproject.toml b/pyproject.toml index a77ad83..4b186c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,14 +131,13 @@ pytest_add_cli_args_test_selection = [ # uv run mutmut run 'netprotocols.checksum.*' 'netprotocols._tlv.*' \ # 'netprotocols.layer7.dns.x__read_name*' \ # 'netprotocols.layer7.dns.x__labels*' -# Mutants mutmut generates elsewhere in dns.py (the record/RDATA -# parsing helpers, ...) are out of scope for this audit and were not -# triaged. mutmut 3.7.0 generates none at all for a decorated function -# other than a lone @staticmethod/@classmethod, or for anything inside -# a decorated class -- so the @lru_cache section parsers and the -# @dataclass DNS/DNSOverTCP classes are never mutated, whatever the -# filters say. Code meant to be audited has to live in undecorated -# module-level functions. +# Mutants mutmut generates elsewhere in dns.py (record/RDATA parsing, +# the DNS/DNSOverTCP dataclass methods, ...) are out of scope for this +# audit and were not triaged. mutmut (3.8.0) generates none at all for +# a decorated function other than a lone @staticmethod/@classmethod, so +# the @lru_cache section parsers are never mutated, whatever the +# filters say; logic meant to be audited belongs in undecorated +# functions. # # Triage record: scripts/mutmut_survivors.txt lists each accepted # survivor with the reason it is equivalent, and diff --git a/scripts/check_mutmut_survivors.py b/scripts/check_mutmut_survivors.py index 40ee148..79b3f6c 100644 --- a/scripts/check_mutmut_survivors.py +++ b/scripts/check_mutmut_survivors.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Fail a mutmut run that left an unaccepted mutant undetected. -``mutmut run`` (3.7.0, pinned in uv.lock) exits 0 however many mutants +``mutmut run`` (3.8.0, pinned in uv.lock) exits 0 however many mutants survive: it only exits non-zero when the stats or clean run itself fails. Without this check the nightly mutation workflow is green no matter what it finds. This script reads the full status listing and From 3f440c4e6a2dbb15969e7602729b77583a70c385 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 17:50:43 +0000 Subject: [PATCH 3/3] fix(mutation): let the triage record accept only `survived` mutants An allowlisted mutant whose status turned into `no tests`, `suspicious` or a status a newer mutmut introduces was still subtracted from the undetected set, so the gate passed despite promising to fail closed (CodeRabbit, #213). The record exists to accept equivalent survivors, so it now suppresses status `survived` and nothing else; any other undetected status is reported once, allowlisted or not. Implemented and reviewed by independent sub-agents, per .claude/skills/steward/SKILL.md. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CEQCZGyuvpMyZwUbDucF3G --- scripts/check_mutmut_survivors.py | 13 +++++++++++-- tests/test_mutmut_gate.py | 12 ++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/scripts/check_mutmut_survivors.py b/scripts/check_mutmut_survivors.py index 79b3f6c..281748a 100644 --- a/scripts/check_mutmut_survivors.py +++ b/scripts/check_mutmut_survivors.py @@ -9,7 +9,9 @@ - a mutant that went undetected (survived, had no covering tests, or ended in a status this script does not recognize, so a mutmut - upgrade that adds one fails closed) and is not in the allowlist; + upgrade that adds one fails closed), unless it is in the allowlist + and its status is exactly ``survived``: the allowlist accepts + equivalent survivors and nothing else; - an allowlist entry that is no longer undetected, so the list cannot quietly accumulate entries for mutants a later test kills, or whose number shifted because the function's source changed (mutant numbers @@ -49,6 +51,9 @@ #: Outside this run's filters, or skipped by mutmut itself. UNCHECKED = frozenset({"not checked", "skipped"}) +#: The one undetected status an allowlist entry may accept. +SURVIVED = "survived" + _LINE = re.compile(r"^\s*(?P\S+): (?P.+?)\s*$") @@ -94,9 +99,13 @@ def check( for name, status in statuses.items() if status not in DETECTED and status not in UNCHECKED } + # The allowlist accepts equivalent survivors only: an allowlisted + # mutant in any other undetected status is still reported (once, + # here, not as stale too). + accepted = {name for name in allowlist if statuses.get(name) == SURVIVED} problems = [ f"undetected mutant: {name} ({statuses[name]})" - for name in sorted(undetected - allowlist) + for name in sorted(undetected - accepted) ] problems += [ f"allowlisted mutant is no longer undetected: {name} " diff --git a/tests/test_mutmut_gate.py b/tests/test_mutmut_gate.py index 0e98f68..8bf63bf 100644 --- a/tests/test_mutmut_gate.py +++ b/tests/test_mutmut_gate.py @@ -94,6 +94,18 @@ def test_an_allowlisted_survivor_passes(self) -> None: allowlist = {"netprotocols._tlv.x_g__mutmut_2"} assert gate.check(results, allowlist, MODULES) == [] + @pytest.mark.parametrize( + "status", ["no tests", "suspicious", "a status from v4"] + ) + def test_the_allowlist_accepts_only_survivors(self, status: str) -> None: + """An allowlisted mutant that is undetected for any reason but + `survived` is reported once, as undetected, not as stale.""" + results = statuses() + name = "netprotocols._tlv.x_g__mutmut_2" + results[name] = status + (problem,) = gate.check(results, {name}, MODULES) + assert f"undetected mutant: {name}" in problem + @pytest.mark.parametrize("status", ["killed", None]) def test_a_stale_allowlist_entry_fails(self, status: str | None) -> None: """An entry whose mutant is now killed, or no longer exists