Fail the benchmark when a baseline case disappears

src/docgen/scene_benchmark.py

@@ -367,7 +367,38 @@ def run_benchmark(
         if not cases:
             known = ", ".join(c.id for c in standard_cases())
             raise ValueError(f"unknown benchmark case {case_id!r}; known: {known}")
-    return [score_case(c) for c in cases]
+    return _mark_case_filter([score_case(c) for c in cases], case_id)
+
+
+def _mark_case_filter(scores: list[CaseScore], case_id: str | None) -> list[CaseScore]:
+    """Remember an explicit ``--case`` filter so other baseline ids are not deletions."""
+    if not case_id:
+        return scores
+    for score in scores:
+        score.filtered_case_id = case_id
+    return scores
+
+
+def _filtered_case_id(scores: list[CaseScore]) -> str | None:
+    if not scores:
+        return None
+    marked = [getattr(score, "filtered_case_id", None) for score in scores]
+    if any(item is None for item in marked):
+        return None
+    if len(set(marked)) != 1:
+        return None
+    return marked[0]
+
+
+def _stored_cases_for_scores(
+    baseline: dict[str, Any],
+    scores: list[CaseScore],
+) -> dict[str, Any]:
+    stored = baseline.get("cases") if isinstance(baseline.get("cases"), dict) else {}
+    case_id = _filtered_case_id(scores)
+    if not case_id:
+        return stored
+    return {key: value for key, value in stored.items() if key == case_id}
 
 
 def load_baseline(path: Path | None = None) -> dict[str, Any]:
@@ -391,13 +422,38 @@ def write_baseline(scores: list[CaseScore], path: Path | None = None) -> Path:
     return p
 
 
+def baseline_scoped_to_case(baseline: dict[str, Any], case_id: str | None) -> dict[str, Any]:
+    """Limit a baseline to one case so a filtered run is not a deleted-corpus failure."""
+    if not case_id:
+        return baseline
+    stored = baseline.get("cases") if isinstance(baseline.get("cases"), dict) else {}
+    scoped = dict(baseline)
+    scoped["cases"] = {key: value for key, value in stored.items() if key == case_id}
+    return scoped
+
+
+def missing_baseline_case_notes(scores: list[CaseScore], stored: dict[str, Any]) -> list[str]:
+    """Baseline ids that this run did not score. Deleting a case must fail."""
+    scored_ids = {score.case_id for score in scores}
+    return [
+        f"{case_id}: missing from current scores"
+        for case_id in stored
+        if case_id not in scored_ids
+    ]
+
+
 def compare_to_baseline(
     scores: list[CaseScore],
     baseline: dict[str, Any],
 ) -> list[str]:
-    """Return regression notes. Empty means the run meets or beats the baseline."""
-    notes: list[str] = []
-    stored = baseline.get("cases") if isinstance(baseline.get("cases"), dict) else {}
+    """Return regression notes. Empty means the run meets or beats the baseline.
+
+    A baseline case id absent from ``scores`` is a failure. Deleting a corpus
+    case must not leave that committed id unchecked. Scores from
+    ``run_benchmark(case_id=...)`` only check that one id.
+    """
+    stored = _stored_cases_for_scores(baseline, scores)
+    notes: list[str] = missing_baseline_case_notes(scores, stored)
     for score in scores:
         prev = stored.get(score.case_id)
         if not isinstance(prev, dict):
@@ -481,7 +537,7 @@ def build_benchmark_report(
     """JSON payload for the CLI, wizard Vue view, and desktop GUI."""
     scores = run_benchmark(case_id=case_id)
     path = baseline_path or default_baseline_path()
-    baseline = load_baseline(path)
+    baseline = baseline_scoped_to_case(load_baseline(path), case_id)
     regressions = compare_to_baseline(scores, baseline)
     report = scores_as_json(scores, regressions=regressions)
     stored = baseline.get("cases") if isinstance(baseline.get("cases"), dict) else {}

tests/test_scene_benchmark.py

@@ -11,6 +11,7 @@
 from docgen.cli import main
 from docgen.scene_benchmark import (
     BenchmarkCase,
+    CaseScore,
     compare_to_baseline,
     default_baseline_path,
     format_table,
@@ -133,6 +134,39 @@ def test_full_corpus_meets_committed_baseline() -> None:
     assert "quality average" in table
 
 
+def _tiny_score(case_id: str) -> CaseScore:
+    return CaseScore(
+        case_id=case_id,
+        title=case_id,
+        role="quality",
+        wait_skips=0,
+        overshoots=0,
+        hold_idle_violations=0,
+        cadence_violations=0,
+        sim_drift=0,
+        mid_hold_pulses=1,
+        box_reveals=1,
+        last_motion_frac=1.0,
+        audio_end=1.0,
+        defect_points=0,
+        quality_points=10,
+        score=100,
+    )
+
+
+def test_compare_flags_baseline_id_missing_from_current_scores() -> None:
+    """A baseline id dropped from the score list must fail (leftover #17)."""
+    scores = [_tiny_score("alpha"), _tiny_score("beta")]
+    baseline = {"version": 1, "cases": {score.case_id: score.snapshot() for score in scores}}
+    assert compare_to_baseline(scores, baseline) == []
+    reduced = [score for score in scores if score.case_id != "beta"]
+    notes = compare_to_baseline(reduced, baseline)
+    assert notes == ["beta: missing from current scores"]
+    kept = next(score for score in scores if score.case_id == "alpha")
+    kept.filtered_case_id = "alpha"
+    assert compare_to_baseline([kept], baseline) == []
+
+
 def test_compare_flags_skip_regression() -> None:
     scores = run_benchmark()
     dump = load_baseline()