diff --git a/src/docgen/scene_benchmark.py b/src/docgen/scene_benchmark.py index 7315778..cad08ab 100644 --- a/src/docgen/scene_benchmark.py +++ b/src/docgen/scene_benchmark.py @@ -367,7 +367,38 @@ def run_benchmark( if not cases: known = ", ".join(c.id for c in standard_cases()) raise ValueError(f"unknown benchmark case {case_id!r}; known: {known}") - return [score_case(c) for c in cases] + return _mark_case_filter([score_case(c) for c in cases], case_id) + + +def _mark_case_filter(scores: list[CaseScore], case_id: str | None) -> list[CaseScore]: + """Remember an explicit ``--case`` filter so other baseline ids are not deletions.""" + if not case_id: + return scores + for score in scores: + score.filtered_case_id = case_id + return scores + + +def _filtered_case_id(scores: list[CaseScore]) -> str | None: + if not scores: + return None + marked = [getattr(score, "filtered_case_id", None) for score in scores] + if any(item is None for item in marked): + return None + if len(set(marked)) != 1: + return None + return marked[0] + + +def _stored_cases_for_scores( + baseline: dict[str, Any], + scores: list[CaseScore], +) -> dict[str, Any]: + stored = baseline.get("cases") if isinstance(baseline.get("cases"), dict) else {} + case_id = _filtered_case_id(scores) + if not case_id: + return stored + return {key: value for key, value in stored.items() if key == case_id} def load_baseline(path: Path | None = None) -> dict[str, Any]: @@ -391,13 +422,38 @@ def write_baseline(scores: list[CaseScore], path: Path | None = None) -> Path: return p +def baseline_scoped_to_case(baseline: dict[str, Any], case_id: str | None) -> dict[str, Any]: + """Limit a baseline to one case so a filtered run is not a deleted-corpus failure.""" + if not case_id: + return baseline + stored = baseline.get("cases") if isinstance(baseline.get("cases"), dict) else {} + scoped = dict(baseline) + scoped["cases"] = {key: value for key, value in stored.items() if key == case_id} + return scoped + + +def missing_baseline_case_notes(scores: list[CaseScore], stored: dict[str, Any]) -> list[str]: + """Baseline ids that this run did not score. Deleting a case must fail.""" + scored_ids = {score.case_id for score in scores} + return [ + f"{case_id}: missing from current scores" + for case_id in stored + if case_id not in scored_ids + ] + + def compare_to_baseline( scores: list[CaseScore], baseline: dict[str, Any], ) -> list[str]: - """Return regression notes. Empty means the run meets or beats the baseline.""" - notes: list[str] = [] - stored = baseline.get("cases") if isinstance(baseline.get("cases"), dict) else {} + """Return regression notes. Empty means the run meets or beats the baseline. + + A baseline case id absent from ``scores`` is a failure. Deleting a corpus + case must not leave that committed id unchecked. Scores from + ``run_benchmark(case_id=...)`` only check that one id. + """ + stored = _stored_cases_for_scores(baseline, scores) + notes: list[str] = missing_baseline_case_notes(scores, stored) for score in scores: prev = stored.get(score.case_id) if not isinstance(prev, dict): @@ -481,7 +537,7 @@ def build_benchmark_report( """JSON payload for the CLI, wizard Vue view, and desktop GUI.""" scores = run_benchmark(case_id=case_id) path = baseline_path or default_baseline_path() - baseline = load_baseline(path) + baseline = baseline_scoped_to_case(load_baseline(path), case_id) regressions = compare_to_baseline(scores, baseline) report = scores_as_json(scores, regressions=regressions) stored = baseline.get("cases") if isinstance(baseline.get("cases"), dict) else {} diff --git a/tests/test_scene_benchmark.py b/tests/test_scene_benchmark.py index 6377b02..65c271c 100644 --- a/tests/test_scene_benchmark.py +++ b/tests/test_scene_benchmark.py @@ -11,6 +11,7 @@ from docgen.cli import main from docgen.scene_benchmark import ( BenchmarkCase, + CaseScore, compare_to_baseline, default_baseline_path, format_table, @@ -133,6 +134,39 @@ def test_full_corpus_meets_committed_baseline() -> None: assert "quality average" in table +def _tiny_score(case_id: str) -> CaseScore: + return CaseScore( + case_id=case_id, + title=case_id, + role="quality", + wait_skips=0, + overshoots=0, + hold_idle_violations=0, + cadence_violations=0, + sim_drift=0, + mid_hold_pulses=1, + box_reveals=1, + last_motion_frac=1.0, + audio_end=1.0, + defect_points=0, + quality_points=10, + score=100, + ) + + +def test_compare_flags_baseline_id_missing_from_current_scores() -> None: + """A baseline id dropped from the score list must fail (leftover #17).""" + scores = [_tiny_score("alpha"), _tiny_score("beta")] + baseline = {"version": 1, "cases": {score.case_id: score.snapshot() for score in scores}} + assert compare_to_baseline(scores, baseline) == [] + reduced = [score for score in scores if score.case_id != "beta"] + notes = compare_to_baseline(reduced, baseline) + assert notes == ["beta: missing from current scores"] + kept = next(score for score in scores if score.case_id == "alpha") + kept.filtered_case_id = "alpha" + assert compare_to_baseline([kept], baseline) == [] + + def test_compare_flags_skip_regression() -> None: scores = run_benchmark() dump = load_baseline()