diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b6216ba..84c441d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,9 +34,11 @@ The runner image bundles the `pred` binary, the agent stack (mini-swe-agent + Li and the problem-reductions source pinned at `v0.6.0`. Any LiteLLM-routable provider key works. -The **target library version is not hardcoded** — it tracks the benchmark. `PR_REF` selects -a tag or commit of problem-reductions; the image records that commit and its matching -`pred`. Pull the published image for that ref, or build it locally when unavailable: +The official round is pinned to the exact tag and commit listed in the +[README](README.md#current-benchmark-round). `PR_REF` selects the corresponding +problem-reductions source when preparing an image; the runner records the resolved commit +and its matching `pred`. Pull the published image for that ref, or build it locally when +unavailable: ```bash make runner-pull PR_REF=v0.6.0 @@ -210,7 +212,7 @@ python -m benchmark.backend_score --local submissions/ results/scored/ For each `PENDING` submission it: -1. validates the current schema, pinned library commit, and clean-run status, then flips +1. validates the current submission structure, pinned library commit, and clean-run status, then flips status `PENDING → RUNNING`, 2. re-runs `benchmark/verify_submission.py` — which calls `verify()` on **every** certificate and re-derives the bundle from `pred`, so a fabricated or tampered diff --git a/README.md b/README.md index c52e719..c43c162 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,18 @@ A benchmark that measures how efficiently AI models find bugs in reduction rules The leaderboard is a static site (`site/`) published to **GitHub Pages**. Submitting uses a CLI (`python -m benchmark.submit`) that uploads your run to a private store; the maintainer re-verifies it with `pred` and publishes only the aggregate. See [CONTRIBUTING.md](CONTRIBUTING.md) to run and submit. +## Current benchmark round + +| Contract | Current value | Ownership | +|---|---|---| +| Submission format | [`benchmark/submission.schema.json`](benchmark/submission.schema.json) | The runner writes the current structure and the scorer parses that structure directly. The payload has no schema-version field. | +| `problem-reductions` target | [`v0.6.0` / `aa2d1a10cffa434871d12a4d6f411147fb7e08a8`](https://github.com/CodingThrust/problem-reductions/commit/aa2d1a10cffa434871d12a4d6f411147fb7e08a8) | Every official result is verified against this exact commit. | +| `pred` | `0.6.0` | The runner/scorer image supplies and verifies the matching binary. | + +Do not hand-edit `library_commit` in `submission.json`. The runner records it; the intake +client performs a structural courtesy check, and the private scorer is the authority for +the current submission structure and round pin. + ## What this measures A reduction rule maps problem A → problem B. A **bug** is a round-trip failure: diff --git a/benchmark/backend_score.py b/benchmark/backend_score.py index baa5317..2a4d503 100644 --- a/benchmark/backend_score.py +++ b/benchmark/backend_score.py @@ -25,10 +25,10 @@ from benchmark.env_setup import pinned_commit from benchmark.submit import validate_submission +from benchmark.submit_ledger import has_submit_ledger from benchmark.verify_submission import leaderboard_entry, score_submission STATUS_SUFFIX = ".status.json" -OFFICIAL_SCHEMA_VERSION = "2.1" class PermanentSubmissionError(ValueError): @@ -180,8 +180,9 @@ def _validate_for_scoring(submission: object, *, official: bool, Basic validation applies to local and production scoring so malformed input is a permanent per-object failure instead of an exception that jams the whole queue. The - official gate additionally fixes the schema and target commit, and rejects partial runs - from the public leaderboard. Test submissions may be partial because they never publish. + official gate additionally requires the current ledger-backed structure and target + commit, and rejects partial runs from the public leaderboard. Test submissions may be + partial because they never publish. """ if not isinstance(submission, dict): raise PermanentSubmissionError("submission is not a JSON object") @@ -189,9 +190,8 @@ def _validate_for_scoring(submission: object, *, official: bool, problems = validate_submission(submission) if official: - if submission.get("schema_version") != OFFICIAL_SCHEMA_VERSION: - problems.append( - f"official submissions require schema_version {OFFICIAL_SCHEMA_VERSION}") + if not has_submit_ledger(submission): + problems.append("official submissions require submit_limit and submit_log") target_commit = expected_commit or pinned_commit() if submission.get("library_commit") != target_commit: problems.append( @@ -320,7 +320,8 @@ def main() -> None: parser.add_argument("--repo-dir", default=None, help="problem-reductions repo (default: pred on PATH)") parser.add_argument( "--official", action="store_true", - help="Require the current schema, pinned library commit, and a clean non-test run") + help="Require the current submission structure, pinned library commit, and a clean " + "non-test run") parser.add_argument( "--expected-commit", default=None, help="Override the official target commit (tests/operations; default: image pin)") diff --git a/benchmark/results.schema.json b/benchmark/results.schema.json index 32ad57f..3e35003 100644 --- a/benchmark/results.schema.json +++ b/benchmark/results.schema.json @@ -6,10 +6,6 @@ "type": "object", "required": ["model", "library_commit", "bugs_found", "total_tokens_k", "efficiency_bugs_per_ktok", "rules_tested", "results"], "properties": { - "schema_version": { - "type": ["string", "null"], - "description": "Submission schema version carried through for private auditability" - }, "model": { "type": "string", "description": "LiteLLM model name (e.g. 'anthropic/claude-sonnet-4-6')" diff --git a/benchmark/run_submission.py b/benchmark/run_submission.py index 9ee5b9a..a7508ba 100644 --- a/benchmark/run_submission.py +++ b/benchmark/run_submission.py @@ -42,7 +42,6 @@ from benchmark.usage import Usage, usage_as_dict, usage_from_dict from benchmark.verify import count_bugs -SCHEMA_VERSION = "2.1" RUNNER_VERSION = "0.8.0" BACKENDS = ("mini-swe", "claude-code", "codex") @@ -89,7 +88,6 @@ def build_submission( submit_attempts = submit_attempts or [] envelope = { - "schema_version": SCHEMA_VERSION, "model": model, "library_commit": library_commit, "bugs_found": bugs, diff --git a/benchmark/submission.schema.json b/benchmark/submission.schema.json index ebc20a6..0a403ca 100644 --- a/benchmark/submission.schema.json +++ b/benchmark/submission.schema.json @@ -4,12 +4,8 @@ "title": "BenchmarkSubmission", "description": "A self-describing, rankable whole-repository benchmark submission. The bounded submit ledger lets the backend rank and re-verify every claimed counterexample.", "type": "object", - "required": ["schema_version", "model", "library_commit", "bugs_found", "total_tokens_k", "rules_tested", "results", "submit_limit", "submit_log"], + "required": ["model", "library_commit", "bugs_found", "total_tokens_k", "rules_tested", "results", "submit_limit", "submit_log"], "properties": { - "schema_version": { - "type": "string", - "description": "Submission schema version (e.g. '1.0')" - }, "model": { "type": "string", "description": "LiteLLM model name (e.g. 'anthropic/claude-sonnet-4-6')" @@ -75,7 +71,7 @@ }, "trajectory": { "type": "array", - "description": "The agent's own message history for this rule. Schema 2.1 uses submit_log for provenance; legacy submissions use this trajectory.", + "description": "The agent's own message history for this rule. Retained for historical data; current submissions use submit_log for provenance.", "items": { "type": "object", "properties": { @@ -122,7 +118,7 @@ "test": {"type": "boolean", "description": "When true, the backend scores and stores the submission privately but EXCLUDES it from the public leaderboard (for end-to-end tests). Set via `prb submit --test`."}, "trajectory": { "type": "array", - "description": "Envelope-level session trajectory for a whole-repo run, stored once instead of on every row. Schema 2.1 provenance comes from submit_log.", + "description": "Envelope-level session trajectory for a whole-repo run, stored once instead of on every row. Current submissions use submit_log for provenance.", "items": { "type": "object", "properties": { diff --git a/benchmark/submit.py b/benchmark/submit.py index 15668bb..f4ade28 100644 --- a/benchmark/submit.py +++ b/benchmark/submit.py @@ -31,10 +31,10 @@ import urllib.request from pathlib import Path -from benchmark.submit_ledger import schema_requires_ledger, submit_ledger_error +from benchmark.submit_ledger import submit_ledger_error -REQUIRED_ENVELOPE = ("schema_version", "model", "library_commit", "bugs_found", - "total_tokens_k", "rules_tested", "results") +REQUIRED_ENVELOPE = ("model", "library_commit", "bugs_found", "total_tokens_k", + "rules_tested", "results") def load_submission(path: Path) -> dict: @@ -110,7 +110,7 @@ def validate_submission(sub: dict) -> list[str]: rule = row.get("rule", "?") if not row.get("certificate"): problems.append(f"results[{i}] ({rule}): bug_found row has no certificate") - if (not schema_requires_ledger(sub.get("schema_version")) + if ("submit_log" not in sub and not row.get("trajectory") and not envelope_traj): problems.append(f"results[{i}] ({rule}): bug_found row has no trajectory " "(required as provenance — on the row or the envelope)") diff --git a/benchmark/submit_ledger.py b/benchmark/submit_ledger.py index e3fc0c7..e43799d 100644 --- a/benchmark/submit_ledger.py +++ b/benchmark/submit_ledger.py @@ -3,17 +3,6 @@ import json -LEDGER_SCHEMA = (2, 1) - - -def schema_requires_ledger(version) -> bool: - try: - parts = tuple(int(part) for part in str(version).split(".")[:2]) - except ValueError: - return False - return parts >= LEDGER_SCHEMA - - def has_submit_ledger(submission: dict) -> bool: return "submit_limit" in submission or "submit_log" in submission @@ -21,8 +10,7 @@ def has_submit_ledger(submission: dict) -> bool: def submit_ledger_error(submission: dict) -> str | None: """Return one structural error, or ``None`` for a valid/legacy submission.""" if not has_submit_ledger(submission): - return ("schema 2.1+ submission is missing submit_limit and submit_log" - if schema_requires_ledger(submission.get("schema_version")) else None) + return None limit, log = submission.get("submit_limit"), submission.get("submit_log") if not isinstance(limit, int) or isinstance(limit, bool) or limit < 0: diff --git a/benchmark/tests/test_backend_score.py b/benchmark/tests/test_backend_score.py index ab923cd..54dd550 100644 --- a/benchmark/tests/test_backend_score.py +++ b/benchmark/tests/test_backend_score.py @@ -23,7 +23,6 @@ def _traj(cert: dict) -> list[dict]: def _write_submission(subs_dir: Path, name: str, results, **over) -> Path: results = [{"tokens_k": 0, **row} for row in results] sub = { - "schema_version": "2.0", "model": over.pop("model", "anthropic/test"), "library_commit": "deadbeef", "bugs_found": over.pop("bugs_found", 0), @@ -161,18 +160,18 @@ def test_main_exits_zero_when_all_scored(self, tmp_path, monkeypatch): monkeypatch.setattr("sys.argv", ["backend_score", "--local", str(subs), str(results)]) bs.main() # returns normally (no SystemExit) → exit 0 - def test_official_gate_rejects_wrong_schema_commit_and_partial_run(self, tmp_path): + def test_official_gate_rejects_missing_ledger_wrong_commit_and_partial_run(self, tmp_path): subs, results = tmp_path / "subs", tmp_path / "results" subs.mkdir() _write_submission( subs, "bad-round.json", [{"rule": "R", "result": "no_bug", "tokens_k": 0}], - schema_version="2.0", library_commit="wrong", run_error="quota exhausted") + library_commit="wrong", run_error="quota exhausted") summary = bs.process_local( str(subs), str(results), official=True, expected_commit="expected") assert summary[0]["status"] == "FAILED" assert summary[0]["retryable"] is False error = summary[0]["error"] - assert "schema_version 2.1" in error + assert "require submit_limit and submit_log" in error assert "library_commit" in error assert "partial run" in error @@ -181,7 +180,7 @@ def test_official_gate_accepts_current_clean_submission(self, tmp_path): subs.mkdir() _write_submission( subs, "ok.json", [{"rule": "R", "result": "no_bug", "tokens_k": 0}], - schema_version="2.1", library_commit="expected", submit_limit=1, + library_commit="expected", submit_limit=1, submit_log=[]) summary = bs.process_local( str(subs), str(results), official=True, expected_commit="expected") @@ -192,7 +191,7 @@ def test_official_test_submission_may_preserve_run_error(self, tmp_path): subs.mkdir() _write_submission( subs, "test.json", [{"rule": "R", "result": "no_bug", "tokens_k": 0}], - schema_version="2.1", library_commit="expected", submit_limit=1, + library_commit="expected", submit_limit=1, submit_log=[], test=True, run_error="intentional smoke failure") summary = bs.process_local( str(subs), str(results), official=True, expected_commit="expected") diff --git a/benchmark/tests/test_docs.py b/benchmark/tests/test_docs.py index c02ff56..0957a8d 100644 --- a/benchmark/tests/test_docs.py +++ b/benchmark/tests/test_docs.py @@ -6,6 +6,8 @@ import pytest from pathlib import Path +from benchmark.env_setup import PINNED_COMMIT, PINNED_PRED_VERSION + pytestmark = pytest.mark.judgment REPO_ROOT = Path(__file__).parent.parent.parent @@ -40,6 +42,12 @@ def test_readme_has_metrics_section(self): t = _text(README) assert "bugs/ktok" in t or "bugs_per_ktok" in t + def test_readme_lists_current_round_contract(self): + text = README.read_text(encoding="utf-8") + assert PINNED_COMMIT in text + assert f"`{PINNED_PRED_VERSION}`" in text + assert "no schema-version field" in text.lower() + class TestGuide: def test_guide_exists(self): diff --git a/benchmark/tests/test_run_submission.py b/benchmark/tests/test_run_submission.py index b195136..a1d2bce 100644 --- a/benchmark/tests/test_run_submission.py +++ b/benchmark/tests/test_run_submission.py @@ -52,7 +52,7 @@ class TestBuildSubmission: def test_envelope_fields_present(self): rows = [{"rule": "r1", "result": "no_certificate", "tokens_k": 2.0}] sub = rs.build_submission("anthropic/x", rows, library_commit="abc123") - for k in ("schema_version", "model", "library_commit", + for k in ("model", "library_commit", "bugs_found", "total_tokens_k", "rules_tested", "results", "efficiency_bugs_per_ktok", "submit_limit", "submit_log"): assert k in sub @@ -91,7 +91,7 @@ class TestRunFake: def test_produces_rankable_submission(self, tmp_path): repo = _fake_repo(tmp_path) sub = rs.run("fake/model", str(repo), fake=True, library_commit="deadbeef") - assert sub["schema_version"] + assert "schema_version" not in sub assert sub["agent_mode"] == "whole-repo" assert sub["rules_tested"] == 0 assert sub["bugs_found"] == 0 @@ -116,3 +116,4 @@ def test_submission_matches_schema_required_fields(self, tmp_path): (Path(rs.__file__).parent / "submission.schema.json").read_text()) for field in schema["required"]: assert field in sub, f"missing required field: {field}" + assert "schema_version" not in schema["properties"] diff --git a/benchmark/tests/test_submit.py b/benchmark/tests/test_submit.py index 012b4d3..5957ae0 100644 --- a/benchmark/tests/test_submit.py +++ b/benchmark/tests/test_submit.py @@ -9,11 +9,13 @@ def _valid(tmp_path: Path, results=None) -> Path: doc = { - "schema_version": "2.0", "model": "anthropic/test", "library_commit": "deadbeef", + "model": "anthropic/test", "library_commit": "deadbeef", "bugs_found": 0, "total_tokens_k": 10.0, "rules_tested": 1, "results": results if results is not None else [ {"rule": "r1", "result": "no_certificate", "tokens_k": 1.0}], + "submit_limit": 100, + "submit_log": [], } p = tmp_path / "submission.json" p.write_text(json.dumps(doc), encoding="utf-8") @@ -36,13 +38,13 @@ def test_clean_passes(self, tmp_path): def test_missing_envelope_field(self): assert any("model" in p for p in sub.validate_submission({"results": []})) - def test_bug_found_needs_certificate_and_trajectory(self): - doc = {**json.loads('{"schema_version":"2","model":"m","library_commit":"c",' - '"total_tokens_k":0,"rules_tested":1}'), + def test_bug_found_needs_certificate(self): + doc = {**json.loads('{"model":"m","library_commit":"c",' + '"total_tokens_k":0,"rules_tested":1,' + '"submit_limit":100,"submit_log":[]}'), "results": [_bug_row(with_cert=False, with_traj=False)]} problems = sub.validate_submission(doc) assert any("no certificate" in p for p in problems) - assert any("no trajectory" in p for p in problems) def test_bug_found_complete_passes(self, tmp_path): p = _valid(tmp_path, results=[_bug_row()]) @@ -61,11 +63,6 @@ def test_valid_submit_ledger_passes(self, tmp_path): doc["submit_log"] = [{"attempt": 1, "accepted": False, "reason": "bad"}] assert sub.validate_submission(doc) == [] - def test_schema_2_1_requires_ledger(self, tmp_path): - doc = sub.load_submission(_valid(tmp_path)) - doc["schema_version"] = "2.1" - assert any("missing submit_limit" in p for p in sub.validate_submission(doc)) - class TestSubmit: def test_dry_run_does_not_send(self, tmp_path, monkeypatch): @@ -75,8 +72,8 @@ def test_dry_run_does_not_send(self, tmp_path, monkeypatch): def test_invalid_submission_raises_before_network(self, tmp_path, monkeypatch): monkeypatch.setattr(sub, "_post", lambda *a, **k: pytest.fail("must not POST invalid")) - p = _valid(tmp_path, results=[_bug_row(with_traj=False)]) - with pytest.raises(ValueError, match="trajectory"): + p = _valid(tmp_path, results=[_bug_row(with_cert=False, with_traj=False)]) + with pytest.raises(ValueError, match="certificate"): sub.submit(p, "https://x/submit", "k") def test_success_returns_body(self, tmp_path, monkeypatch): diff --git a/benchmark/tests/test_verify_submission.py b/benchmark/tests/test_verify_submission.py index df5e2b4..ba48d58 100644 --- a/benchmark/tests/test_verify_submission.py +++ b/benchmark/tests/test_verify_submission.py @@ -30,7 +30,6 @@ def _bug_row(cert: dict, **over) -> dict: def _submission(results, **over) -> dict: base = { - "schema_version": "2.0", "model": "anthropic/test", "library_commit": "deadbeef", "bugs_found": 999, # deliberately wrong — the scorer must ignore it @@ -125,7 +124,7 @@ def test_new_submission_requires_bounded_submit_ledger(self, monkeypatch): monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok")) cert = {"rule": "r1", "source": {"n": 1}, "bundle": {}} sub = _submission( - [_bug_row(cert)], schema_version="2.1", + [_bug_row(cert)], submit_limit=100, submit_log=[{"attempt": 1, "accepted": True, "rule": "r1", "reason": "ok", "certificate": cert}], @@ -134,19 +133,12 @@ def test_new_submission_requires_bounded_submit_ledger(self, monkeypatch): assert scored["bugs_found"] == 1 assert "bounded submit command" in report[0]["provenance"] - def test_schema_2_1_cannot_downgrade_by_deleting_ledger(self, monkeypatch): - monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok")) - cert = {"rule": "r1", "source": {"n": 1}, "bundle": {}} - scored, _ = vs.score_submission(_submission([_bug_row(cert)], schema_version="2.1")) - assert scored["bugs_found"] == 0 - assert "missing submit_limit" in scored["results"][0]["reject_reason"] - def test_final_answer_certificate_not_in_submit_ledger_is_rejected(self, monkeypatch): monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok")) submitted = {"rule": "r1", "source": {"n": 1}, "bundle": {}} only_in_prose = {"rule": "r2", "source": {"n": 2}, "bundle": {}} sub = _submission( - [_bug_row(only_in_prose)], schema_version="2.1", + [_bug_row(only_in_prose)], submit_limit=100, submit_log=[{"attempt": 1, "accepted": True, "rule": "r1", "reason": "ok", "certificate": submitted}], @@ -159,7 +151,7 @@ def test_tampered_budget_log_fails_closed(self, monkeypatch): monkeypatch.setattr(vs, "verify", lambda c, r=None: Verdict(True, "ok")) cert = {"rule": "r1", "source": {"n": 1}, "bundle": {}} sub = _submission( - [_bug_row(cert)], schema_version="2.1", + [_bug_row(cert)], submit_limit=0, submit_log=[{"attempt": 1, "accepted": True, "rule": "r1", "reason": "ok", "certificate": cert}], @@ -173,7 +165,7 @@ def test_ledger_certificate_cannot_be_counted_under_another_row_rule(self, monke cert = {"rule": "r1", "source": {"n": 1}, "bundle": {}} forged_row = _bug_row(cert, rule="r2") sub = _submission( - [forged_row], schema_version="2.1", submit_limit=1, + [forged_row], submit_limit=1, submit_log=[{"attempt": 1, "accepted": True, "rule": "r1", "reason": "ok", "certificate": cert}], ) diff --git a/benchmark/verify_submission.py b/benchmark/verify_submission.py index f8e4f8a..fb8ea1e 100644 --- a/benchmark/verify_submission.py +++ b/benchmark/verify_submission.py @@ -24,8 +24,7 @@ from pathlib import Path from benchmark.submit_ledger import (accepted_certificate_index, certificate_key, - has_submit_ledger, schema_requires_ledger, - submit_ledger_error) + has_submit_ledger, submit_ledger_error) from benchmark.usage import usage_from_dict from benchmark.verify import count_bugs, verify @@ -76,15 +75,14 @@ def score_submission(submission: dict, repo_dir: str | None = None) -> tuple[dic """Re-verify every certificate and recompute the score. A bug counts only when pred confirms the round-trip failure and provenance succeeds: - bounded submit ledger for schema 2.1+, trajectory certificate for legacy submissions. + bounded submit ledger for current submissions, trajectory certificate for legacy data. Returns (scored, report); ``scored`` is results.schema.json-shaped and ``report`` is a per-certificate list of {rule, violation, accepted, reason, provenance}. """ rescored: list[dict] = [] report: list[dict] = [] ledger_problem = submit_ledger_error(submission) - uses_ledger = (has_submit_ledger(submission) - or schema_requires_ledger(submission.get("schema_version"))) + uses_ledger = has_submit_ledger(submission) accepted_keys = (accepted_certificate_index(submission) if uses_ledger and ledger_problem is None else set()) # Only legacy submissions use trajectory provenance. New runs use the ledger directly. @@ -139,7 +137,6 @@ def score_submission(submission: dict, repo_dir: str | None = None) -> tuple[dic total_tokens = usage_from_dict(submission.get("usage_totals")).total_tokens tokens_k = total_tokens / 1000 if total_tokens else (submission.get("total_tokens_k", 0.0) or 0.0) scored = { - "schema_version": submission.get("schema_version"), "model": submission.get("model", "unknown"), "library_commit": submission.get("library_commit", "unknown"), # Test submissions (end-to-end checks) are scored and stored privately like any