Remove submission schema version field

CONTRIBUTING.md

@@ -34,9 +34,11 @@ The runner image bundles the `pred` binary, the agent stack (mini-swe-agent + Li
 and the problem-reductions source pinned at `v0.6.0`. Any LiteLLM-routable provider key
 works.
 
-The **target library version is not hardcoded** — it tracks the benchmark. `PR_REF` selects
-a tag or commit of problem-reductions; the image records that commit and its matching
-`pred`. Pull the published image for that ref, or build it locally when unavailable:
+The official round is pinned to the exact tag and commit listed in the
+[README](README.md#current-benchmark-round). `PR_REF` selects the corresponding
+problem-reductions source when preparing an image; the runner records the resolved commit
+and its matching `pred`. Pull the published image for that ref, or build it locally when
+unavailable:
 
 ```bash
 make runner-pull PR_REF=v0.6.0
@@ -210,7 +212,7 @@ python -m benchmark.backend_score --local submissions/ results/scored/
 
 For each `PENDING` submission it:
 
-1. validates the current schema, pinned library commit, and clean-run status, then flips
+1. validates the current submission structure, pinned library commit, and clean-run status, then flips
    status `PENDING → RUNNING`,
 2. re-runs `benchmark/verify_submission.py` — which calls `verify()` on **every**
    certificate and re-derives the bundle from `pred`, so a fabricated or tampered

README.md

@@ -4,6 +4,18 @@ A benchmark that measures how efficiently AI models find bugs in reduction rules
 
 The leaderboard is a static site (`site/`) published to **GitHub Pages**. Submitting uses a CLI (`python -m benchmark.submit`) that uploads your run to a private store; the maintainer re-verifies it with `pred` and publishes only the aggregate. See [CONTRIBUTING.md](CONTRIBUTING.md) to run and submit.
 
+## Current benchmark round
+
+| Contract | Current value | Ownership |
+|---|---|---|
+| Submission format | [`benchmark/submission.schema.json`](benchmark/submission.schema.json) | The runner writes the current structure and the scorer parses that structure directly. The payload has no schema-version field. |
+| `problem-reductions` target | [`v0.6.0` / `aa2d1a10cffa434871d12a4d6f411147fb7e08a8`](https://github.com/CodingThrust/problem-reductions/commit/aa2d1a10cffa434871d12a4d6f411147fb7e08a8) | Every official result is verified against this exact commit. |
+| `pred` | `0.6.0` | The runner/scorer image supplies and verifies the matching binary. |
+
+Do not hand-edit `library_commit` in `submission.json`. The runner records it; the intake
+client performs a structural courtesy check, and the private scorer is the authority for
+the current submission structure and round pin.
+
 ## What this measures
 
 A reduction rule maps problem A → problem B. A **bug** is a round-trip failure:

benchmark/backend_score.py

@@ -25,10 +25,10 @@
 
 from benchmark.env_setup import pinned_commit
 from benchmark.submit import validate_submission
+from benchmark.submit_ledger import has_submit_ledger
 from benchmark.verify_submission import leaderboard_entry, score_submission
 
 STATUS_SUFFIX = ".status.json"
-OFFICIAL_SCHEMA_VERSION = "2.1"
 
 
 class PermanentSubmissionError(ValueError):
@@ -180,18 +180,18 @@ def _validate_for_scoring(submission: object, *, official: bool,
 
     Basic validation applies to local and production scoring so malformed input is a
     permanent per-object failure instead of an exception that jams the whole queue.  The
-    official gate additionally fixes the schema and target commit, and rejects partial runs
-    from the public leaderboard. Test submissions may be partial because they never publish.
+    official gate additionally requires the current ledger-backed structure and target
+    commit, and rejects partial runs from the public leaderboard. Test submissions may be
+    partial because they never publish.
     """
     if not isinstance(submission, dict):
         raise PermanentSubmissionError("submission is not a JSON object")
 
     problems = validate_submission(submission)
 
     if official:
-        if submission.get("schema_version") != OFFICIAL_SCHEMA_VERSION:
-            problems.append(
-                f"official submissions require schema_version {OFFICIAL_SCHEMA_VERSION}")
+        if not has_submit_ledger(submission):
+            problems.append("official submissions require submit_limit and submit_log")
         target_commit = expected_commit or pinned_commit()
         if submission.get("library_commit") != target_commit:
             problems.append(
@@ -320,7 +320,8 @@ def main() -> None:
     parser.add_argument("--repo-dir", default=None, help="problem-reductions repo (default: pred on PATH)")
     parser.add_argument(
         "--official", action="store_true",
-        help="Require the current schema, pinned library commit, and a clean non-test run")
+        help="Require the current submission structure, pinned library commit, and a clean "
+             "non-test run")
     parser.add_argument(
         "--expected-commit", default=None,
         help="Override the official target commit (tests/operations; default: image pin)")

benchmark/results.schema.json

@@ -6,10 +6,6 @@
   "type": "object",
   "required": ["model", "library_commit", "bugs_found", "total_tokens_k", "efficiency_bugs_per_ktok", "rules_tested", "results"],
   "properties": {
-    "schema_version": {
-      "type": ["string", "null"],
-      "description": "Submission schema version carried through for private auditability"
-    },
     "model": {
       "type": "string",
       "description": "LiteLLM model name (e.g. 'anthropic/claude-sonnet-4-6')"

benchmark/run_submission.py

@@ -42,7 +42,6 @@
 from benchmark.usage import Usage, usage_as_dict, usage_from_dict
 from benchmark.verify import count_bugs
 
-SCHEMA_VERSION = "2.1"
 RUNNER_VERSION = "0.8.0"
 BACKENDS = ("mini-swe", "claude-code", "codex")
 
@@ -89,7 +88,6 @@ def build_submission(
 
     submit_attempts = submit_attempts or []
     envelope = {
-        "schema_version": SCHEMA_VERSION,
         "model": model,
         "library_commit": library_commit,
         "bugs_found": bugs,

benchmark/submission.schema.json

@@ -4,12 +4,8 @@
   "title": "BenchmarkSubmission",
   "description": "A self-describing, rankable whole-repository benchmark submission. The bounded submit ledger lets the backend rank and re-verify every claimed counterexample.",
   "type": "object",
-  "required": ["schema_version", "model", "library_commit", "bugs_found", "total_tokens_k", "rules_tested", "results", "submit_limit", "submit_log"],
+  "required": ["model", "library_commit", "bugs_found", "total_tokens_k", "rules_tested", "results", "submit_limit", "submit_log"],
   "properties": {
-    "schema_version": {
-      "type": "string",
-      "description": "Submission schema version (e.g. '1.0')"
-    },
     "model": {
       "type": "string",
       "description": "LiteLLM model name (e.g. 'anthropic/claude-sonnet-4-6')"
@@ -75,7 +71,7 @@
           },
           "trajectory": {
             "type": "array",
-            "description": "The agent's own message history for this rule. Schema 2.1 uses submit_log for provenance; legacy submissions use this trajectory.",
+            "description": "The agent's own message history for this rule. Retained for historical data; current submissions use submit_log for provenance.",
             "items": {
               "type": "object",
               "properties": {
@@ -122,7 +118,7 @@
     "test": {"type": "boolean", "description": "When true, the backend scores and stores the submission privately but EXCLUDES it from the public leaderboard (for end-to-end tests). Set via `prb submit --test`."},
     "trajectory": {
       "type": "array",
-      "description": "Envelope-level session trajectory for a whole-repo run, stored once instead of on every row. Schema 2.1 provenance comes from submit_log.",
+      "description": "Envelope-level session trajectory for a whole-repo run, stored once instead of on every row. Current submissions use submit_log for provenance.",
       "items": {
         "type": "object",
         "properties": {

benchmark/submit.py

@@ -31,10 +31,10 @@
 import urllib.request
 from pathlib import Path
 
-from benchmark.submit_ledger import schema_requires_ledger, submit_ledger_error
+from benchmark.submit_ledger import submit_ledger_error
 
-REQUIRED_ENVELOPE = ("schema_version", "model", "library_commit", "bugs_found",
-                     "total_tokens_k", "rules_tested", "results")
+REQUIRED_ENVELOPE = ("model", "library_commit", "bugs_found", "total_tokens_k",
+                     "rules_tested", "results")
 
 
 def load_submission(path: Path) -> dict:
@@ -110,7 +110,7 @@ def validate_submission(sub: dict) -> list[str]:
             rule = row.get("rule", "?")
             if not row.get("certificate"):
                 problems.append(f"results[{i}] ({rule}): bug_found row has no certificate")
-            if (not schema_requires_ledger(sub.get("schema_version"))
+            if ("submit_log" not in sub
                     and not row.get("trajectory") and not envelope_traj):
                 problems.append(f"results[{i}] ({rule}): bug_found row has no trajectory "
                                 "(required as provenance — on the row or the envelope)")

benchmark/submit_ledger.py

@@ -3,26 +3,14 @@
 
 import json
 
-LEDGER_SCHEMA = (2, 1)
-
-
-def schema_requires_ledger(version) -> bool:
-    try:
-        parts = tuple(int(part) for part in str(version).split(".")[:2])
-    except ValueError:
-        return False
-    return parts >= LEDGER_SCHEMA
-
-
 def has_submit_ledger(submission: dict) -> bool:
     return "submit_limit" in submission or "submit_log" in submission
 
 
 def submit_ledger_error(submission: dict) -> str | None:
     """Return one structural error, or ``None`` for a valid/legacy submission."""
     if not has_submit_ledger(submission):
-        return ("schema 2.1+ submission is missing submit_limit and submit_log"
-                if schema_requires_ledger(submission.get("schema_version")) else None)
+        return None
 
     limit, log = submission.get("submit_limit"), submission.get("submit_log")
     if not isinstance(limit, int) or isinstance(limit, bool) or limit < 0:

benchmark/tests/test_backend_score.py

@@ -23,7 +23,6 @@ def _traj(cert: dict) -> list[dict]:
 def _write_submission(subs_dir: Path, name: str, results, **over) -> Path:
     results = [{"tokens_k": 0, **row} for row in results]
     sub = {
-        "schema_version": "2.0",
         "model": over.pop("model", "anthropic/test"),
         "library_commit": "deadbeef",
         "bugs_found": over.pop("bugs_found", 0),
@@ -161,18 +160,18 @@ def test_main_exits_zero_when_all_scored(self, tmp_path, monkeypatch):
         monkeypatch.setattr("sys.argv", ["backend_score", "--local", str(subs), str(results)])
         bs.main()  # returns normally (no SystemExit) → exit 0
 
-    def test_official_gate_rejects_wrong_schema_commit_and_partial_run(self, tmp_path):
+    def test_official_gate_rejects_missing_ledger_wrong_commit_and_partial_run(self, tmp_path):
         subs, results = tmp_path / "subs", tmp_path / "results"
         subs.mkdir()
         _write_submission(
             subs, "bad-round.json", [{"rule": "R", "result": "no_bug", "tokens_k": 0}],
-            schema_version="2.0", library_commit="wrong", run_error="quota exhausted")
+            library_commit="wrong", run_error="quota exhausted")
         summary = bs.process_local(
             str(subs), str(results), official=True, expected_commit="expected")
         assert summary[0]["status"] == "FAILED"
         assert summary[0]["retryable"] is False
         error = summary[0]["error"]
-        assert "schema_version 2.1" in error
+        assert "require submit_limit and submit_log" in error
         assert "library_commit" in error
         assert "partial run" in error
 
@@ -181,7 +180,7 @@ def test_official_gate_accepts_current_clean_submission(self, tmp_path):
         subs.mkdir()
         _write_submission(
             subs, "ok.json", [{"rule": "R", "result": "no_bug", "tokens_k": 0}],
-            schema_version="2.1", library_commit="expected", submit_limit=1,
+            library_commit="expected", submit_limit=1,
             submit_log=[])
         summary = bs.process_local(
             str(subs), str(results), official=True, expected_commit="expected")
@@ -192,7 +191,7 @@ def test_official_test_submission_may_preserve_run_error(self, tmp_path):
         subs.mkdir()
         _write_submission(
             subs, "test.json", [{"rule": "R", "result": "no_bug", "tokens_k": 0}],
-            schema_version="2.1", library_commit="expected", submit_limit=1,
+            library_commit="expected", submit_limit=1,
             submit_log=[], test=True, run_error="intentional smoke failure")
         summary = bs.process_local(
             str(subs), str(results), official=True, expected_commit="expected")

benchmark/tests/test_docs.py

@@ -6,6 +6,8 @@
 import pytest
 from pathlib import Path
 
+from benchmark.env_setup import PINNED_COMMIT, PINNED_PRED_VERSION
+
 pytestmark = pytest.mark.judgment
 
 REPO_ROOT = Path(__file__).parent.parent.parent
@@ -40,6 +42,12 @@ def test_readme_has_metrics_section(self):
         t = _text(README)
         assert "bugs/ktok" in t or "bugs_per_ktok" in t
 
+    def test_readme_lists_current_round_contract(self):
+        text = README.read_text(encoding="utf-8")
+        assert PINNED_COMMIT in text
+        assert f"`{PINNED_PRED_VERSION}`" in text
+        assert "no schema-version field" in text.lower()
+
 
 class TestGuide:
     def test_guide_exists(self):

benchmark/tests/test_run_submission.py

@@ -52,7 +52,7 @@ class TestBuildSubmission:
     def test_envelope_fields_present(self):
         rows = [{"rule": "r1", "result": "no_certificate", "tokens_k": 2.0}]
         sub = rs.build_submission("anthropic/x", rows, library_commit="abc123")
-        for k in ("schema_version", "model", "library_commit",
+        for k in ("model", "library_commit",
                   "bugs_found", "total_tokens_k", "rules_tested",
                   "results", "efficiency_bugs_per_ktok", "submit_limit", "submit_log"):
             assert k in sub
@@ -91,7 +91,7 @@ class TestRunFake:
     def test_produces_rankable_submission(self, tmp_path):
         repo = _fake_repo(tmp_path)
         sub = rs.run("fake/model", str(repo), fake=True, library_commit="deadbeef")
-        assert sub["schema_version"]
+        assert "schema_version" not in sub
         assert sub["agent_mode"] == "whole-repo"
         assert sub["rules_tested"] == 0
         assert sub["bugs_found"] == 0
@@ -116,3 +116,4 @@ def test_submission_matches_schema_required_fields(self, tmp_path):
             (Path(rs.__file__).parent / "submission.schema.json").read_text())
         for field in schema["required"]:
             assert field in sub, f"missing required field: {field}"
+        assert "schema_version" not in schema["properties"]

benchmark/tests/test_submit.py

@@ -9,11 +9,13 @@
 
 def _valid(tmp_path: Path, results=None) -> Path:
     doc = {
-        "schema_version": "2.0", "model": "anthropic/test", "library_commit": "deadbeef",
+        "model": "anthropic/test", "library_commit": "deadbeef",
         "bugs_found": 0,
         "total_tokens_k": 10.0,
         "rules_tested": 1, "results": results if results is not None else [
             {"rule": "r1", "result": "no_certificate", "tokens_k": 1.0}],
+        "submit_limit": 100,
+        "submit_log": [],
     }
     p = tmp_path / "submission.json"
     p.write_text(json.dumps(doc), encoding="utf-8")
@@ -36,13 +38,13 @@ def test_clean_passes(self, tmp_path):
     def test_missing_envelope_field(self):
         assert any("model" in p for p in sub.validate_submission({"results": []}))
 
-    def test_bug_found_needs_certificate_and_trajectory(self):
-        doc = {**json.loads('{"schema_version":"2","model":"m","library_commit":"c",'
-                            '"total_tokens_k":0,"rules_tested":1}'),
+    def test_bug_found_needs_certificate(self):
+        doc = {**json.loads('{"model":"m","library_commit":"c",'
+                            '"total_tokens_k":0,"rules_tested":1,'
+                            '"submit_limit":100,"submit_log":[]}'),
                "results": [_bug_row(with_cert=False, with_traj=False)]}
         problems = sub.validate_submission(doc)
         assert any("no certificate" in p for p in problems)
-        assert any("no trajectory" in p for p in problems)
 
     def test_bug_found_complete_passes(self, tmp_path):
         p = _valid(tmp_path, results=[_bug_row()])
@@ -61,11 +63,6 @@ def test_valid_submit_ledger_passes(self, tmp_path):
         doc["submit_log"] = [{"attempt": 1, "accepted": False, "reason": "bad"}]
         assert sub.validate_submission(doc) == []
 
-    def test_schema_2_1_requires_ledger(self, tmp_path):
-        doc = sub.load_submission(_valid(tmp_path))
-        doc["schema_version"] = "2.1"
-        assert any("missing submit_limit" in p for p in sub.validate_submission(doc))
-
 
 class TestSubmit:
     def test_dry_run_does_not_send(self, tmp_path, monkeypatch):
@@ -75,8 +72,8 @@ def test_dry_run_does_not_send(self, tmp_path, monkeypatch):
 
     def test_invalid_submission_raises_before_network(self, tmp_path, monkeypatch):
         monkeypatch.setattr(sub, "_post", lambda *a, **k: pytest.fail("must not POST invalid"))
-        p = _valid(tmp_path, results=[_bug_row(with_traj=False)])
-        with pytest.raises(ValueError, match="trajectory"):
+        p = _valid(tmp_path, results=[_bug_row(with_cert=False, with_traj=False)])
+        with pytest.raises(ValueError, match="certificate"):
             sub.submit(p, "https://x/submit", "k")
 
     def test_success_returns_body(self, tmp_path, monkeypatch):