Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 180 additions & 0 deletions .github/workflows/publish-proof-eval-image.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
name: publish-proof-eval-image

# Publishes ghcr.io/cortexlm/proof-eval and prints the pushed sha256 digest.
# That digest is what the control plane pins in config/proof-pin.toml as
# eval_image_digest; a tag is never the pin. Do not invent a sha256.
#
# Two images, two Dockerfiles:
#
# :<sha> scoring — CUDA base + torch. Pin this, and only after
# this job has pulled THAT digest and run the harvest-PATH
# check + selftest against it.
# :<sha>-contract slim contract only. Fast, cannot score, not the pin.

on:
push:
branches: ["main", "cursor/**"]
paths:
- "eval/**"
- ".github/workflows/publish-proof-eval-image.yml"
workflow_dispatch:

permissions:
contents: read
packages: write

concurrency:
group: publish-proof-eval-${{ github.ref }}
cancel-in-progress: false

jobs:
contract:
runs-on: ubuntu-latest
outputs:
digest: ${{ steps.push.outputs.digest }}
steps:
- uses: actions/checkout@v4
- name: resolve the image name
run: echo "IMAGE=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/proof-eval" >> "${GITHUB_ENV}"
- uses: docker/setup-buildx-action@v3
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: build and push the contract image
id: push
uses: docker/build-push-action@v6
with:
context: .
file: eval/Dockerfile
push: true
provenance: false
platforms: linux/amd64
tags: |
${{ env.IMAGE }}:${{ github.sha }}-contract
build-args: |
WITH_RUNTIME=0
PROOF_GIT_SHA=${{ github.sha }}
cache-from: type=gha,scope=proof-eval-contract
cache-to: type=gha,mode=max,scope=proof-eval-contract
- name: report
run: |
{
echo "### proof-eval (contract only)"
echo
echo '```'
echo "image = \"${IMAGE}\""
echo "digest = \"${{ steps.push.outputs.digest }}\""
echo '```'
echo
echo "Contract layer only: slim, no model runtime, so it refuses to score."
echo "Do not pin this digest."
} >> "${GITHUB_STEP_SUMMARY}"

runtime:
runs-on: ubuntu-latest
timeout-minutes: 120
outputs:
digest: ${{ steps.push.outputs.digest }}
steps:
- uses: actions/checkout@v4
- name: resolve the image name
run: echo "IMAGE=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/proof-eval" >> "${GITHUB_ENV}"
- name: free disk for the CUDA base and runtime wheels
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
/opt/hostedtoolcache /usr/local/share/boost \
/opt/hostedtoolcache/CodeQL /usr/local/lib/node_modules \
/usr/share/swift /opt/az || true
df -h /
- uses: docker/setup-buildx-action@v3
- uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: build and push the scoring image
id: push
uses: docker/build-push-action@v6
with:
context: .
file: eval/Dockerfile.scoring
push: true
provenance: false
platforms: linux/amd64
tags: |
${{ env.IMAGE }}:${{ github.sha }}
build-args: |
WITH_RUNTIME=1
PROOF_GIT_SHA=${{ github.sha }}
- name: pull the published digest and prove it can score
env:
DIGEST: ${{ steps.push.outputs.digest }}
run: |
set -eu
case "${DIGEST}" in
sha256:*) ;;
*) echo "no sha256 digest was published"; exit 1 ;;
esac
ref="${IMAGE}@${DIGEST}"
docker pull "${ref}"
docker run --rm --entrypoint /bin/sh "${ref}" -c \
'test -f /usr/bin/proof-eval && test -x /usr/bin/proof-eval && env -i PATH=/usr/bin:/bin /usr/bin/proof-eval --help'
docker run --rm --entrypoint /bin/sh "${ref}" -c \
'test ! -L /usr/bin/proof-eval'
docker run --rm --entrypoint /bin/sh "${ref}" -c \
'env -i PATH=/usr/bin:/bin /usr/bin/proof-eval score --help'
docker run --rm --entrypoint /opt/proof-eval-venv/bin/python "${ref}" \
-c 'import torch, transformers'
docker run --rm --entrypoint /bin/sh "${ref}" -c \
'test -f /opt/proof-eval/baked_proxies.json && grep -q "Qwen/Qwen3.8-0.6B" /opt/proof-eval/baked_proxies.json'
docker run --rm --entrypoint /bin/sh "${ref}" -c \
'env -i PATH=/usr/bin:/bin HOME=/root PROOF_SELFTEST_REQUIRE_RUNTIME=1 /usr/bin/proof-eval selftest'
- name: publicize the GHCR package
env:
GH_TOKEN: ${{ github.token }}
OWNER: ${{ github.repository_owner }}
run: |
set -euo pipefail
enc=$(python3 -c 'import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1], safe=""))' "proof-eval")
code=$(curl -sS -o /tmp/vis.json -w "%{http_code}" \
-X PUT \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${GH_TOKEN}" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/orgs/${OWNER}/packages/container/${enc}/visibility" \
-d '{"visibility":"public"}' || true)
echo "pkg=proof-eval http=${code} body=$(head -c 200 /tmp/vis.json || true)"
- name: report the digest to pin
env:
DIGEST: ${{ steps.push.outputs.digest }}
run: |
set -eu
case "${DIGEST}" in
sha256:*) ;;
*) echo "no sha256 digest was published"; exit 1 ;;
esac
{
echo "### proof-eval (scoring image, CUDA base)"
echo
echo "Pulled this digest after push, ran the harvest-PATH check,"
echo "proved \`import torch, transformers\`, baked proxy Qwen/Qwen3.8-0.6B,"
echo "and ran \`proof-eval selftest\` (fabric 12.5 Gbit/s / no IB-NVLink-NCCL)."
echo "Paste into the control plane's \`config/proof-pin.toml\`:"
echo
echo '```toml'
echo "eval_image = \"${IMAGE}\""
echo "eval_image_digest = \"${DIGEST}\""
echo "proxy_model = \"Qwen/Qwen3.8-0.6B\""
echo "proxy_models = [\"Qwen/Qwen3.8-0.6B\"]"
echo '```'
} >> "${GITHUB_STEP_SUMMARY}"
echo "eval_image_digest = \"${DIGEST}\""
echo "${DIGEST}" > /tmp/proof-eval.digest
- name: upload digest
uses: actions/upload-artifact@v4
with:
name: proof-eval-digest
path: /tmp/proof-eval.digest
if-no-files-found: error
2 changes: 1 addition & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

Short contract for agents and operators. Prefer linking over restating runbooks.

**Product:** Cortex ([`CortexLM/cortex`](https://github.com/CortexLM/cortex)) — Bittensor subnet control plane. **Two live challenges:** `bounty` (2000 bps) and `proof` (8000 bps). Proof-weighted 20%/80% lock regardless of eval digest. Proof's `eval_image_digest` is empty (submits 503); do not invent a sha256. Sum is 10000. `relearn`, `relearn-image`, `relearn-agent`, `relearn-mm`, `design`, and `prism` are **off** — no trust-root row, so they have no emission and no leaf may verify. Relearn* code stays behind the `relearn` / `mm` compose profiles. Proof scores operator-published research topics (dynamic `topic_id`, digest-pinned RLM judge, `wta` or `discovery` payout). Naming split (Cortex vs leftover `base` / `BASE_*`): [`docs/NAMING.md`](docs/NAMING.md).
**Product:** Cortex ([`CortexLM/cortex`](https://github.com/CortexLM/cortex)) — Bittensor subnet control plane. **Two live challenges:** `bounty` (2000 bps) and `proof` (8000 bps). Proof-weighted 20%/80% lock regardless of eval digest. Proof eval digest is pinned (`ghcr.io/cortexlm/proof-eval@sha256:78b614a1…`, proxy `Qwen/Qwen3.8-0.6B`); live submits still 503 until harvest is wired, a baseline is sealed, and ≥1 topic is open. Empty digest stays fail-closed (do not invent a sha256). Sum is 10000. `relearn`, `relearn-image`, `relearn-agent`, `relearn-mm`, `design`, and `prism` are **off** — no trust-root row, so they have no emission and no leaf may verify. Relearn* code stays behind the `relearn` / `mm` compose profiles. Proof scores operator-published research topics (dynamic `topic_id`, digest-pinned RLM judge, `wta` or `discovery` payout). Naming split (Cortex vs leftover `base` / `BASE_*`): [`docs/NAMING.md`](docs/NAMING.md).

PRs require a [Greptile](https://greptile.com) review (`.greptile/`). If the bot is silent, comment `@greptileai review`.

Expand Down
19 changes: 12 additions & 7 deletions config/proof-pin.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@
#
# Deploy = bump eval_image_digest after proof-eval CI is green. Empty digest
# is the pre-launch state: live submits answer 503. Do not invent a sha256.
# proxy_model must be an id the image actually bakes (see proxy_models).
#
# Scoring image published by .github/workflows/publish-proof-eval-image.yml
# run 33892650063 (commit 51f937c7818f0eb1e3ed1412972de98b6994952b).
# Pulled that digest after push; harvest-PATH + selftest + baked proxy
# Qwen/Qwen3.8-0.6B + 12.5 Gbit/s fabric enforcement proved on those bytes.
#
# Topic documents are signed by the `proof` row key in config/challenges.toml
# (sr25519, domain `base-proof-topic-v1`). `topic_pubkey` must match that row.
Expand All @@ -24,14 +30,13 @@
challenge_id = "proof"
scoring_version = 1
base_model_family = "Qwen/Qwen3.8"
# Empty until the eval image exists. A topic naming a proxy the image does
# not bake is a publish 400.
proxy_model = ""
proxy_models = []
# Baked into ghcr.io/cortexlm/proof-eval@sha256:78b614a1… (see baked_proxies.json).
proxy_model = "Qwen/Qwen3.8-0.6B"
proxy_models = ["Qwen/Qwen3.8-0.6B"]
eval_image = "ghcr.io/cortexlm/proof-eval"
eval_image_digest = ""
proof_git = "https://github.com/CortexLM/relearn"
proof_git_sha = ""
eval_image_digest = "sha256:78b614a1f51ce5dd80076c4e343a2b31b85d6c36025e02836cb83929867e7009"
proof_git = "https://github.com/CortexLM/cortex"
proof_git_sha = "51f937c7818f0eb1e3ed1412972de98b6994952b"
topic_pubkey = "3e7f70f09165e265ab89ab04a4fc91dc0531d54a100c538fb14c6f008421c375"
flops_budget_max = 2000000000000000000
epsilon_nll_min = 0.02
Expand Down
16 changes: 12 additions & 4 deletions crates/proof-task/tests/committed_pin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,19 +51,27 @@ fn proof_row_pubkey() -> String {
}

#[test]
fn committed_pin_is_proof_with_empty_eval_digest() {
fn committed_pin_is_proof_with_a_real_eval_digest() {
let p = pin();
assert_eq!(p.challenge_id, CHALLENGE_ID);
assert_eq!(p.eval_image, EVAL_IMAGE);
assert!(
!p.eval_image.contains(':'),
"the tag belongs in eval_image_digest, not eval_image"
);
let digest = p.eval_image_digest.trim();
assert!(
p.eval_image_digest.trim().is_empty(),
"do not invent a sha256; empty digest is the pre-launch 503"
digest.starts_with("sha256:") && digest.len() == 71,
"committed digest must be a real sha256 pin, not invented or empty: {digest:?}"
);
assert!(!p.can_rent(), "empty digest cannot rent");
let hex = digest.trim_start_matches("sha256:");
assert!(
hex.len() == 64 && hex.chars().all(|c| c.is_ascii_hexdigit()),
"{digest}"
);
assert!(p.can_rent(), "pinned digest must be rentable");
assert_eq!(p.proxy_model, "Qwen/Qwen3.8-0.6B");
assert!(p.bakes_proxy("Qwen/Qwen3.8-0.6B"));
assert_eq!(p.holdout_size, HOLDOUT_SIZE);
assert_eq!(p.stratum_size, STRATUM_SIZE);
}
Expand Down
93 changes: 93 additions & 0 deletions deploy/scripts/proof-operator-path.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
#!/usr/bin/env bash
# Proof operator path: holdout → seal baseline on the pinned image → sign → publish.
#
# Secrets stay off git. This script prints the commands; it never writes under
# config/ or docs/, and it never prints a mini-secret.
#
# Topic schema (payout_mode, validation, discovery shares) is owned by the
# control-plane PR. This path only runs holdout / baseline / admin inject
# against whatever signed document the operator drafted.
#
# can_score becomes true only when ALL of: real digest pin, wired harvest
# (LIUM_API_KEY + SSH pub), ≥1 open signed topic, verified holdout, sealed
# baseline. Empty digest stays 503 (never invent a sha256).
set -euo pipefail

TOPIC_ID="${PROOF_TOPIC_ID:-dt-no-ib-v0}"
SECRETS="${PROOF_SECRETS_DIR:-$HOME/.base-secrets/proof}"
IMAGE="ghcr.io/cortexlm/proof-eval"
PROXY="Qwen/Qwen3.8-0.6B"

digest=$(python3 - <<'PY' || true
import tomllib, pathlib, sys
p = pathlib.Path("config/proof-pin.toml")
if not p.is_file():
sys.exit(0)
print(tomllib.loads(p.read_text()).get("eval_image_digest","").strip())
PY
)
digest="${PROOF_EVAL_IMAGE_DIGEST:-$digest}"

cat <<EOF
# Proof operator ceremony (${TOPIC_ID})

# 0. Pin must already carry a real sha256. Empty digest → every submit 503.
# Current eval_image_digest=${digest:-<empty>}
# Proxy the image bakes: ${PROXY}

mkdir -p '${SECRETS}'
chmod 700 '${SECRETS}'

# 1. Select a stratified holdout (records NEVER enter git).
cargo run -p xtask -- proof-holdout \\
--topic-id '${TOPIC_ID}' \\
--synthetic \\
--salt "\$PROOF_HOLDOUT_SALT" \\
--size 120 \\
--out '${SECRETS}/holdouts.json'

# Production: drop --synthetic and pass --catalog <private.json>.

# 2. Stage shard bytes the image will score (content-addressed).
# PROOF_HOLDOUT_STORE/\${content_sha256} ← packed shard text.
# The request carries fingerprints only.

# 3. Seal the AdamW / comms baseline ON THE PINNED IMAGE (not sim).
# The image enforces 12.5 Gbit/s / no IB / no NVLink / no NCCL fast path
# before it will emit numbers.
if [ -n "${digest}" ]; then
echo "docker run --rm --entrypoint /usr/bin/proof-eval ${IMAGE}@${digest} baseline --request /tmp/proof_eval/request.json --out /tmp/proof_eval/baseline.json"
else
echo "# digest still empty — do not invent a sha256; wait for publish-proof-eval-image"
fi
# Put the measurement JSON at ${SECRETS}/baselines.json keyed by topic id.
# script_sha256 = sha256(/opt/proof-eval/baselines/adamw.py) from that image.
# metrics_commitment = BaselineMeasurement.commitment() over the vector.

# 4. Sign the operator draft (YAML or JSON). Schema is payout_mode +
# validation.{score_on,accept_if,reject_if} + metric; this helper does
# not invent those fields. --synthetic is local/dev; production uses
# --holdout so the commitment matches the host file.
cargo run -p xtask -- proof-topic \\
--input '${SECRETS}/${TOPIC_ID}.yaml' \\
--secret deploy/secrets/proof_sk \\
--holdout '${SECRETS}/holdouts.json' \\
--out '${SECRETS}/topics.json'

# 5. Publish (dynamic inject). Admin bearer from PROOF_ADMIN_TOKENS_FILE.
# curl -sS -X POST "\$PROOF_BASE/v1/admin/proof/topics" \\
# -H "authorization: Bearer \$PROOF_ADMIN_TOKEN" \\
# -H 'content-type: application/json' \\
# --data-binary @${SECRETS}/topics.json

# 6. Point the host at the operator files (never in git):
# PROOF_TOPICS_FILE=${SECRETS}/topics.json
# PROOF_HOLDOUT_FILE=${SECRETS}/holdouts.json
# PROOF_BASELINE_FILE=${SECRETS}/baselines.json
# LIUM_API_KEY=… LIUM_SSH_PUBLIC_KEY_FILE=…
# Restart proof-challenge, then:
# curl -sS "\$PROOF_BASE/v1/status" | jq '{can_score,eval_image_digest,open_topics,live_harvest_wired,baseline_sealed}'

# can_score is true only with: real digest + harvest wired + open topic +
# sealed baseline + verified holdout. Empty digest stays 503.
EOF
6 changes: 3 additions & 3 deletions docs/COMPLETENESS.md
Original file line number Diff line number Diff line change
Expand Up @@ -127,13 +127,13 @@ Removed as **live products**. Shared rails (`prism-lium*`, `prism-competition` p
| Challenge id | **done** | `proof` on the wire. Topics are operator-published signed documents; git carries no catalog. |
| Crates (`crates/proof-*`) | **done** | task (signed topics, holdout commitments, global pin, `payout_mode` / English `validation`), score (per-topic pass + WTA/discovery sum), store, eval (RLM judge, fail-closed readiness), harvest, http, challenge. |
| Binary (`bins/proof-challenge`) | **done** | HTTP API on `:8100`. |
| Miner CLI (`bins/ctx`) | **done** | `ctx proof submit|show|status|topics`. Empty `eval_image_digest` → 503. |
| Miner CLI (`bins/ctx`) | **done** | `ctx proof submit|show|status|topics`. Unpinned digest / unwired harvest / no open topic → 503. |
| Compose / images | **done** | Default compose + `images.yml` target `proof-challenge`. |
| Eval pin | **v0** | `config/proof-pin.toml` — `eval_image` `ghcr.io/cortexlm/proof-eval`, `eval_image_digest` empty until first green proof-eval CI. Empty digest live submits **503**. Do not invent a sha256. |
| Eval pin | **done** | `config/proof-pin.toml` — `eval_image` `ghcr.io/cortexlm/proof-eval`, digest `sha256:78b614a1…` (publish-proof-eval-image run 33892650063, commit `51f937c7`). Baked proxy `Qwen/Qwen3.8-0.6B`. Empty digest is gone; live submits still **503** until harvest is wired, a baseline is sealed, and ≥1 topic is open. Do not re-pin a guessed sha256. |
| Topics | **done** | sr25519 under the `proof` trust-root key (`base-proof-topic-v1`). Admin `POST /v1/admin/proof/topics`. A topic must be sealed to `open`. |
| Holdout | **done** | Per-topic operator file (`PROOF_HOLDOUT_FILE`). Commitment in the topic document, never in the pin. `xtask proof-holdout --topic-id`. |
| Live harvest | **done** | `crates/proof-harvest` over `harvest-pod`; `PROOF_FORCE_SIM` is local-only. |
| Emission | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Empty `eval_image_digest` → 503. Split equally across currently `open` topics, then `wta` or `discovery`. Empty open set → `NoScore(ChallengeInternal)`. |
| Emission | **8000 bps** | Proof-weighted 20%/80% regardless of digest. Unwired harvest / unsealed baseline / empty open set → 503 / `NoScore(ChallengeInternal)`. Split equally across currently `open` topics, then `wta` or `discovery`. Empty digest still 503s (never invent a sha256). |
| Spec | live | [`PROOF.md`](PROOF.md). |

## Infrastructure
Expand Down
Loading
Loading