From bb1080ca4724d81f9696b108fa7dda2ec1b1df1a Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 9 Sep 2026 00:40:17 +0000 Subject: [PATCH 01/29] Add self-hosted CI: tests and A/B benches on PRs Workflow runs on the self-hosted runner: a test job (release build, cargo test with default and arena_compact+random features, docs) and a bench job that benchmarks the PR head against its base on the same machine. bench_ab.sh builds both sides once, runs them in alternating order pinned to one core, averages the rounds with bench_avg_files.py and posts the bench_cmp.py table to the step summary and the PR. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QXEKcXZn55RYJY7BSJuYeS --- .github/scripts/bench_ab.sh | 85 ++++++++++++++++++++++++++++ .github/workflows/ci.yml | 109 ++++++++++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+) create mode 100755 .github/scripts/bench_ab.sh create mode 100644 .github/workflows/ci.yml diff --git a/.github/scripts/bench_ab.sh b/.github/scripts/bench_ab.sh new file mode 100755 index 00000000..bfc29627 --- /dev/null +++ b/.github/scripts/bench_ab.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# A/B benchmark of two commits on the same machine. +# +# Builds the bench binaries for BASE and HEAD once each (separate target +# dirs), then runs them for ROUNDS rounds, alternating which side goes first +# and pinning every run to one core. Per-bench results are averaged over the +# rounds with benches/bench_avg_files.py and compared with benches/bench_cmp.py. +# +# usage: bench_ab.sh +# +# env: BENCH_ROUNDS rounds per side (default 3) +# BENCHES space separated bench targets (default: the set used in BENCH_BUGFIXES) +# BENCH_CPU core to pin to (default 3) +# BENCH_OUT output directory (default ./bench-out) +# DIVAN_SAMPLE_COUNT sample count for benches that do not set their own (default 40) +# CARGO_TARGET_DIR parent of the two per-side target dirs (default ./target) +set -euo pipefail + +BASE_SHA=${1:?usage: bench_ab.sh } +HEAD_SHA=${2:?usage: bench_ab.sh } +ROUNDS=${BENCH_ROUNDS:-3} +BENCHES=${BENCHES:-"shakespeare cities sparse_keys binary_keys superdense_keys act_paths zipper_head_owned product_zipper"} +CPU=${BENCH_CPU:-3} +OUT=$(realpath -m "${BENCH_OUT:-$PWD/bench-out}") +TARGET=$(realpath -m "${CARGO_TARGET_DIR:-$PWD/target}") +export DIVAN_SAMPLE_COUNT=${DIVAN_SAMPLE_COUNT:-40} + +repo=$PWD +base_src=$OUT/src-base +mkdir -p "$OUT" +rm -f "$OUT"/*.txt "$OUT"/*.log + +cleanup() { git -C "$repo" worktree remove --force "$base_src" 2>/dev/null || true; } +trap cleanup EXIT +cleanup +git worktree add --detach "$base_src" "$BASE_SHA" >/dev/null + +# build_side : writes " " lines to $OUT/bins-.txt +build_side() { + local side=$1 src=$2 args=() + for b in $BENCHES; do args+=(--bench "$b"); done + echo "== building $side ($(git -C "$src" rev-parse --short HEAD)) into $TARGET/ab-$side" + (cd "$src" && cargo bench --no-run --message-format=json "${args[@]}" \ + --target-dir "$TARGET/ab-$side" 2>"$OUT/build-$side.log") \ + | python3 -c ' +import json, sys +for line in sys.stdin: + m = json.loads(line) + if m.get("reason") == "compiler-artifact" and m.get("executable") and "bench" in m["target"]["kind"]: + print(m["target"]["name"], m["executable"]) +' > "$OUT/bins-$side.txt" + for b in $BENCHES; do + grep -q "^$b " "$OUT/bins-$side.txt" || { echo "no executable for bench $b on $side" >&2; exit 1; } + done +} + +build_side base "$base_src" +build_side head "$repo" + +exe_for() { awk -v n="$2" '$1 == n { print $2 }' "$OUT/bins-$1.txt"; } + +for ((r = 1; r <= ROUNDS; r++)); do + if (( r % 2 )); then order="base head"; else order="head base"; fi + for b in $BENCHES; do + for side in $order; do + echo "== round $r/$ROUNDS $b $side" + taskset -c "$CPU" "$(exe_for "$side" "$b")" --bench \ + > "$OUT/$side-$b-r$r.txt" 2>> "$OUT/run-$side.log" + done + done +done + +strip_ansi() { sed 's/\x1b\[[0-9;]*m//g'; } +: > "$OUT/compare.txt" +for b in $BENCHES; do + for side in base head; do + python3 "$repo/benches/bench_avg_files.py" "$OUT/$side-$b-r"*.txt -o "$OUT/$side-$b-avg.txt" + done + { + echo "$b (base $(git rev-parse --short "$BASE_SHA") head $(git rev-parse --short "$HEAD_SHA") rounds $ROUNDS median ns)" + python3 "$repo/benches/bench_cmp.py" --base "$OUT/base-$b-avg.txt" --other "$OUT/head-$b-avg.txt" | strip_ansi + echo + } >> "$OUT/compare.txt" +done +cat "$OUT/compare.txt" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..31c55c95 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,109 @@ +name: CI + +on: + pull_request: + push: + branches: [master] + workflow_dispatch: + inputs: + base: + description: base ref to benchmark against + default: master + rounds: + description: bench rounds per side + default: "3" + benches: + description: space separated bench targets (empty = default set) + default: "" + +permissions: + contents: read + +# One workflow run per PR / branch; a new push cancels the one in progress. +concurrency: + group: ci-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: never + # Persistent across jobs on the runner (outside its _work dir), so builds are incremental. + CARGO_TARGET_DIR: /home/gh-runner/cache/target + +jobs: + test: + name: tests + runs-on: [self-hosted, linux, x64] + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - name: toolchain + run: | + rustup toolchain install stable --profile minimal + rustup default stable + rustc --version && cargo --version + - name: build + run: cargo build --release --all-targets + - name: unit + integration tests + run: cargo test --release + - name: tests with arena_compact + random + run: cargo test --release --features arena_compact,random + - name: doc tests + docs + run: cargo doc --no-deps + + bench: + name: bench A/B vs base + # PRs and manual runs only; a push to master has nothing to compare against. + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + runs-on: [self-hosted, linux, x64, bench] + timeout-minutes: 300 + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: toolchain + run: | + rustup toolchain install stable --profile minimal + rustup default stable + - name: resolve base + id: base + run: | + if [ "${{ github.event_name }}" = pull_request ]; then + sha=${{ github.event.pull_request.base.sha }} + else + sha=$(git rev-parse "origin/${{ inputs.base }}" 2>/dev/null || git rev-parse "${{ inputs.base }}") + fi + echo "sha=$sha" >> "$GITHUB_OUTPUT" + - name: A/B bench + env: + BENCH_ROUNDS: ${{ inputs.rounds || '3' }} + BENCHES: ${{ inputs.benches }} + BENCH_OUT: ${{ runner.temp }}/bench-out + run: | + # an empty BENCHES must fall through to the script's default set + [ -n "$BENCHES" ] || unset BENCHES + .github/scripts/bench_ab.sh "${{ steps.base.outputs.sha }}" "${{ github.sha }}" + - name: summary + if: always() + run: | + out="${{ runner.temp }}/bench-out" + if [ -s "$out/compare.txt" ]; then + { echo '```'; cat "$out/compare.txt"; echo '```'; } > "$out/compare.md" + cat "$out/compare.md" >> "$GITHUB_STEP_SUMMARY" + fi + - uses: actions/upload-artifact@v4 + if: always() + with: + name: bench-out + path: ${{ runner.temp }}/bench-out/*.txt + if-no-files-found: ignore + - name: comment on PR + # The token is read-only for PRs from forks, so only comment on same-repo PRs. + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + continue-on-error: true + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: bench + path: ${{ runner.temp }}/bench-out/compare.md From 7e3fa339bbe2723a4d97440e2b707d5a11e5e041 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 9 Sep 2026 12:32:04 +0000 Subject: [PATCH 02/29] CI bench: pass required-features, report progress to the PR while running bench_ab.sh failed on the runner with exit 101 because act_paths declares required-features (arena_compact, serialization) and the script built without them. It now reads each side's Cargo.toml and passes the features the requested benches need, and prints the cargo error into the step log when a build fails instead of leaving it in a temp file. The script also appends to progress.txt after every run and rewrites compare.txt after every completed round. The workflow runs it in the background and, once a minute, posts progress plus the compare table of the rounds finished so far to one PR comment via pr_comment.py (stdlib only). The comment is found by a marker so re-runs and later pushes reuse it, and its id is cached per run so the periodic updates skip the lookup. This replaces the sticky-comment action. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QXEKcXZn55RYJY7BSJuYeS --- .github/scripts/bench_ab.sh | 75 ++++++++++++++++++++++++++--------- .github/scripts/pr_comment.py | 71 +++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 20 +++++----- 3 files changed, 137 insertions(+), 29 deletions(-) create mode 100755 .github/scripts/pr_comment.py diff --git a/.github/scripts/bench_ab.sh b/.github/scripts/bench_ab.sh index bfc29627..95940a19 100755 --- a/.github/scripts/bench_ab.sh +++ b/.github/scripts/bench_ab.sh @@ -14,6 +14,10 @@ # BENCH_OUT output directory (default ./bench-out) # DIVAN_SAMPLE_COUNT sample count for benches that do not set their own (default 40) # CARGO_TARGET_DIR parent of the two per-side target dirs (default ./target) +# +# Progress is appended to $BENCH_OUT/progress.txt after every run, and +# $BENCH_OUT/compare.txt is rewritten after every completed round, so a watcher +# (see pr_comment.sh) can show partial results while the script runs. set -euo pipefail BASE_SHA=${1:?usage: bench_ab.sh } @@ -28,33 +32,74 @@ export DIVAN_SAMPLE_COUNT=${DIVAN_SAMPLE_COUNT:-40} repo=$PWD base_src=$OUT/src-base mkdir -p "$OUT" -rm -f "$OUT"/*.txt "$OUT"/*.log +rm -f "$OUT"/*.txt "$OUT"/*.log "$OUT"/*.json + +progress() { echo "$(date -u +%H:%M:%S) $*" >> "$OUT/progress.txt"; } +strip_ansi() { sed 's/\x1b\[[0-9;]*m//g'; } + +# compare_rounds : average the rounds finished so far per bench and side, then compare; writes $OUT/compare.txt +compare_rounds() { + local tmp=$OUT/compare.tmp + : > "$tmp" + for b in $BENCHES; do + for side in base head; do + python3 "$repo/benches/bench_avg_files.py" "$OUT/$side-$b-r"*.txt -o "$OUT/$side-$b-avg.txt" + done + { + echo "$b (base $(git rev-parse --short "$BASE_SHA") head $(git rev-parse --short "$HEAD_SHA") rounds $1 median ns)" + python3 "$repo/benches/bench_cmp.py" --base "$OUT/base-$b-avg.txt" --other "$OUT/head-$b-avg.txt" | strip_ansi + echo + } >> "$tmp" + done + mv "$tmp" "$OUT/compare.txt" +} cleanup() { git -C "$repo" worktree remove --force "$base_src" 2>/dev/null || true; } trap cleanup EXIT cleanup git worktree add --detach "$base_src" "$BASE_SHA" >/dev/null +progress "plan: $ROUNDS round(s) x base/head x [$BENCHES], core $CPU" # build_side : writes " " lines to $OUT/bins-.txt build_side() { - local side=$1 src=$2 args=() + local side=$1 src=$2 args=() feats for b in $BENCHES; do args+=(--bench "$b"); done - echo "== building $side ($(git -C "$src" rev-parse --short HEAD)) into $TARGET/ab-$side" - (cd "$src" && cargo bench --no-run --message-format=json "${args[@]}" \ - --target-dir "$TARGET/ab-$side" 2>"$OUT/build-$side.log") \ - | python3 -c ' + # features the requested benches declare via required-features (only those the side's Cargo.toml has) + feats=$(cd "$src" && python3 - "$BENCHES" <<'PY' +import sys, tomllib +t = tomllib.load(open('Cargo.toml', 'rb')) +want = set(sys.argv[1].split()) +have = set(t.get('features', {})) +need = set() +for b in t.get('bench', []): + if b.get('name') in want: + need |= set(b.get('required-features', [])) +print(','.join(sorted(need & have))) +PY +) + [[ -n $feats ]] && args+=(--features "$feats") + echo "== building $side ($(git -C "$src" rev-parse --short HEAD)) into $TARGET/ab-$side${feats:+ with features $feats}" + if ! (cd "$src" && cargo bench --no-run --message-format=json "${args[@]}" --target-dir "$TARGET/ab-$side" \ + > "$OUT/build-$side.json" 2> "$OUT/build-$side.log"); then + echo "build of $side failed; tail of $OUT/build-$side.log:" >&2 + tail -30 "$OUT/build-$side.log" >&2 + exit 1 + fi + python3 -c ' import json, sys for line in sys.stdin: m = json.loads(line) if m.get("reason") == "compiler-artifact" and m.get("executable") and "bench" in m["target"]["kind"]: print(m["target"]["name"], m["executable"]) -' > "$OUT/bins-$side.txt" +' < "$OUT/build-$side.json" > "$OUT/bins-$side.txt" for b in $BENCHES; do grep -q "^$b " "$OUT/bins-$side.txt" || { echo "no executable for bench $b on $side" >&2; exit 1; } done } +progress "building base $(git rev-parse --short "$BASE_SHA")" build_side base "$base_src" +progress "building head $(git rev-parse --short "$HEAD_SHA")" build_side head "$repo" exe_for() { awk -v n="$2" '$1 == n { print $2 }' "$OUT/bins-$1.txt"; } @@ -64,22 +109,14 @@ for ((r = 1; r <= ROUNDS; r++)); do for b in $BENCHES; do for side in $order; do echo "== round $r/$ROUNDS $b $side" + t0=$SECONDS taskset -c "$CPU" "$(exe_for "$side" "$b")" --bench \ > "$OUT/$side-$b-r$r.txt" 2>> "$OUT/run-$side.log" + progress "round $r/$ROUNDS $b $side $((SECONDS - t0))s" done done + compare_rounds "$r" + progress "round $r/$ROUNDS done, compare.txt refreshed" done -strip_ansi() { sed 's/\x1b\[[0-9;]*m//g'; } -: > "$OUT/compare.txt" -for b in $BENCHES; do - for side in base head; do - python3 "$repo/benches/bench_avg_files.py" "$OUT/$side-$b-r"*.txt -o "$OUT/$side-$b-avg.txt" - done - { - echo "$b (base $(git rev-parse --short "$BASE_SHA") head $(git rev-parse --short "$HEAD_SHA") rounds $ROUNDS median ns)" - python3 "$repo/benches/bench_cmp.py" --base "$OUT/base-$b-avg.txt" --other "$OUT/head-$b-avg.txt" | strip_ansi - echo - } >> "$OUT/compare.txt" -done cat "$OUT/compare.txt" diff --git a/.github/scripts/pr_comment.py b/.github/scripts/pr_comment.py new file mode 100755 index 00000000..d26d4a7d --- /dev/null +++ b/.github/scripts/pr_comment.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Create or update the single bench comment on a pull request. + +One comment per pull request, edited in place. The first call of a run finds +the PR's existing bench comment by the hidden marker on its first line (so a +re-run or a new push reuses it) or creates it, and records the id in +$BENCH_OUT/comment_id; later calls in the same run go straight to that id. +Standard library only. + +usage: pr_comment.py +env: GITHUB_TOKEN GITHUB_REPOSITORY (provided by Actions) + BENCH_OUT dir holding progress.txt / compare.txt from bench_ab.sh + GITHUB_SERVER_URL GITHUB_RUN_ID for the run link, optional +""" +import json, os, sys, time, urllib.request +from pathlib import Path + +MARKER = '' +LIMIT = 65536 # GitHub's comment body cap +PROGRESS_LINES = 40 + +pr, status = sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else '' +out = Path(os.environ['BENCH_OUT']) +repo = os.environ['GITHUB_REPOSITORY'] +api = f'https://api.github.com/repos/{repo}' +headers = {'Authorization': f"Bearer {os.environ['GITHUB_TOKEN']}", + 'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json'} +run_url = f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/{repo}/actions/runs/{os.environ.get('GITHUB_RUN_ID', '')}" + + +def call(method, url, data=None): + req = urllib.request.Request(url, method=method, headers=headers, + data=json.dumps(data).encode() if data is not None else None) + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + + +def read(name): + p = out / name + return p.read_text() if p.is_file() else '' + + +parts = [MARKER, f'### Bench A/B vs base: {status}', '', + f"[run log]({run_url}) · updated {time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime())} UTC"] +progress = read('progress.txt').splitlines() +if progress: + parts += ['', f'
progress (last {PROGRESS_LINES} lines)', '', '```', + *progress[-PROGRESS_LINES:], '```', '
'] +compare = read('compare.txt') +if compare: + head = '\n'.join(parts) + room = LIMIT - len(head) - 200 + if len(compare) > room: + compare = compare[:room] + '\n… truncated; the full table is in the bench-out artifact\n' + parts += ['', '```', compare.rstrip(), '```'] +body = '\n'.join(parts) + +id_file = out / 'comment_id' +if id_file.is_file(): + cid = id_file.read_text().strip() + how = 'updated' +else: + found = [c['id'] for c in call('GET', f'{api}/issues/{pr}/comments?per_page=100') if c['body'].startswith(MARKER)] + cid = found[0] if found else None + how = 'reused' if found else 'created' +if cid is None: + cid = call('POST', f'{api}/issues/{pr}/comments', {'body': body})['id'] +else: + call('PATCH', f'{api}/issues/comments/{cid}', {'body': body}) +id_file.write_text(str(cid)) +print(f'comment {cid} {how}: {len(body)} chars') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31c55c95..084aa170 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: timeout-minutes: 300 permissions: contents: read - pull-requests: write + pull-requests: write # for the progress comment; read-only on fork PRs, where commenting is skipped steps: - uses: actions/checkout@v4 with: @@ -81,10 +81,18 @@ jobs: BENCH_ROUNDS: ${{ inputs.rounds || '3' }} BENCHES: ${{ inputs.benches }} BENCH_OUT: ${{ runner.temp }}/bench-out + GITHUB_TOKEN: ${{ github.token }} + # Only same-repo PRs get a writable token; on fork PRs the comment calls fail and are ignored. + PR: ${{ github.event.pull_request.number }} run: | # an empty BENCHES must fall through to the script's default set [ -n "$BENCHES" ] || unset BENCHES - .github/scripts/bench_ab.sh "${{ steps.base.outputs.sha }}" "${{ github.sha }}" + mkdir -p "$BENCH_OUT" + comment() { [ -z "$PR" ] || python3 .github/scripts/pr_comment.py "$PR" "$1" || true; } + .github/scripts/bench_ab.sh "${{ steps.base.outputs.sha }}" "${{ github.sha }}" & bench=$! + # refresh the PR comment once a minute while the bench runs: progress + the compare table of finished rounds + while kill -0 $bench 2>/dev/null; do comment "running"; sleep 60; done + if wait $bench; then comment "done"; else comment "failed, see run log"; exit 1; fi - name: summary if: always() run: | @@ -99,11 +107,3 @@ jobs: name: bench-out path: ${{ runner.temp }}/bench-out/*.txt if-no-files-found: ignore - - name: comment on PR - # The token is read-only for PRs from forks, so only comment on same-repo PRs. - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository - continue-on-error: true - uses: marocchino/sticky-pull-request-comment@v2 - with: - header: bench - path: ${{ runner.temp }}/bench-out/compare.md From dd8b7997e34a289c54115301abde1006007c0f21 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 9 Sep 2026 12:38:28 +0000 Subject: [PATCH 03/29] CI bench: comment once at start with the job link, once at the end with results The per-minute progress updates duplicated the job log. The comment now carries a link to the specific job (looked up from the run's jobs as the one in progress on this runner, falling back to the run link) and, when the bench finishes, the compare table. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QXEKcXZn55RYJY7BSJuYeS --- .github/scripts/pr_comment.py | 44 +++++++++++++++++++++++------------ .github/workflows/ci.yml | 10 ++++---- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/.github/scripts/pr_comment.py b/.github/scripts/pr_comment.py index d26d4a7d..2f7024c7 100755 --- a/.github/scripts/pr_comment.py +++ b/.github/scripts/pr_comment.py @@ -1,23 +1,23 @@ #!/usr/bin/env python3 """Create or update the single bench comment on a pull request. -One comment per pull request, edited in place. The first call of a run finds -the PR's existing bench comment by the hidden marker on its first line (so a -re-run or a new push reuses it) or creates it, and records the id in -$BENCH_OUT/comment_id; later calls in the same run go straight to that id. -Standard library only. +One comment per pull request, edited in place: a status line, a link to the +job that produced it, and bench_ab.sh's compare table once it exists. The +first call of a run finds the PR's existing bench comment by the hidden marker +on its first line (so a re-run or a new push reuses it) or creates it, and +records the id in $BENCH_OUT/comment_id; later calls in the same run go +straight to that id. Standard library only. usage: pr_comment.py -env: GITHUB_TOKEN GITHUB_REPOSITORY (provided by Actions) - BENCH_OUT dir holding progress.txt / compare.txt from bench_ab.sh - GITHUB_SERVER_URL GITHUB_RUN_ID for the run link, optional +env: GITHUB_TOKEN GITHUB_REPOSITORY GITHUB_RUN_ID RUNNER_NAME (provided by Actions) + BENCH_OUT dir holding compare.txt from bench_ab.sh + GITHUB_SERVER_URL optional """ import json, os, sys, time, urllib.request from pathlib import Path MARKER = '' LIMIT = 65536 # GitHub's comment body cap -PROGRESS_LINES = 40 pr, status = sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else '' out = Path(os.environ['BENCH_OUT']) @@ -25,7 +25,8 @@ api = f'https://api.github.com/repos/{repo}' headers = {'Authorization': f"Bearer {os.environ['GITHUB_TOKEN']}", 'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json'} -run_url = f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/{repo}/actions/runs/{os.environ.get('GITHUB_RUN_ID', '')}" +run_id = os.environ.get('GITHUB_RUN_ID', '') +run_url = f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/{repo}/actions/runs/{run_id}" def call(method, url, data=None): @@ -40,12 +41,25 @@ def read(name): return p.read_text() if p.is_file() else '' +def job_url(): + """Link to this job's log: the job in progress on this runner within the run. Cached per run.""" + cache = out / 'job_url' + if cache.is_file(): + return cache.read_text().strip() + url = run_url + try: + jobs = call('GET', f'{api}/actions/runs/{run_id}/jobs?per_page=100')['jobs'] + mine = [j for j in jobs if j.get('runner_name') == os.environ.get('RUNNER_NAME') and j.get('status') == 'in_progress'] + if mine: + url = mine[0]['html_url'] + cache.write_text(url) + except Exception as e: # the run link is a fine fallback + print(f'job lookup failed, using run link: {e}', file=sys.stderr) + return url + + parts = [MARKER, f'### Bench A/B vs base: {status}', '', - f"[run log]({run_url}) · updated {time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime())} UTC"] -progress = read('progress.txt').splitlines() -if progress: - parts += ['', f'
progress (last {PROGRESS_LINES} lines)', '', '```', - *progress[-PROGRESS_LINES:], '```', '
'] + f"[job log]({job_url()}) · {time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime())} UTC"] compare = read('compare.txt') if compare: head = '\n'.join(parts) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 084aa170..ef353406 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,10 +89,12 @@ jobs: [ -n "$BENCHES" ] || unset BENCHES mkdir -p "$BENCH_OUT" comment() { [ -z "$PR" ] || python3 .github/scripts/pr_comment.py "$PR" "$1" || true; } - .github/scripts/bench_ab.sh "${{ steps.base.outputs.sha }}" "${{ github.sha }}" & bench=$! - # refresh the PR comment once a minute while the bench runs: progress + the compare table of finished rounds - while kill -0 $bench 2>/dev/null; do comment "running"; sleep 60; done - if wait $bench; then comment "done"; else comment "failed, see run log"; exit 1; fi + comment "running" + if .github/scripts/bench_ab.sh "${{ steps.base.outputs.sha }}" "${{ github.sha }}"; then + comment "done" + else + comment "failed, see the job log"; exit 1 + fi - name: summary if: always() run: | From ed83cc00a0e42798b938a61b7a3e48fefc29fec7 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 9 Sep 2026 12:49:44 +0000 Subject: [PATCH 04/29] CI bench: print each bench's compare table as soon as both sides have run The table for a bench is averaged over the rounds finished so far and saved as cmp-.txt; compare.txt is the concatenation, rewritten at the end of every round. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QXEKcXZn55RYJY7BSJuYeS --- .github/scripts/bench_ab.sh | 41 ++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/.github/scripts/bench_ab.sh b/.github/scripts/bench_ab.sh index 95940a19..559a7605 100755 --- a/.github/scripts/bench_ab.sh +++ b/.github/scripts/bench_ab.sh @@ -15,9 +15,11 @@ # DIVAN_SAMPLE_COUNT sample count for benches that do not set their own (default 40) # CARGO_TARGET_DIR parent of the two per-side target dirs (default ./target) # -# Progress is appended to $BENCH_OUT/progress.txt after every run, and -# $BENCH_OUT/compare.txt is rewritten after every completed round, so a watcher -# (see pr_comment.sh) can show partial results while the script runs. +# As soon as both sides of a bench have run in a round, its compare table +# (averaged over the rounds finished so far) is printed and saved as +# $BENCH_OUT/cmp-.txt; $BENCH_OUT/compare.txt, the concatenation, is +# rewritten after every completed round. Progress lines go to +# $BENCH_OUT/progress.txt. pr_comment.py posts compare.txt to the PR. set -euo pipefail BASE_SHA=${1:?usage: bench_ab.sh } @@ -37,20 +39,26 @@ rm -f "$OUT"/*.txt "$OUT"/*.log "$OUT"/*.json progress() { echo "$(date -u +%H:%M:%S) $*" >> "$OUT/progress.txt"; } strip_ansi() { sed 's/\x1b\[[0-9;]*m//g'; } -# compare_rounds : average the rounds finished so far per bench and side, then compare; writes $OUT/compare.txt +# compare_bench : average each side over the rounds run so far and +# compare; writes $OUT/cmp-.txt and prints it +compare_bench() { + local b=$1 + for side in base head; do + python3 "$repo/benches/bench_avg_files.py" "$OUT/$side-$b-r"*.txt -o "$OUT/$side-$b-avg.txt" + done + { + echo "$b (base $(git rev-parse --short "$BASE_SHA") head $(git rev-parse --short "$HEAD_SHA") rounds $2 median ns)" + python3 "$repo/benches/bench_cmp.py" --base "$OUT/base-$b-avg.txt" --other "$OUT/head-$b-avg.txt" | strip_ansi + echo + } > "$OUT/cmp-$b.txt" + cat "$OUT/cmp-$b.txt" +} + +# compare_rounds : concatenate the per-bench tables into $OUT/compare.txt compare_rounds() { local tmp=$OUT/compare.tmp : > "$tmp" - for b in $BENCHES; do - for side in base head; do - python3 "$repo/benches/bench_avg_files.py" "$OUT/$side-$b-r"*.txt -o "$OUT/$side-$b-avg.txt" - done - { - echo "$b (base $(git rev-parse --short "$BASE_SHA") head $(git rev-parse --short "$HEAD_SHA") rounds $1 median ns)" - python3 "$repo/benches/bench_cmp.py" --base "$OUT/base-$b-avg.txt" --other "$OUT/head-$b-avg.txt" | strip_ansi - echo - } >> "$tmp" - done + for b in $BENCHES; do cat "$OUT/cmp-$b.txt" >> "$tmp"; done mv "$tmp" "$OUT/compare.txt" } @@ -114,9 +122,10 @@ for ((r = 1; r <= ROUNDS; r++)); do > "$OUT/$side-$b-r$r.txt" 2>> "$OUT/run-$side.log" progress "round $r/$ROUNDS $b $side $((SECONDS - t0))s" done + compare_bench "$b" "$r" done - compare_rounds "$r" + compare_rounds progress "round $r/$ROUNDS done, compare.txt refreshed" done -cat "$OUT/compare.txt" +echo "== final compare over $ROUNDS round(s): $OUT/compare.txt" From 945e87d10f4be301a45d93e098cfd0ef54b7891b Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 9 Sep 2026 12:58:50 +0000 Subject: [PATCH 05/29] CI: run the bench job after the test job passes Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QXEKcXZn55RYJY7BSJuYeS --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef353406..b1b42326 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,9 @@ jobs: bench: name: bench A/B vs base - # PRs and manual runs only; a push to master has nothing to compare against. + # Runs after the tests pass (skipped when they fail), on PRs and manual runs only; + # a push to master has nothing to compare against. + needs: test if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' runs-on: [self-hosted, linux, x64, bench] timeout-minutes: 300 From 8d9c35582a7cea817bee0425759644bda6667f99 Mon Sep 17 00:00:00 2001 From: Igor Malovitsa Date: Wed, 9 Sep 2026 13:08:28 +0000 Subject: [PATCH 06/29] CI: differential fuzz job as a regression gate against the base commit The crate on master has 44 new divergences from the Lean model on the seed-7 corpus, so "zero divergences" cannot be the bar. fuzz_ab.sh runs head and base on identical inputs (model vs crate, 20000; ACT read side, 5000) and fails only when head diverges on an input base did not. Both sides use head's differential/ and lean/, so only the crate under test differs; if base cannot be built with head's harness its own is tried, and with no baseline at all the job reports that and does not gate. Two enabling fixes: the differential harness did not compile since the ZipperValues/ZipperValuesAt split (ReadSource now requires ZipperValuesAt), and differential.py accepts PATHMAP_TRACE / PATHMAP_ACT_TRACE to find binaries built into another target dir. The fuzz job runs after the tests, on PRs and manual runs. Lean is installed through elan for the runner user; lake's build dir is kept in the runner's cache since checkout wipes ignored files. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01QXEKcXZn55RYJY7BSJuYeS --- .github/scripts/fuzz_ab.sh | 151 ++++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 53 +++++++++++++ differential/src/harness.rs | 2 +- lean/differential.py | 8 +- 4 files changed, 210 insertions(+), 4 deletions(-) create mode 100755 .github/scripts/fuzz_ab.sh diff --git a/.github/scripts/fuzz_ab.sh b/.github/scripts/fuzz_ab.sh new file mode 100755 index 00000000..37799723 --- /dev/null +++ b/.github/scripts/fuzz_ab.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Differential fuzz of two commits against the Lean model, as a regression gate. +# +# The crate at HEAD has known divergences from the model, so "zero divergences" +# cannot be the bar. Instead both commits are run on identical inputs and the +# job fails only when HEAD diverges on an input BASE did not. The harness +# (differential/) and the model (lean/) are taken from HEAD for both sides, so +# the only thing that differs is the crate under test in src/. If BASE cannot +# be built with HEAD's harness, BASE's own harness is tried; if that fails too +# there is no baseline, which is reported loudly and does not fail the job. +# +# usage: fuzz_ab.sh +# +# env: FUZZ_INPUTS random programs, model vs crate (default 20000) +# FUZZ_ACT_INPUTS random programs with the ACT read side (default 5000; 0 skips) +# FUZZ_SEED (default 7) +# FUZZ_JOBS worker processes (default 16) +# FUZZ_OUT output dir (default ./fuzz-out) +# CARGO_TARGET_DIR parent of the per-side target dirs (default ./target) +# LAKE_CACHE optional dir to keep lean's .lake build dirs across runs +set -euo pipefail + +BASE_SHA=${1:?usage: fuzz_ab.sh } +HEAD_SHA=${2:?usage: fuzz_ab.sh } +INPUTS=${FUZZ_INPUTS:-20000} +ACT_INPUTS=${FUZZ_ACT_INPUTS:-5000} +SEED=${FUZZ_SEED:-7} +JOBS=${FUZZ_JOBS:-16} +OUT=$(realpath -m "${FUZZ_OUT:-$PWD/fuzz-out}") +TARGET=$(realpath -m "${CARGO_TARGET_DIR:-$PWD/target}") +LAKE_CACHE=${LAKE_CACHE:-} + +repo=$PWD +base_src=$OUT/src-base +mkdir -p "$OUT" +rm -f "$OUT"/*.txt "$OUT"/*.log "$OUT"/*.md + +cleanup() { git -C "$repo" worktree remove --force "$base_src" 2>/dev/null || true; } +trap cleanup EXIT +cleanup +git worktree add --detach "$base_src" "$BASE_SHA" >/dev/null + +# build_side : lake build + cargo build into this side's target dir +build_side() { + local side=$1 src=$2 + if [[ -n $LAKE_CACHE ]]; then + mkdir -p "$LAKE_CACHE/$side" + rm -rf "$src/lean/.lake"; ln -sfn "$LAKE_CACHE/$side" "$src/lean/.lake" + fi + (cd "$src/lean" && lake build) > "$OUT/lake-$side.log" 2>&1 || { tail -30 "$OUT/lake-$side.log" >&2; return 1; } + (cd "$src" && cargo build --release -p differential --target-dir "$TARGET/fuzz-$side") > "$OUT/build-$side.log" 2>&1 \ + || { tail -30 "$OUT/build-$side.log" >&2; return 1; } +} + +echo "== building head ($(git rev-parse --short "$HEAD_SHA"))" +build_side head "$repo" + +echo "== building base ($(git rev-parse --short "$BASE_SHA")) with head's differential/ and lean/" +rm -rf "$base_src/differential" "$base_src/lean" +cp -r "$repo/differential" "$base_src/differential" +cp -r "$repo/lean" "$base_src/lean" && rm -rf "$base_src/lean/.lake" # head's model and harness, not its build dir +baseline=head-harness +if ! build_side base "$base_src"; then + echo "== head's harness does not build against base; trying base's own" + git -C "$base_src" checkout -- differential lean + git -C "$base_src" clean -fdq -- differential lean + if build_side base "$base_src"; then baseline=base-harness; else baseline=none; fi +fi + +# run_side