fix(ci): rebuild cluster host keys per run so a node rebuild self-heals

.github/scripts/refresh-cluster-known-hosts.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# Build a PER-RUN SSH known_hosts for the bench cluster.
+#
+# Why this exists (2026-08-19): msa2-client was rebuilt onto a new SSD and so
+# presents a new host key. The inventory sets StrictHostKeyChecking=accept-new,
+# which accepts UNKNOWN hosts but correctly REFUSES CHANGED ones — so every run
+# after the rebuild died with:
+#
+#     fatal: [msa2-client]: UNREACHABLE!
+#     Host key verification failed.
+#
+# while the other two hosts passed. The symptom is badly misleading: a perfectly
+# healthy machine looks dead. The stale entry lives in the persistent
+# ~/.ssh/known_hosts of whichever host ran the job, and because Ubuntu hashes
+# known_hosts by default `grep` finds nothing — only `ssh-keygen -F` reveals it.
+#
+# Rather than mutate the operator's persistent known_hosts, scan the cluster
+# fresh into a file under the runner root (on /tmp, i.e. tmpfs, discarded on
+# reboot) and point ansible at that. A rebuilt node is picked up automatically
+# and host-key checking still applies WITHIN the run.
+#
+# Host list comes from ansible/inventory.yml — one source of truth for topology.
+#
+# Portability: no `mapfile`, no nested heredocs. macOS ships bash 3.2 and this
+# script must stay testable on the dev machine, not just on the Linux runners.
+set -euo pipefail
+
+INVENTORY="${1:-ansible/inventory.yml}"
+KH="${2:?usage: refresh-cluster-known-hosts.sh <inventory> <known_hosts path>}"
+
+TARGETS=$(python3 -c '
+import sys, yaml
+doc = yaml.safe_load(open(sys.argv[1]))
+out = []
+def walk(node):
+    for name, v in (node.get("hosts") or {}).items():
+        out.append(name)
+        if v and v.get("lan_ip"):
+            out.append(str(v["lan_ip"]))
+    for child in (node.get("children") or {}).values():
+        walk(child or {})
+walk(doc["all"])
+seen = set()
+for t in out:
+    if t not in seen:
+        seen.add(t)
+        print(t)
+' "$INVENTORY")
+
+if [ -z "$TARGETS" ]; then
+  echo "refresh-cluster-known-hosts: no hosts parsed from $INVENTORY" >&2
+  exit 1
+fi
+
+mkdir -p "$(dirname "$KH")"
+: > "$KH"
+chmod 600 "$KH"
+
+echo "refresh-cluster-known-hosts: scanning $(echo "$TARGETS" | wc -w | tr -d ' ') target(s) -> $KH"
+for t in $TARGETS; do
+  # -T 5 bounds an offline host to 5s. A host that is genuinely down is NOT
+  # fatal here: ansible then reports a real UNREACHABLE with a real reason,
+  # which is far more diagnosable than a host-key mismatch.
+  if ssh-keyscan -T 5 "$t" 2>/dev/null >> "$KH"; then
+    printf '  %-16s ok\n' "$t"
+  else
+    printf '  %-16s no response (will surface as a real UNREACHABLE, not a key mismatch)\n' "$t"
+  fi
+done
+
+echo "refresh-cluster-known-hosts: $(wc -l < "$KH" | tr -d ' ') key line(s) written"

.github/workflows/benchmark-tier.yml

@@ -179,7 +179,14 @@ jobs:
             echo "ANSIBLE_PRIVATE_KEY_FILE=${RUNNER_ROOT}/.ssh/id_ed25519"
             echo "PROBATORIUM_LOCAL_HOST=$(hostname -s)"
             echo "CLUSTER_USE_LAN=1"
+            echo "CLUSTER_KNOWN_HOSTS=${RUNNER_ROOT}/.ssh/known_hosts"
           } >> "$GITHUB_ENV"
+          # Rebuild the cluster host-key file for THIS run. Without it, a node
+          # that has been reinstalled presents a new key, accept-new refuses the
+          # change, and ansible reports UNREACHABLE — a healthy machine that
+          # looks dead. See the script header.
+          bash .github/scripts/refresh-cluster-known-hosts.sh \
+            ansible/inventory.yml "${RUNNER_ROOT}/.ssh/known_hosts"
 
       - name: Publish preflight (fail fast on missing docs token)
         # When this run will publish (BENCH_PUBLISH != 0), refuse to start the

.github/workflows/matrix-nightly-tier.yml

@@ -97,7 +97,14 @@ jobs:
             echo "ANSIBLE_PRIVATE_KEY_FILE=${RUNNER_ROOT}/.ssh/id_ed25519"
             echo "PROBATORIUM_LOCAL_HOST=$(hostname -s)"
             echo "CLUSTER_USE_LAN=1"
+            echo "CLUSTER_KNOWN_HOSTS=${RUNNER_ROOT}/.ssh/known_hosts"
           } >> "$GITHUB_ENV"
+          # Rebuild the cluster host-key file for THIS run. Without it, a node
+          # that has been reinstalled presents a new key, accept-new refuses the
+          # change, and ansible reports UNREACHABLE — a healthy machine that
+          # looks dead. See the script header.
+          bash .github/scripts/refresh-cluster-known-hosts.sh \
+            ansible/inventory.yml "${RUNNER_ROOT}/.ssh/known_hosts"
 
       - name: mage Deploy (stage refapps + validator)
         env:

.github/workflows/matrix-pr-tier.yml

@@ -164,7 +164,14 @@ jobs:
             echo "ANSIBLE_PRIVATE_KEY_FILE=${RUNNER_ROOT}/.ssh/id_ed25519"
             echo "PROBATORIUM_LOCAL_HOST=$(hostname -s)"
             echo "CLUSTER_USE_LAN=1"
+            echo "CLUSTER_KNOWN_HOSTS=${RUNNER_ROOT}/.ssh/known_hosts"
           } >> "$GITHUB_ENV"
+          # Rebuild the cluster host-key file for THIS run. Without it, a node
+          # that has been reinstalled presents a new key, accept-new refuses the
+          # change, and ansible reports UNREACHABLE — a healthy machine that
+          # looks dead. See the script header.
+          bash .github/scripts/refresh-cluster-known-hosts.sh \
+            ansible/inventory.yml "${RUNNER_ROOT}/.ssh/known_hosts"
 
       - name: mage Deploy (stage refapps + validator)
         env:

.github/workflows/matrix-weekend-tier.yml

@@ -98,7 +98,14 @@ jobs:
             echo "ANSIBLE_PRIVATE_KEY_FILE=${RUNNER_ROOT}/.ssh/id_ed25519"
             echo "PROBATORIUM_LOCAL_HOST=$(hostname -s)"
             echo "CLUSTER_USE_LAN=1"
+            echo "CLUSTER_KNOWN_HOSTS=${RUNNER_ROOT}/.ssh/known_hosts"
           } >> "$GITHUB_ENV"
+          # Rebuild the cluster host-key file for THIS run. Without it, a node
+          # that has been reinstalled presents a new key, accept-new refuses the
+          # change, and ansible reports UNREACHABLE — a healthy machine that
+          # looks dead. See the script header.
+          bash .github/scripts/refresh-cluster-known-hosts.sh \
+            ansible/inventory.yml "${RUNNER_ROOT}/.ssh/known_hosts"
 
       - name: mage Deploy (stage refapps + validator)
         env:

ansible/inventory.yml

@@ -30,6 +30,12 @@ all:
     #   ServerAliveCountMax=18    — survive ~3 minutes of dropped heartbeats
     #   ControlPersist=600s       — long-lived ControlMaster across cells
     #   TCPKeepAlive=yes          — kernel-level keepalive too
+    #   UserKnownHostsFile        — CI points this at a per-run file rebuilt by
+    #     .github/scripts/refresh-cluster-known-hosts.sh, so a REBUILT node's new
+    #     host key never collides with a stale entry. accept-new refuses CHANGED
+    #     keys (correctly), which after msa2-client's 2026-08-18 rebuild made a
+    #     healthy machine look dead in every run. Unset outside CI → normal
+    #     ~/.ssh/known_hosts, so dev-machine behaviour is unchanged.
     ansible_ssh_common_args: >-
       -o ControlMaster=auto
       -o ControlPersist=600s
@@ -38,6 +44,7 @@ all:
       -o ServerAliveCountMax=18
       -o TCPKeepAlive=yes
       -o StrictHostKeyChecking=accept-new
+      -o UserKnownHostsFile={{ lookup('env', 'CLUSTER_KNOWN_HOSTS') | default('~/.ssh/known_hosts', true) }}
     ansible_command_timeout: 120
     # Privilege escalation must tolerate a saturated host: ansible's
     # default 12s become-prompt timeout dropped msa2-server as