fix(ci-queue-watch): a blind watchdog must be RED, not green

.github/workflows/ci-queue-watch.yml

@@ -66,24 +66,42 @@ jobs:
 
       # Reading another repository's Actions queue needs a token with fleet
       # scope; the job's own GITHUB_TOKEN is scoped to this repository alone and
-      # would report a fleet of one. No token means this watchdog cannot do its
-      # job -- say so loudly and exit 0. Never red a scheduled run on a missing
-      # secret: an environmental condition must not look like a fleet outage,
-      # which is the same inversion the actor gate caused in a2a-maintain.
-      - name: Check for a fleet-scoped token
-        id: tok
+      # would report a fleet of one.
+      #
+      # THIS STEP USED TO WARN AND EXIT 0, AND THAT WAS THE DEFECT. The job below
+      # was gated on its output, so a scheduled run with no token skipped the
+      # observation and the job went GREEN. It did that ten times, every one
+      # green, including runs at 07:20Z and 07:57Z while the fleet was 40 hours
+      # into an outage. Green read as "the fleet is fine"; it actually meant "I
+      # looked at nothing" -- the exact vacuous pass this watchdog exists to
+      # detect, reproduced inside the detector.
+      #
+      # The original reasoning was half right, and that half is preserved: an
+      # environmental gap must not masquerade as a FLEET OUTAGE. The resolution
+      # is not to go green, it is to fail with a DIFFERENT SIGNAL. This error
+      # says the watchdog is blind. ci_queue_watch.py's stall error says the
+      # fleet is stuck. If those two were confusable the red would be as
+      # uninformative as the green was -- hence the distinct title and wording.
+      #
+      # There is no legitimate no-token run of this job. The job-level `if`
+      # excludes pull_request, leaving schedule (always the default branch, full
+      # secrets) and workflow_dispatch (requires write access). A missing secret
+      # here is misconfiguration, never a transient.
+      #
+      # scripts/ci_queue_watch.py deliberately still exits 0 with no token, and
+      # test_no_token_skips_cleanly pins that: for a CLI run by hand, no
+      # credential means nothing to misconfigure. The difference is RUN CONTEXT,
+      # which only this workflow knows. Do not "align" them.
+      - name: Require a fleet-scoped token
         env:
           FLEET_READ_PAT: ${{ secrets.FLEET_READ_PAT }}
         run: |
           if [ -z "$FLEET_READ_PAT" ]; then
-            echo "::warning::FLEET_READ_PAT is not set — the watchdog can only see this repo, so it is skipping. Set a PAT with read access to Actions across the fleet (scope: repo, or fine-grained Actions:read on the Fuze* repos)."
-            echo "ok=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "ok=true" >> "$GITHUB_OUTPUT"
+            echo "::error title=Watchdog not configured::FLEET_READ_PAT is unset, so this watchdog can see only its own repository and cannot observe the fleet at all. This is a CONFIGURATION failure and NOT a fleet outage — do not read it as one. Fix: add a PAT with Actions:read across the Fuze* repos as the FLEET_READ_PAT secret on this repository."
+            exit 1
           fi
 
       - name: Observe the fleet CI queue
-        if: steps.tok.outputs.ok == 'true'
         env:
           GITHUB_TOKEN: ${{ secrets.FLEET_READ_PAT }}
           STALL: ${{ inputs.stall_minutes || '30' }}

scripts/__tests__/test_ci_queue_watch.py

@@ -175,5 +175,66 @@ def test_source_excludes_zero_runner_id(self):
                       "never picked anything up reports recent starts and reads healthy")
 
 
+class BlindWatchdogMustBeRedNotGreen(unittest.TestCase):
+    """The WORKFLOW-level half of the same defect, invisible to the tests above.
+
+    scripts/ci_queue_watch.py exiting 0 with no token is correct, and
+    test_no_token_skips_cleanly pins it. The bug lived one layer up: the workflow
+    warned, set an output, and gated the observation step on it — so a scheduled
+    run with no token skipped the work and the JOB went green. Ten consecutive
+    green runs observed nothing, two of them while the fleet was 40 hours into an
+    outage.
+
+    These assertions read the REAL workflow file, not a fixture. A fixture would
+    have to encode the structure under test and could therefore agree with the
+    bug — the failure mode that let a hardcoded default_branch survive twelve
+    green tests elsewhere in this repo.
+    """
+
+    WORKFLOW = os.path.join(REPO, ".github", "workflows", "ci-queue-watch.yml")
+
+    def _watch_job(self):
+        try:
+            import yaml
+        except ImportError:  # pragma: no cover
+            self.skipTest("PyYAML unavailable")
+        with open(self.WORKFLOW, encoding="utf-8") as fh:
+            return yaml.safe_load(fh)["jobs"]["watch"]
+
+    def _guard(self):
+        steps = self._watch_job()["steps"]
+        g = [s for s in steps if "token" in (s.get("name") or "").lower()]
+        self.assertEqual(len(g), 1, "expected exactly one token guard step")
+        return g[0]
+
+    def test_missing_token_fails_the_job_rather_than_skipping_it(self):
+        """The regression: the guard must exit non-zero, not set a flag."""
+        run = self._guard().get("run", "")
+        self.assertIn("exit 1", run,
+                      "the token guard must FAIL when FLEET_READ_PAT is unset — "
+                      "warning and continuing is what produced ten green runs "
+                      "that had observed nothing")
+        self.assertNotIn("GITHUB_OUTPUT", run,
+                         "the guard must not export a skip flag; that flag is how "
+                         "the observation step got bypassed silently")
+
+    def test_no_step_is_gated_on_token_presence(self):
+        """A leftover `if:` would re-open the hole even with the guard failing."""
+        for s in self._watch_job()["steps"]:
+            self.assertNotIn("steps.tok", str(s.get("if", "")),
+                             f"step {s.get('name')!r} is still gated on a token "
+                             f"output — the observation must be unconditional "
+                             f"once the guard has passed")
+
+    def test_blindness_and_stall_are_distinguishable_signals(self):
+        """A red indistinguishable from a fleet stall is as useless as the green
+        was. The guard must name itself a CONFIGURATION failure."""
+        run = self._guard().get("run", "")
+        self.assertIn("::error", run)
+        self.assertIn("NOT a fleet outage", run,
+                      "the error must explicitly disclaim the fleet-outage "
+                      "reading, or an operator acts on the wrong incident")
+
+
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main(verbosity=2)