ci-queue-watch.yml

name: CI Queue Watchdog

# Answers one question, every 15 minutes, for every runner pool in the fleet:
# is it PROGRESSING? See scripts/ci_queue_watch.py for why that is the measure
# and not "how long has this job been waiting".
#
# RUNS ON ubuntu-latest, DELIBERATELY. A watchdog scheduled onto the runner
# fleet it monitors cannot report that fleet being down -- it just queues with
# everything else and stays silent, which is the exact failure it exists to
# catch. This repo is public, so GitHub-hosted minutes are unmetered and this
# job is independent of ARC and of the account's Actions budget.

on:
  schedule:
    - cron: '*/15 * * * *'
  pull_request:
    paths:
      - 'scripts/ci_queue_watch.py'
      - 'scripts/__tests__/test_ci_queue_watch.py'
      - '.github/workflows/ci-queue-watch.yml'
  workflow_dispatch:
    inputs:
      stall_minutes:
        description: 'Queued work plus no start within this many minutes = STALLED'
        required: false
        default: '30'

permissions:
  contents: read

concurrency:
  group: ci-queue-watch
  cancel-in-progress: true

jobs:
  # Self-tests run on any PR touching the watchdog, and they assert the script
  # REFUSES when it cannot see the fleet -- not merely that it reports correctly
  # when it can. A watchdog wired to nothing passes every "does it work" test
  # ever written, which is exactly how the vacuous gates in this family survived.
  selftest:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
        with:
          python-version: '3.12'
      - name: Run watchdog self-tests
        run: |
          test -f scripts/__tests__/test_ci_queue_watch.py || {
            echo "::error::watchdog self-tests are missing — refusing to ship the watchdog without them"
            exit 1
          }
          python3 -m unittest discover -s scripts/__tests__ -p 'test_ci_queue_watch*.py' -v

  watch:
    if: github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4

      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97  # v7.0.0
        with:
          python-version: '3.12'

      # Reading another repository's Actions queue needs a token with fleet
      # scope; the job's own GITHUB_TOKEN is scoped to this repository alone and
      # would report a fleet of one.
      #
      # THIS STEP USED TO WARN AND EXIT 0, AND THAT WAS THE DEFECT. The job below
      # was gated on its output, so a scheduled run with no token skipped the
      # observation and the job went GREEN. It did that ten times, every one
      # green, including runs at 07:20Z and 07:57Z while the fleet was 40 hours
      # into an outage. Green read as "the fleet is fine"; it actually meant "I
      # looked at nothing" -- the exact vacuous pass this watchdog exists to
      # detect, reproduced inside the detector.
      #
      # The original reasoning was half right, and that half is preserved: an
      # environmental gap must not masquerade as a FLEET OUTAGE. The resolution
      # is not to go green, it is to fail with a DIFFERENT SIGNAL. This error
      # says the watchdog is blind. ci_queue_watch.py's stall error says the
      # fleet is stuck. If those two were confusable the red would be as
      # uninformative as the green was -- hence the distinct title and wording.
      #
      # There is no legitimate no-token run of this job. The job-level `if`
      # excludes pull_request, leaving schedule (always the default branch, full
      # secrets) and workflow_dispatch (requires write access). A missing secret
      # here is misconfiguration, never a transient.
      #
      # scripts/ci_queue_watch.py deliberately still exits 0 with no token, and
      # test_no_token_skips_cleanly pins that: for a CLI run by hand, no
      # credential means nothing to misconfigure. The difference is RUN CONTEXT,
      # which only this workflow knows. Do not "align" them.
      - name: Require a fleet-scoped token
        env:
          FLEET_READ_PAT: ${{ secrets.FLEET_READ_PAT }}
        run: |
          if [ -z "$FLEET_READ_PAT" ]; then
            echo "::error title=Watchdog not configured::FLEET_READ_PAT is unset, so this watchdog can see only its own repository and cannot observe the fleet at all. This is a CONFIGURATION failure and NOT a fleet outage — do not read it as one. Fix: add a PAT with Actions:read across the Fuze* repos as the FLEET_READ_PAT secret on this repository."
            exit 1
          fi

      - name: Observe the fleet CI queue
        env:
          GITHUB_TOKEN: ${{ secrets.FLEET_READ_PAT }}
          STALL: ${{ inputs.stall_minutes || '30' }}
        run: |
          set -o pipefail
          python3 scripts/ci_queue_watch.py \
            --stall-minutes "$STALL" \
            --fail-on-stall \
            | tee queue.txt
          {
            echo '## Fleet CI queue'
            echo
            echo '```'
            cat queue.txt
            echo '```'
          } >> "$GITHUB_STEP_SUMMARY"