diff --git a/config/components/alerts/reconcile-storm.yaml b/config/components/alerts/reconcile-storm.yaml index d158ab66..311b07aa 100644 --- a/config/components/alerts/reconcile-storm.yaml +++ b/config/components/alerts/reconcile-storm.yaml @@ -15,12 +15,13 @@ spec: # controller re-reconciles the same objects in a self-perpetuating loop. # Reconciles keep succeeding, so reconcile_errors_total stays flat and # error-based alerting misses it entirely. The fingerprint is a high - # reconcile rate while the workqueue stays drained. Evaluated per + # reconcile rate while the workqueue stays mostly drained. Evaluated per # controller across every controller compute-manager reports (job label # scopes to compute-manager; the controller/name label identifies which # one is storming when it fires) so it also covers cell-side controllers - # and any controller added later. The drained-queue guard keeps a - # legitimately busy controller (deep backlog) from tripping it. + # and any controller added later. The average-depth guard keeps a + # legitimately busy controller (sustained backlog) from tripping it + # without clearing the alert on a brief queue-depth spike. - alert: ComputeControllerReconcileStorm expr: | ( @@ -30,8 +31,8 @@ spec: ) and on (controller) ( - max by (controller) ( - max_over_time(workqueue_depth{job="compute-metrics"}[5m]) + avg by (controller) ( + avg_over_time(workqueue_depth{job="compute-metrics"}[5m]) ) < 5 ) for: 15m @@ -45,7 +46,7 @@ spec: description: | The compute controller "{{ $labels.controller }}" has been reconciling at {{ $value | humanize }}/s (>30/min) for 15 minutes - while its workqueue has stayed drained. The same objects are being + while its workqueue has stayed mostly drained on average. The same objects are being re-reconciled in a self-perpetuating loop, burning control-plane capacity without doing real work. @@ -57,39 +58,3 @@ spec: The firing controller label tells you where to look. See the runbook for confirmation queries, known causes, and remediation. runbook_url: https://github.com/datum-cloud/compute/blob/main/docs/runbooks/compute-management-controller-reconcile-storm.md - - # Corroborating signal on the same fingerprint from the workqueue side: - # a high add rate against a drained queue is what feeds the loop. - # workqueue metrics label the controller as "name" rather than - # "controller". - - alert: ComputeControllerWorkqueueAddStorm - expr: | - ( - sum by (name) ( - rate(workqueue_adds_total{job="compute-metrics"}[5m]) - ) > 0.5 - ) - and on (name) - ( - max by (name) ( - max_over_time(workqueue_depth{job="compute-metrics"}[5m]) - ) < 5 - ) - for: 15m - labels: - severity: warning - service: compute-controllers - namespace: compute-system - team: compute - annotations: - summary: Compute {{ $labels.name }} controller workqueue is churning while drained - description: | - The compute controller "{{ $labels.name }}" has been enqueuing work - at {{ $value | humanize }}/s (>30/min) for 15 minutes while its - workqueue depth has stayed near zero. Items are being re-added as - fast as they drain, which is the enqueue-side signature of a - reconcile loop. - - The firing controller label tells you where to look. See the - runbook for confirmation queries, known causes, and remediation. - runbook_url: https://github.com/datum-cloud/compute/blob/main/docs/runbooks/compute-management-controller-reconcile-storm.md diff --git a/docs/runbooks/compute-management-controller-reconcile-storm.md b/docs/runbooks/compute-management-controller-reconcile-storm.md index aa600fdc..d5a90b6f 100644 --- a/docs/runbooks/compute-management-controller-reconcile-storm.md +++ b/docs/runbooks/compute-management-controller-reconcile-storm.md @@ -1,6 +1,6 @@ # Runbook: Compute controller reconcile storm -**Alerts:** `ComputeControllerReconcileStorm`, `ComputeControllerWorkqueueAddStorm` +**Alert:** `ComputeControllerReconcileStorm` **Severity:** warning **Component:** compute controllers (`compute-system/compute-manager`) @@ -19,7 +19,7 @@ including any controller added later. **The firing `controller` / `name` label tells you which controller is storming and therefore where to look.** Each reconcile **succeeds**, so error-based alerting stays silent. The tell is a -high reconcile / enqueue rate while the workqueue stays drained (depth ~0) and +high reconcile rate while the workqueue stays mostly drained (low average depth) and the number of objects for that controller's resource is small and stable. Left unchecked, this wastes control-plane CPU and apiserver capacity and adds @@ -54,8 +54,8 @@ to focus on the storming controller. ) ``` -2. **Enqueue rate vs. queue depth** — the smoking gun. A high add rate with a - depth pinned near zero means items are re-added as fast as they drain. Note +2. **Enqueue rate vs. queue depth** — the corroborating signal. A high add rate + with a low average depth means items are re-added as fast as they drain. Note the workqueue metrics label the controller as `name`, not `controller`: ```promql @@ -66,8 +66,8 @@ to focus on the storming controller. ) * 60 ) - # depth stays ~0 the whole time - max by (name) (workqueue_depth{job="compute-metrics"}) + # average depth stays low; brief spikes do not invalidate the signal + avg by (name) (avg_over_time(workqueue_depth{job="compute-metrics"}[5m])) ``` 3. **Object count is tiny and stable** — confirms the rate is not just real work