From 92cb8287d219f787a0bf3a2d0153b5a463c837b1 Mon Sep 17 00:00:00 2001 From: Evan Nemerson Date: Tue, 14 Jul 2026 09:47:24 -0400 Subject: [PATCH] CP-44602: Distinguish RBAC failure from a missing VWC in webhook config collection anaximander reported "No ValidatingWebhookConfigurations found" whenever the cluster-scoped list failed -- for example when the collecting user lacks permission on validatingwebhookconfigurations -- because the error was swallowed with "2>/dev/null || echo """. That produced a misleading "the webhook receives no traffic" conclusion even when the webhook was working normally. Capture the kubectl exit status and, on failure, emit a warning that this is a permissions or API limitation rather than proof the webhook is unconfigured. Match the webhook config by chart label first, then by any webhook entry whose service is in the agent namespace (not just the first entry), and read the caBundle from the first non-empty entry. Only report a genuine absence when the listing succeeds and returns nothing. Verified: bash -n passes; fixtures for label-not-first, service-only, genuine absence, and a Forbidden error are all classified correctly. Co-Authored-By: Claude opus-4.8 --- scripts/anaximander.sh | 92 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 82 insertions(+), 10 deletions(-) diff --git a/scripts/anaximander.sh b/scripts/anaximander.sh index fa1e15b3..968b6b08 100755 --- a/scripts/anaximander.sh +++ b/scripts/anaximander.sh @@ -539,18 +539,86 @@ info "Gathering webhook configuration..." echo "===================================" echo "" - # Find VWCs that reference a service in the agent namespace. - # This catches the CloudZero webhook regardless of release name or - # label changes across chart versions. - VWC_NAMES=$(kubectl --context "${KUBECTX}" get validatingwebhookconfigurations \ - -o jsonpath="{range .items[?(@.webhooks[0].clientConfig.service.namespace==\"${KUBENS}\")]}{.metadata.name}{\"\\n\"}{end}" 2>/dev/null || echo "") + # List the cluster's ValidatingWebhookConfigurations as JSON. stdout and + # stderr are kept in SEPARATE streams (stderr is redirected to a temp file, + # never merged into the JSON) and the exit status is captured, so that a + # permission error, a parse failure, and a genuine absence are three + # DISTINCT diagnoses. This matters because the collection user often lacks + # list access on cluster-scoped resources; the previous implementation + # swallowed such errors ("2>/dev/null || echo """) and reported them + # identically to "no webhook configured", producing a misleading "the + # webhook receives no traffic" conclusion on a healthy deployment. stderr + # must not be folded into the JSON, because a kubectl warning emitted on an + # otherwise-successful call (e.g. a deprecation or API Warning header) would + # corrupt the JSON and re-introduce the same false negative. + VWC_ERR_FILE=$(mktemp) + VWC_RC=0 + VWC_JQ_RC=0 + VWC_NAMES="" + # This script runs under "set -e", so an unguarded "VAR=$(cmd)" whose command + # exits non-zero would abort the entire script before its exit status could be + # inspected -- which would defeat the RBAC-vs-absence distinction below (a + # denied kubectl call would kill the whole collection instead of warning). + # The "|| VWC_RC=$?" form keeps each assignment in a conditional context so + # errexit does not fire and the real exit code is captured for branching. + VWC_JSON=$(kubectl --context "${KUBECTX}" get validatingwebhookconfigurations -o json 2>"${VWC_ERR_FILE}") || VWC_RC=$? + + if [ "${VWC_RC}" -eq 0 ]; then + # Match the CloudZero webhook by chart label first (most reliable across + # release names), then fall back to any VWC that points ANY of its + # webhook entries at a service in the agent namespace. Checking all + # webhooks[*] -- not just webhooks[0] -- avoids missing the config when + # the CloudZero entries are not first, and jq tolerates other + # controllers' webhook configs that lack a service clientConfig. jq's + # exit status is captured on its own so a parse failure (e.g. jq not + # installed, or unexpected output) is not mistaken for "none found". + VWC_NAMES=$(printf '%s' "${VWC_JSON}" | jq -r --arg ns "${KUBENS}" ' + .items[] + | select( + (.metadata.labels["app.kubernetes.io/part-of"] == "cloudzero-agent") + or (any(.webhooks[]?; .clientConfig.service.namespace == $ns)) + ) + | .metadata.name') || VWC_JQ_RC=$? + if [ "${VWC_JQ_RC}" -eq 0 ]; then + VWC_NAMES=$(printf '%s\n' "${VWC_NAMES}" | sort -u) + fi + fi - if [ -z "${VWC_NAMES}" ]; then + if [ "${VWC_RC}" -ne 0 ]; then + echo "WARNING: Could not list ValidatingWebhookConfigurations (kubectl exit ${VWC_RC})." + echo "" + echo "This is almost always a permissions limitation of the collection" + echo "user on cluster-scoped resources, or a transient API error. It is" + echo "NOT evidence that the webhook is unconfigured. If the webhook pods" + echo "are logging \"Pushing records to remote write endpoint\" with" + echo "non-zero counts, the webhook IS receiving admission traffic." + echo "" + echo "To verify manually with sufficient privileges:" + echo " kubectl get validatingwebhookconfigurations -l app.kubernetes.io/part-of=cloudzero-agent" + echo "" + echo "kubectl error output:" + sed 's/^/ /' "${VWC_ERR_FILE}" + elif [ "${VWC_JQ_RC}" -ne 0 ]; then + echo "WARNING: Could not parse the ValidatingWebhookConfiguration list (jq exit ${VWC_JQ_RC})." + echo "" + echo "The listing itself succeeded, but its output could not be parsed --" + echo "most commonly because jq is not installed on the collection machine." + echo "This is NOT evidence that the webhook is unconfigured." + echo "" + echo "To verify manually:" + echo " kubectl get validatingwebhookconfigurations -l app.kubernetes.io/part-of=cloudzero-agent" + elif [ -z "${VWC_NAMES}" ]; then echo "No ValidatingWebhookConfigurations found referencing namespace ${KUBENS}" + echo "(and none labeled app.kubernetes.io/part-of=cloudzero-agent)." echo "" - echo "This means the Kubernetes API server is not configured to send" - echo "admission requests to the CloudZero webhook. The webhook pods" - echo "will run but receive no traffic." + echo "The cluster listing succeeded and parsed cleanly, so this is a" + echo "genuine result: the Kubernetes API server is not configured to send" + echo "admission requests to the CloudZero webhook. The webhook pods will" + echo "run but receive no admission traffic." + echo "" + echo "Cross-check: if the webhook pods are still logging \"Pushing records" + echo "to remote write endpoint\" with non-zero counts, they ARE receiving" + echo "traffic and the label/service match above should be re-examined." else for VWC_NAME in ${VWC_NAMES}; do echo "--- ${VWC_NAME} ---" @@ -562,8 +630,10 @@ info "Gathering webhook configuration..." echo "" echo "--- caBundle status ---" + # Take the first non-empty caBundle across all webhook entries, + # rather than assuming it lives on webhooks[0]. CA_BUNDLE=$(kubectl --context "${KUBECTX}" get validatingwebhookconfiguration "${VWC_NAME}" \ - -o jsonpath='{.webhooks[0].clientConfig.caBundle}' 2>/dev/null || echo "") + -o json 2>/dev/null | jq -r '[.webhooks[].clientConfig.caBundle // empty] | first // ""' 2>/dev/null || echo "") if [ -z "${CA_BUNDLE}" ]; then echo "WARNING: caBundle is EMPTY. The API server cannot verify the" @@ -582,6 +652,8 @@ info "Gathering webhook configuration..." echo "" done fi + + rm -f "${VWC_ERR_FILE}" } > "${OUTPUT_DIR}/webhook-config.txt" 2>&1 echo "[SUCCESS] Gather webhook configuration" >> "${COMMAND_RESULTS_FILE}"