fix: fail closed edge CI monitor Slack notifications

ci-operator/step-registry/openshift/edge-tooling/ci-monitor/openshift-edge-tooling-ci-monitor-commands.sh

@@ -7,6 +7,19 @@ set -o pipefail
 echo "=== Edge Enablement CI Monitor ==="
 echo "Started at $(date -u '+%Y-%m-%d %H:%M UTC')"
 
+# Let the post step distinguish a monitor that never started from one that
+# started but could not produce a usable report.
+touch "${SHARED_DIR}/monitor-started"
+
+# Do not let data from an earlier attempt be evaluated as this monitor's
+# result. The post step only accepts data published after validation below.
+rm -f "${SHARED_DIR}/monitor-completed" \
+    "${SHARED_DIR}/monitor-report-ready" \
+    "${SHARED_DIR}/monitor-data-ready" \
+    "${SHARED_DIR}/failing-jobs.txt" \
+    "${SHARED_DIR}"/.failing-jobs.* \
+    "${SHARED_DIR}"/.monitor-data-ready.*
+
 # ---------------------------------------------------------------------------
 # Load secrets (xtrace disabled to prevent leaking credentials in logs)
 # ---------------------------------------------------------------------------
@@ -109,13 +122,105 @@ echo "Claude permissions configured."
 # ---------------------------------------------------------------------------
 WORKDIR=$(mktemp -d /tmp/ci-monitor-XXXXXX)
 cd "${WORKDIR}"
+REPORT_START_MARKER="${WORKDIR}/report-started"
+touch "${REPORT_START_MARKER}"
+
+DATA_LINE_REGEX='^(BLOCKING|INFORMING)[|][^|]+[|]https://[^|]+[|][^|]+[|][0-9]+[.][0-9]+[|][a-zA-Z0-9._-]+$'
+
+clear_monitor_data() {
+    rm -f "${SHARED_DIR}/monitor-data-ready" \
+        "${SHARED_DIR}/failing-jobs.txt" \
+        "${SHARED_DIR}"/.failing-jobs.* \
+        "${SHARED_DIR}"/.monitor-data-ready.*
+}
+
+# Extract one complete, explicitly delimited section. Empty sections are valid;
+# malformed data, repeated/out-of-order delimiters, and truncated sections are
+# not. Stream-json records can have surrounding text, so emit only the matched
+# pipe-delimited record.
+extract_job_section() {
+    local section="$1"
+    local log_file="$2"
+
+    awk -v section="${section}" '
+        BEGIN {
+            start = section "_JOBS_START"
+            end = section "_JOBS_END"
+            record = section "[|][^|]+[|]https://[^|]+[|][^|]+[|][0-9]+[.][0-9]+[|][a-zA-Z0-9._-]+"
+        }
+        index($0, start) {
+            if (started || in_section || ended) exit 1
+            started = 1
+            in_section = 1
+            next
+        }
+        index($0, end) {
+            if (!in_section || ended) exit 1
+            in_section = 0
+            ended = 1
+            next
+        }
+        in_section && index($0, section "|") {
+            if (!match($0, record)) exit 1
+            entry = substr($0, RSTART, RLENGTH)
+            remainder = substr($0, RSTART + RLENGTH)
+            if (entry !~ ("^" record "$") || remainder ~ /^[|]/) exit 1
+            print entry
+        }
+        END {
+            if (!started || !ended || in_section) exit 1
+        }
+    ' "${log_file}"
+}
+
+publish_extracted_jobs() {
+    local log_file="$1"
+    local jobs_tmp
+    local ready_tmp
+
+    [[ -r "${log_file}" ]] || return 1
+    jobs_tmp=$(mktemp "${SHARED_DIR}/.failing-jobs.XXXXXX") || return 1
+
+    if ! extract_job_section BLOCKING "${log_file}" > "${jobs_tmp}" \
+        || ! extract_job_section INFORMING "${log_file}" >> "${jobs_tmp}" \
+        || ! sort -u "${jobs_tmp}" -o "${jobs_tmp}"; then
+        rm -f "${jobs_tmp}"
+        return 1
+    fi
+
+    # A zero-byte result is valid only after both sections above validated and
+    # the separate data-ready marker is atomically published below.
+    if [[ -s "${jobs_tmp}" ]] && grep -qvE "${DATA_LINE_REGEX}" "${jobs_tmp}"; then
+        rm -f "${jobs_tmp}"
+        return 1
+    fi
+
+    if ! mv -f "${jobs_tmp}" "${SHARED_DIR}/failing-jobs.txt"; then
+        rm -f "${jobs_tmp}"
+        return 1
+    fi
+
+    ready_tmp=$(mktemp "${SHARED_DIR}/.monitor-data-ready.XXXXXX") || {
+        rm -f "${SHARED_DIR}/failing-jobs.txt"
+        return 1
+    }
+    if ! mv -f "${ready_tmp}" "${SHARED_DIR}/monitor-data-ready"; then
+        rm -f "${ready_tmp}" "${SHARED_DIR}/failing-jobs.txt"
+        return 1
+    fi
+}
 
 copy_artifacts() {
     echo "Copying artifacts to ${ARTIFACT_DIR}..."
+    rm -f "${SHARED_DIR}/monitor-report-ready"
+    clear_monitor_data
+
     local report_dir="${EDGE_TOOLING_DIR}/payload-monitor/reports"
     if [[ -d "${report_dir}" ]]; then
         local latest_html
-        latest_html=$(ls -t "${report_dir}"/*.html 2>/dev/null | head -1)
+        latest_html=$(find "${report_dir}" -maxdepth 1 -type f -name '*.html' \
+            -newer "${REPORT_START_MARKER}" -printf '%T@ %p\n' \
+            | sort -nr | head -1 | cut -d' ' -f2-)
         if [[ -n "${latest_html}" ]]; then
             cp "${latest_html}" "${ARTIFACT_DIR}/edge-ci-monitor-summary.html"
         fi
@@ -126,13 +231,28 @@ copy_artifacts() {
     # downstream steps.  Each line is prefixed BLOCKING| or INFORMING|.
     # SHARED_DIR is backed by a K8s Secret (1 MB limit) so only the
     # extracted data is shared — not the full multi-MB stream-JSON log.
-    if [[ -r "${ARTIFACT_DIR}/claude-analysis.log" ]]; then
-        {
-            sed -n '/BLOCKING_JOBS_START/,/BLOCKING_JOBS_END/p' "${ARTIFACT_DIR}/claude-analysis.log" \
-                | grep -oE 'BLOCKING\|[^|]+\|https://[^|]+\|[^|]+\|[0-9]+\.[0-9]+\|[a-zA-Z0-9._-]+'
-            sed -n '/INFORMING_JOBS_START/,/INFORMING_JOBS_END/p' "${ARTIFACT_DIR}/claude-analysis.log" \
-                | grep -oE 'INFORMING\|[^|]+\|https://[^|]+\|[^|]+\|[0-9]+\.[0-9]+\|[a-zA-Z0-9._-]+'
-        } | sort -u > "${SHARED_DIR}/failing-jobs.txt" || true
+    if publish_extracted_jobs "${ARTIFACT_DIR}/claude-analysis.log"; then
+        echo "Validated job data is ready for Slack notification."
+    else
+        clear_monitor_data
+        echo "WARNING: Extracted job data is unavailable or incomplete."
+    fi
+
+    # A report is ready only when this run generated a complete dashboard with
+    # complete data. The Slack post step must not infer success from an empty
+    # extracted jobs file alone.
+    local dashboard="${ARTIFACT_DIR}/edge-ci-monitor-summary.html"
+    if [[ -s "${dashboard}" ]] \
+        && grep -qF '<title>Edge OCP Payload Monitor</title>' "${dashboard}" \
+        && grep -qF '</html>' "${dashboard}" \
+        && ! grep -qF 'class="error-banner"' "${dashboard}" \
+        && ! grep -qF 'class="skip-banner"' "${dashboard}" \
+        && grep -qE 'Versions: \[[^]]*[0-9]' "${ARTIFACT_DIR}/claude-analysis.log" \
+        && ! grep -qF 'No payload data for versions:' "${ARTIFACT_DIR}/claude-analysis.log"; then
+        touch "${SHARED_DIR}/monitor-report-ready"
+        echo "Dashboard report is ready for Slack notification."
+    else
+        echo "WARNING: Dashboard report is unavailable or incomplete."
     fi
 
     # Archive Claude session for local continuation

ci-operator/step-registry/openshift/edge-tooling/ci-monitor/send-slack/openshift-edge-tooling-ci-monitor-send-slack-commands.sh

@@ -2,10 +2,10 @@
 set -euo pipefail
 
 # ---------------------------------------------------------------------------
-# Guards — skip sending in certain conditions
+# Guard — do nothing only when the monitor step never started
 # ---------------------------------------------------------------------------
-if [[ ! -f "${SHARED_DIR}/monitor-completed" ]]; then
-    echo "Monitor step did not complete — skipping Slack notification."
+if [[ ! -f "${SHARED_DIR}/monitor-started" ]]; then
+    echo "Monitor step did not start — skipping Slack notification."
     exit 0
 fi
 
@@ -35,17 +35,26 @@ BLOCKING_COUNT=0
 INFORMING_COUNT=0
 BLOCKING_LINES=""
 INFORMING_LINES=""
-DATA_AVAILABLE=false
+NOTIFICATION_READY=false
+UNAVAILABLE_REASON=""
 
-if [[ -f "${JOBS_FILE}" ]]; then
-    DATA_AVAILABLE=true
+if [[ ! -f "${SHARED_DIR}/monitor-completed" ]]; then
+    UNAVAILABLE_REASON="monitor did not complete"
+elif [[ ! -f "${SHARED_DIR}/monitor-report-ready" ]]; then
+    UNAVAILABLE_REASON="dashboard report is unavailable or incomplete"
+elif [[ ! -f "${SHARED_DIR}/monitor-data-ready" ]]; then
+    UNAVAILABLE_REASON="extracted job data is unavailable or incomplete"
+elif [[ ! -f "${JOBS_FILE}" ]]; then
+    UNAVAILABLE_REASON="extracted job data is unavailable"
+elif [[ -s "${JOBS_FILE}" ]] && grep -qvE '^(BLOCKING|INFORMING)[|][^|]+[|]https://[^|]+[|][^|]+[|][0-9]+[.][0-9]+[|][a-zA-Z0-9._-]+$' "${JOBS_FILE}"; then
+    UNAVAILABLE_REASON="extracted job data is invalid"
+else
+    NOTIFICATION_READY=true
     BLOCKING_LINES=$(grep '^BLOCKING|' "${JOBS_FILE}" || true)
     INFORMING_LINES=$(grep '^INFORMING|' "${JOBS_FILE}" || true)
 
     [[ -n "${BLOCKING_LINES}" ]] && BLOCKING_COUNT=$(echo "${BLOCKING_LINES}" | wc -l)
     [[ -n "${INFORMING_LINES}" ]] && INFORMING_COUNT=$(echo "${INFORMING_LINES}" | wc -l)
-else
-    echo "Warning: ${JOBS_FILE} not found."
 fi
 
 # ---------------------------------------------------------------------------
@@ -78,9 +87,9 @@ version_summary() {
 # ---------------------------------------------------------------------------
 NL=$'\n'
 
-if [[ "${DATA_AVAILABLE}" != "true" ]]; then
+if [[ "${NOTIFICATION_READY}" != "true" ]]; then
     ICON=":warning:"
-    MESSAGE="${ICON} *Edge OCP CI Monitor* — Data unavailable. Please investigate the artifacts."
+    MESSAGE="${ICON} *Edge OCP CI Monitor* — Data unavailable: ${UNAVAILABLE_REASON}. Please investigate the Prow logs."
 elif [[ "${BLOCKING_COUNT}" -eq 0 ]] && [[ "${INFORMING_COUNT}" -eq 0 ]]; then
     ICON=":large_green_circle:"
     MESSAGE="${ICON} *Edge OCP CI Monitor* — No failing jobs found."
@@ -104,7 +113,11 @@ else
     fi
 fi
 
-MESSAGE+="${NL}<${DASHBOARD_URL}|View Dashboard> | <${JOB_URL}|Prow Logs> | @edge-enablement-payload-manager"
+if [[ "${NOTIFICATION_READY}" == "true" ]]; then
+    MESSAGE+="${NL}<${DASHBOARD_URL}|View Dashboard> | <${JOB_URL}|Prow Logs> | @edge-enablement-payload-manager"
+else
+    MESSAGE+="${NL}<${JOB_URL}|Prow Logs> | @edge-enablement-payload-manager"
+fi
 
 # ---------------------------------------------------------------------------
 # Send to Slack (or dry-run on PRs)