diff --git a/ci-operator/step-registry/openshift/edge-tooling/ci-monitor/openshift-edge-tooling-ci-monitor-commands.sh b/ci-operator/step-registry/openshift/edge-tooling/ci-monitor/openshift-edge-tooling-ci-monitor-commands.sh index 70af53074aabd..7c1ffa87510b8 100644 --- a/ci-operator/step-registry/openshift/edge-tooling/ci-monitor/openshift-edge-tooling-ci-monitor-commands.sh +++ b/ci-operator/step-registry/openshift/edge-tooling/ci-monitor/openshift-edge-tooling-ci-monitor-commands.sh @@ -7,6 +7,19 @@ set -o pipefail echo "=== Edge Enablement CI Monitor ===" echo "Started at $(date -u '+%Y-%m-%d %H:%M UTC')" +# Let the post step distinguish a monitor that never started from one that +# started but could not produce a usable report. +touch "${SHARED_DIR}/monitor-started" + +# Do not let data from an earlier attempt be evaluated as this monitor's +# result. The post step only accepts data published after validation below. +rm -f "${SHARED_DIR}/monitor-completed" \ + "${SHARED_DIR}/monitor-report-ready" \ + "${SHARED_DIR}/monitor-data-ready" \ + "${SHARED_DIR}/failing-jobs.txt" \ + "${SHARED_DIR}"/.failing-jobs.* \ + "${SHARED_DIR}"/.monitor-data-ready.* + # --------------------------------------------------------------------------- # Load secrets (xtrace disabled to prevent leaking credentials in logs) # --------------------------------------------------------------------------- @@ -109,13 +122,105 @@ echo "Claude permissions configured." # --------------------------------------------------------------------------- WORKDIR=$(mktemp -d /tmp/ci-monitor-XXXXXX) cd "${WORKDIR}" +REPORT_START_MARKER="${WORKDIR}/report-started" +touch "${REPORT_START_MARKER}" + +DATA_LINE_REGEX='^(BLOCKING|INFORMING)[|][^|]+[|]https://[^|]+[|][^|]+[|][0-9]+[.][0-9]+[|][a-zA-Z0-9._-]+$' + +clear_monitor_data() { + rm -f "${SHARED_DIR}/monitor-data-ready" \ + "${SHARED_DIR}/failing-jobs.txt" \ + "${SHARED_DIR}"/.failing-jobs.* \ + "${SHARED_DIR}"/.monitor-data-ready.* +} + +# Extract one complete, explicitly delimited section. Empty sections are valid; +# malformed data, repeated/out-of-order delimiters, and truncated sections are +# not. Stream-json records can have surrounding text, so emit only the matched +# pipe-delimited record. +extract_job_section() { + local section="$1" + local log_file="$2" + + awk -v section="${section}" ' + BEGIN { + start = section "_JOBS_START" + end = section "_JOBS_END" + record = section "[|][^|]+[|]https://[^|]+[|][^|]+[|][0-9]+[.][0-9]+[|][a-zA-Z0-9._-]+" + } + index($0, start) { + if (started || in_section || ended) exit 1 + started = 1 + in_section = 1 + next + } + index($0, end) { + if (!in_section || ended) exit 1 + in_section = 0 + ended = 1 + next + } + in_section && index($0, section "|") { + if (!match($0, record)) exit 1 + entry = substr($0, RSTART, RLENGTH) + remainder = substr($0, RSTART + RLENGTH) + if (entry !~ ("^" record "$") || remainder ~ /^[|]/) exit 1 + print entry + } + END { + if (!started || !ended || in_section) exit 1 + } + ' "${log_file}" +} + +publish_extracted_jobs() { + local log_file="$1" + local jobs_tmp + local ready_tmp + + [[ -r "${log_file}" ]] || return 1 + jobs_tmp=$(mktemp "${SHARED_DIR}/.failing-jobs.XXXXXX") || return 1 + + if ! extract_job_section BLOCKING "${log_file}" > "${jobs_tmp}" \ + || ! extract_job_section INFORMING "${log_file}" >> "${jobs_tmp}" \ + || ! sort -u "${jobs_tmp}" -o "${jobs_tmp}"; then + rm -f "${jobs_tmp}" + return 1 + fi + + # A zero-byte result is valid only after both sections above validated and + # the separate data-ready marker is atomically published below. + if [[ -s "${jobs_tmp}" ]] && grep -qvE "${DATA_LINE_REGEX}" "${jobs_tmp}"; then + rm -f "${jobs_tmp}" + return 1 + fi + + if ! mv -f "${jobs_tmp}" "${SHARED_DIR}/failing-jobs.txt"; then + rm -f "${jobs_tmp}" + return 1 + fi + + ready_tmp=$(mktemp "${SHARED_DIR}/.monitor-data-ready.XXXXXX") || { + rm -f "${SHARED_DIR}/failing-jobs.txt" + return 1 + } + if ! mv -f "${ready_tmp}" "${SHARED_DIR}/monitor-data-ready"; then + rm -f "${ready_tmp}" "${SHARED_DIR}/failing-jobs.txt" + return 1 + fi +} copy_artifacts() { echo "Copying artifacts to ${ARTIFACT_DIR}..." + rm -f "${SHARED_DIR}/monitor-report-ready" + clear_monitor_data + local report_dir="${EDGE_TOOLING_DIR}/payload-monitor/reports" if [[ -d "${report_dir}" ]]; then local latest_html - latest_html=$(ls -t "${report_dir}"/*.html 2>/dev/null | head -1) + latest_html=$(find "${report_dir}" -maxdepth 1 -type f -name '*.html' \ + -newer "${REPORT_START_MARKER}" -printf '%T@ %p\n' \ + | sort -nr | head -1 | cut -d' ' -f2-) if [[ -n "${latest_html}" ]]; then cp "${latest_html}" "${ARTIFACT_DIR}/edge-ci-monitor-summary.html" fi @@ -126,13 +231,28 @@ copy_artifacts() { # downstream steps. Each line is prefixed BLOCKING| or INFORMING|. # SHARED_DIR is backed by a K8s Secret (1 MB limit) so only the # extracted data is shared — not the full multi-MB stream-JSON log. - if [[ -r "${ARTIFACT_DIR}/claude-analysis.log" ]]; then - { - sed -n '/BLOCKING_JOBS_START/,/BLOCKING_JOBS_END/p' "${ARTIFACT_DIR}/claude-analysis.log" \ - | grep -oE 'BLOCKING\|[^|]+\|https://[^|]+\|[^|]+\|[0-9]+\.[0-9]+\|[a-zA-Z0-9._-]+' - sed -n '/INFORMING_JOBS_START/,/INFORMING_JOBS_END/p' "${ARTIFACT_DIR}/claude-analysis.log" \ - | grep -oE 'INFORMING\|[^|]+\|https://[^|]+\|[^|]+\|[0-9]+\.[0-9]+\|[a-zA-Z0-9._-]+' - } | sort -u > "${SHARED_DIR}/failing-jobs.txt" || true + if publish_extracted_jobs "${ARTIFACT_DIR}/claude-analysis.log"; then + echo "Validated job data is ready for Slack notification." + else + clear_monitor_data + echo "WARNING: Extracted job data is unavailable or incomplete." + fi + + # A report is ready only when this run generated a complete dashboard with + # complete data. The Slack post step must not infer success from an empty + # extracted jobs file alone. + local dashboard="${ARTIFACT_DIR}/edge-ci-monitor-summary.html" + if [[ -s "${dashboard}" ]] \ + && grep -qF 'Edge OCP Payload Monitor' "${dashboard}" \ + && grep -qF '' "${dashboard}" \ + && ! grep -qF 'class="error-banner"' "${dashboard}" \ + && ! grep -qF 'class="skip-banner"' "${dashboard}" \ + && grep -qE 'Versions: \[[^]]*[0-9]' "${ARTIFACT_DIR}/claude-analysis.log" \ + && ! grep -qF 'No payload data for versions:' "${ARTIFACT_DIR}/claude-analysis.log"; then + touch "${SHARED_DIR}/monitor-report-ready" + echo "Dashboard report is ready for Slack notification." + else + echo "WARNING: Dashboard report is unavailable or incomplete." fi # Archive Claude session for local continuation diff --git a/ci-operator/step-registry/openshift/edge-tooling/ci-monitor/send-slack/openshift-edge-tooling-ci-monitor-send-slack-commands.sh b/ci-operator/step-registry/openshift/edge-tooling/ci-monitor/send-slack/openshift-edge-tooling-ci-monitor-send-slack-commands.sh index d30894c4de442..79a763ab03707 100755 --- a/ci-operator/step-registry/openshift/edge-tooling/ci-monitor/send-slack/openshift-edge-tooling-ci-monitor-send-slack-commands.sh +++ b/ci-operator/step-registry/openshift/edge-tooling/ci-monitor/send-slack/openshift-edge-tooling-ci-monitor-send-slack-commands.sh @@ -2,10 +2,10 @@ set -euo pipefail # --------------------------------------------------------------------------- -# Guards — skip sending in certain conditions +# Guard — do nothing only when the monitor step never started # --------------------------------------------------------------------------- -if [[ ! -f "${SHARED_DIR}/monitor-completed" ]]; then - echo "Monitor step did not complete — skipping Slack notification." +if [[ ! -f "${SHARED_DIR}/monitor-started" ]]; then + echo "Monitor step did not start — skipping Slack notification." exit 0 fi @@ -35,17 +35,26 @@ BLOCKING_COUNT=0 INFORMING_COUNT=0 BLOCKING_LINES="" INFORMING_LINES="" -DATA_AVAILABLE=false +NOTIFICATION_READY=false +UNAVAILABLE_REASON="" -if [[ -f "${JOBS_FILE}" ]]; then - DATA_AVAILABLE=true +if [[ ! -f "${SHARED_DIR}/monitor-completed" ]]; then + UNAVAILABLE_REASON="monitor did not complete" +elif [[ ! -f "${SHARED_DIR}/monitor-report-ready" ]]; then + UNAVAILABLE_REASON="dashboard report is unavailable or incomplete" +elif [[ ! -f "${SHARED_DIR}/monitor-data-ready" ]]; then + UNAVAILABLE_REASON="extracted job data is unavailable or incomplete" +elif [[ ! -f "${JOBS_FILE}" ]]; then + UNAVAILABLE_REASON="extracted job data is unavailable" +elif [[ -s "${JOBS_FILE}" ]] && grep -qvE '^(BLOCKING|INFORMING)[|][^|]+[|]https://[^|]+[|][^|]+[|][0-9]+[.][0-9]+[|][a-zA-Z0-9._-]+$' "${JOBS_FILE}"; then + UNAVAILABLE_REASON="extracted job data is invalid" +else + NOTIFICATION_READY=true BLOCKING_LINES=$(grep '^BLOCKING|' "${JOBS_FILE}" || true) INFORMING_LINES=$(grep '^INFORMING|' "${JOBS_FILE}" || true) [[ -n "${BLOCKING_LINES}" ]] && BLOCKING_COUNT=$(echo "${BLOCKING_LINES}" | wc -l) [[ -n "${INFORMING_LINES}" ]] && INFORMING_COUNT=$(echo "${INFORMING_LINES}" | wc -l) -else - echo "Warning: ${JOBS_FILE} not found." fi # --------------------------------------------------------------------------- @@ -78,9 +87,9 @@ version_summary() { # --------------------------------------------------------------------------- NL=$'\n' -if [[ "${DATA_AVAILABLE}" != "true" ]]; then +if [[ "${NOTIFICATION_READY}" != "true" ]]; then ICON=":warning:" - MESSAGE="${ICON} *Edge OCP CI Monitor* — Data unavailable. Please investigate the artifacts." + MESSAGE="${ICON} *Edge OCP CI Monitor* — Data unavailable: ${UNAVAILABLE_REASON}. Please investigate the Prow logs." elif [[ "${BLOCKING_COUNT}" -eq 0 ]] && [[ "${INFORMING_COUNT}" -eq 0 ]]; then ICON=":large_green_circle:" MESSAGE="${ICON} *Edge OCP CI Monitor* — No failing jobs found." @@ -104,7 +113,11 @@ else fi fi -MESSAGE+="${NL}<${DASHBOARD_URL}|View Dashboard> | <${JOB_URL}|Prow Logs> | @edge-enablement-payload-manager" +if [[ "${NOTIFICATION_READY}" == "true" ]]; then + MESSAGE+="${NL}<${DASHBOARD_URL}|View Dashboard> | <${JOB_URL}|Prow Logs> | @edge-enablement-payload-manager" +else + MESSAGE+="${NL}<${JOB_URL}|Prow Logs> | @edge-enablement-payload-manager" +fi # --------------------------------------------------------------------------- # Send to Slack (or dry-run on PRs)