diff --git a/collection/stages/roles/lb_tests/tasks/cleanup_lb_test_leftovers.yml b/collection/stages/roles/lb_tests/tasks/cleanup_lb_test_leftovers.yml index b14f7b0f..bc3f7609 100644 --- a/collection/stages/roles/lb_tests/tasks/cleanup_lb_test_leftovers.yml +++ b/collection/stages/roles/lb_tests/tasks/cleanup_lb_test_leftovers.yml @@ -67,3 +67,23 @@ OS_CLOUD: "{{ user_cloud }}" changed_when: false failed_when: false + +# Orphan lb-sg-* Neutron SGs left on node ports trip ProviderSpec SG matching +# when lb_tests runs before openstack_test (seen on 4.21 warm re-runs). +- name: Best-effort remove orphan lb-sg-* security group attachments and SGs + ansible.builtin.shell: | + set -o pipefail + openstack security group list -f value -c ID -c Name 2>/dev/null | while read -r sg_id sg_name; do + case "$sg_name" in + lb-sg-*) + for port_id in $(openstack port list --security-group "$sg_id" -f value -c ID 2>/dev/null); do + openstack port unset --security-group "$sg_id" "$port_id" 2>/dev/null || true + done + openstack security group delete "$sg_id" 2>/dev/null || true + ;; + esac + done + environment: + OS_CLOUD: "{{ user_cloud }}" + changed_when: false + failed_when: false diff --git a/collection/stages/roles/lb_tests/tasks/main.yml b/collection/stages/roles/lb_tests/tasks/main.yml index 4c5581dc..ba1b687b 100644 --- a/collection/stages/roles/lb_tests/tasks/main.yml +++ b/collection/stages/roles/lb_tests/tasks/main.yml @@ -101,3 +101,7 @@ openstack_test_ote_run_serial: true openstack_testsuite_name: openstack_tests_lb_ovn openstack_reset_result_dir: no # As we want to keep the logs generated in the previous step + +- name: Best-effort cleanup after LB tests (before openstack_test) + ansible.builtin.include_tasks: cleanup_lb_test_leftovers.yml + when: lb_tests_guest_cleanup | bool diff --git a/collection/stages/roles/openstack_test/files/ote_resolve_results.py b/collection/stages/roles/openstack_test/files/ote_resolve_results.py index aa561a84..b6c8761f 100644 --- a/collection/stages/roles/openstack_test/files/ote_resolve_results.py +++ b/collection/stages/roles/openstack_test/files/ote_resolve_results.py @@ -10,7 +10,7 @@ or in ginkgo summary lines (``SUCCESS!`` / ``FAIL!``). Usage: - ote_resolve_results.py count passed|failed|skipped + ote_resolve_results.py count passed|failed|skipped|unknown ote_resolve_results.py junit """ @@ -23,26 +23,47 @@ from typing import Any +def _is_ote_shaped(entry: Any) -> bool: + return isinstance(entry, dict) and "result" in entry + + def _load_outer_results(raw: str) -> list[dict[str, Any]]: raw = raw.strip() if not raw: return [] - # Skip leading klog / noise before the JSON array/object. - start_candidates = [i for i, ch in enumerate(raw) if ch in "[{"] - for start in start_candidates: - chunk = raw[start:] + # Scan top-level JSON values. Skip empty lists / non-OTE-shaped JSON + # (e.g. bare ``[]`` from ginkgo text like ``map[]``) so we reach the real + # result array(s) at the end of the log. Advance past each decoded value + # to avoid re-parsing nested JSON inside ``output`` fields. Serial + # run-test appends multiple arrays — collect all OTE-shaped entries. + results: list[dict[str, Any]] = [] + decoder = json.JSONDecoder() + i = 0 + n = len(raw) + while i < n: + while i < n and raw[i] not in "[{": + i += 1 + if i >= n: + break try: - data, _ = json.JSONDecoder().raw_decode(chunk) + data, end = decoder.raw_decode(raw[i:]) except json.JSONDecodeError: + i += 1 continue + i = i + end if isinstance(data, list): - return [r for r in data if isinstance(r, dict)] - if isinstance(data, dict): - return [data] + ote = [r for r in data if _is_ote_shaped(r)] + if ote: + results.extend(ote) + continue + if _is_ote_shaped(data): + results.append(data) + + if results: + return results # NDJSON fallback - results: list[dict[str, Any]] = [] for line in raw.splitlines(): line = line.strip() if not line or not line.startswith("{"): @@ -51,7 +72,7 @@ def _load_outer_results(raw: str) -> list[dict[str, Any]]: obj = json.loads(line) except json.JSONDecodeError: continue - if isinstance(obj, dict): + if _is_ote_shaped(obj): results.append(obj) return results @@ -206,7 +227,7 @@ def main(argv: list[str]) -> int: cmd = argv[1] if cmd == "count": if len(argv) != 4: - print("usage: count passed|failed|skipped", file=sys.stderr) + print("usage: count passed|failed|skipped|unknown", file=sys.stderr) return 2 print(cmd_count(argv[2], argv[3])) return 0 diff --git a/collection/stages/roles/openstack_test/tasks/cleanup_bogus_machines.yml b/collection/stages/roles/openstack_test/tasks/cleanup_bogus_machines.yml new file mode 100644 index 00000000..752af039 --- /dev/null +++ b/collection/stages/roles/openstack_test/tasks/cleanup_bogus_machines.yml @@ -0,0 +1,90 @@ +--- +# Purge leftover bogus-* MachineSets/Machines before openstack_test. +# Pairs with openstack-test bz_2073398 cleanup; suite-level hygiene if that It left +# stuck objects that poison later Machine / MachineSet / ProviderSpec specs. +# After delete, wait until gone; fail if leftovers remain so openstack_test does not start dirty. +- name: List bogus MachineSets in openshift-machine-api + ansible.builtin.shell: | + set -o pipefail + oc get machineset -n openshift-machine-api -o name 2>/dev/null \ + | grep -E 'bogus-' || true + environment: + KUBECONFIG: "{{ kubeconfig }}" + register: bogus_machinesets_cmd + changed_when: false + failed_when: false + +- name: List bogus Machines in openshift-machine-api + ansible.builtin.shell: | + set -o pipefail + oc get machine -n openshift-machine-api -o name 2>/dev/null \ + | grep -E 'bogus-' || true + environment: + KUBECONFIG: "{{ kubeconfig }}" + register: bogus_machines_cmd + changed_when: false + failed_when: false + +- name: Delete leftover bogus MachineSets + ansible.builtin.command: + argv: + - oc + - delete + - "{{ item }}" + - -n + - openshift-machine-api + - --wait=false + environment: + KUBECONFIG: "{{ kubeconfig }}" + loop: "{{ bogus_machinesets_cmd.stdout_lines | default([]) }}" + loop_control: + label: "{{ item }}" + register: bogus_ms_delete + changed_when: bogus_ms_delete.rc == 0 + failed_when: false + when: bogus_machinesets_cmd.stdout_lines | default([]) | length > 0 + +- name: Delete leftover bogus Machines + ansible.builtin.command: + argv: + - oc + - delete + - "{{ item }}" + - -n + - openshift-machine-api + - --wait=false + environment: + KUBECONFIG: "{{ kubeconfig }}" + loop: "{{ bogus_machines_cmd.stdout_lines | default([]) }}" + loop_control: + label: "{{ item }}" + register: bogus_machine_delete + changed_when: bogus_machine_delete.rc == 0 + failed_when: false + when: bogus_machines_cmd.stdout_lines | default([]) | length > 0 + +# Deletes used --wait=false; poll until matching objects are gone so openstack_test +# does not start while bogus-* resources are still terminating. Bounded timeout. +# Fail when leftovers remain after the timeout so openstack_test cannot start dirty. +- name: Wait until bogus Machines and MachineSets are gone + ansible.builtin.shell: | + set -o pipefail + ms=$(oc get machineset -n openshift-machine-api -o name 2>/dev/null | grep -E 'bogus-' || true) + m=$(oc get machine -n openshift-machine-api -o name 2>/dev/null | grep -E 'bogus-' || true) + if [ -n "$ms" ] || [ -n "$m" ]; then + echo "bogus leftovers still present:" + printf '%s\n' "$ms" "$m" | sed '/^$/d' + exit 1 + fi + echo "no bogus Machines or MachineSets remain" + exit 0 + environment: + KUBECONFIG: "{{ kubeconfig }}" + register: bogus_gone + until: bogus_gone.rc == 0 + retries: 30 + delay: 10 + changed_when: false + when: >- + (bogus_machinesets_cmd.stdout_lines | default([]) | length > 0) + or (bogus_machines_cmd.stdout_lines | default([]) | length > 0) diff --git a/collection/stages/roles/openstack_test/tasks/main.yml b/collection/stages/roles/openstack_test/tasks/main.yml index 0eaa57ce..bfb0418c 100644 --- a/collection/stages/roles/openstack_test/tasks/main.yml +++ b/collection/stages/roles/openstack_test/tasks/main.yml @@ -32,6 +32,9 @@ delegate_to: "{{ hypervisor }}" remote_user: root + - name: Best-effort purge leftover bogus Machines before openstack-test + ansible.builtin.include_tasks: cleanup_bogus_machines.yml + - name: Include Openstack-Test tasks ansible.builtin.include_tasks: run_openstack_test.yml diff --git a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml index c0f23b38..179da042 100644 --- a/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml +++ b/collection/stages/roles/openstack_test/tasks/run_openstack_test.yml @@ -188,18 +188,57 @@ ansible.builtin.shell: | set -o pipefail : > {{ openstack_test_log_path }} - run_exit=0 + suite_exit=0 py_exit=0 + expected=0 + resolve="{{ openstack_test_ote_resolve_script }}" while IFS= read -r test || [[ -n "$test" ]]; do [[ -z "${test// }" ]] && continue - if ! {{ openstack_test_executable }} run-test --output=json "$test" \ - >> {{ openstack_test_log_path }} 2>&1; then - run_exit=1 + expected=$((expected + 1)) + tmp=$(mktemp) + rc=0 + {{ openstack_test_executable }} run-test --output=json "$test" \ + > "$tmp" 2>&1 || rc=$? + cat "$tmp" >> {{ openstack_test_log_path }} + # Skip-only (and pass) often exit non-zero from run-test; ignore that + # only when the resolver confirms skipped/passed. Failed/unknown keep + # the suite failed. Empty resolve on this chunk alone is deferred to + # the end-of-suite resolvable==expected check (OTE often exits + # non-zero before JSON is parseable as a standalone result). + if [ "$rc" -ne 0 ]; then + failed=$(python3 "$resolve" count "$tmp" failed || echo 1) + skipped=$(python3 "$resolve" count "$tmp" skipped || echo 0) + passed=$(python3 "$resolve" count "$tmp" passed || echo 0) + unknown=$(python3 "$resolve" count "$tmp" unknown || echo 0) + if [ "$failed" -gt 0 ] || [ "$unknown" -gt 0 ]; then + suite_exit=1 + elif [ $((failed + skipped + passed + unknown)) -eq 0 ]; then + if grep -q 'FAIL!' "$tmp" 2>/dev/null; then + suite_exit=1 + elif ! grep -qE 'SUCCESS!|SKIP' "$tmp" 2>/dev/null; then + # No resolvable outcome and no ginkgo success/skip signal. + suite_exit=1 + fi + fi fi + rm -f "$tmp" done < {{ tests_to_run_path }} - python3 "{{ openstack_test_ote_resolve_script }}" junit \ + python3 "$resolve" junit \ "{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$? - if [ "$run_exit" -ne 0 ] || [ "$py_exit" -ne 0 ]; then + failed_count=$(python3 "$resolve" count \ + "{{ openstack_test_log_path }}" failed || echo 0) + skipped_count=$(python3 "$resolve" count \ + "{{ openstack_test_log_path }}" skipped || echo 0) + passed_count=$(python3 "$resolve" count \ + "{{ openstack_test_log_path }}" passed || echo 0) + unknown_count=$(python3 "$resolve" count \ + "{{ openstack_test_log_path }}" unknown || echo 0) + # Require every requested test to resolve as pass/skip/fail (no unknown, + # no missing entries). Exit 0 with empty/unresolvable output must fail. + resolvable=$(( ${passed_count:-0} + ${skipped_count:-0} + ${failed_count:-0} )) + if [ "$suite_exit" -ne 0 ] || [ "$py_exit" -ne 0 ] \ + || [ "${failed_count:-0}" -gt 0 ] || [ "${unknown_count:-0}" -gt 0 ] \ + || [ "$resolvable" -ne "$expected" ]; then exit 1 fi exit 0 diff --git a/jobs_definitions/osp_verification_4.21_nightly.yaml b/jobs_definitions/osp_verification_4.21_nightly.yaml index 2ce479a1..fe8d0a5b 100644 --- a/jobs_definitions/osp_verification_4.21_nightly.yaml +++ b/jobs_definitions/osp_verification_4.21_nightly.yaml @@ -13,6 +13,7 @@ openshift_release: "4.21" openshift_build_name: "" # Empty resolves to latest nightly via 4.21.0-0.nightly/latest installation_type: ipi +lb_tests_guest_cleanup: true stages: - prepare - install diff --git a/jobs_definitions/osp_verification_4.22_nightly.yaml b/jobs_definitions/osp_verification_4.22_nightly.yaml index 8f47b430..a6ee3f1a 100644 --- a/jobs_definitions/osp_verification_4.22_nightly.yaml +++ b/jobs_definitions/osp_verification_4.22_nightly.yaml @@ -13,6 +13,7 @@ openshift_release: "4.22" openshift_build_name: "" # Empty resolves to latest nightly via 4.22.0-0.nightly/latest installation_type: ipi +lb_tests_guest_cleanup: true stages: - prepare - install diff --git a/jobs_definitions/osp_verification_4.23_nightly.yaml b/jobs_definitions/osp_verification_4.23_nightly.yaml index 7f883b08..8e3732b0 100644 --- a/jobs_definitions/osp_verification_4.23_nightly.yaml +++ b/jobs_definitions/osp_verification_4.23_nightly.yaml @@ -13,6 +13,7 @@ openshift_release: "4.23" openshift_build_name: "" # Empty resolves to latest nightly via 4.23.0-0.nightly/latest installation_type: ipi +lb_tests_guest_cleanup: true stages: - prepare - install diff --git a/jobs_definitions/osp_verification_5.0_nightly.yaml b/jobs_definitions/osp_verification_5.0_nightly.yaml index 57708b11..1bfbe2c9 100644 --- a/jobs_definitions/osp_verification_5.0_nightly.yaml +++ b/jobs_definitions/osp_verification_5.0_nightly.yaml @@ -16,6 +16,7 @@ openshift_release: "5.0" openshift_build_name: "" # Empty resolves to latest nightly via 5.0.0-0.nightly/latest installation_type: ipi +lb_tests_guest_cleanup: true stages: - prepare - install diff --git a/jobs_definitions/osp_verification_5.1_nightly.yaml b/jobs_definitions/osp_verification_5.1_nightly.yaml index 359d8dc2..41f5a138 100644 --- a/jobs_definitions/osp_verification_5.1_nightly.yaml +++ b/jobs_definitions/osp_verification_5.1_nightly.yaml @@ -14,6 +14,7 @@ openshift_release: "5.1" openshift_build_name: "" # Empty resolves to latest nightly via 5.1.0-0.nightly/latest installation_type: ipi +lb_tests_guest_cleanup: true stages: - prepare - install