osp_verification: stabilize LB tests in periodic CI jobs

collection/stages/roles/lb_tests/defaults/main.yml

@@ -1,3 +1,13 @@
 ---
 # defaults file for lb_tests
 lb_allowlist_file: 'files/openstack-test-lb-allowlist.yaml'
+lb_amphora_allowlist_file: "{{ role_path }}/files/openstack-test-lb-amphora-allowlist.yaml"
+lb_ovn_allowlist_file: "{{ role_path }}/files/openstack-test-lb-ovn-allowlist.yaml"
+lb_blocklist_file: "{{ role_path }}/files/openstack-test-lb-ingress-blocklist.yaml"
+lb_amphora_isolated_allowlist_file: "{{ role_path }}/files/openstack-test-lb-amphora-isolated-allowlist.yaml"
+lb_amphora_sourceranges_allowlist_file: "{{ role_path }}/files/openstack-test-lb-amphora-sourceranges-allowlist.yaml"
+# Two-phase amphora: run isolation-sensitive LB cases first, then the rest.
+# sourceRanges runs in a third late Amphora phase (see main.yml).
+lb_amphora_two_phase_tests: true
+# Broad guest cleanup (e2e projects, shard ICs, ERROR LBs). Off by default on shared labs.
+lb_tests_guest_cleanup: false

collection/stages/roles/lb_tests/files/openstack-test-lb-amphora-allowlist.yaml

@@ -0,0 +1,3 @@
+---
+".*Amphora.*":
+  note: "Amphora Load Balancer tests (lb_tests stage only)"

collection/stages/roles/lb_tests/files/openstack-test-lb-amphora-isolated-allowlist.yaml

@@ -0,0 +1,11 @@
+---
+# Run first on a clean cluster (see lb_tests two-phase amphora flow).
+# Order of keys defines serial run order when filter_tests_list applies this allowlist.
+# UDP sourceRanges is intentionally NOT here — it runs in a late Amphora phase after
+# other Amphora cases so OCCM/Octavia are less contended (allowed_cidrs clear flake).
+".*create a UDP Amphora LoadBalancer when a UDP svc with type:LoadBalancer is created on Openshift":
+  note: "UDP Amphora basic (before other cases allocate shared LBs)"
+".*apply lb-method on TCP Amphora LoadBalancer when a TCP svc with monitors and ETP:Local is created on Openshift":
+  note: "TCP Amphora lb-method + monitors"
+".*apply lb-method on UDP Amphora LoadBalancer when a UDP svc with monitors and ETP:Local is created on Openshift":
+  note: "UDP Amphora lb-method + monitors"

collection/stages/roles/lb_tests/files/openstack-test-lb-amphora-sourceranges-allowlist.yaml

@@ -0,0 +1,6 @@
+---
+# Late Amphora-only phase (after isolated + remaining amphora, before OVN).
+# sourceRanges clearing allowed_cidrs is sensitive to OCCM load; running it last
+# under Amphora config improves odds without a dedicated ansible stage.
+".*limit service access on an UDP Amphora LoadBalancer when an UDP LoadBalancer svc setting the loadBalancerSourceRanges spec is created on Openshift":
+  note: "UDP Amphora loadBalancerSourceRanges (after other Amphora LB tests)"

collection/stages/roles/lb_tests/files/openstack-test-lb-ingress-blocklist.yaml

@@ -0,0 +1,3 @@
+---
+".*LoadBalancerService ingressController is created on Openshift.*":
+  note: "Deferred — OSASINFRA-2412 / IngressController admission (run 7); re-enable when fixed."

collection/stages/roles/lb_tests/files/openstack-test-lb-ovn-allowlist.yaml

@@ -0,0 +1,3 @@
+---
+".*OVN.*":
+  note: "OVN Load Balancer tests (lb_tests stage only)"

collection/stages/roles/lb_tests/tasks/cleanup_lb_test_leftovers.yml

@@ -0,0 +1,69 @@
+---
+# Best-effort guest cleanup between LB test phases (max-shared-lb / Octavia slots).
+- name: List e2e-test-openstack projects
+  ansible.builtin.shell: |
+    set -o pipefail
+    oc get projects -o name 2>/dev/null | sed 's|project/||' | grep -E '^e2e-test-openstack-' || true
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  register: lb_cleanup_e2e_projects_cmd
+  changed_when: false
+  failed_when: false
+
+- name: List shard IngressControllers
+  ansible.builtin.shell: |
+    set -o pipefail
+    oc get ingresscontroller -n openshift-ingress-operator -o name 2>/dev/null | grep shard- || true
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  register: lb_cleanup_shard_ics_cmd
+  changed_when: false
+  failed_when: false
+
+- name: Delete stale e2e-test-openstack projects (async)
+  ansible.builtin.command:
+    argv:
+      - oc
+      - delete
+      - project
+      - "{{ item }}"
+      - --wait=false
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ lb_cleanup_e2e_projects_cmd.stdout_lines | default([]) }}"
+  loop_control:
+    label: "{{ item }}"
+  register: lb_cleanup_delete_projects
+  changed_when: lb_cleanup_delete_projects.rc == 0
+  failed_when: false
+  when: lb_cleanup_e2e_projects_cmd.stdout_lines | default([]) | length > 0
+
+- name: Delete shard IngressControllers left by ingress LB tests (async)
+  ansible.builtin.command:
+    argv:
+      - oc
+      - delete
+      - "{{ item }}"
+      - --wait=false
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ lb_cleanup_shard_ics_cmd.stdout_lines | default([]) }}"
+  loop_control:
+    label: "{{ item }}"
+  register: lb_cleanup_delete_ics
+  changed_when: lb_cleanup_delete_ics.rc == 0
+  failed_when: false
+  when: lb_cleanup_shard_ics_cmd.stdout_lines | default([]) | length > 0
+
+- name: Delete Octavia load balancers in ERROR state
+  ansible.builtin.shell: |
+    set -o pipefail
+    openstack loadbalancer list -f value -c id -c provisioning_status 2>/dev/null | while read -r id st; do
+      if [ "$st" = "ERROR" ]; then
+        openstack loadbalancer delete "$id" --wait 2>/dev/null || true
+      fi
+    done
+  environment:
+    OS_CLOUD: "{{ user_cloud }}"
+  changed_when: false
+  failed_when: false

collection/stages/roles/lb_tests/tasks/main.yml

@@ -1,4 +1,17 @@
 ---
+# include_role(openstack_test) re-templates vars; role_path there is openstack_test, not lb_tests.
+- name: Best-effort cleanup before LB tests
+  ansible.builtin.include_tasks: cleanup_lb_test_leftovers.yml
+  when: lb_tests_guest_cleanup | bool
+
+- name: Materialize LB allow/block paths for nested openstack_test role
+  ansible.builtin.set_fact:
+    lb_ingress_blocklist_resolved: "{{ lb_blocklist_file }}"
+    lb_amphora_allowlist_resolved: "{{ lb_amphora_allowlist_file }}"
+    lb_amphora_isolated_allowlist_resolved: "{{ lb_amphora_isolated_allowlist_file }}"
+    lb_amphora_sourceranges_allowlist_resolved: "{{ lb_amphora_sourceranges_allowlist_file }}"
+    lb_ovn_allowlist_resolved: "{{ lb_ovn_allowlist_file }}"
+
 - name: Get install-config content to obtain the 1st machineCIDR
   ansible.builtin.include_role:
     name: shiftstack.tools.tools_get_deploy_info
@@ -24,14 +37,50 @@
     max_shared_lb: "2"
     log_directory: "{{ artifacts_dir }}/lb_amphora_sourceip"
 
-- name: Run Openstack-Test on OCP with Amphora LoadBalancer provider and sourceIP lb method
-  ansible.builtin.include_role:
-    name: shiftstack.stages.openstack_test
+- name: Amphora LB tests — phase 1 (isolation-sensitive cases on clean slate)
+  when: lb_amphora_two_phase_tests | bool
+  block:
+    - name: Run amphora isolated LB openstack-test phase
+      ansible.builtin.include_tasks: run_amphora_openstack_test.yml
+      vars:
+        lb_amphora_openstack_test_results_dir: "{{ artifacts_dir }}/lb_amphora_sourceip_isolated"
+        lb_amphora_openstack_test_allowlist: "{{ lb_amphora_isolated_allowlist_resolved }}"
+        lb_amphora_openstack_test_blocklist: "{{ lb_ingress_blocklist_resolved }}"
+        lb_amphora_openstack_testsuite_name: openstack_tests_lb_amphora_sourceip_isolated
+        lb_amphora_openstack_reset_result_dir: true
+
+    - name: Cleanup guest LB leftovers between amphora phases
+      ansible.builtin.include_tasks: cleanup_lb_test_leftovers.yml
+      when: lb_tests_guest_cleanup | bool
+
+- name: Amphora LB tests — phase 2 (remaining amphora cases)
+  ansible.builtin.include_tasks: run_amphora_openstack_test.yml
   vars:
-    openstack_test_results_dir: "{{ artifacts_dir }}/lb_amphora_sourceip"
-    openstack_tests_allowlist_file: "{{ lb_allowlist_file }}"
-    openstack_testsuite_name: openstack_tests_lb_amphora_sourceip
-    openstack_reset_result_dir: no # As we want to keep the logs generated in the previous step
+    lb_amphora_openstack_test_results_dir: "{{ artifacts_dir }}/lb_amphora_sourceip"
+    lb_amphora_openstack_test_allowlist: "{{ lb_amphora_allowlist_resolved }}"
+    # Exclude isolated + late sourceRanges so those patterns are not double-run.
+    lb_amphora_openstack_test_blocklist: >-
+      {{ lb_ingress_blocklist_resolved ~ ',' ~ lb_amphora_isolated_allowlist_resolved ~ ',' ~ lb_amphora_sourceranges_allowlist_resolved
+         if (lb_amphora_two_phase_tests | bool)
+         else lb_ingress_blocklist_resolved }}
+    lb_amphora_openstack_testsuite_name: openstack_tests_lb_amphora_sourceip
+    lb_amphora_openstack_reset_result_dir: false
+
+- name: Amphora LB tests — phase 3 (UDP sourceRanges after other Amphora cases)
+  when: lb_amphora_two_phase_tests | bool
+  block:
+    - name: Cleanup guest LB leftovers before late sourceRanges
+      ansible.builtin.include_tasks: cleanup_lb_test_leftovers.yml
+      when: lb_tests_guest_cleanup | bool
+
+    - name: Run amphora UDP sourceRanges openstack-test phase
+      ansible.builtin.include_tasks: run_amphora_openstack_test.yml
+      vars:
+        lb_amphora_openstack_test_results_dir: "{{ artifacts_dir }}/lb_amphora_sourceip_sourceranges"
+        lb_amphora_openstack_test_allowlist: "{{ lb_amphora_sourceranges_allowlist_resolved }}"
+        lb_amphora_openstack_test_blocklist: "{{ lb_ingress_blocklist_resolved }}"
+        lb_amphora_openstack_testsuite_name: openstack_tests_lb_amphora_sourceip_sourceranges
+        lb_amphora_openstack_reset_result_dir: true
 
 - name: Enable OVN provider on cloud-provider-config
   ansible.builtin.include_tasks: change_cloud_cm.yml
@@ -47,6 +96,8 @@
     name: shiftstack.stages.openstack_test
   vars:
     openstack_test_results_dir: "{{ artifacts_dir }}/lb_ovn"
-    openstack_tests_allowlist_file: "{{ lb_allowlist_file }}"
+    openstack_tests_allowlist_file: "{{ lb_ovn_allowlist_resolved }}"
+    openstack_tests_blocklist_file: "{{ lb_ingress_blocklist_resolved }}"
+    openstack_test_ote_run_serial: true
     openstack_testsuite_name: openstack_tests_lb_ovn
     openstack_reset_result_dir: no # As we want to keep the logs generated in the previous step

collection/stages/roles/lb_tests/tasks/run_amphora_openstack_test.yml

@@ -0,0 +1,12 @@
+---
+# Shared openstack_test invocation for amphora LB (single or two-phase).
+- name: Run Openstack-Test on OCP with Amphora LoadBalancer provider and sourceIP lb method
+  ansible.builtin.include_role:
+    name: shiftstack.stages.openstack_test
+  vars:
+    openstack_test_results_dir: "{{ lb_amphora_openstack_test_results_dir }}"
+    openstack_tests_allowlist_file: "{{ lb_amphora_openstack_test_allowlist }}"
+    openstack_tests_blocklist_file: "{{ lb_amphora_openstack_test_blocklist }}"
+    openstack_test_ote_run_serial: true
+    openstack_testsuite_name: "{{ lb_amphora_openstack_testsuite_name }}"
+    openstack_reset_result_dir: "{{ lb_amphora_openstack_reset_result_dir | default(false) }}"

collection/stages/roles/openstack_test/defaults/main.yml

@@ -7,8 +7,10 @@ openstack_test_dir: "{{ artifacts_dir }}/{{ openstack_test_name }}"
 openstack_test_executable: "{{ home_dir }}/openstack-tests"
 openstack_test_results_dir: "{{ artifacts_dir }}/{{ openstack_test_name }}-results"
 openstack_tests_allowlist_file: ""
-openstack_tests_blocklist_file: ""
+openstack_tests_blocklist_file: "{{ role_path }}/files/openstack-test-lb-blocklist.yaml"
 openstack_tests_go_version: "{{ tests.default_go_version_target }}"
 # OTE (OpenShift Tests Extension) — used when discovered_openshift_release >= 4.20
 openstack_test_suite: openstack-test/all
 openstack_test_ote_binary_relpath: bin/openstack-test-tests-ext
+# Run filtered OTE tests one-by-one (required for [lb][Serial] under Octavia limits).
+openstack_test_ote_run_serial: false

collection/stages/roles/openstack_test/files/openstack-test-lb-blocklist.yaml

@@ -0,0 +1,3 @@
+---
+".*[lb].*":
+  note: "Load Balancer tests run only in lb_tests after cloud.conf patch (change_cloud_cm)."

collection/stages/roles/openstack_test/tasks/run_openstack_test.yml

@@ -96,6 +96,12 @@
   changed_when: true
   when: not (openstack_test_use_ote | bool)
 
+- name: Remove stale allowlist txt before YAML conversion
+  ansible.builtin.file:
+    path: "{{ allowlist_path }}"
+    state: absent
+  when: openstack_tests_allowlist_file != ''
+
 - name: Convert the allowlist YAML files to TXT
   ansible.builtin.include_role:
     name: tools_openshift_tests
@@ -106,14 +112,21 @@
   with_items: "{{ openstack_tests_allowlist_file.split(',') }}"
   when: openstack_tests_allowlist_file != ''
 
+- name: Remove stale blocklist txt before YAML conversion
+  ansible.builtin.file:
+    path: "{{ blocklist_path }}"
+    state: absent
+  when: openstack_tests_blocklist_file != ''
+
 - name: Convert the blocklist YAML files to TXT
   ansible.builtin.include_role:
     name: tools_openshift_tests
     tasks_from: convert_yaml_tests_file_to_txt.yml
   vars:
     input_tests_list: "{{ item | trim }}"
     output_tests_list: "{{ blocklist_path }}"
-    yaml_format_based_on_ocp_version: true
+    # Flat regex map (same shape as lb allowlist); not conformance version-keyed YAML.
+    yaml_format_based_on_ocp_version: false
   with_items: "{{ openstack_tests_blocklist_file.split(',') }}"
   when: openstack_tests_blocklist_file != ''
 
@@ -168,16 +181,52 @@
               - "{{ openstack_test_junit_path }}"
           changed_when: true
 
-    # OTE run-test has no --junit-path; pipe filtered names via stdin and
-    # synthesize junit from resolved log outcomes for post_openshift_tests.
-    - name: Run OTE filtered tests via run-test and write junit
+    # OTE run-test has no --junit-path; synthesize junit via ote_resolve_results.py.
+    # lb_tests: run serially — piping the list runs many LBs in parallel and hits
+    # Octavia/max-shared-lb timeouts despite [lb][Serial] in test names.
+    - name: Run OTE filtered tests serially via run-test and write junit
+      ansible.builtin.shell: |
+        set -o pipefail
+        : > {{ openstack_test_log_path }}
+        run_exit=0
+        py_exit=0
+        while IFS= read -r test || [[ -n "$test" ]]; do
+          [[ -z "${test// }" ]] && continue
+          if ! {{ openstack_test_executable }} run-test --output=json "$test" \
+            >> {{ openstack_test_log_path }} 2>&1; then
+            run_exit=1
+          fi
+        done < {{ tests_to_run_path }}
+        python3 "{{ openstack_test_ote_resolve_script }}" junit \
+          "{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$?
+        if [ "$run_exit" -ne 0 ] || [ "$py_exit" -ne 0 ]; then
+          exit 1
+        fi
+        exit 0
+      environment:
+        OS_CLOUD: "{{ user_cloud }}"
+        KUBECONFIG: "{{ kubeconfig }}"
+        RHOSO_KUBECONFIG: "{{ rhoso_kubeconfig }}"
+        SHIFTSTACK_PASS_FILE: "{{ kubeadmin_password }}"
+      changed_when: true
+      when:
+        - openstack_test_use_ote | bool
+        - openstack_test_filtering | bool
+        - openstack_test_ote_run_serial | bool
+
+    - name: Run OTE filtered tests via run-test (stdin batch) and write junit
       ansible.builtin.shell: |
         set -o pipefail
+        run_exit=0
+        py_exit=0
         cat {{ tests_to_run_path }} | {{ openstack_test_executable }} run-test \
           > {{ openstack_test_log_path }} || run_exit=$?
         python3 "{{ openstack_test_ote_resolve_script }}" junit \
           "{{ openstack_test_log_path }}" "{{ openstack_test_junit_path }}" || py_exit=$?
-        exit ${run_exit:-${py_exit:-0}}
+        if [ "$run_exit" -ne 0 ] || [ "$py_exit" -ne 0 ]; then
+          exit 1
+        fi
+        exit 0
       environment:
         OS_CLOUD: "{{ user_cloud }}"
         KUBECONFIG: "{{ kubeconfig }}"
@@ -187,6 +236,7 @@
       when:
         - openstack_test_use_ote | bool
         - openstack_test_filtering | bool
+        - not (openstack_test_ote_run_serial | bool)
 
     - name: Run legacy openstack tests
       ansible.builtin.shell: >