lb_tests: two-phase amphora runs with guest cleanup between phases

Run isolation-sensitive Amphora LB cases first on a clean cluster, then the
remainder, with best-effort e2e/LB cleanup between phases. Truncate converted
allowlist/blocklist files so manual reruns cannot accumulate stale regex lines.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: tusharjadhav3302

collection/stages/roles/lb_tests/defaults/main.yml

@@ -4,3 +4,6 @@ lb_allowlist_file: 'files/openstack-test-lb-allowlist.yaml'
 lb_amphora_allowlist_file: "{{ role_path }}/files/openstack-test-lb-amphora-allowlist.yaml"
 lb_ovn_allowlist_file: "{{ role_path }}/files/openstack-test-lb-ovn-allowlist.yaml"
 lb_blocklist_file: "{{ role_path }}/files/openstack-test-lb-ingress-blocklist.yaml"
+lb_amphora_isolated_allowlist_file: "{{ role_path }}/files/openstack-test-lb-amphora-isolated-allowlist.yaml"
+# Two-phase amphora: run isolation-sensitive LB cases first, then the rest.
+lb_amphora_two_phase_tests: true

collection/stages/roles/lb_tests/files/openstack-test-lb-amphora-isolated-allowlist.yaml

@@ -0,0 +1,11 @@
+---
+# Run first on a clean cluster (see lb_tests two-phase amphora flow).
+# Order of keys defines serial run order when filter_tests_list applies this allowlist.
+".*create a UDP Amphora LoadBalancer when a UDP svc with type:LoadBalancer is created on Openshift":
+  note: "UDP Amphora basic (before other cases allocate shared LBs)"
+".*apply lb-method on TCP Amphora LoadBalancer when a TCP svc with monitors and ETP:Local is created on Openshift":
+  note: "TCP Amphora lb-method + monitors"
+".*apply lb-method on UDP Amphora LoadBalancer when a UDP svc with monitors and ETP:Local is created on Openshift":
+  note: "UDP Amphora lb-method + monitors"
+".*limit service access on an UDP Amphora LoadBalancer when an UDP LoadBalancer svc setting the loadBalancerSourceRanges spec is created on Openshift":
+  note: "UDP Amphora loadBalancerSourceRanges"

collection/stages/roles/lb_tests/tasks/cleanup_lb_test_leftovers.yml

@@ -0,0 +1,69 @@
+---
+# Best-effort guest cleanup between LB test phases (max-shared-lb / Octavia slots).
+- name: List e2e-test-openstack projects
+  ansible.builtin.shell: |
+    set -o pipefail
+    oc get projects -o name 2>/dev/null | sed 's|project/||' | grep -E '^e2e-test-openstack-' || true
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  register: lb_cleanup_e2e_projects_cmd
+  changed_when: false
+  failed_when: false
+
+- name: List shard IngressControllers
+  ansible.builtin.shell: |
+    set -o pipefail
+    oc get ingresscontroller -n openshift-ingress-operator -o name 2>/dev/null | grep shard- || true
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  register: lb_cleanup_shard_ics_cmd
+  changed_when: false
+  failed_when: false
+
+- name: Delete stale e2e-test-openstack projects (async)
+  ansible.builtin.command:
+    argv:
+      - oc
+      - delete
+      - project
+      - "{{ item }}"
+      - --wait=false
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ lb_cleanup_e2e_projects_cmd.stdout_lines | default([]) }}"
+  loop_control:
+    label: "{{ item }}"
+  register: lb_cleanup_delete_projects
+  changed_when: lb_cleanup_delete_projects.rc == 0
+  failed_when: false
+  when: lb_cleanup_e2e_projects_cmd.stdout_lines | default([]) | length > 0
+
+- name: Delete shard IngressControllers left by ingress LB tests (async)
+  ansible.builtin.command:
+    argv:
+      - oc
+      - delete
+      - "{{ item }}"
+      - --wait=false
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ lb_cleanup_shard_ics_cmd.stdout_lines | default([]) }}"
+  loop_control:
+    label: "{{ item }}"
+  register: lb_cleanup_delete_ics
+  changed_when: lb_cleanup_delete_ics.rc == 0
+  failed_when: false
+  when: lb_cleanup_shard_ics_cmd.stdout_lines | default([]) | length > 0
+
+- name: Delete Octavia load balancers in ERROR state
+  ansible.builtin.shell: |
+    set -o pipefail
+    openstack loadbalancer list -f value -c id -c provisioning_status 2>/dev/null | while read -r id st; do
+      if [ "$st" = "ERROR" ]; then
+        openstack loadbalancer delete "$id" --wait 2>/dev/null || true
+      fi
+    done
+  environment:
+    OS_CLOUD: "{{ user_cloud }}"
+  changed_when: false
+  failed_when: false

collection/stages/roles/lb_tests/tasks/main.yml

@@ -1,9 +1,13 @@
 ---
 # include_role(openstack_test) re-templates vars; role_path there is openstack_test, not lb_tests.
+- name: Best-effort cleanup before LB tests
+  ansible.builtin.include_tasks: cleanup_lb_test_leftovers.yml
+
 - name: Materialize LB allow/block paths for nested openstack_test role
   ansible.builtin.set_fact:
     lb_ingress_blocklist_resolved: "{{ lb_blocklist_file }}"
     lb_amphora_allowlist_resolved: "{{ lb_amphora_allowlist_file }}"
+    lb_amphora_isolated_allowlist_resolved: "{{ lb_amphora_isolated_allowlist_file }}"
     lb_ovn_allowlist_resolved: "{{ lb_ovn_allowlist_file }}"
 
 - name: Get install-config content to obtain the 1st machineCIDR
@@ -31,16 +35,32 @@
     max_shared_lb: "2"
     log_directory: "{{ artifacts_dir }}/lb_amphora_sourceip"
 
-- name: Run Openstack-Test on OCP with Amphora LoadBalancer provider and sourceIP lb method
-  ansible.builtin.include_role:
-    name: shiftstack.stages.openstack_test
+- name: Amphora LB tests — phase 1 (isolation-sensitive cases on clean slate)
+  when: lb_amphora_two_phase_tests | bool
+  block:
+    - name: Run amphora isolated LB openstack-test phase
+      ansible.builtin.include_tasks: run_amphora_openstack_test.yml
+      vars:
+        lb_amphora_openstack_test_results_dir: "{{ artifacts_dir }}/lb_amphora_sourceip_isolated"
+        lb_amphora_openstack_test_allowlist: "{{ lb_amphora_isolated_allowlist_resolved }}"
+        lb_amphora_openstack_test_blocklist: "{{ lb_ingress_blocklist_resolved }}"
+        lb_amphora_openstack_testsuite_name: openstack_tests_lb_amphora_sourceip_isolated
+        lb_amphora_openstack_reset_result_dir: true
+
+    - name: Cleanup guest LB leftovers between amphora phases
+      ansible.builtin.include_tasks: cleanup_lb_test_leftovers.yml
+
+- name: Amphora LB tests — phase 2 (remaining amphora cases)
+  ansible.builtin.include_tasks: run_amphora_openstack_test.yml
   vars:
-    openstack_test_results_dir: "{{ artifacts_dir }}/lb_amphora_sourceip"
-    openstack_tests_allowlist_file: "{{ lb_amphora_allowlist_resolved }}"
-    openstack_tests_blocklist_file: "{{ lb_ingress_blocklist_resolved }}"
-    openstack_test_ote_run_serial: true
-    openstack_testsuite_name: openstack_tests_lb_amphora_sourceip
-    openstack_reset_result_dir: no # As we want to keep the logs generated in the previous step
+    lb_amphora_openstack_test_results_dir: "{{ artifacts_dir }}/lb_amphora_sourceip"
+    lb_amphora_openstack_test_allowlist: "{{ lb_amphora_allowlist_resolved }}"
+    lb_amphora_openstack_test_blocklist: >-
+      {{ lb_ingress_blocklist_resolved ~ ',' ~ lb_amphora_isolated_allowlist_resolved
+         if (lb_amphora_two_phase_tests | bool)
+         else lb_ingress_blocklist_resolved }}
+    lb_amphora_openstack_testsuite_name: openstack_tests_lb_amphora_sourceip
+    lb_amphora_openstack_reset_result_dir: false
 
 - name: Enable OVN provider on cloud-provider-config
   ansible.builtin.include_tasks: change_cloud_cm.yml

collection/stages/roles/lb_tests/tasks/run_amphora_openstack_test.yml

@@ -0,0 +1,12 @@
+---
+# Shared openstack_test invocation for amphora LB (single or two-phase).
+- name: Run Openstack-Test on OCP with Amphora LoadBalancer provider and sourceIP lb method
+  ansible.builtin.include_role:
+    name: shiftstack.stages.openstack_test
+  vars:
+    openstack_test_results_dir: "{{ lb_amphora_openstack_test_results_dir }}"
+    openstack_tests_allowlist_file: "{{ lb_amphora_openstack_test_allowlist }}"
+    openstack_tests_blocklist_file: "{{ lb_amphora_openstack_test_blocklist }}"
+    openstack_test_ote_run_serial: true
+    openstack_testsuite_name: "{{ lb_amphora_openstack_testsuite_name }}"
+    openstack_reset_result_dir: "{{ lb_amphora_openstack_reset_result_dir | default(false) }}"

collection/tools/roles/tools_openshift_tests/tasks/convert_yaml_tests_file_to_txt.yml

@@ -8,6 +8,11 @@
     yaml_testslist: "{{ yaml_testslist[discovered_openshift_release][discovered_ocp_network_type] }}"
   when: yaml_format_based_on_ocp_version|default(False)
 
+- name: Remove stale "{{ output_tests_list }}" before rewrite
+  ansible.builtin.file:
+    path: "{{ output_tests_list }}"
+    state: absent
+
 - name: Create the "{{ output_tests_list }}" file
   ansible.builtin.file:
     path: "{{ output_tests_list }}"