Merge pull request #41 from shiftstack/osp-verification-lb-ci-stability

osp_verification: stabilize LB tests in periodic CI jobs

Author: openshift-merge-bot[bot]

collection/stages/roles/lb_tests/defaults/main.yml

@@ -1,3 +1,13 @@
 ---
 # defaults file for lb_tests
 lb_allowlist_file: 'files/openstack-test-lb-allowlist.yaml'
+lb_amphora_allowlist_file: "{{ role_path }}/files/openstack-test-lb-amphora-allowlist.yaml"
+lb_ovn_allowlist_file: "{{ role_path }}/files/openstack-test-lb-ovn-allowlist.yaml"
+lb_blocklist_file: "{{ role_path }}/files/openstack-test-lb-ingress-blocklist.yaml"
+lb_amphora_isolated_allowlist_file: "{{ role_path }}/files/openstack-test-lb-amphora-isolated-allowlist.yaml"
+lb_amphora_sourceranges_allowlist_file: "{{ role_path }}/files/openstack-test-lb-amphora-sourceranges-allowlist.yaml"
+# Two-phase amphora: run isolation-sensitive LB cases first, then the rest.
+# sourceRanges runs in a third late Amphora phase (see main.yml).
+lb_amphora_two_phase_tests: true
+# Broad guest cleanup (e2e projects, shard ICs, ERROR LBs). Off by default on shared labs.
+lb_tests_guest_cleanup: false

collection/stages/roles/lb_tests/files/openstack-test-lb-amphora-allowlist.yaml

@@ -0,0 +1,3 @@
+---
+".*Amphora.*":
+  note: "Amphora Load Balancer tests (lb_tests stage only)"

collection/stages/roles/lb_tests/files/openstack-test-lb-amphora-isolated-allowlist.yaml

@@ -0,0 +1,11 @@
+---
+# Run first on a clean cluster (see lb_tests two-phase amphora flow).
+# Order of keys defines serial run order when filter_tests_list applies this allowlist.
+# UDP sourceRanges is intentionally NOT here — it runs in a late Amphora phase after
+# other Amphora cases so OCCM/Octavia are less contended (allowed_cidrs clear flake).
+".*create a UDP Amphora LoadBalancer when a UDP svc with type:LoadBalancer is created on Openshift":
+  note: "UDP Amphora basic (before other cases allocate shared LBs)"
+".*apply lb-method on TCP Amphora LoadBalancer when a TCP svc with monitors and ETP:Local is created on Openshift":
+  note: "TCP Amphora lb-method + monitors"
+".*apply lb-method on UDP Amphora LoadBalancer when a UDP svc with monitors and ETP:Local is created on Openshift":
+  note: "UDP Amphora lb-method + monitors"

collection/stages/roles/lb_tests/files/openstack-test-lb-amphora-sourceranges-allowlist.yaml

@@ -0,0 +1,6 @@
+---
+# Late Amphora-only phase (after isolated + remaining amphora, before OVN).
+# sourceRanges clearing allowed_cidrs is sensitive to OCCM load; running it last
+# under Amphora config improves odds without a dedicated ansible stage.
+".*limit service access on an UDP Amphora LoadBalancer when an UDP LoadBalancer svc setting the loadBalancerSourceRanges spec is created on Openshift":
+  note: "UDP Amphora loadBalancerSourceRanges (after other Amphora LB tests)"

collection/stages/roles/lb_tests/files/openstack-test-lb-ingress-blocklist.yaml

@@ -0,0 +1,3 @@
+---
+".*LoadBalancerService ingressController is created on Openshift.*":
+  note: "Deferred — OSASINFRA-2412 / IngressController admission (run 7); re-enable when fixed."

collection/stages/roles/lb_tests/files/openstack-test-lb-ovn-allowlist.yaml

@@ -0,0 +1,3 @@
+---
+".*OVN.*":
+  note: "OVN Load Balancer tests (lb_tests stage only)"

collection/stages/roles/lb_tests/tasks/cleanup_lb_test_leftovers.yml

@@ -0,0 +1,69 @@
+---
+# Best-effort guest cleanup between LB test phases (max-shared-lb / Octavia slots).
+- name: List e2e-test-openstack projects
+  ansible.builtin.shell: |
+    set -o pipefail
+    oc get projects -o name 2>/dev/null | sed 's|project/||' | grep -E '^e2e-test-openstack-' || true
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  register: lb_cleanup_e2e_projects_cmd
+  changed_when: false
+  failed_when: false
+
+- name: List shard IngressControllers
+  ansible.builtin.shell: |
+    set -o pipefail
+    oc get ingresscontroller -n openshift-ingress-operator -o name 2>/dev/null | grep shard- || true
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  register: lb_cleanup_shard_ics_cmd
+  changed_when: false
+  failed_when: false
+
+- name: Delete stale e2e-test-openstack projects (async)
+  ansible.builtin.command:
+    argv:
+      - oc
+      - delete
+      - project
+      - "{{ item }}"
+      - --wait=false
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ lb_cleanup_e2e_projects_cmd.stdout_lines | default([]) }}"
+  loop_control:
+    label: "{{ item }}"
+  register: lb_cleanup_delete_projects
+  changed_when: lb_cleanup_delete_projects.rc == 0
+  failed_when: false
+  when: lb_cleanup_e2e_projects_cmd.stdout_lines | default([]) | length > 0
+
+- name: Delete shard IngressControllers left by ingress LB tests (async)
+  ansible.builtin.command:
+    argv:
+      - oc
+      - delete
+      - "{{ item }}"
+      - --wait=false
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  loop: "{{ lb_cleanup_shard_ics_cmd.stdout_lines | default([]) }}"
+  loop_control:
+    label: "{{ item }}"
+  register: lb_cleanup_delete_ics
+  changed_when: lb_cleanup_delete_ics.rc == 0
+  failed_when: false
+  when: lb_cleanup_shard_ics_cmd.stdout_lines | default([]) | length > 0
+
+- name: Delete Octavia load balancers in ERROR state
+  ansible.builtin.shell: |
+    set -o pipefail
+    openstack loadbalancer list -f value -c id -c provisioning_status 2>/dev/null | while read -r id st; do
+      if [ "$st" = "ERROR" ]; then
+        openstack loadbalancer delete "$id" --wait 2>/dev/null || true
+      fi
+    done
+  environment:
+    OS_CLOUD: "{{ user_cloud }}"
+  changed_when: false
+  failed_when: false

collection/stages/roles/lb_tests/tasks/main.yml

@@ -1,4 +1,17 @@
 ---
+# include_role(openstack_test) re-templates vars; role_path there is openstack_test, not lb_tests.
+- name: Best-effort cleanup before LB tests
+  ansible.builtin.include_tasks: cleanup_lb_test_leftovers.yml
+  when: lb_tests_guest_cleanup | bool
+
+- name: Materialize LB allow/block paths for nested openstack_test role
+  ansible.builtin.set_fact:
+    lb_ingress_blocklist_resolved: "{{ lb_blocklist_file }}"
+    lb_amphora_allowlist_resolved: "{{ lb_amphora_allowlist_file }}"
+    lb_amphora_isolated_allowlist_resolved: "{{ lb_amphora_isolated_allowlist_file }}"
+    lb_amphora_sourceranges_allowlist_resolved: "{{ lb_amphora_sourceranges_allowlist_file }}"
+    lb_ovn_allowlist_resolved: "{{ lb_ovn_allowlist_file }}"
+
 - name: Get install-config content to obtain the 1st machineCIDR
   ansible.builtin.include_role:
     name: shiftstack.tools.tools_get_deploy_info
@@ -24,14 +37,50 @@
     max_shared_lb: "2"
     log_directory: "{{ artifacts_dir }}/lb_amphora_sourceip"
 
-- name: Run Openstack-Test on OCP with Amphora LoadBalancer provider and sourceIP lb method
-  ansible.builtin.include_role:
-    name: shiftstack.stages.openstack_test
+- name: Amphora LB tests — phase 1 (isolation-sensitive cases on clean slate)
+  when: lb_amphora_two_phase_tests | bool
+  block:
+    - name: Run amphora isolated LB openstack-test phase
+      ansible.builtin.include_tasks: run_amphora_openstack_test.yml
+      vars:
+        lb_amphora_openstack_test_results_dir: "{{ artifacts_dir }}/lb_amphora_sourceip_isolated"
+        lb_amphora_openstack_test_allowlist: "{{ lb_amphora_isolated_allowlist_resolved }}"
+        lb_amphora_openstack_test_blocklist: "{{ lb_ingress_blocklist_resolved }}"
+        lb_amphora_openstack_testsuite_name: openstack_tests_lb_amphora_sourceip_isolated
+        lb_amphora_openstack_reset_result_dir: true
+
+    - name: Cleanup guest LB leftovers between amphora phases
+      ansible.builtin.include_tasks: cleanup_lb_test_leftovers.yml
+      when: lb_tests_guest_cleanup | bool
+
+- name: Amphora LB tests — phase 2 (remaining amphora cases)
+  ansible.builtin.include_tasks: run_amphora_openstack_test.yml
   vars:
-    openstack_test_results_dir: "{{ artifacts_dir }}/lb_amphora_sourceip"
-    openstack_tests_allowlist_file: "{{ lb_allowlist_file }}"
-    openstack_testsuite_name: openstack_tests_lb_amphora_sourceip
-    openstack_reset_result_dir: no # As we want to keep the logs generated in the previous step
+    lb_amphora_openstack_test_results_dir: "{{ artifacts_dir }}/lb_amphora_sourceip"
+    lb_amphora_openstack_test_allowlist: "{{ lb_amphora_allowlist_resolved }}"
+    # Exclude isolated + late sourceRanges so those patterns are not double-run.
+    lb_amphora_openstack_test_blocklist: >-
+      {{ lb_ingress_blocklist_resolved ~ ',' ~ lb_amphora_isolated_allowlist_resolved ~ ',' ~ lb_amphora_sourceranges_allowlist_resolved
+         if (lb_amphora_two_phase_tests | bool)
+         else lb_ingress_blocklist_resolved }}
+    lb_amphora_openstack_testsuite_name: openstack_tests_lb_amphora_sourceip
+    lb_amphora_openstack_reset_result_dir: false
+
+- name: Amphora LB tests — phase 3 (UDP sourceRanges after other Amphora cases)
+  when: lb_amphora_two_phase_tests | bool
+  block:
+    - name: Cleanup guest LB leftovers before late sourceRanges
+      ansible.builtin.include_tasks: cleanup_lb_test_leftovers.yml
+      when: lb_tests_guest_cleanup | bool
+
+    - name: Run amphora UDP sourceRanges openstack-test phase
+      ansible.builtin.include_tasks: run_amphora_openstack_test.yml
+      vars:
+        lb_amphora_openstack_test_results_dir: "{{ artifacts_dir }}/lb_amphora_sourceip_sourceranges"
+        lb_amphora_openstack_test_allowlist: "{{ lb_amphora_sourceranges_allowlist_resolved }}"
+        lb_amphora_openstack_test_blocklist: "{{ lb_ingress_blocklist_resolved }}"
+        lb_amphora_openstack_testsuite_name: openstack_tests_lb_amphora_sourceip_sourceranges
+        lb_amphora_openstack_reset_result_dir: true
 
 - name: Enable OVN provider on cloud-provider-config
   ansible.builtin.include_tasks: change_cloud_cm.yml
@@ -47,6 +96,8 @@
     name: shiftstack.stages.openstack_test
   vars:
     openstack_test_results_dir: "{{ artifacts_dir }}/lb_ovn"
-    openstack_tests_allowlist_file: "{{ lb_allowlist_file }}"
+    openstack_tests_allowlist_file: "{{ lb_ovn_allowlist_resolved }}"
+    openstack_tests_blocklist_file: "{{ lb_ingress_blocklist_resolved }}"
+    openstack_test_ote_run_serial: true
     openstack_testsuite_name: openstack_tests_lb_ovn
     openstack_reset_result_dir: no # As we want to keep the logs generated in the previous step

collection/stages/roles/lb_tests/tasks/run_amphora_openstack_test.yml

@@ -0,0 +1,12 @@
+---
+# Shared openstack_test invocation for amphora LB (single or two-phase).
+- name: Run Openstack-Test on OCP with Amphora LoadBalancer provider and sourceIP lb method
+  ansible.builtin.include_role:
+    name: shiftstack.stages.openstack_test
+  vars:
+    openstack_test_results_dir: "{{ lb_amphora_openstack_test_results_dir }}"
+    openstack_tests_allowlist_file: "{{ lb_amphora_openstack_test_allowlist }}"
+    openstack_tests_blocklist_file: "{{ lb_amphora_openstack_test_blocklist }}"
+    openstack_test_ote_run_serial: true
+    openstack_testsuite_name: "{{ lb_amphora_openstack_testsuite_name }}"
+    openstack_reset_result_dir: "{{ lb_amphora_openstack_reset_result_dir | default(false) }}"

collection/stages/roles/openstack_test/defaults/main.yml

@@ -7,8 +7,10 @@ openstack_test_dir: "{{ artifacts_dir }}/{{ openstack_test_name }}"
 openstack_test_executable: "{{ home_dir }}/openstack-tests"
 openstack_test_results_dir: "{{ artifacts_dir }}/{{ openstack_test_name }}-results"
 openstack_tests_allowlist_file: ""
-openstack_tests_blocklist_file: ""
+openstack_tests_blocklist_file: "{{ role_path }}/files/openstack-test-lb-blocklist.yaml"
 openstack_tests_go_version: "{{ tests.default_go_version_target }}"
 # OTE (OpenShift Tests Extension) — used when discovered_openshift_release >= 4.20
 openstack_test_suite: openstack-test/all
 openstack_test_ote_binary_relpath: bin/openstack-test-tests-ext
+# Run filtered OTE tests one-by-one (required for [lb][Serial] under Octavia limits).
+openstack_test_ote_run_serial: false