osp_verification: stabilize LB tests in periodic CI jobs

Run lb_tests before openstack_test, block [lb] specs from the default openstack
suite, and exclude deferred IngressController LB cases from lb_tests. Add
pre-Amphora cleanup and allow combined allowlist+blocklist filtering.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: tusharjadhav3302

collection/stages/roles/lb_tests/defaults/main.yml

@@ -1,3 +1,4 @@
 ---
 # defaults file for lb_tests
 lb_allowlist_file: 'files/openstack-test-lb-allowlist.yaml'
+lb_blocklist_file: "{{ role_path }}/files/openstack-test-lb-ingress-blocklist.yaml"

collection/stages/roles/lb_tests/files/openstack-test-lb-ingress-blocklist.yaml

@@ -0,0 +1,3 @@
+---
+".*LoadBalancerService ingressController is created on Openshift.*":
+  note: "Deferred — OSASINFRA-2412 / IngressController admission (run 7); re-enable when fixed."

collection/stages/roles/lb_tests/tasks/cleanup_lb_test_leftovers.yml

@@ -0,0 +1,36 @@
+---
+- name: Delete stale e2e-test-openstack projects (best-effort)
+  ansible.builtin.shell: |
+    set -euo pipefail
+    for ns in $(oc get projects -o name 2>/dev/null | sed 's|project/||' | grep -E '^e2e-test-openstack-' || true); do
+      oc delete project "$ns" --wait=false 2>/dev/null || true
+    done
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  changed_when: false
+  failed_when: false
+
+- name: Delete stale test IngressControllers shard-* (best-effort)
+  ansible.builtin.shell: |
+    set -euo pipefail
+    for ic in $(oc get ingresscontroller -n openshift-ingress-operator -o name 2>/dev/null | grep 'shard-' || true); do
+      oc delete "$ic" --wait=false 2>/dev/null || true
+    done
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  changed_when: false
+  failed_when: false
+
+- name: Delete Octavia load balancers stuck in ERROR (best-effort)
+  ansible.builtin.shell: |
+    set -euo pipefail
+    openstack loadbalancer list -f value -c id -c provisioning_status 2>/dev/null | \
+    while read -r id status; do
+      if [ "$status" = "ERROR" ]; then
+        openstack loadbalancer delete "$id" --wait 2>/dev/null || true
+      fi
+    done
+  environment:
+    OS_CLOUD: "{{ user_cloud }}"
+  changed_when: false
+  failed_when: false

collection/stages/roles/lb_tests/tasks/main.yml

@@ -1,4 +1,7 @@
 ---
+- name: Best-effort cleanup of stale LB / e2e resources before Amphora tests
+  ansible.builtin.include_tasks: cleanup_lb_test_leftovers.yml
+
 - name: Get install-config content to obtain the 1st machineCIDR
   ansible.builtin.include_role:
     name: shiftstack.tools.tools_get_deploy_info
@@ -30,6 +33,7 @@
   vars:
     openstack_test_results_dir: "{{ artifacts_dir }}/lb_amphora_sourceip"
     openstack_tests_allowlist_file: "{{ lb_allowlist_file }}"
+    openstack_tests_blocklist_file: "{{ lb_blocklist_file }}"
     openstack_testsuite_name: openstack_tests_lb_amphora_sourceip
     openstack_reset_result_dir: no # As we want to keep the logs generated in the previous step
 
@@ -48,5 +52,6 @@
   vars:
     openstack_test_results_dir: "{{ artifacts_dir }}/lb_ovn"
     openstack_tests_allowlist_file: "{{ lb_allowlist_file }}"
+    openstack_tests_blocklist_file: "{{ lb_blocklist_file }}"
     openstack_testsuite_name: openstack_tests_lb_ovn
     openstack_reset_result_dir: no # As we want to keep the logs generated in the previous step

collection/stages/roles/openstack_test/defaults/main.yml

@@ -7,7 +7,7 @@ openstack_test_dir: "{{ artifacts_dir }}/{{ openstack_test_name }}"
 openstack_test_executable: "{{ home_dir }}/openstack-tests"
 openstack_test_results_dir: "{{ artifacts_dir }}/{{ openstack_test_name }}-results"
 openstack_tests_allowlist_file: ""
-openstack_tests_blocklist_file: ""
+openstack_tests_blocklist_file: "{{ role_path }}/files/openstack-test-lb-blocklist.yaml"
 openstack_tests_go_version: "{{ tests.default_go_version_target }}"
 # OTE (OpenShift Tests Extension) — used when discovered_openshift_release >= 4.20
 openstack_test_suite: openstack-test/all

collection/stages/roles/openstack_test/files/openstack-test-lb-blocklist.yaml

@@ -0,0 +1,3 @@
+---
+".*[lb].*":
+  note: "Load Balancer tests run only in lb_tests after cloud.conf patch (change_cloud_cm)."

collection/stages/roles/openstack_test/tasks/run_openstack_test.yml

@@ -113,7 +113,8 @@
   vars:
     input_tests_list: "{{ item | trim }}"
     output_tests_list: "{{ blocklist_path }}"
-    yaml_format_based_on_ocp_version: true
+    # Flat regex map (same shape as lb allowlist); not conformance version-keyed YAML.
+    yaml_format_based_on_ocp_version: false
   with_items: "{{ openstack_tests_blocklist_file.split(',') }}"
   when: openstack_tests_blocklist_file != ''
 

collection/tools/plugins/modules/filter_tests_list.py

@@ -159,8 +159,67 @@ def run_module():
             input_tests.append((normalized, line if line.endswith('\n') else line + '\n'))
 
     if allowlist_file and blocklist_file:
-        module.fail_json(msg="parameters are mutually exclusive: "
-                         "allowlist_file|blocklist_file", **result)
+        # Apply allowlist first, then blocklist (used by lb_tests: [lb] allow + ingress block).
+        tests_to_run = []
+        try:
+            with open(allowlist_file, 'r') as f:
+                allowlist = [line for line in f]
+        except IOError:
+            module.fail_json(msg="Error opening the allowlist file")
+
+        for allowlist_test in allowlist:
+            allow_norm = normalize_test_line(allowlist_test)
+            if allow_norm is None:
+                continue
+            allowlist_test_in_input_tests = False
+            escaped_allow_test = escape_special_characters(allow_norm)
+
+            for test_norm, test_raw in input_tests:
+                if re.fullmatch(escaped_allow_test, test_norm):
+                    tests_to_run.append((test_norm, test_raw))
+                    allowlist_test_in_input_tests = True
+
+            if not allowlist_test_in_input_tests:
+                module.fail_json(msg="Error: Found a test that exists in {} but not in {} -"
+                                 " '{}'".format(allowlist_file,
+                                                input_tests_file,
+                                                allowlist_test), **result)
+
+        try:
+            with open(blocklist_file, 'r') as f:
+                blocklist = [line for line in f]
+        except IOError:
+            module.fail_json(msg="Error opening the blocklist file")
+
+        blocklist_norms = []
+        for blocklist_test in blocklist:
+            block_norm = normalize_test_line(blocklist_test)
+            if block_norm is not None:
+                blocklist_norms.append((block_norm, blocklist_test))
+
+        filtered_tests = []
+        blocked_tests = []
+        for test_norm, test_raw in tests_to_run:
+            test_in_blocklist = False
+            for block_norm, _block_raw in blocklist_norms:
+                escaped_block_test = escape_special_characters(block_norm)
+                if re.fullmatch(escaped_block_test, test_norm):
+                    test_in_blocklist = True
+                    break
+            if test_in_blocklist:
+                blocked_tests.append(test_raw)
+            else:
+                filtered_tests.append(test_raw)
+
+        try:
+            with open(output_file, 'w') as f:
+                f.writelines(filtered_tests)
+        except IOError:
+            module.fail_json(msg="Error writing to output file")
+
+        result['filter_type'] = 'allowlist+blocklist'
+        result['filter_tests_file'] = allowlist_file
+        result['changed'] = True
 
     elif allowlist_file:
         tests_to_run = []

jobs_definitions/osp_verification.yaml

@@ -20,8 +20,8 @@ stages:
   - post
   - verification
   - day2ops
-  - openstack_test
   - lb_tests
+  - openstack_test
   - egressip_tests
 
 day2ops_procedures:

jobs_definitions/osp_verification_4.21_nightly.yaml

@@ -19,8 +19,8 @@ stages:
   - post
   - verification
   - day2ops
-  - openstack_test
   - lb_tests
+  - openstack_test
   - egressip_tests
 
 day2ops_procedures: