test(breakfix): validate NICo and Kubernetes node maintenance (BFX01-02)

docs/guides/remote-deployment.md

@@ -85,6 +85,25 @@ Pass extra pytest arguments after `--`:
 uv run isvctl deploy run <target-ip> -f isvctl/configs/suites/slurm.yaml -- -v -s -k "test_name"
 ```
 
+### Node Maintenance Validation
+
+The BFX01-02 reference uses the NVIDIA Maintenance Operator API on the
+target's active Kubernetes context. It never selects a node implicitly and
+drains only its uniquely labelled probe workload.
+
+```bash
+cat >node-maintenance-overrides.yaml <<'EOF'
+tests:
+  settings:
+    breakfix_node_maintenance_allow_mutation: true
+    breakfix_node_maintenance_node: <dedicated-test-node>
+EOF
+
+ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl deploy run <target-ip> \
+  -f isvctl/configs/suites/k8s.yaml -f node-maintenance-overrides.yaml \
+  -- -v -s -k K8sReturnNodeMaintenanceCheck
+```
+
 ### With ISV Lab Service Integration
 
 Upload results to the ISV Lab Service:

docs/test-plan.yaml

@@ -2052,6 +2052,7 @@ domains:
                 labels:
                   - bare_metal
                   - breakfix
+                  - kubernetes
                   - min_req
                 priority: P1
                 milestone: M5

isvctl/configs/providers/aws/config/eks.yaml

@@ -206,6 +206,22 @@ commands:
           TF_AUTO_APPROVE: "true"
           NODE_POOL_STATE_FILE: "terraform-delete.tfstate"
 
+      # BFX01-02: request and verify Kubernetes node maintenance through the
+      # shared reference. The operation stays disabled until explicitly
+      # authorized and targeted at a dedicated node.
+      - name: return_node_maintenance
+        phase: test
+        command: "python3 ../../shared/breakfix/return_node_maintenance.py"
+        args:
+          - "--allow-mutation={{ breakfix_node_maintenance_allow_mutation | default(false, true) }}"
+          - "--node={{ breakfix_node_maintenance_node | default('', true) }}"
+          - "--timeout-seconds=120"
+        timeout: 1200
+        requires_available_validations:
+          - K8sReturnNodeMaintenanceCheck
+        requires_settings:
+          breakfix_node_maintenance_allow_mutation: true
+
       - name: teardown
         phase: teardown
         command: "../scripts/eks/teardown.sh"
@@ -216,6 +232,10 @@ commands:
 tests:
   description: "AWS EKS GPU cluster validation"
 
+  settings:
+    breakfix_node_maintenance_allow_mutation: false
+    breakfix_node_maintenance_node: ""
+
   validations:
     storage_provider_api:
       checks:

isvctl/configs/providers/minikube.yaml

@@ -34,6 +34,18 @@ commands:
         phase: setup
         command: "my-isv/scripts/k8s/setup_minikube.sh"
         timeout: 60
+      - name: return_node_maintenance
+        phase: test
+        command: "python shared/breakfix/return_node_maintenance.py"
+        args:
+          - "--allow-mutation={{ breakfix_node_maintenance_allow_mutation | default(false, true) }}"
+          - "--node={{ breakfix_node_maintenance_node | default('', true) }}"
+          - "--timeout-seconds=120"
+        timeout: 1200
+        requires_available_validations:
+          - K8sReturnNodeMaintenanceCheck
+        requires_settings:
+          breakfix_node_maintenance_allow_mutation: true
       - name: teardown
         phase: teardown
         command: "my-isv/scripts/k8s/teardown_minikube.sh"
@@ -44,6 +56,8 @@ tests:
 
   settings:
     show_skipped_tests: true
+    breakfix_node_maintenance_allow_mutation: false
+    breakfix_node_maintenance_node: ""
 
   validations:
     # Host-level GPU checks (not in k8s.yaml)

isvctl/configs/providers/my-isv/config/k8s.yaml

@@ -73,8 +73,23 @@ commands:
         args: ["--region", "{{region}}"]
         timeout: 600
 
+      - name: return_node_maintenance
+        phase: test
+        command: "python ../../shared/breakfix/return_node_maintenance.py"
+        args:
+          - "--allow-mutation={{ breakfix_node_maintenance_allow_mutation | default(false, true) }}"
+          - "--node={{ breakfix_node_maintenance_node | default('', true) }}"
+          - "--timeout-seconds=120"
+        timeout: 1200
+        requires_available_validations:
+          - K8sReturnNodeMaintenanceCheck
+        requires_settings:
+          breakfix_node_maintenance_allow_mutation: true
+
 tests:
   description: "my-isv Kubernetes platform suite (real cluster required)"
   settings:
     region: ""
     machine_id: ""
+    breakfix_node_maintenance_allow_mutation: false
+    breakfix_node_maintenance_node: ""

isvctl/configs/providers/my-isv/scripts/breakfix/return_node_maintenance.py

@@ -30,7 +30,8 @@ def main() -> int:
             "requested": True,
             "accepted": True,
             "machine_id": machine_id,
-            "maintenance_mode": True,
+            "maintenance_mode": "Maintenance",
+            "restored": True,
         },
     )
 

isvctl/configs/providers/nico/config/bare_metal.yaml

@@ -85,6 +85,9 @@
 #   - Expected machines pre-registered; machines ingested and DPUs initialized
 #   - NICO_ORGANIZATION and NICO_SITE_ID environment variables set
 #   - Optional NICO_INSTANCE_ID narrows inventory checks to a known instance
+#   - BFX01-02 is staging-only and mutating. It structured-skips unless both
+#     NICO_BREAKFIX_MACHINE_ID names a dedicated fixture and
+#     NICO_BREAKFIX_ALLOW_MUTATION=1 explicitly opts in.
 #   - ssh-keygen on the host (used by query_key_access to mint the throwaway key)
 #
 # Usage:
@@ -439,17 +442,19 @@ commands:
       - name: return_node_maintenance
         phase: test
         continue_on_failure: true
-        command: "python ../scripts/breakfix/gap_stub.py"
+        command: "python ../scripts/breakfix/return_node_maintenance.py"
         args:
           - "--org"
           - "{{org}}"
           - "--site-id"
           - "{{site_id}}"
           - "--api-base"
           - "{{nico_api_base}}"
-          - "--gap"
-          - "BFX01-02"
-        timeout: 600
+          - "--machine-id={{breakfix_machine_id}}"
+          - "--allow-mutation={{breakfix_allow_mutation}}"
+        # The subprocess must outlive both bounded PATCH calls and restoration
+        # polling so its finally block cannot be killed before cleanup.
+        timeout: 1200
 
       - name: return_rack_maintenance
         phase: test
@@ -592,6 +597,11 @@ tests:
     site_id: "{{env.NICO_SITE_ID}}"
     nico_api_base: "{{env.NICO_API_BASE}}"
     instance_id: "{{env.NICO_INSTANCE_ID}}"
+    # BFX01-02 is mutating and never selects a Machine automatically. Set this
+    # only to a dedicated staging fixture that the test may enter and leave
+    # maintenance mode.
+    breakfix_machine_id: "{{env.NICO_BREAKFIX_MACHINE_ID}}"
+    breakfix_allow_mutation: "{{env.NICO_BREAKFIX_ALLOW_MUTATION}}"
     # The imported bare_metal suite defines these for the full instance
     # lifecycle, which NICo does not run yet. Blank them so the suite's
     # {{instance_type}} self-reference does not emit missing-variable warnings.

isvctl/configs/providers/nico/scripts/breakfix/gap_stub.py

@@ -5,7 +5,7 @@
 """Emit the documented NICo gap payload for a break-fix requirement.
 
 Several break-fix requirements have no NICo tenant REST surface to exercise:
-the mutating BFX01 workflows run through Maestro/repair fixtures, and the
+the remaining mutating BFX01 workflows run through Maestro/repair fixtures, and the
 BFX02-02/BFX03-02/BFX04-01/BFX05/BFX06 signals are not exposed at all. Each of
 those steps emits a structured skip naming the gap rather than a hard failure,
 so the suite reports "not available on this platform" instead of "broken".
@@ -28,10 +28,6 @@
 
 # gap id -> (skip reason, contract fields the bound validation still expects)
 GAPS: dict[str, tuple[str, dict[str, Any]]] = {
-    "BFX01-02": (
-        "Return-node-for-maintenance is a mutating NICo repair workflow requiring lab fixtures (BFX01-02 gap)",
-        {"operation": {"requested": False, "accepted": False}},
-    ),
     "BFX01-03": (
         "Rack-level maintenance return API is not exposed on NICo tenant REST (BFX01-03 gap)",
         {"operation": {"requested": False, "accepted": False}},