From 2669b767826d2127c002bc570bceb764af7ef35d Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Wed, 5 Aug 2026 20:22:57 +0200 Subject: [PATCH 1/4] feat: integrate Kubernetes Launch Kit validation Add a generic Launch Kit provider and six Network Operator east-west networking use cases. The production Network Operator workflow validates an ISV-provisioned cluster through preflight, discover, generate, and validate without invoking Launch Kit deploy or clean. Preserve Launch Kit evidence while exposing reusable semantic checks and use-case-level reporting. Consume GPUDirect DMA-BUF results with endpoint GPU, PCI, bandwidth, and threshold diagnostics, and register the applicable checks in the catalog. Extend orchestration and reporting with named phases, validation-aware workflow pruning, structured composite subtests, process-group timeouts, recursive suite discovery, and accurate JUnit failures for command-stage errors. Keep lifecycle commands available in the generic provider while allowing prerequisite checks and evidence validation to follow the command subset selected by a consuming workflow. Document the provider contract, the validation-only ownership boundary, prerequisites, catalog metadata, PRD coverage, evidence handling, and remaining integration gaps. Signed-off-by: Alexander Maslennikov --- AGENTS.md | 127 +- docs/README.md | 1 + docs/guides/configuration.md | 213 ++- .../guides/k8s-launch-kit/network-operator.md | 316 +++++ docs/packages/isvctl.md | 21 + docs/packages/isvtest.md | 19 + docs/requirements/README.md | 19 +- ...network-operator-readiness-requirements.md | 51 + ...twork-operator-readiness-requirements.yaml | 83 ++ .../test-requirements-matrix.adoc | 540 ++++++++ .../test-requirements-matrix.yaml | 262 +++- docs/test-plan.adoc | 286 +++- docs/test-plan.yaml | 330 +++++ .../providers/k8s-launch-kit/README.md | 87 ++ .../config/network-operator.yaml | 697 ++++++++++ .../k8s-launch-kit/config/provider.yaml | 219 ++++ .../k8s-launch-kit/scripts/adapter.py | 724 ++++++++++ isvctl/configs/suites/README.md | 72 +- .../network-operator-use-cases.yaml | 181 +++ .../k8s-launch-kit/network-operator.yaml | 122 ++ isvctl/src/isvctl/cli/test.py | 79 +- isvctl/src/isvctl/config/output_schemas.py | 44 + isvctl/src/isvctl/config/schema.py | 93 +- isvctl/src/isvctl/config/suite_resolution.py | 7 +- isvctl/src/isvctl/doctor/checks/config.py | 2 +- isvctl/src/isvctl/orchestrator/commands.py | 5 +- isvctl/src/isvctl/orchestrator/loop.py | 246 +++- isvctl/src/isvctl/orchestrator/process.py | 105 ++ .../src/isvctl/orchestrator/step_executor.py | 10 +- .../providers/k8s_launch_kit/__init__.py | 4 + .../fixtures/launch_kit_scenarios.json | 104 ++ .../k8s_launch_kit/fixtures/mock_kubectl.py | 61 + .../k8s_launch_kit/fixtures/mock_l8k.py | 706 ++++++++++ .../providers/k8s_launch_kit/test_provider.py | 1164 +++++++++++++++++ .../k8s_launch_kit/test_timeout_config.py | 33 + isvctl/tests/test_orchestrator_loop.py | 411 ++++++ isvctl/tests/test_orchestrator_process.py | 102 ++ isvctl/tests/test_schema.py | 106 ++ isvctl/tests/test_stub_contracts.py | 2 +- isvctl/tests/test_suite_resolution.py | 15 + isvctl/tests/test_test_cli_labels.py | 103 ++ isvtest/src/isvtest/catalog.py | 4 +- isvtest/src/isvtest/core/composite.py | 20 +- isvtest/src/isvtest/core/resolution.py | 120 +- isvtest/src/isvtest/main.py | 12 +- isvtest/src/isvtest/testing/subtests.py | 26 +- isvtest/src/isvtest/tests/test_validations.py | 12 + .../validations/k8s_launch_kit/__init__.py | 4 + .../validations/k8s_launch_kit/checks.py | 721 ++++++++++ isvtest/tests/k8s_launch_kit/test_checks.py | 530 ++++++++ isvtest/tests/test_catalog.py | 3 + isvtest/tests/test_composite.py | 48 + isvtest/tests/test_main.py | 22 + isvtest/tests/test_subtests_junit.py | 22 +- isvtest/tests/test_validation.py | 20 + scripts/requirements_source_to_md.py | 48 +- scripts/test_plan_coverage.py | 6 +- .../tests/test_requirements_source_to_md.py | 67 + scripts/tests/test_validate_suite_wiring.py | 13 + scripts/validate_suite_wiring.py | 4 +- 60 files changed, 9323 insertions(+), 151 deletions(-) create mode 100644 docs/guides/k8s-launch-kit/network-operator.md create mode 100644 docs/requirements/network-operator-readiness-requirements.md create mode 100644 docs/requirements/network-operator-readiness-requirements.yaml create mode 100644 isvctl/configs/providers/k8s-launch-kit/README.md create mode 100644 isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml create mode 100644 isvctl/configs/providers/k8s-launch-kit/config/provider.yaml create mode 100644 isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py create mode 100644 isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml create mode 100644 isvctl/configs/suites/k8s-launch-kit/network-operator.yaml create mode 100644 isvctl/src/isvctl/orchestrator/process.py create mode 100644 isvctl/tests/providers/k8s_launch_kit/__init__.py create mode 100644 isvctl/tests/providers/k8s_launch_kit/fixtures/launch_kit_scenarios.json create mode 100755 isvctl/tests/providers/k8s_launch_kit/fixtures/mock_kubectl.py create mode 100755 isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py create mode 100644 isvctl/tests/providers/k8s_launch_kit/test_provider.py create mode 100644 isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py create mode 100644 isvctl/tests/test_orchestrator_process.py create mode 100644 isvtest/src/isvtest/validations/k8s_launch_kit/__init__.py create mode 100644 isvtest/src/isvtest/validations/k8s_launch_kit/checks.py create mode 100644 isvtest/tests/k8s_launch_kit/test_checks.py create mode 100644 scripts/tests/test_requirements_source_to_md.py diff --git a/AGENTS.md b/AGENTS.md index ee7720a00..de73dd3ec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,7 +77,8 @@ Entry point: `isvctl/src/isvctl/main.py` (Typer). - `cli/` - subcommands (`test`, `deploy`, `clean`, `docs`, `report`) - `orchestrator/` - `loop.py` (phase loop), `step_executor.py` (step + validation - execution, supports `best_effort` mode), `commands.py` (timeouts), `context.py` + execution, supports `best_effort` mode), `commands.py` (legacy command model), + `process.py` (shared subprocess and process-group timeout handling), `context.py` (Jinja2 with missing-reference warnings) - `config/` - `schema.py` (Pydantic), `output_schemas.py` (per-step JSON schemas), `merger.py` (multi-file merge) @@ -139,7 +140,9 @@ forwarded env vars → optional isvreporter upload. - Workspace root `pyproject.toml` defines members; each package has its own `pyproject.toml`; all source under `src/`. -- `isvctl/configs/suites/` - provider-agnostic test contracts. +- `isvctl/configs/suites/` - provider-agnostic test contracts. Discovery is + recursive, so related domain suites may be grouped in a subdirectory; YAML + filename stems must remain globally unique. - `isvctl/configs/providers//` - one folder per provider (`aws/`, `my-isv/`, ...): - `config/` - YAML wiring (imports a suite, supplies commands) - `scripts/` - executable scripts (Python/Bash) that do the work, organized by @@ -160,6 +163,126 @@ forwarded env vars → optional isvreporter upload. `aws/scripts/common/` provides `ec2`, `errors` (with `delete_with_retry`), `ssh_utils.wait_for_ssh`, `serial_console`, `vpc`. +### Network Operator / Kubernetes Launch Kit + +- All provider-owned Launch Kit files live under + `isvctl/configs/providers/k8s-launch-kit/`: generic and Network Operator YAML + in `config/`, executable transport in `scripts/`, and provider documentation + in `README.md`. +- `isvctl/configs/providers/k8s-launch-kit/config/provider.yaml` is the generic provider. Its + public API mirrors the CLI: `prepare`, `verify`, Kubernetes preflight, + `discover`, `generate`, `deploy`, `validate`, and `clean`. Workflow settings are raw + argument arrays; do not model or duplicate Launch Kit flags/defaults here. + The single file-level input, `user_config`, points to a complete Launch Kit + configuration. Discovery copies it to `/user-config.yaml` and + writes the resolved result to `/cluster-config.yaml`, preserving + the source file and the default paths used by subsequent commands. + Its `validate` step uses `timeout: null` so l8k owns the automatically + calculated or user-supplied matrix deadline; all other workflow steps retain + finite outer isvctl watchdogs. Any provider may use a null `StepConfig` + timeout when its invoked command owns a bounded deadline. +- Launch Kit-specific transport code belongs under + `isvctl/configs/providers/k8s-launch-kit/`, not `providers/shared/` (which is + reserved for scripts reused by unrelated providers). Production code lives + in `scripts/`. Executable mocks and pinned fixtures are test-only and live in + `isvctl/tests/providers/k8s_launch_kit/fixtures/`; product configuration must + never reference them. +- `prepare` supports `verify` and explicit `install` modes. Install mode + downloads and records the official Launch Kit installer, then delegates + archive selection, checksum verification, and installation to it. Both modes + verify `l8k version --output json` and `l8k schema`. The configured string + environment is shared by install, verification, preflight, and workflows. +- The Kubernetes preflight is mandatory before each normal test use case. It + accepts the non-empty subset of Launch Kit commands selected by the caller, + derives a single explicit kubeconfig from their raw arguments (and rejects + conflicts), verifies API access, requires a non-empty node inventory, and + requires at least one Ready node. A failure stops the remaining steps in that + workflow/use case. +- `isvctl/configs/suites/k8s-launch-kit/network-operator.yaml` owns only catalog wiring and + interpretation for the globally selectable PRD checks. Each check binds to + the real step that produced its evidence. +- `isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml` composes those shared + check classes into six concrete validation tests: RoCE and InfiniBand across + SR-IOV, RDMA Shared, and host-device modes. Include only checks applicable to + a use case; do not run all checks and hide mismatches as interleaved skips. +- `isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml` is the + production six-use-case configuration. It defaults to real `l8k` and + `kubectl`, executes each supported fabric/deployment combination as a named + custom phase, and gives every use case isolated working and evidence + directories. ISVs own deployment: each use case runs preflight, discover, + generate, and validate only. It must never invoke `l8k deploy`, `l8k clean`, + or a cleanup finalizer. Its fabric/deployment arguments define test identity; + Launch Kit continues to own runtime defaults and users extend raw argv in overlays. + A global `user_config` is staged independently for each selected use case; + when it is set, raw discovery arguments cannot also select user/save config paths. + Keep default config/deployment path flags out of all grouped phase arguments; + overlays may add them only when intentionally overriding Launch Kit's paths. +- Mock-backed coverage loads that same production YAML and injects test-owned + executables only in `isvctl/tests/providers/k8s_launch_kit/test_provider.py`. + Result-check tests live under `isvtest/tests/k8s_launch_kit/`. +- Independent use-case phases are listed in `continue_after_failure` so a failed + case does not suppress later evidence. The failed phase still fails the final + run. Never use that option for shared setup or dependent phases. +- `StepConfig.finalizer_for` links cleanup to a mutating target. Provider + cleanup belongs in `phase: teardown`; the orchestrator runs it immediately + after the target phase validations and reports `-teardown`, + including for `--phase test`. It only activates when the target process + started, while `--phase teardown` runs it unconditionally as recovery. Use + this instead of unconditional cleanup when a preflight failure must not + delete pre-existing state. Schema validation requires matching capability + and validation-selection gates. +- Lifecycle steps associated with a selectable test declare + `requires_selected_validations`. The gate applies release, capability, label, + and suite exclusions before command execution. It is also the reporting + ownership edge: a failed selected step makes each named validation a + `step_failed` error in structured results and JUnit rather than allowing a + later missing-output skip. Keep + `requires_available_validations` for release-only gating; pytest `-k`/`-m` + selection remains too late to prune lifecycle commands. +- `CompositeCheck` predates the Launch Kit work and is framework machinery for + `compose:` entries. It now forwards member probes as `MemberName/probe-name`. + A member-level `pytest.skip` is reported as a skipped member while the + composite continues; skipped members neither pass nor fail the parent. + Successful validations with subtests are compacted by the shared isvctl + renderer; do not add suite-specific output flags. +- Launch Kit areas are separate validation classes in + `isvtest/validations/k8s_launch_kit/checks.py`; detailed probes use `report_subtest()` so + all manifest and connectivity rows reach JUnit output before the parent fails. +- Do not invent a `selfValidation` field in l8k output. Current `discover`, + `generate`, and `clean` emit one `ui.JSONResult`, successful standalone + `deploy` emits no stdout, and `validate` emits a JSON stream (static state, + connectivity matrix, then report path). The provider wraps these unmodified documents in a transport + envelope and keeps semantic assertions in pytest. The envelope records the + absolute command working directory so validations can resolve Launch Kit's + relative evidence paths without rewriting its output. +- Current l8k base check selection remains ICMP, `rping`, and `ib_write_bw`. + When `validation.gpuDirect.enabled` is true, GPUDirect DMA-BUF follows + `ib_write_bw` and is emitted as the distinct `gpudirect_dmabuf` result family. + Consume that family without adding an AI Cloud Validation default or a fourth + `--validation-checks` value. +- `l8k clean` remains the generic provider's only supported deletion path; do + not reproduce its CR/finalizer/Helm logic with kubectl. The Network Operator + validation suite intentionally has no deletion path because the ISV owns the + pre-existing deployment. Any future state-mutating test requires an explicit + transactional restore and verification contract before it is added. +- Use `--label ethernet` or `--label infiniband` to prune the grouped run to one + fabric's three workflows. Use `--label sriov`, `--label rdma_shared`, or + `--label host_device` for one deployment-mode pair; labels compose to select + one concrete use case. `-k`/marker selection still happens after lifecycle + commands and does not prune Launch Kit workflows. +- Use `--label gpudirect` to select the six GPU-capable use-case definitions. + The semantic member skips when Launch Kit emits no `gpudirect_dmabuf` rows; + emitted failed rows must fail the parent with endpoint GPU evidence. +- Current reporting uploads JUnit/log/catalog only. Files under + `_output/k8s-launch-kit` are local evidence until the reporter gains an + explicit, redacted attachment contract. +- Design, prerequisites, unit-test boundaries, PRD mapping, and production gaps live in + `docs/guides/k8s-launch-kit/network-operator.md`. +- The structured PRD source is + `docs/requirements/network-operator-readiness-requirements.yaml`; keep its + `ENT-REQ-*` edges in `docs/requirements/test-requirements-matrix.yaml` and + regenerate committed views with `make plan`. + ## Environment Variables | Variable | Description | Used by | diff --git a/docs/README.md b/docs/README.md index 052375a85..ed828d15d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,6 +19,7 @@ Welcome to the documentation for NVIDIA AI Cloud Validation suite - a collection - [Configuration](guides/configuration.md) - Configuration file format and options - [External Validation Guide](guides/external-validation-guide.md) - Create custom validations without modifying the repo +- [Network Operator Launch Kit integration](guides/k8s-launch-kit/network-operator.md) - Production provider, Kubernetes preflight, mock-backed unit coverage, evidence, and limitations - [Remote Deployment](guides/remote-deployment.md) - Deploy and run tests on remote machines - [Local Development](guides/local-development.md) - MicroK8s setup for local testing - [Troubleshooting: Test runs stuck in STARTED](guides/troubleshooting-started-tests.md) - Why runs stay STARTED in the portal and how to fix it diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index 9e5547144..86d9bdbc4 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -56,7 +56,10 @@ Pre-built configs are provided in `isvctl/configs/`: | `providers/aws/config/vm.yaml` | AWS EC2 GPU instance tests | | `providers/aws/config/iam.yaml` | AWS IAM user lifecycle | | `providers/aws/config/eks.yaml` | AWS EKS with GPU nodes | +| `providers/k8s-launch-kit/config/provider.yaml` | Generic Kubernetes Launch Kit workflow | +| `providers/k8s-launch-kit/config/network-operator.yaml` | Six Network Operator Launch Kit use cases | | `suites/k8s.yaml` | Standard Kubernetes cluster | +| `suites/k8s-launch-kit/*.yaml` | Launch Kit-specific Network Operator catalog wiring | | `suites/slurm.yaml` | Slurm HPC cluster | ## Basic Usage @@ -155,17 +158,60 @@ Each platform defines phases and steps: commands: network: phases: ["setup", "test", "teardown"] # Execution order + continue_after_failure: [] # Optional independent test phases steps: [...] # Steps grouped by phase ``` | Field | Required | Description | | ----- | -------- | ----------- | -| `phases` | No | Ordered list of phases (default: `["setup", "test", "teardown"]`) | +| `phases` | No | Ordered list of phases (default: `["setup", "teardown"]`) | +| `continue_after_failure` | No | Phase names whose failure records a failed run but does not prevent later phases from running | | `steps` | Yes | List of step configurations | | `skip` | No | Skip this entire platform | **Important:** If a step's `phase` is not in the `phases` list, an error is raised. +Phase names are not limited to `setup`, `test`, and `teardown`. Any other name +is a custom test phase: it runs in the declared order, appears under its own +name in the orchestration summary, and is selected by `--phase test`. A +validation bound to a step runs after that step's custom phase. + +By default, a failed phase prevents later non-teardown phases from running. Use +`continue_after_failure` only when the named phases are independent test cases +and collecting every result in one invocation is more useful than stopping at +the first failure: + +```yaml +commands: + network_operator: + phases: [setup, roce-sriov, infiniband-sriov, roce-host-device] + continue_after_failure: [roce-sriov, infiniband-sriov, roce-host-device] + steps: + - name: prepare + phase: setup + command: ./prepare.sh + - name: test_roce_sriov + phase: roce-sriov + command: ./run-use-case.sh + args: [roce-sriov] + - name: test_infiniband_sriov + phase: infiniband-sriov + command: ./run-use-case.sh + args: [infiniband-sriov] + - name: test_roce_host_device + phase: roce-host-device + command: ./run-use-case.sh + args: [roce-host-device] +``` + +This setting changes continuation, not the verdict: if `roce-sriov` fails, +later listed use cases still run, but the final orchestration result remains +failed. Every continuation name must also appear once in `phases`; +configuration validation rejects unknown or duplicate names and forbids +`setup` and `teardown`. Do not list prerequisites shared by later phases or +phases that leave state on which later phases depend. Teardown retains its +existing `teardown_on_failure` behavior. + ### Step Configuration Each step defines a command to execute: @@ -180,6 +226,7 @@ Each step defines a command to execute: AWS_PROFILE: "production" skip: false continue_on_failure: false + finalizer_for: null output_schema: vpc ``` @@ -189,12 +236,84 @@ Each step defines a command to execute: | `phase` | No | Phase this step belongs to (default: `setup`) | | `command` | Yes | Script/command to execute | | `args` | No | Arguments (supports Jinja2 templates) | -| `timeout` | No | Timeout in seconds (default: 300) | +| `timeout` | No | Orchestration watchdog in seconds (default: 300); `null` disables it | | `env` | No | Environment variables | | `skip` | No | Skip this step | | `continue_on_failure` | No | Continue even if this step fails | +| `finalizer_for` | No | Run as linked teardown after the named step's phase when that command was attempted | | `output_schema` | No | Schema name for output validation | | `requires` | No | Capability contexts this step runs in (see [Capabilities](#capabilities-and-requires)) | +| `requires_available_validations` | No | Validation names that must be available after release filtering | +| `requires_selected_validations` | No | Configured validation names that must remain selected after release, capability, label, and suite-exclusion filtering; failed steps become errors on these owning validations | + +The timeout is an orchestration watchdog, not a provider-specific setting. Set +it to `null` only when the invoked tool owns a bounded deadline; isvctl will +then wait for the command to exit. On POSIX systems, isvctl starts each step in +a separate process group. When the +watchdog expires, it sends `SIGTERM` to the entire group, waits briefly, then +uses `SIGKILL` if needed. This prevents a wrapper's child CLI from continuing +to modify infrastructure after the wrapper step has been reported as timed +out. On non-POSIX systems, isvctl terminates the direct child process. + +#### Linked teardown finalizers + +Use `finalizer_for` when cleanup must run after the validations for one custom +test phase, including when the mutating step or a validation failed. Declare +cleanup in `phase: teardown`; the orchestrator executes it directly after its +target's test phase instead of waiting until every test case has finished: + +```yaml +commands: + network: + phases: [setup, use-case-one, use-case-two, teardown] + continue_after_failure: [use-case-one] + steps: + - name: deploy_fixture + phase: use-case-one + command: ./deploy.sh + + - name: clean_fixture + phase: teardown + command: ./clean.sh + finalizer_for: deploy_fixture +``` + +The finalizer target must resolve to one unique step, precede the configured +`teardown` phase, and cannot itself be a finalizer. The finalizer must use the +same capability and validation-selection gates as its target. Configuration +validation rejects violations of these rules. + +The orchestrator withholds linked teardown from normal phase execution, runs +the target phase validations, and then executes the eligible cleanup in +best-effort mode. The result is reported separately as +`-teardown`. This interleaving applies even to `--phase test`, so +multiple independent cases cannot leave deployments overlapping until the end +of the suite. A target activates cleanup only when its command process actually +started, whether it passed or failed. If an earlier prerequisite stopped the +phase, a template could not be rendered, or the executable could not be +started, cleanup is reported as skipped; this prevents deletion of pre-existing +state the current run never mutated. + +An explicit `--phase teardown` run executes linked teardown steps without an +in-memory target attempt. This is the standalone recovery path for resources +left by an interrupted earlier run. When target test phases and teardown are +part of the same invocation, already-linked cleanup is not run again in the +final teardown position. + +An ordinary use-case failure may still honor `continue_after_failure` after its +finalizers succeed. A failed finalizer always blocks later non-teardown phases, +because the fixture can no longer be assumed clean. Finalizer command output +and failure details are recorded in the teardown phase result. Keep finalizers +lifecycle-only rather than binding validations to their output, because target +phase validations intentionally run before cleanup. A same-phase finalizer is +still supported for compatibility, but a destructive provider cleanup should +normally be declared in `phase: teardown` so its lifecycle role and reporting +are explicit. + +Finalizers are an orchestration guarantee, not a recovery service. An abrupt +isvctl process termination, host failure, or `SIGKILL` can prevent them from +running. Provider cleanup commands should therefore be idempotent and usable as +standalone recovery commands. #### Gating a step with `requires` @@ -217,6 +336,58 @@ same one, so setup and teardown always move together. A step that survives the gate must not reference a gated-off step's output; use `default(...)` if it legitimately might be absent. +#### Gating Mutating Steps by Test Selection + +Use `requires_selected_validations` when a lifecycle step exists only to serve +specific validation entries. This applies selection before the command runs, +so `--label` and `--exclude-label` do not execute an unrelated deployment and +then discard its result: + +```yaml +commands: + network: + steps: + - name: deploy_ethernet_fixture + phase: ethernet + command: ./deploy-ethernet.sh + requires_selected_validations: [EthernetConnectivityCheck] + +tests: + validations: + network: + checks: + EthernetConnectivityCheck: + step: deploy_ethernet_fixture + labels: [ethernet] +``` + +With no label filter, the validation is selected and the step runs. With +`--label ethernet`, it also runs; with `--label infiniband`, the step is +skipped before execution. Every listed validation must be configured and +selected. The gate also honors the release manifest, capability requirements, +`tests.exclude.tests`, and effective label exclusions. + +The same list is the reporting ownership edge for the lifecycle step. If a +selected step fails before its validation can run, each listed validation is +reported as `error` with reason `step_failed`, including in JUnit. This prevents +an early deploy or setup failure from being misreported as a harmless +`step_no_output` skip merely because a later validation step was never reached. +The error message names the failed step and retains its redacted command +diagnostic. + +`requires_available_validations` is narrower: it only prevents a step from +running when its named checks are absent from the release manifest. Retain it +for providers that only need release gating. + +Pytest `-k` and `-m` expressions are evaluated inside pytest and therefore do +not drive `requires_selected_validations`. Use framework `--label` filtering +for lifecycle pruning in mutating suites. + +Selection-filtered validations remain in the structured result and JUnit +report. With the default `tests.settings.show_skipped_tests: false`, terminal +output omits summary phases containing only those filtered validations. Set it +to `true` when the skipped selection decisions should be visible interactively. + ### Validation Configuration Validations are centralized in `tests.validations`, grouped by category. Each group binds to a step and lists checks as a dict: @@ -351,6 +522,13 @@ Capability names and plain-suite names share one namespace, so a plain suite may not be named after a capability. `catalog_document` and `scripts/validate_suite_wiring.py` both reject the collision. +Suite discovery is recursive under `isvctl/configs/suites/`. A domain with +multiple related suites may therefore use a subdirectory such as +`suites/k8s-launch-kit/`; catalog generation, `--suite` resolution, doctor, +wiring validation, and test-plan coverage all discover the nested YAMLs. Suite +identity is still the YAML filename stem, so stems must remain unique across +the complete suite tree. + ## Import and Override Provider configs can import a canonical test suite and override command definitions while inheriting validations (unless explicitly overridden): @@ -584,6 +762,37 @@ checks: fields: ["network_id"] ``` +`CompositeCheck` is the existing framework runner behind `compose:`; authors do +not register or invoke that class directly. The YAML key creates one catalog +test and runs every listed validation member. Each member is reported as a +subtest. If a member reports its own probes through `report_subtest()`, those +probes are retained with qualified names such as +`LaunchKitRdmaConnectivityCheck/rping/worker-a->worker-b/rail-0->rail-1`. +This avoids collisions between members and keeps the full probe tree in pytest +and JUnit output. + +A member may call `pytest.skip` when it is not applicable to the current +environment. `CompositeCheck` records that member as a skipped subtest and +continues with the remaining members. The skip neither passes nor fails the +member, and the parent composite passes when every non-skipped member passes. +This is different from skipping the step output or the composite itself, both +of which skip the entire parent validation. + +The orchestration summary automatically abbreviates a successful validation +that reported subtests: + +```text +MyUseCase: PASSED - 12 subtests passed +``` + +If optional probes were skipped, the summary includes passed, failed, and +skipped counts. Failed and errored validations keep their original diagnostic +message instead of being abbreviated. There is no YAML presentation flag; +this behavior applies to composites and ordinary validation classes alike. +After subtest testcase nodes are injected into JUnit, the suite's tests, +failures, errors, and skipped counters are recalculated from those serialized +nodes so reports do not double-count pytest's pre-counted subtest events. + `SchemaValidation` remains directly wireable, but is catalog-excluded because the step executor runs schema checks automatically. diff --git a/docs/guides/k8s-launch-kit/network-operator.md b/docs/guides/k8s-launch-kit/network-operator.md new file mode 100644 index 000000000..b9a69fd09 --- /dev/null +++ b/docs/guides/k8s-launch-kit/network-operator.md @@ -0,0 +1,316 @@ + + + +# Network Operator validation through Kubernetes Launch Kit + +## Safety and ownership boundary + +The production Network Operator suite validates an installation that the ISV +has already deployed and configured. Its workflow is: + +```text +verify l8k + -> verify Kubernetes access and at least one Ready node + -> l8k discover + -> l8k generate + -> l8k validate + -> AI Cloud Validation checks and reports +``` + +It does **not** invoke `l8k deploy` or `l8k clean`. AI Cloud Validation therefore +does not install, replace, reconfigure, or remove the ISV-managed Network +Operator deployment. A validation or orchestration failure also cannot activate +a cleanup finalizer. + +Discovery may label nodes and validation creates Launch Kit's temporary test +workloads. Those operations remain part of Launch Kit itself. The important +ownership boundary is that this suite never applies the generated Network +Operator deployment manifests and never removes the existing installation. + +The generic provider still exposes `discover`, `generate`, `deploy`, `validate`, +and `clean`. That API mirrors the complete Launch Kit CLI and remains available +to other suites that explicitly own deployment lifecycle. The validation-only +behavior is defined by `config/network-operator.yaml`, not by removing features +from the generic provider. + +## Architecture + +AI Cloud Validation separates command execution from result interpretation: + +```text +provider configuration + -> ordered isvctl phases and steps + -> adapter.py executes real l8k and kubectl + -> raw command envelopes become step outputs and evidence + -> suite configuration + -> binds one use-case check to each validate step + -> supplies discover/generate/preflight outputs as context + -> isvtest validation classes + -> interpret unmodified l8k JSON + -> report member and probe-level subtests + -> console, JUnit, retained artifacts, and optional Labs upload +``` + +The main files are: + +| Layer | File | +|---|---| +| Generic provider | `isvctl/configs/providers/k8s-launch-kit/config/provider.yaml` | +| Network Operator validation workflow | `isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml` | +| CLI transport | `isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py` | +| Individual PRD checks | `isvctl/configs/suites/k8s-launch-kit/network-operator.yaml` | +| Concrete use cases | `isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml` | +| Result interpretation | `isvtest/src/isvtest/validations/k8s_launch_kit/checks.py` | +| Provider tests and mock CLI | `isvctl/tests/providers/k8s_launch_kit/` | +| Result-check tests | `isvtest/tests/k8s_launch_kit/test_checks.py` | +| PRD and traceability | `docs/requirements/` | + +The provider and suite files are intentionally separate. The provider owns +process execution and evidence. The suite owns catalog identity, selection, +and the mapping from step outputs to reusable validation checks. + +## Network Operator workflow + +For every selected use case, the production provider executes these steps in +order: + +1. `launch_kit__preflight` checks Kubernetes access; +2. `launch_kit__discover` runs `l8k discover` with that use case's + fabric and deployment type; +3. `launch_kit__generate` runs `l8k generate` to reconstruct the + expected manifests used by Launch Kit validation; +4. `launch_kit__validate` runs `l8k validate` against the already + deployed cluster state. + +Generation is intentional even though deployment is external. Launch Kit +validation compares the live installation with the desired resources generated +for the selected topology. The generated files are evidence; this suite does +not apply them. + +The six concrete use cases are: + +| Test | Fabric labels | Deployment label | +|---|---|---| +| `EastWestNetworkRoceSriovCheck` | `ethernet`, `roce` | `sriov` | +| `EastWestNetworkInfiniBandSriovCheck` | `infiniband` | `sriov` | +| `EastWestNetworkRoceRdmaSharedCheck` | `ethernet`, `roce` | `rdma_shared` | +| `EastWestNetworkInfiniBandRdmaSharedCheck` | `infiniband` | `rdma_shared` | +| `EastWestNetworkRoceHostDeviceCheck` | `ethernet`, `roce` | `host_device` | +| `EastWestNetworkInfiniBandHostDeviceCheck` | `infiniband` | `host_device` | + +All applicable use cases run by default. Labels can select a fabric, a +deployment mode, or their intersection. `continue_after_failure` lets later +independent use cases collect evidence after an earlier use case fails; the +overall run still fails. + +## Provider API + +The adapter accepts raw argument arrays instead of reproducing Launch Kit's +domain configuration: + +| Key | Meaning | +|---|---| +| `executable` | Existing `l8k` command or absolute path | +| `installation.mode` | `verify` by default, or explicit `install` | +| `installation.version` | Optional exact Launch Kit version | +| `installation.installer_ref` | Immutable installer commit used in install mode | +| `installation.installer_sha256` | Trusted installer digest used in install mode | +| `installation.prefix` | Optional installation prefix | +| `user_config` | Optional path to a complete Launch Kit configuration | +| `kubectl_command` | Optional kubectl-compatible argv prefix | +| `working_dir` | Per-workflow Launch Kit working directory | +| `artifact_dir` | Per-workflow evidence directory | +| `environment` | String environment entries forwarded to all commands | +| `.arguments` | Raw arguments for that Launch Kit command | + +AI Cloud Validation defines no defaults for namespaces, node selectors, +Network Operator versions, driver modes, rails, resource names, IP pools, GPU +counts, validation modes/checks, bandwidth thresholds, or Launch Kit timeouts. +Omitted values are resolved by the installed Launch Kit release. The adapter +adds only `--output json` and rejects a conflicting user output mode. + +The generic configuration includes argument arrays for all five Launch Kit +workflow commands. The Network Operator configuration includes only `discover`, +`generate`, and `validate`, because those are the commands it actually invokes. + +### Complete user configuration + +Set `context.k8s_launch_kit.user_config` when a cluster needs settings that are +not exposed as CLI flags. This must be a complete Launch Kit configuration; +the provider does not merge partial YAML. + +For each selected use case, the adapter: + +1. copies the source to `/user-config.yaml` with mode `0600`; +2. adds `--user-config ` and + `--save-cluster-config /cluster-config.yaml` to discovery; +3. removes the staged copy as soon as discovery exits; +4. records only source path, size, and SHA-256 provenance in evidence. + +The source file is never modified or retained as an uploaded artifact. Do not +put it inside a retained working or evidence directory, and do not repeat the +provider-owned `--user-config` or `--save-cluster-config` flags in discovery +arguments. + +Example overlay: + +```yaml +import: + - /path/to/ai-cloud-validation/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml + +context: + k8s_launch_kit: + user_config: /secure/path/cluster-config.yaml + environment: + KUBECONFIG: /secure/path/kubeconfig.yaml +``` + +The supplied configuration must describe the desired use case consistently +with the selected labels. The use-case discovery flags select the authoritative +fabric and deployment type. + +## Installation verification and Kubernetes prerequisite + +`installation.mode: verify` resolves the configured executable and captures: + +- `l8k version --output json`; +- `l8k schema --output json`. + +The generic provider contract requires the schema to advertise all five +workflow commands, including deploy and clean. This does not cause the Network +Operator suite to call those commands. An optional exact version is checked in +both setup and test-phase verification, so `--phase test` is safe when setup is +skipped. + +Install mode requires an immutable full Git commit for the official installer +and a caller-supplied SHA-256. The adapter verifies the downloaded installer +before executing it, then verifies the installed binary. + +The preflight helper accepts the non-empty subset of Launch Kit commands used +by its caller. It extracts `--kubeconfig` from those command arguments, +rejects inconsistent kubeconfigs, and runs kubectl probes for API access and at +least one Ready node. If no command argument selects a kubeconfig, kubectl and +l8k inherit the same forwarded `KUBECONFIG` environment or normal client +defaults. + +## Timeouts + +Provider step timeouts are outer isvctl watchdogs. Discovery and generation +have finite watchdogs. Network Operator validate steps use `timeout: null`, so +isvctl does not preempt a valid large connectivity matrix. Launch Kit computes +and logs its bounded validation budget by default, or uses a user-supplied +Launch Kit timeout argument. + +Other providers may use `timeout: null` only when the child tool owns a bounded +deadline. An enclosing CI job can still impose a total job timeout. + +## Result and error reporting + +The adapter preserves Launch Kit JSON documents without renaming fields and +records the exact argv, cwd, stdout, stderr, exit code, and timing for every +command. A non-zero process result remains attached to its step even when the +CLI emitted partial JSON. + +The composite use-case check converts Launch Kit resource and connectivity rows +into pytest subtests. Names identify the member check and the individual probe, +so failures point to a concrete resource, rail, source/destination pair, or +bandwidth result. A member that is inapplicable may skip without skipping the +whole use case; failed members fail the parent. + +If preflight, discovery, or generation fails before validate produces output, +the owning use-case validation is emitted as a `step_failed` error in JUnit. A +normal Launch Kit validation failure is emitted as a failed testcase with its +probe diagnostics. Neither path invokes deploy or cleanup. + +`LaunchKitEvidenceCaptureCheck` accepts both lifecycle-owning and +validation-only workflows. Verify, preflight, discover, generate, and validate +evidence is required. Deploy evidence is checked only when a suite actually +provides `deploy_output`. + +## Running the suite + +Prerequisites: + +- the selected kubeconfig reaches a Kubernetes cluster with at least one Ready + worker node; +- Network Operator and the resources required for the selected profile are + already deployed and reconciled; +- the installed `l8k` release supports JSON version, schema, discovery, + generation, and validation output; +- the execution identity can discover topology, create validation workloads, + exec into them, inspect events/resources, and collect logs; +- the cluster satisfies Launch Kit prerequisites for the selected SR-IOV, + RDMA Shared, host-device, RoCE, InfiniBand, and GPUDirect checks. + +Run all use cases: + +```bash +ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl test run \ + -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ + --capability kubernetes --no-upload -- -v +``` + +Select a subset by adding labels after `--`: + +```bash +# Ethernet/RoCE only +... --capability kubernetes --no-upload -- -v --label ethernet + +# All SR-IOV profiles +... --capability kubernetes --no-upload -- -v --label sriov + +# RoCE SR-IOV only +... --capability kubernetes --no-upload -- -v \ + --label ethernet --label sriov +``` + +Do not use `--phase teardown` for this Network Operator configuration; it has +no teardown phase because deployment lifecycle belongs to the ISV. + +## Evidence and upload + +Each use case writes to separate working and evidence directories under: + +```text +_output/k8s-launch-kit/network-operator/use-cases// + work/ + cluster-config.yaml + deployment/ + k8s-launch-kit-validation-report.html (location may vary by l8k release) + evidence/ + kubernetes-preflight/ + commands/discover/ + commands/generate/ + commands/validate/ +``` + +Shared setup and verification evidence is stored under +`_output/k8s-launch-kit/network-operator/shared-evidence/`. Generated manifests, +resource status, events, connectivity and bandwidth documents, stdout/stderr, +and the HTML report are registered in provider step outputs and retained +locally. Current AI Cloud Labs upload sends JUnit, the combined run log, and +catalog metadata; it does not yet upload these evidence files as attachments. + +User configuration contents are intentionally excluded; only provenance is +retained. + +## PRD coverage and remaining work + +The suite provides selectable and grouped Network Operator checks for RoCE and +InfiniBand across SR-IOV, RDMA Shared, and host-device profiles. It reuses Launch +Kit topology, manifest readiness, ICMP, rping, RDMA bandwidth, multi-rail, and +GPUDirect validation output. Catalog metadata and requirement mappings identify +ownership, labels, dependencies, applicability, and prerequisites. + +The validation-only boundary changes the interpretation of ENT-REQ-010: this +integration does not modify the Network Operator deployment, so it has no +pre-test operator state to restore. If a future test intentionally changes +operator state, that test needs a separate, explicit Launch Kit transaction or +snapshot/restore contract before it can be added here. + +Live qualification is still required for every supported hardware/fabric/ +deployment combination. Unit fixtures prove integration and reporting behavior, +not partner certification. Additional Launch Kit improvements worth considering +are stable machine-readable result schemas, explicit artifact manifests, and a +first-class API for validating an existing deployment without regenerating +desired manifests when the site already has an authoritative complete config. diff --git a/docs/packages/isvctl.md b/docs/packages/isvctl.md index 10ba7f3a4..348082ab6 100644 --- a/docs/packages/isvctl.md +++ b/docs/packages/isvctl.md @@ -133,6 +133,27 @@ isvctl test validate -f isvctl/configs/suites/k8s.yaml See [Configuration Guide](../guides/configuration.md) for full details. +All lifecycle and step commands run with captured stdout/stderr and an outer +watchdog. On POSIX, a timeout terminates the command's complete process group +(`SIGTERM`, then `SIGKILL` after a short grace period), which prevents a child +provider CLI from continuing after its wrapper step has timed out. See +[Step Configuration](../guides/configuration.md#step-configuration). + +Cleanup steps may use `phase: teardown` with +`finalizer_for: `. The linked teardown runs directly after the +target's phase validations whenever that command started, including after +target or validation failure, and is reported as `-teardown`. +An explicit teardown-only run executes it as standalone recovery. Cleanup +failure blocks later non-teardown phases. See +[Linked teardown finalizers](../guides/configuration.md#linked-teardown-finalizers) +for activation, ordering, and process-failure limitations. + +Steps gated with `requires_selected_validations` also declare which validation +owns their lifecycle result. If one of those steps fails, the named validation +is emitted as a `step_failed` error in structured results and JUnit even when a +later validation-producing step never runs. See +[Gating mutating steps by test selection](../guides/configuration.md#gating-mutating-steps-by-test-selection). + ### Unified Config Structure ```yaml diff --git a/docs/packages/isvtest.md b/docs/packages/isvtest.md index 32c65483f..8d93e2914 100644 --- a/docs/packages/isvtest.md +++ b/docs/packages/isvtest.md @@ -67,6 +67,25 @@ Utility checks that work with any step output. `SchemaValidation` remains directly wireable, but is catalog-excluded because the step executor runs schema checks automatically. +### Composite checks and nested results + +`CompositeCheck` is existing internal framework machinery used when a suite +declares `compose:`. It is excluded from discovery and the catalog; the named +YAML entry is the test identity. Every member runs, even after an earlier member +fails, and is reported as a parent subtest. Subtests reported by a member are +also forwarded with `MemberName/probe-name` names, so their detailed messages +and timing remain available in pytest and JUnit. If a member calls +`pytest.skip`, `CompositeCheck` records that member as skipped and continues +with the remaining members. A skipped member neither passes nor fails the +composite; the composite passes when every non-skipped member passes. + +The isvctl orchestration summary uses the structured subtest counts to render a +concise line for successful parents. Failure and error messages are never +replaced by that summary. JUnit suite counters are reconciled with the emitted +parent and subtest testcase nodes after injection. See the +[configuration guide](../guides/configuration.md#available-validations) for +YAML and output examples. + | Validation | Platforms | Description | | ---------- | --------- | ----------- | | `StepSuccessCheck` | all | Compose-only: check step completed successfully | diff --git a/docs/requirements/README.md b/docs/requirements/README.md index d7e3de1b6..d81fbbabb 100644 --- a/docs/requirements/README.md +++ b/docs/requirements/README.md @@ -15,6 +15,8 @@ reconciles all of these different goals. | `software-reference-requirements.md` | Generated rendering of the reference YAML (`make plan`); one contributing requirements doc among several. | | `storage-acceptance-requirements.yaml` | **Source of record** for the DGXC Storage Acceptance Test requirements (PRD-ref namespace). | | `storage-acceptance-requirements.md` | Generated rendering of the storage YAML (`make plan`). | +| `network-operator-readiness-requirements.yaml` | **Source of record** for the Enterprise RA Network Operator self-validation integration PRD. | +| `network-operator-readiness-requirements.md` | Generated rendering of the Network Operator PRD YAML (`make plan`). | | `test-requirements-matrix.yaml` | The **traceability matrix (index)**: which requirement(s) each test relates to, across documents (`source`). | | `test-requirements-matrix.adoc` | Generated and committed traceability matrix, viewable in github (or renderable to html) | | `../../scripts/reqtrace.py` | Integrity checks (`reqtrace validate`; `make reqcheck`). | @@ -96,6 +98,7 @@ record. | `BFX` (04+) | reference | break-fix health (continues offtake `BFX`) | | `BENCH` | reference | exemplar benchmarking | | `N-*` | storage | Storage Acceptance test IDs | +| `ENT-REQ-*` | network-operator-prd | Enterprise Network Operator self-validation integration | ## 3. `legacy_ids` @@ -146,16 +149,22 @@ belonging to a given requirements document*. We keep the data flexible for this: When a new team's requirements document is blessed, reconcile it here: -1. **Register prefix(es)** for the new doc in the registry (sec. 2). Resolve +1. **Add a structured requirements source.** Give it a globally unique + top-level `source`, because that value is the matrix join key. For a project + PRD, set `format: project-prd` to reuse the generic section/area renderer; + do not add a source-specific renderer branch. Add the file to + `DEFAULT_SOURCES` in `requirements_source_to_md.py` when `make plan` should + render it by default. +2. **Register prefix(es)** for the new doc in the registry (sec. 2). Resolve any overload before proceeding (see the `CP` lesson). -2. **Assign IDs.** Prefer mirroring the upstream requirement IDs. On collision +3. **Assign IDs.** Prefer mirroring the upstream requirement IDs. On collision with an existing prefix, apply the collision policy (sec. 2): continue the number space, or (selectively) choose another resolution and record why. -3. **Add/adjust tests** in `test-plan.yaml` (the canonical truth). Use +4. **Add/adjust tests** in `test-plan.yaml` (the canonical truth). Use `legacy_ids` for any renames. -4. **Update the matrix** (`test-requirements-matrix.yaml`): add each +5. **Update the matrix** (`test-requirements-matrix.yaml`): add each test->requirement edge with the new `source`, plus `annotations`/`notes`. -5. **Validate**: `make reqcheck` must pass; **regenerate**: `make plan`. +6. **Validate**: `make reqcheck` must pass; **regenerate**: `make plan`. > Kept as a subsection for now; promote to its own `ONBOARDING.md` if it grows. diff --git a/docs/requirements/network-operator-readiness-requirements.md b/docs/requirements/network-operator-readiness-requirements.md new file mode 100644 index 000000000..12e0a8485 --- /dev/null +++ b/docs/requirements/network-operator-readiness-requirements.md @@ -0,0 +1,51 @@ + + +# Enterprise RA Network Operator Self-Validation Integration PRD + +> Structured source of record: `network-operator-readiness-requirements.yaml` (version prd-snapshot-2026-08-04). +> Owner: NVIDIA Network Operator team. +> Edit the YAML, not this file. + +## Ownership + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-000 | Integration ownership | The Network Operator team owns and maintains the integration solution, including compatibility updates for the underlying tests. | active | + +## Framework Integration + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-001 | Standard validation workflow | Integrate Network Operator self-validation tests into AI Cloud Validation so Enterprise and AI Cloud Ready users can run them through the standard workflow. | active | +| ENT-REQ-002 | Launch Kit reuse | Reuse applicable Kubernetes Launch Kit validation components, including topology discovery, manifest readiness, RDMA connectivity, and RDMA bandwidth. | active | +| ENT-REQ-003 | Selection and program profiles | Support individual and grouped Network Operator validation, with Enterprise and AI Cloud Ready profiles able to mark checks required or optional. | active | +| ENT-REQ-004 | Runtime parameters | Expose applicable runtime parameters such as namespace, node selector, network and driver modes, rail and network names, resource and IP pool names, GPU count, and timeout. | active | + +## Network Validation + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-005 | Ethernet and RoCE | Validate SR-IOV Network RDMA and RDMA Shared scenarios, including secondary network attachment, RDMA device availability, pod-to-pod RDMA or RoCE connectivity, and basic bandwidth. | active | +| ENT-REQ-006 | InfiniBand | Validate InfiniBand SR-IOV and RDMA Shared with IPoIB scenarios, including IB device availability, pod network attachment, and pod-to-pod InfiniBand connectivity. | active | +| ENT-REQ-007 | Host-device networking | Validate host-device networking for Kubernetes workers running in virtual machines, covering both Ethernet or RoCE and InfiniBand. | active | +| ENT-REQ-008 | GPUDirect RDMA | Validate GPUDirect RDMA peer-to-peer connectivity between GPU-enabled pods across supported worker nodes. | active | +| ENT-REQ-009 | Deployment health | Validate Network Operator deployment health and required resources for the selected mode, including policies, secondary networks, IP pools, Multus and CNI components, and drivers. | active | + +## Lifecycle Safety + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-010 | State restoration | For tests that modify Network Operator or cluster state, capture the pre-test configuration and restore the original Network Operator state after success or failure. | active | + +## Catalog and Documentation + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-011 | Catalog metadata | Add catalog entries for all tests with owner, labels, dependencies, descriptions, and required YAML updates while following repository contribution standards. | active | +| ENT-REQ-012 | Prerequisites | Document the required cluster prerequisites for each validation area. | active | + +## Reporting + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-013 | Results and evidence | Integrate pass or fail status, logs, Launch Kit reports, generated manifests, Kubernetes state, connectivity results, and bandwidth results into AI Cloud Validation reporting, catalog, and AI Cloud Labs artifacts. | active | diff --git a/docs/requirements/network-operator-readiness-requirements.yaml b/docs/requirements/network-operator-readiness-requirements.yaml new file mode 100644 index 000000000..5dc2aabef --- /dev/null +++ b/docs/requirements/network-operator-readiness-requirements.yaml @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Enterprise RA Network Operator self-validation integration requirements. +# Structured from the PRD supplied for the Kubernetes Launch Kit integration. +# Render the publishable Markdown view with `make plan`. + +source: network-operator-prd +format: project-prd +title: Enterprise RA Network Operator Self-Validation Integration PRD +version: prd-snapshot-2026-08-04 +owner: NVIDIA Network Operator team +requirements: + - req_id: ENT-REQ-000 + section: Ownership + area: Integration ownership + description: The Network Operator team owns and maintains the integration solution, including compatibility updates for the underlying tests. + status: active + - req_id: ENT-REQ-001 + section: Framework Integration + area: Standard validation workflow + description: Integrate Network Operator self-validation tests into AI Cloud Validation so Enterprise and AI Cloud Ready users can run them through the standard workflow. + status: active + - req_id: ENT-REQ-002 + section: Framework Integration + area: Launch Kit reuse + description: Reuse applicable Kubernetes Launch Kit validation components, including topology discovery, manifest readiness, RDMA connectivity, and RDMA bandwidth. + status: active + - req_id: ENT-REQ-003 + section: Framework Integration + area: Selection and program profiles + description: Support individual and grouped Network Operator validation, with Enterprise and AI Cloud Ready profiles able to mark checks required or optional. + status: active + - req_id: ENT-REQ-004 + section: Framework Integration + area: Runtime parameters + description: Expose applicable runtime parameters such as namespace, node selector, network and driver modes, rail and network names, resource and IP pool names, GPU count, and timeout. + status: active + - req_id: ENT-REQ-005 + section: Network Validation + area: Ethernet and RoCE + description: Validate SR-IOV Network RDMA and RDMA Shared scenarios, including secondary network attachment, RDMA device availability, pod-to-pod RDMA or RoCE connectivity, and basic bandwidth. + status: active + - req_id: ENT-REQ-006 + section: Network Validation + area: InfiniBand + description: Validate InfiniBand SR-IOV and RDMA Shared with IPoIB scenarios, including IB device availability, pod network attachment, and pod-to-pod InfiniBand connectivity. + status: active + - req_id: ENT-REQ-007 + section: Network Validation + area: Host-device networking + description: Validate host-device networking for Kubernetes workers running in virtual machines, covering both Ethernet or RoCE and InfiniBand. + status: active + - req_id: ENT-REQ-008 + section: Network Validation + area: GPUDirect RDMA + description: Validate GPUDirect RDMA peer-to-peer connectivity between GPU-enabled pods across supported worker nodes. + status: active + - req_id: ENT-REQ-009 + section: Network Validation + area: Deployment health + description: Validate Network Operator deployment health and required resources for the selected mode, including policies, secondary networks, IP pools, Multus and CNI components, and drivers. + status: active + - req_id: ENT-REQ-010 + section: Lifecycle Safety + area: State restoration + description: For tests that modify Network Operator or cluster state, capture the pre-test configuration and restore the original Network Operator state after success or failure. + status: active + - req_id: ENT-REQ-011 + section: Catalog and Documentation + area: Catalog metadata + description: Add catalog entries for all tests with owner, labels, dependencies, descriptions, and required YAML updates while following repository contribution standards. + status: active + - req_id: ENT-REQ-012 + section: Catalog and Documentation + area: Prerequisites + description: Document the required cluster prerequisites for each validation area. + status: active + - req_id: ENT-REQ-013 + section: Reporting + area: Results and evidence + description: Integrate pass or fail status, logs, Launch Kit reports, generated manifests, Kubernetes state, connectivity results, and bandwidth results into AI Cloud Validation reporting, catalog, and AI Cloud Labs artifacts. + status: active diff --git a/docs/requirements/test-requirements-matrix.adoc b/docs/requirements/test-requirements-matrix.adoc index 49745c250..06d2ca229 100644 --- a/docs/requirements/test-requirements-matrix.adoc +++ b/docs/requirements/test-requirements-matrix.adoc @@ -3504,4 +3504,544 @@ docs/requirements/test-requirements-matrix.yaml. Run `make plan` to regenerate. | full | +| [[K8S42-01]]K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools +| ENT-REQ-000 +| network-operator-prd +| partial +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools +| ENT-REQ-001 +| network-operator-prd +| full +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools +| ENT-REQ-003 +| network-operator-prd +| partial +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools +| ENT-REQ-009 +| network-operator-prd +| full +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools +| ENT-REQ-011 +| network-operator-prd +| partial +| + +| [[K8S42-02]]K8S42-02 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate the SR-IOV policy and profile-specific secondary network resources reported by Launch Kit +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| [[K8S42-03]]K8S42-03 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pod-to-pod RDMA-CM connectivity across selected rails +| ENT-REQ-002 +| network-operator-prd +| full +| + +| K8S42-03 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pod-to-pod RDMA-CM connectivity across selected rails +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-03 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pod-to-pod RDMA-CM connectivity across selected rails +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| [[K8S42-04]]K8S42-04 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate the exact secondary network resource selected for an Ethernet or RoCE profile +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| [[K8S42-05]]K8S42-05 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate the exact secondary network resource selected for an InfiniBand profile +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| [[K8S42-06]]K8S42-06 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate HostDeviceNetwork readiness for an applicable Ethernet or InfiniBand profile +| ENT-REQ-007 +| network-operator-prd +| partial +| + +| [[K8S42-07]]K8S42-07 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate GPUDirect RDMA DMA-BUF bandwidth between GPU-enabled pod endpoints +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| [[K8S42-08]]K8S42-08 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Launch Kit discovery completed and resolved a fabric and deployment profile +| ENT-REQ-002 +| network-operator-prd +| partial +| + +| K8S42-08 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Launch Kit discovery completed and resolved a fabric and deployment profile +| ENT-REQ-004 +| network-operator-prd +| partial +| + +| K8S42-08 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Launch Kit discovery completed and resolved a fabric and deployment profile +| ENT-REQ-012 +| network-operator-prd +| partial +| + +| [[K8S42-09]]K8S42-09 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-09 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| K8S42-09 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness +| ENT-REQ-009 +| network-operator-prd +| partial +| + +| [[K8S42-10]]K8S42-10 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Macvlan or IPoIB network readiness for an applicable RDMA Shared profile +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-10 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate Macvlan or IPoIB network readiness for an applicable RDMA Shared profile +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| [[K8S42-11]]K8S42-11 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation +| ENT-REQ-002 +| network-operator-prd +| partial +| + +| K8S42-11 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-11 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| [[K8S42-12]]K8S42-12 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth +| ENT-REQ-002 +| network-operator-prd +| partial +| + +| K8S42-12 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-12 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| [[K8S42-13]]K8S42-13 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate same-rail and cross-rail coverage for a multi-rail Launch Kit profile +| ENT-REQ-002 +| network-operator-prd +| partial +| + +| [[K8S42-15]]K8S42-15 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Verify raw phase output, generated files, connectivity results, and Launch Kit reports were preserved +| ENT-REQ-011 +| network-operator-prd +| partial +| + +| K8S42-15 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Verify raw phase output, generated files, connectivity results, and Launch Kit reports were preserved +| ENT-REQ-013 +| network-operator-prd +| partial +| + +| [[K8S42-16]]K8S42-16 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned Ethernet or RoCE SR-IOV Network RDMA profile through Launch Kit +| ENT-REQ-001 +| network-operator-prd +| partial +| + +| K8S42-16 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned Ethernet or RoCE SR-IOV Network RDMA profile through Launch Kit +| ENT-REQ-003 +| network-operator-prd +| partial +| + +| K8S42-16 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned Ethernet or RoCE SR-IOV Network RDMA profile through Launch Kit +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-16 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned Ethernet or RoCE SR-IOV Network RDMA profile through Launch Kit +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| K8S42-16 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned Ethernet or RoCE SR-IOV Network RDMA profile through Launch Kit +| ENT-REQ-013 +| network-operator-prd +| partial +| + +| [[K8S42-17]]K8S42-17 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned InfiniBand SR-IOV Network RDMA profile through Launch Kit +| ENT-REQ-001 +| network-operator-prd +| partial +| + +| K8S42-17 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned InfiniBand SR-IOV Network RDMA profile through Launch Kit +| ENT-REQ-003 +| network-operator-prd +| partial +| + +| K8S42-17 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned InfiniBand SR-IOV Network RDMA profile through Launch Kit +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| K8S42-17 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned InfiniBand SR-IOV Network RDMA profile through Launch Kit +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| K8S42-17 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned InfiniBand SR-IOV Network RDMA profile through Launch Kit +| ENT-REQ-013 +| network-operator-prd +| partial +| + +| [[K8S42-18]]K8S42-18 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned Ethernet or RoCE RDMA Shared profile through Launch Kit +| ENT-REQ-001 +| network-operator-prd +| partial +| + +| K8S42-18 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned Ethernet or RoCE RDMA Shared profile through Launch Kit +| ENT-REQ-003 +| network-operator-prd +| partial +| + +| K8S42-18 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned Ethernet or RoCE RDMA Shared profile through Launch Kit +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-18 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned Ethernet or RoCE RDMA Shared profile through Launch Kit +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| K8S42-18 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned Ethernet or RoCE RDMA Shared profile through Launch Kit +| ENT-REQ-013 +| network-operator-prd +| partial +| + +| [[K8S42-19]]K8S42-19 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned InfiniBand or IPoIB RDMA Shared profile through Launch Kit +| ENT-REQ-001 +| network-operator-prd +| partial +| + +| K8S42-19 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned InfiniBand or IPoIB RDMA Shared profile through Launch Kit +| ENT-REQ-003 +| network-operator-prd +| partial +| + +| K8S42-19 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned InfiniBand or IPoIB RDMA Shared profile through Launch Kit +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| K8S42-19 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned InfiniBand or IPoIB RDMA Shared profile through Launch Kit +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| K8S42-19 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate a pre-provisioned InfiniBand or IPoIB RDMA Shared profile through Launch Kit +| ENT-REQ-013 +| network-operator-prd +| partial +| + +| [[K8S42-20]]K8S42-20 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pre-provisioned Ethernet or RoCE host-device networking on worker VMs through Launch Kit +| ENT-REQ-001 +| network-operator-prd +| partial +| + +| K8S42-20 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pre-provisioned Ethernet or RoCE host-device networking on worker VMs through Launch Kit +| ENT-REQ-003 +| network-operator-prd +| partial +| + +| K8S42-20 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pre-provisioned Ethernet or RoCE host-device networking on worker VMs through Launch Kit +| ENT-REQ-007 +| network-operator-prd +| partial +| + +| K8S42-20 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pre-provisioned Ethernet or RoCE host-device networking on worker VMs through Launch Kit +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| K8S42-20 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pre-provisioned Ethernet or RoCE host-device networking on worker VMs through Launch Kit +| ENT-REQ-013 +| network-operator-prd +| partial +| + +| [[K8S42-21]]K8S42-21 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pre-provisioned InfiniBand host-device networking on worker VMs through Launch Kit +| ENT-REQ-001 +| network-operator-prd +| partial +| + +| K8S42-21 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pre-provisioned InfiniBand host-device networking on worker VMs through Launch Kit +| ENT-REQ-003 +| network-operator-prd +| partial +| + +| K8S42-21 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pre-provisioned InfiniBand host-device networking on worker VMs through Launch Kit +| ENT-REQ-007 +| network-operator-prd +| partial +| + +| K8S42-21 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pre-provisioned InfiniBand host-device networking on worker VMs through Launch Kit +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| K8S42-21 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Validate pre-provisioned InfiniBand host-device networking on worker VMs through Launch Kit +| ENT-REQ-013 +| network-operator-prd +| partial +| + |=== diff --git a/docs/requirements/test-requirements-matrix.yaml b/docs/requirements/test-requirements-matrix.yaml index 64b889c7c..e6dad8103 100644 --- a/docs/requirements/test-requirements-matrix.yaml +++ b/docs/requirements/test-requirements-matrix.yaml @@ -8,7 +8,7 @@ # # Per mapping: # test_id - matches docs/test-plan.yaml -# requirements - list of { req_id, source: offtake|reference, coverage: full|partial } +# requirements - list of { req_id, source: , coverage: full|partial } # annotations - free-form (e.g. how this relationship was decided) # notes - free-form scratch # @@ -2568,3 +2568,263 @@ mappings: coverage: full annotations: '' notes: '' + - test_id: K8S42-01 + requirements: + - req_id: ENT-REQ-000 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: full + - req_id: ENT-REQ-003 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-009 + source: network-operator-prd + coverage: full + - req_id: ENT-REQ-011 + source: network-operator-prd + coverage: partial + annotations: 'Production provider establishes the integration and catalog boundary; mock-backed unit tests exercise it; long-term ownership and typed program policy metadata are not runtime assertions.' + notes: '' + - test_id: K8S42-02 + requirements: + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + annotations: 'Covers SR-IOV attachment and device readiness; connectivity and bandwidth are separate selectable checks.' + notes: '' + - test_id: K8S42-03 + requirements: + - req_id: ENT-REQ-002 + source: network-operator-prd + coverage: full + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + annotations: 'Reuses the Launch Kit rping matrix for both RoCE and InfiniBand profiles.' + notes: '' + - test_id: K8S42-04 + requirements: + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + annotations: 'Aggregates the Ethernet and RoCE profile results.' + notes: '' + - test_id: K8S42-05 + requirements: + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + annotations: 'Checks the profile-specific InfiniBand network kind; device, attachment, and connectivity evidence is split across other checks.' + notes: '' + - test_id: K8S42-06 + requirements: + - req_id: ENT-REQ-007 + source: network-operator-prd + coverage: partial + annotations: 'Production wiring and unit fixtures cover both host-device profile contracts; live VM qualification remains required.' + notes: '' + - test_id: K8S42-07 + requirements: + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + annotations: 'Consumes Launch Kit gpudirect_dmabuf matrix verdicts with endpoint GPU indices, PCI addresses, bandwidth, threshold, and errors; mock-qualified only and still requires live GPUDirect hardware qualification.' + notes: '' + - test_id: K8S42-08 + requirements: + - req_id: ENT-REQ-002 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-004 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-012 + source: network-operator-prd + coverage: partial + annotations: 'Forwards user-owned discover arguments without copying Launch Kit defaults; live prerequisites are documented separately.' + notes: '' + - test_id: K8S42-09 + requirements: + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-009 + source: network-operator-prd + coverage: partial + annotations: 'Checks generated secondary-network, IP pool, Multus attachment, and pod-address evidence.' + notes: '' + - test_id: K8S42-10 + requirements: + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + annotations: 'Covers Macvlan and IPoIB RDMA Shared profiles.' + notes: '' + - test_id: K8S42-11 + requirements: + - req_id: ENT-REQ-002 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + annotations: 'Reuses Launch Kit strict ICMP same-rail reachability and cross-rail isolation results.' + notes: '' + - test_id: K8S42-12 + requirements: + - req_id: ENT-REQ-002 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + annotations: 'Reuses Launch Kit ib_write_bw verdicts and reports the observed and Launch Kit-resolved minimum bandwidth.' + notes: '' + - test_id: K8S42-13 + requirements: + - req_id: ENT-REQ-002 + source: network-operator-prd + coverage: partial + annotations: 'Checks the complete two-rail strict connectivity matrix.' + notes: '' + - test_id: K8S42-15 + requirements: + - req_id: ENT-REQ-011 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Local evidence and framework reporting are implemented; typed catalog ownership and Labs binary attachment upload remain gaps.' + notes: '' + - test_id: K8S42-16 + requirements: + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-003 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Concrete validation-only RoCE SR-IOV discover-generate-validate use case for an ISV-provisioned deployment; the validation path was live-qualified on a two-node Ubuntu 24.04 single-rail cluster, with the multi-rail member skipped as inapplicable.' + notes: '' + - test_id: K8S42-17 + requirements: + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-003 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Concrete validation-only InfiniBand SR-IOV discover-generate-validate use case for an ISV-provisioned deployment; mock-qualified only.' + notes: '' + - test_id: K8S42-18 + requirements: + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-003 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Concrete validation-only RoCE RDMA Shared discover-generate-validate use case for an ISV-provisioned deployment; mock-qualified only.' + notes: '' + - test_id: K8S42-19 + requirements: + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-003 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Concrete validation-only InfiniBand and IPoIB RDMA Shared discover-generate-validate use case for an ISV-provisioned deployment; mock-qualified only.' + notes: '' + - test_id: K8S42-20 + requirements: + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-003 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-007 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Concrete validation-only RoCE host-device workflow for an ISV-provisioned deployment; mock-qualified and not yet proven on worker VMs.' + notes: '' + - test_id: K8S42-21 + requirements: + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-003 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-007 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Concrete validation-only InfiniBand host-device workflow for an ISV-provisioned deployment; mock-qualified and not yet proven on worker VMs.' + notes: '' diff --git a/docs/test-plan.adoc b/docs/test-plan.adoc index fabd52576..b87e9bb6d 100644 --- a/docs/test-plan.adoc +++ b/docs/test-plan.adoc @@ -3456,7 +3456,7 @@ a| | pending | -.58+| Workload Orchestration +.78+| Workload Orchestration | Backup and Recovery | Centralized managed service to automate and govern data backup across services | AWS backup @@ -3646,7 +3646,7 @@ a| | published | -.45+| Managed Kubernetes Control Plane +.65+| Managed Kubernetes Control Plane | Tenant-isolated Kubernetes control planes for managing k8s workloads. does placement, networking, lifecyle mgmt. | AWS EKS, GCP GKE | [[K8S05-01]]K8S05-01 @@ -4033,6 +4033,288 @@ a| https://github.com/NVIDIA/ai-cloud-validation/issues/220[#220] | pending | +.20+| Network Operator self-validation through Kubernetes Launch Kit +.20+| +| [[K8S42-01]]K8S42-01 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Production provider integration and unit coverage for ENT-REQ-001, ENT-REQ-009, and ENT-REQ-011 +| P0 +a| +| LimitedEnv +| +| +| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools +| pending +| + +| [[K8S42-02]]K8S42-02 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-005 +| P0 +a| +| LimitedEnv +| +| +| Validate the SR-IOV policy and profile-specific secondary network resources reported by Launch Kit +| pending +| + +| [[K8S42-03]]K8S42-03 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-002 and ENT-REQ-005 +| P0 +a| +| LimitedEnv +| +| +| Validate pod-to-pod RDMA-CM connectivity across selected rails +| pending +| + +| [[K8S42-04]]K8S42-04 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-005 +| P0 +a| +| LimitedEnv +| +| +| Validate the exact secondary network resource selected for an Ethernet or RoCE profile +| pending +| + +| [[K8S42-05]]K8S42-05 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-006 +| P0 +a| +| LimitedEnv +| +| +| Validate the exact secondary network resource selected for an InfiniBand profile +| pending +| + +| [[K8S42-06]]K8S42-06 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-007 +| P0 +a| +| LimitedEnv +| +| +| Validate HostDeviceNetwork readiness for an applicable Ethernet or InfiniBand profile +| pending +| + +| [[K8S42-07]]K8S42-07 +| K8S42 +| +| era, gpudirect, kubernetes, ncp, network_operator, slow +| Launch Kit gpudirect_dmabuf result-family integration and mock-backed unit coverage for ENT-REQ-008; live GPU qualification remains required +| P0 +a| +| LimitedEnv +| +| +| Validate GPUDirect RDMA DMA-BUF bandwidth between GPU-enabled pod endpoints +| pending +| + +| [[K8S42-08]]K8S42-08 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-004, and ENT-REQ-012 +| P0 +a| +| LimitedEnv +| +| +| Validate Launch Kit discovery completed and resolved a fabric and deployment profile +| pending +| + +| [[K8S42-09]]K8S42-09 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-005, ENT-REQ-006, and ENT-REQ-009 +| P0 +a| +| LimitedEnv +| +| +| Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness +| pending +| + +| [[K8S42-10]]K8S42-10 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-005 and ENT-REQ-006 +| P0 +a| +| LimitedEnv +| +| +| Validate Macvlan or IPoIB network readiness for an applicable RDMA Shared profile +| pending +| + +| [[K8S42-11]]K8S42-11 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-005, and ENT-REQ-006 +| P0 +a| +| LimitedEnv +| +| +| Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation +| pending +| + +| [[K8S42-12]]K8S42-12 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-005, and ENT-REQ-006 +| P0 +a| +| LimitedEnv +| +| +| Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth +| pending +| + +| [[K8S42-13]]K8S42-13 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Provider wiring and unit coverage for Enterprise multi-rail coverage in ENT-REQ-002 +| P0 +a| +| LimitedEnv +| +| +| Validate same-rail and cross-rail coverage for a multi-rail Launch Kit profile +| pending +| + +| [[K8S42-15]]K8S42-15 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| Local evidence and framework reporting cover ENT-REQ-013; Labs attachment upload remains required +| P0 +a| +| LimitedEnv +| +| +| Verify raw phase output, generated files, connectivity results, and Launch Kit reports were preserved +| pending +| + +| [[K8S42-16]]K8S42-16 +| K8S42 +| +| era, ethernet, gpudirect, kubernetes, ncp, network_operator, network_operator_use_cases, roce, slow, sriov +| Validation-only composite use case; the validation path was live-qualified on a two-node Ubuntu 24.04 single-rail cluster, with multi-rail skipped as inapplicable +| P0 +a| +| LimitedEnv +| +| +| Validate a pre-provisioned Ethernet or RoCE SR-IOV Network RDMA profile through Launch Kit +| pending +| + +| [[K8S42-17]]K8S42-17 +| K8S42 +| +| era, gpudirect, infiniband, kubernetes, ncp, network_operator, network_operator_use_cases, slow, sriov +| Validation-only composite use case: preflight, discover, generate, and validate +| P0 +a| +| LimitedEnv +| +| +| Validate a pre-provisioned InfiniBand SR-IOV Network RDMA profile through Launch Kit +| pending +| + +| [[K8S42-18]]K8S42-18 +| K8S42 +| +| era, ethernet, gpudirect, kubernetes, ncp, network_operator, network_operator_use_cases, rdma_shared, roce, slow +| Validation-only composite use case: preflight, discover, generate, and validate +| P0 +a| +| LimitedEnv +| +| +| Validate a pre-provisioned Ethernet or RoCE RDMA Shared profile through Launch Kit +| pending +| + +| [[K8S42-19]]K8S42-19 +| K8S42 +| +| era, gpudirect, infiniband, kubernetes, ncp, network_operator, network_operator_use_cases, rdma_shared, slow +| Validation-only composite use case: preflight, discover, generate, and validate +| P0 +a| +| LimitedEnv +| +| +| Validate a pre-provisioned InfiniBand or IPoIB RDMA Shared profile through Launch Kit +| pending +| + +| [[K8S42-20]]K8S42-20 +| K8S42 +| +| era, ethernet, gpudirect, host_device, kubernetes, ncp, network_operator, network_operator_use_cases, roce, slow +| Validation-only composite use case: preflight, discover, generate, and validate; live VM qualification remains required +| P0 +a| +| LimitedEnv +| +| +| Validate pre-provisioned Ethernet or RoCE host-device networking on worker VMs through Launch Kit +| pending +| + +| [[K8S42-21]]K8S42-21 +| K8S42 +| +| era, gpudirect, host_device, infiniband, kubernetes, ncp, network_operator, network_operator_use_cases, slow +| Validation-only composite use case: preflight, discover, generate, and validate; live VM qualification remains required +| P0 +a| +| LimitedEnv +| +| +| Validate pre-provisioned InfiniBand host-device networking on worker VMs through Launch Kit +| pending +| + .2+| K8s Versioning & Compliance .2+| | [[K8S02-01]]K8S02-01 diff --git a/docs/test-plan.yaml b/docs/test-plan.yaml index bf209c912..ef7b8f0cf 100644 --- a/docs/test-plan.yaml +++ b/docs/test-plan.yaml @@ -3636,6 +3636,336 @@ domains: milestone: M5 github_issues: - "#220" + - description: Network Operator self-validation through Kubernetes Launch Kit + tests: + - summary: Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-01 + status: pending + notes: "Production provider integration and unit coverage for ENT-REQ-001, ENT-REQ-009, and ENT-REQ-011" + - summary: Validate the SR-IOV policy and profile-specific secondary network resources reported by Launch Kit + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-02 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-005" + - summary: Validate pod-to-pod RDMA-CM connectivity across selected rails + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-03 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-002 and ENT-REQ-005" + - summary: Validate the exact secondary network resource selected for an Ethernet or RoCE profile + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-04 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-005" + - summary: Validate the exact secondary network resource selected for an InfiniBand profile + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-05 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-006" + - summary: Validate HostDeviceNetwork readiness for an applicable Ethernet or InfiniBand profile + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-06 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-007" + - summary: Validate GPUDirect RDMA DMA-BUF bandwidth between GPU-enabled pod endpoints + labels: + - era + - gpudirect + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-07 + status: pending + notes: "Launch Kit gpudirect_dmabuf result-family integration and mock-backed unit coverage for ENT-REQ-008; live GPU qualification remains required" + - summary: Validate Launch Kit discovery completed and resolved a fabric and deployment profile + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-08 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-004, and ENT-REQ-012" + - summary: Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-09 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-005, ENT-REQ-006, and ENT-REQ-009" + - summary: Validate Macvlan or IPoIB network readiness for an applicable RDMA Shared profile + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-10 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-005 and ENT-REQ-006" + - summary: Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-11 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-005, and ENT-REQ-006" + - summary: Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-12 + status: pending + notes: "Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-005, and ENT-REQ-006" + - summary: Validate same-rail and cross-rail coverage for a multi-rail Launch Kit profile + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-13 + status: pending + notes: "Provider wiring and unit coverage for Enterprise multi-rail coverage in ENT-REQ-002" + - summary: Verify raw phase output, generated files, connectivity results, and Launch Kit reports were preserved + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-15 + status: pending + notes: "Local evidence and framework reporting cover ENT-REQ-013; Labs attachment upload remains required" + - summary: Validate a pre-provisioned Ethernet or RoCE SR-IOV Network RDMA profile through Launch Kit + labels: + - era + - ethernet + - gpudirect + - kubernetes + - ncp + - network_operator + - network_operator_use_cases + - roce + - slow + - sriov + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-16 + status: pending + notes: "Validation-only composite use case; the validation path was live-qualified on a two-node Ubuntu 24.04 single-rail cluster, with multi-rail skipped as inapplicable" + - summary: Validate a pre-provisioned InfiniBand SR-IOV Network RDMA profile through Launch Kit + labels: + - era + - gpudirect + - infiniband + - kubernetes + - ncp + - network_operator + - network_operator_use_cases + - slow + - sriov + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-17 + status: pending + notes: "Validation-only composite use case: preflight, discover, generate, and validate" + - summary: Validate a pre-provisioned Ethernet or RoCE RDMA Shared profile through Launch Kit + labels: + - era + - ethernet + - gpudirect + - kubernetes + - ncp + - network_operator + - network_operator_use_cases + - rdma_shared + - roce + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-18 + status: pending + notes: "Validation-only composite use case: preflight, discover, generate, and validate" + - summary: Validate a pre-provisioned InfiniBand or IPoIB RDMA Shared profile through Launch Kit + labels: + - era + - gpudirect + - infiniband + - kubernetes + - ncp + - network_operator + - network_operator_use_cases + - rdma_shared + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-19 + status: pending + notes: "Validation-only composite use case: preflight, discover, generate, and validate" + - summary: Validate pre-provisioned Ethernet or RoCE host-device networking on worker VMs through Launch Kit + labels: + - era + - ethernet + - gpudirect + - host_device + - kubernetes + - ncp + - network_operator + - network_operator_use_cases + - roce + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-20 + status: pending + notes: "Validation-only composite use case: preflight, discover, generate, and validate; live VM qualification remains required" + - summary: Validate pre-provisioned InfiniBand host-device networking on worker VMs through Launch Kit + labels: + - era + - gpudirect + - host_device + - infiniband + - kubernetes + - ncp + - network_operator + - network_operator_use_cases + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-21 + status: pending + notes: "Validation-only composite use case: preflight, discover, generate, and validate; live VM qualification remains required" - description: "K8s Versioning & Compliance" tests: - summary: Verify support for the three most recent minor releases (N-2) diff --git a/isvctl/configs/providers/k8s-launch-kit/README.md b/isvctl/configs/providers/k8s-launch-kit/README.md new file mode 100644 index 000000000..7eed57ff7 --- /dev/null +++ b/isvctl/configs/providers/k8s-launch-kit/README.md @@ -0,0 +1,87 @@ + + + +# Kubernetes Launch Kit provider internals + +This directory owns the implementation behind +`config/provider.yaml`. It is provider-specific code, not a cross-provider +helper. + +## Layout + +| Path | Purpose | +|---|---| +| `config/provider.yaml` | Generic single-workflow provider using real `l8k` and `kubectl` by default | +| `config/network-operator.yaml` | Production six-use-case Network Operator workflow | +| `scripts/adapter.py` | Transport for install/verify, Kubernetes preflight, and one `l8k` workflow command | + +Test doubles and pinned scenario data intentionally live outside the shipped +provider under `isvctl/tests/providers/k8s_launch_kit/fixtures/`. The provider +tests load the production YAML and inject those paths in memory. + +The adapter must remain thin. It accepts raw argument arrays for `discover`, +`generate`, `deploy`, `validate`, and `clean`, appends `--output json`, executes the +configured `l8k` executable, and preserves the CLI's JSON documents without +renaming or interpreting fields. The one file-level input is `user_config`, a +path to a complete Launch Kit configuration. Before discovery, the adapter +copies it to the workflow as a mode-`0600` `user-config.yaml`, explicitly writes +the discovered result to `cluster-config.yaml`, and removes the staged input as +soon as discovery exits. The original is never modified, and evidence retains +only its path, size, and SHA-256 provenance rather than its potentially +sensitive contents. Launch Kit still owns the file schema, domain flags, and +defaults. Semantic assertions belong in `isvtest.validations.k8s_launch_kit`. + +Launch Kit `validate` steps use `timeout: null` so the CLI owns its deadline. +l8k calculates and logs a bounded matrix budget by default and honors a user's +explicit `--connectivity-timeout`. The remaining workflow steps keep finite +isvctl watchdogs. Other providers may also use `timeout: null`, but only when +their child command has its own bounded timeout. + +The grouped Network Operator workflow is validation-only. ISVs install and +configure Network Operator before running it. Each selected use case executes +`preflight -> discover -> generate -> validate`; it never invokes `l8k deploy` +or `l8k clean`, so AI Cloud Validation cannot replace or delete the ISV-managed +installation. The generic `provider.yaml` deliberately retains deploy and +clean as public Launch Kit operations for other consumers. + +The grouped workflow passes only its fabric and deployment identity during +discovery. With no `user_config`, Launch Kit resolves the default +`./cluster-config.yaml` and `./deployment` paths throughout the validation +workflow. With `user_config`, every selected use case stages an independent +copy, and the adapter owns `--user-config` plus `--save-cluster-config` for +discovery. Each transient copy is deleted after its discovery command. Do not +repeat either flag in the raw discovery argument array or place the source +inside the retained provider working directory. + +Each workflow envelope records the absolute working directory while retaining +Launch Kit's JSON documents unchanged. Validations use that metadata to resolve +relative `generatedFiles` paths emitted by the CLI. + +Install mode accepts only an immutable full Git commit for the official +`scripts/install.sh` plus a caller-supplied SHA-256, verifies that digest before +writing or executing the script, delegates archive selection and checksum +handling to Launch Kit, then verifies the binary at the install prefix. +When the user pins `installation.version`, both setup and test-phase +verification require `l8k version --output json` to report that exact version. +The captured schema must advertise all five generic provider commands, +including `deploy` and `clean`. This verifies that the installation satisfies +the generic provider contract even though the Network Operator suite invokes +only discover, generate, and validate. + +The preflight accepts the non-empty subset of Launch Kit commands used by the +calling workflow. It uses their explicit kubeconfig and forwarded environment, +rejects conflicting `--kubeconfig` arguments, and requires Kubernetes API +access plus at least one Ready node before validation starts. + +The same string-only environment mapping is also passed to the installer and +version/schema verification, so proxy and executable runtime settings do not +change between setup and test phases. + +The production adapter executes `executable` directly. There is no Python-file +special case: a test double must be an executable with a valid shebang, just +like any other CLI implementation. This keeps mock behavior out of the public +provider contract. + +See the [integration guide](../../../../docs/guides/k8s-launch-kit/network-operator.md) +for configuration, use cases, evidence, prerequisites, and current production +gaps. diff --git a/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml b/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml new file mode 100644 index 000000000..b80872807 --- /dev/null +++ b/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml @@ -0,0 +1,697 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Production Network Operator workflow. It invokes the real l8k and kubectl +# executables inherited from provider.yaml. Test doubles live only under tests/. +# +# Usage: +# ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl test run \ +# -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ +# --capability kubernetes --no-upload -- -v + +import: + - provider.yaml + - ../../../suites/k8s-launch-kit/network-operator-use-cases.yaml + +version: "1.0" + +context: + k8s_launch_kit: + executable: l8k + installation: + mode: verify + version: "" + installer_ref: "" + installer_sha256: "" + prefix: "" + # Optional complete Launch Kit configuration. Every selected use case + # stages its own copy before discovery; the source file is never modified. + user_config: "" + # An empty override means the adapter invokes kubectl from PATH. Users may + # replace this with any kubectl-compatible argv list in an overlay. + kubectl_command: [] + environment: {} + shared_artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/shared-evidence + use_cases: + roce_sriov: + working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-sriov/work + artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-sriov/evidence + discover: + arguments: + - --fabric + - ethernet + - --deployment-type + - sriov + generate: + arguments: [] + validate: + arguments: [] + infiniband_sriov: + working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-sriov/work + artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-sriov/evidence + discover: + arguments: + - --fabric + - infiniband + - --deployment-type + - sriov + generate: + arguments: [] + validate: + arguments: [] + roce_rdma_shared: + working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-rdma-shared/work + artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-rdma-shared/evidence + discover: + arguments: + - --fabric + - ethernet + - --deployment-type + - rdma_shared + generate: + arguments: [] + validate: + arguments: [] + infiniband_rdma_shared: + working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-rdma-shared/work + artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-rdma-shared/evidence + discover: + arguments: + - --fabric + - infiniband + - --deployment-type + - rdma_shared + generate: + arguments: [] + validate: + arguments: [] + roce_host_device: + working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-host-device/work + artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-host-device/evidence + discover: + arguments: + - --fabric + - ethernet + - --deployment-type + - host_device + generate: + arguments: [] + validate: + arguments: [] + infiniband_host_device: + working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-host-device/work + artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-host-device/evidence + discover: + arguments: + - --fabric + - infiniband + - --deployment-type + - host_device + generate: + arguments: [] + validate: + arguments: [] + +commands: + network_operator: + phases: + - setup + - launch-kit-verification + - roce-sriov + - infiniband-sriov + - roce-rdma-shared + - infiniband-rdma-shared + - roce-host-device + - infiniband-host-device + continue_after_failure: + - roce-sriov + - infiniband-sriov + - roce-rdma-shared + - infiniband-rdma-shared + - roce-host-device + - infiniband-host-device + steps: + - name: launch_kit_prepare + phase: setup + command: python3 ../scripts/adapter.py + args: + - prepare + - --mode + - "{{ context.k8s_launch_kit.installation.mode }}" + - --executable + - "{{ context.k8s_launch_kit.executable }}" + - "--version={{ context.k8s_launch_kit.installation.version }}" + - "--installer-ref={{ context.k8s_launch_kit.installation.installer_ref }}" + - "--installer-sha256={{ context.k8s_launch_kit.installation.installer_sha256 }}" + - "--prefix={{ context.k8s_launch_kit.installation.prefix }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.shared_artifact_dir }}" + timeout: 900 + output_schema: k8s_launch_kit + requires: [kubernetes] + + - name: launch_kit_verify + phase: launch-kit-verification + command: python3 ../scripts/adapter.py + args: + - verify + - --executable + - "{{ steps.launch_kit_prepare.executable | default(context.k8s_launch_kit.executable) }}" + - "--expected-version={{ context.k8s_launch_kit.installation.version }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.shared_artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + + # roce-sriov: preflight -> discover -> generate -> validate + - name: launch_kit_roce_sriov_preflight + phase: roce-sriov + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.use_cases.roce_sriov.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.roce_sriov.generate.arguments, 'validate': context.k8s_launch_kit.use_cases.roce_sriov.validate.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceSriovCheck] + + - name: launch_kit_roce_sriov_discover + phase: roce-sriov + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceSriovCheck] + + - name: launch_kit_roce_sriov_generate + phase: roce-sriov + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceSriovCheck] + + - name: launch_kit_roce_sriov_validate + phase: roce-sriov + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_sriov.artifact_dir }}" + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceSriovCheck] + + # infiniband-sriov: preflight -> discover -> generate -> validate + - name: launch_kit_infiniband_sriov_preflight + phase: infiniband-sriov + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.use_cases.infiniband_sriov.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.infiniband_sriov.generate.arguments, 'validate': context.k8s_launch_kit.use_cases.infiniband_sriov.validate.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandSriovCheck] + + - name: launch_kit_infiniband_sriov_discover + phase: infiniband-sriov + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandSriovCheck] + + - name: launch_kit_infiniband_sriov_generate + phase: infiniband-sriov + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandSriovCheck] + + - name: launch_kit_infiniband_sriov_validate + phase: infiniband-sriov + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.artifact_dir }}" + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandSriovCheck] + + # roce-rdma-shared: preflight -> discover -> generate -> validate + - name: launch_kit_roce_rdma_shared_preflight + phase: roce-rdma-shared + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.use_cases.roce_rdma_shared.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.roce_rdma_shared.generate.arguments, 'validate': context.k8s_launch_kit.use_cases.roce_rdma_shared.validate.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceRdmaSharedCheck] + + - name: launch_kit_roce_rdma_shared_discover + phase: roce-rdma-shared + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceRdmaSharedCheck] + + - name: launch_kit_roce_rdma_shared_generate + phase: roce-rdma-shared + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceRdmaSharedCheck] + + - name: launch_kit_roce_rdma_shared_validate + phase: roce-rdma-shared + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.artifact_dir }}" + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceRdmaSharedCheck] + + # infiniband-rdma-shared: preflight -> discover -> generate -> validate + - name: launch_kit_infiniband_rdma_shared_preflight + phase: infiniband-rdma-shared + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.use_cases.infiniband_rdma_shared.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.infiniband_rdma_shared.generate.arguments, 'validate': context.k8s_launch_kit.use_cases.infiniband_rdma_shared.validate.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandRdmaSharedCheck] + + - name: launch_kit_infiniband_rdma_shared_discover + phase: infiniband-rdma-shared + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandRdmaSharedCheck] + + - name: launch_kit_infiniband_rdma_shared_generate + phase: infiniband-rdma-shared + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandRdmaSharedCheck] + + - name: launch_kit_infiniband_rdma_shared_validate + phase: infiniband-rdma-shared + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.artifact_dir }}" + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandRdmaSharedCheck] + + # roce-host-device: preflight -> discover -> generate -> validate + - name: launch_kit_roce_host_device_preflight + phase: roce-host-device + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.use_cases.roce_host_device.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.roce_host_device.generate.arguments, 'validate': context.k8s_launch_kit.use_cases.roce_host_device.validate.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceHostDeviceCheck] + + - name: launch_kit_roce_host_device_discover + phase: roce-host-device + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceHostDeviceCheck] + + - name: launch_kit_roce_host_device_generate + phase: roce-host-device + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceHostDeviceCheck] + + - name: launch_kit_roce_host_device_validate + phase: roce-host-device + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.roce_host_device.artifact_dir }}" + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkRoceHostDeviceCheck] + + # infiniband-host-device: preflight -> discover -> generate -> validate + - name: launch_kit_infiniband_host_device_preflight + phase: infiniband-host-device + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.use_cases.infiniband_host_device.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.infiniband_host_device.generate.arguments, 'validate': context.k8s_launch_kit.use_cases.infiniband_host_device.validate.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandHostDeviceCheck] + + - name: launch_kit_infiniband_host_device_discover + phase: infiniband-host-device + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandHostDeviceCheck] + + - name: launch_kit_infiniband_host_device_generate + phase: infiniband-host-device + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandHostDeviceCheck] + + - name: launch_kit_infiniband_host_device_validate + phase: infiniband-host-device + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.artifact_dir }}" + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [EastWestNetworkInfiniBandHostDeviceCheck] diff --git a/isvctl/configs/providers/k8s-launch-kit/config/provider.yaml b/isvctl/configs/providers/k8s-launch-kit/config/provider.yaml new file mode 100644 index 000000000..716208adb --- /dev/null +++ b/isvctl/configs/providers/k8s-launch-kit/config/provider.yaml @@ -0,0 +1,219 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Generic Kubernetes Launch Kit provider. This production configuration invokes +# l8k and kubectl from PATH unless a user overlay explicitly replaces them. +# +# This file intentionally contains no Network Operator, topology, resource, +# validation, or l8k timeout defaults. The arguments for every command are +# passed directly to l8k. Omitted values are resolved by Launch Kit itself. + +version: "1.0" + +context: + k8s_launch_kit: + executable: l8k + installation: + mode: verify + version: "" + installer_ref: "" + installer_sha256: "" + prefix: "" + # Optional complete Launch Kit configuration. Discovery stages a copy as + # ./user-config.yaml and writes its resolved output to ./cluster-config.yaml. + user_config: "" + kubectl_command: [] + working_dir: ../../../../../_output/k8s-launch-kit/work + artifact_dir: ../../../../../_output/k8s-launch-kit/evidence + environment: {} + discover: + arguments: [] + generate: + arguments: [] + deploy: + arguments: [] + validate: + arguments: [] + clean: + arguments: [] + +commands: + network_operator: + phases: [setup, test, teardown] + steps: + - name: launch_kit_prepare + phase: setup + command: python3 ../scripts/adapter.py + args: + - prepare + - --mode + - "{{ context.k8s_launch_kit.installation.mode }}" + - --executable + - "{{ context.k8s_launch_kit.executable }}" + - "--version={{ context.k8s_launch_kit.installation.version }}" + - "--installer-ref={{ context.k8s_launch_kit.installation.installer_ref }}" + - "--installer-sha256={{ context.k8s_launch_kit.installation.installer_sha256 }}" + - "--prefix={{ context.k8s_launch_kit.installation.prefix }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 900 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitKubernetesPrerequisiteCheck] + + # Test-phase verification is intentional: --phase test may bypass setup. + - name: launch_kit_verify + phase: test + command: python3 ../scripts/adapter.py + args: + - verify + - --executable + - "{{ steps.launch_kit_prepare.executable | default(context.k8s_launch_kit.executable) }}" + - "--expected-version={{ context.k8s_launch_kit.installation.version }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitKubernetesPrerequisiteCheck] + + # This gate always runs in the test phase, before l8k can mutate a cluster. + - name: launch_kit_kubernetes_preflight + phase: test + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.discover.arguments, 'generate': context.k8s_launch_kit.generate.arguments, 'deploy': context.k8s_launch_kit.deploy.arguments, 'validate': context.k8s_launch_kit.validate.arguments, 'clean': context.k8s_launch_kit.clean.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitKubernetesPrerequisiteCheck] + + - name: launch_kit_discover + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitTopologyDiscoveryCheck] + + - name: launch_kit_generate + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitDeploymentHealthCheck] + + - name: launch_kit_deploy + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - deploy + - --arguments-json + - "{{ context.k8s_launch_kit.deploy.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitDeploymentHealthCheck] + + - name: launch_kit_validate + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + # l8k calculates a bounded matrix budget or honors the user's explicit + # connectivity timeout, so it owns the deadline for this command. + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitDeploymentHealthCheck] + + # Runs after phase validations when deploy was attempted, even if deploy, + # validate, or a validation check failed. + - name: launch_kit_clean + phase: teardown + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable | default(context.k8s_launch_kit.executable) }}" + - --command + - clean + - --arguments-json + - "{{ context.k8s_launch_kit.clean.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_available_validations: [LaunchKitDeploymentHealthCheck] + finalizer_for: launch_kit_deploy diff --git a/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py b/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py new file mode 100644 index 000000000..3db36ed28 --- /dev/null +++ b/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py @@ -0,0 +1,724 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Thin AI Cloud Validation transport for the Kubernetes Launch Kit CLI. + +The provider deliberately exposes the real Launch Kit operations. It forwards +user-supplied arguments verbatim and adds ``--output json`` so stdout can be +preserved as structured evidence. When a complete user config is supplied, the +discover operation also stages it transiently in the working directory, binds +Launch Kit's native ``--user-config`` and ``--save-cluster-config`` flags, and +removes the staged input after discovery. Launch Kit remains the owner of +command flags, configuration schema, and defaults. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.request +from json import JSONDecoder +from pathlib import Path +from typing import Any + +_WORKFLOW_COMMANDS = ("discover", "generate", "deploy", "validate", "clean") +_INSTALLER_URL = "https://raw.githubusercontent.com/NVIDIA/k8s-launch-kit/{ref}/scripts/install.sh" +_STAGED_USER_CONFIG = "user-config.yaml" +_DISCOVERED_CLUSTER_CONFIG = "cluster-config.yaml" + + +def _parse_json_value(raw: str, source: str, expected_type: type[Any]) -> Any: + """Parse a JSON CLI value and enforce its root type.""" + try: + value = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"{source} is not valid JSON: {exc}") from exc + if not isinstance(value, expected_type): + raise ValueError(f"{source} must contain a {expected_type.__name__}") + return value + + +def _parse_json_stream(raw: str, source: str) -> list[dict[str, Any]]: + """Parse zero or more concatenated JSON objects from ``raw``.""" + decoder = JSONDecoder() + documents: list[dict[str, Any]] = [] + offset = 0 + while offset < len(raw): + while offset < len(raw) and raw[offset].isspace(): + offset += 1 + if offset >= len(raw): + break + try: + value, offset = decoder.raw_decode(raw, offset) + except json.JSONDecodeError as exc: + raise ValueError(f"{source} contains invalid JSON at byte {exc.pos}: {exc.msg}") from exc + if not isinstance(value, dict): + raise ValueError(f"{source} document #{len(documents) + 1} is not an object") + documents.append(value) + return documents + + +def _write_json(path: Path, value: Any) -> None: + """Write deterministic structured evidence.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + + +def _resolve_executable(value: str) -> Path: + """Resolve an explicit path or a command available on ``PATH``.""" + candidate = Path(value).expanduser() + if candidate.is_absolute() or candidate.parent != Path("."): + resolved = candidate.resolve() + if not resolved.is_file(): + raise FileNotFoundError(f"Launch Kit executable not found: {resolved}") + return resolved + found = shutil.which(value) + if found is None: + raise FileNotFoundError(f"Launch Kit executable not found on PATH: {value}") + return Path(found).resolve() + + +def _with_json_output(arguments: list[str]) -> list[str]: + """Return workflow arguments that request Launch Kit's automation output.""" + result = list(arguments) + for index, token in enumerate(result): + if token == "--output": + if index + 1 >= len(result): + raise ValueError("--output requires a value") + if result[index + 1] != "json": + raise ValueError("the Launch Kit provider requires --output json") + return result + if token.startswith("--output="): + if token.partition("=")[2] != "json": + raise ValueError("the Launch Kit provider requires --output json") + return result + result.extend(["--output", "json"]) + return result + + +def _structured_error(documents: list[dict[str, Any]]) -> str | None: + """Extract the most actionable Launch Kit structured error.""" + for document in reversed(documents): + error = document.get("error") + if not isinstance(error, dict): + continue + message = error.get("message") + if not isinstance(message, str) or not message: + continue + suggestion = error.get("suggestion") + if isinstance(suggestion, str) and suggestion: + return f"{message}; {suggestion}" + return message + return None + + +def _stderr_excerpt(stderr: str) -> str | None: + """Return the last non-empty stderr line without flooding the envelope.""" + lines = [line.strip() for line in stderr.splitlines() if line.strip()] + return lines[-1] if lines else None + + +def _run_process(argv: list[str], *, cwd: Path, env: dict[str, str]) -> dict[str, Any]: + """Execute a child process and retain both output streams.""" + started = time.monotonic() + try: + completed = subprocess.run( + argv, + cwd=cwd, + env=env, + check=False, + capture_output=True, + text=True, + ) + return { + "exit_code": completed.returncode, + "stdout": completed.stdout, + "stderr": completed.stderr, + "duration_seconds": time.monotonic() - started, + } + except OSError as exc: + return { + "exit_code": -1, + "stdout": "", + "stderr": str(exc), + "duration_seconds": time.monotonic() - started, + } + + +def _record_process(directory: Path, argv: list[str], result: dict[str, Any]) -> dict[str, str]: + """Persist one command, stdout, and stderr as evidence.""" + directory.mkdir(parents=True, exist_ok=True) + stdout_path = directory / "stdout.txt" + stderr_path = directory / "stderr.log" + command_path = directory / "command.json" + stdout_path.write_text(str(result["stdout"]), encoding="utf-8") + stderr_path.write_text(str(result["stderr"]), encoding="utf-8") + _write_json( + command_path, + { + "argv": argv, + "exit_code": result["exit_code"], + "duration_seconds": result["duration_seconds"], + }, + ) + return { + "stdout": str(stdout_path.resolve()), + "stderr": str(stderr_path.resolve()), + "command": str(command_path.resolve()), + } + + +def _environment(raw: str) -> dict[str, str]: + """Merge user-supplied string environment entries with the process environment.""" + supplied = _parse_json_value(raw, "--environment-json", dict) + invalid = [str(key) for key, value in supplied.items() if not isinstance(key, str) or not isinstance(value, str)] + if invalid: + raise ValueError("--environment-json keys and values must be strings") + env = os.environ.copy() + env.update(supplied) + return env + + +def _stage_user_config( + source_value: str, + working_dir: Path, + arguments: list[str], +) -> tuple[list[str], Path | None, dict[str, Any] | None]: + """Stage a complete user config for discovery and return safe provenance.""" + if not source_value: + return arguments, None, None + + conflicting_flags = [ + flag + for flag in ("--user-config", "--save-cluster-config") + if any(token == flag or token.startswith(f"{flag}=") for token in arguments) + ] + if conflicting_flags: + raise ValueError( + "context.k8s_launch_kit.user_config cannot be combined with raw discovery flag(s): " + + ", ".join(conflicting_flags) + ) + + source = Path(source_value).expanduser().resolve() + if not source.is_file(): + raise FileNotFoundError(f"Launch Kit user config not found: {source}") + try: + source.relative_to(working_dir) + except ValueError: + pass + else: + raise ValueError("Launch Kit user config must be outside the retained provider working directory") + + staged = working_dir / _STAGED_USER_CONFIG + content = source.read_bytes() + staged.unlink(missing_ok=True) + try: + staged.write_bytes(content) + staged.chmod(0o600) + except OSError: + staged.unlink(missing_ok=True) + raise + + discovered = working_dir / _DISCOVERED_CLUSTER_CONFIG + return ( + [ + *arguments, + "--user-config", + str(staged), + "--save-cluster-config", + str(discovered), + ], + staged, + { + "source_path": str(source), + "staged_path": str(staged), + "sha256": hashlib.sha256(content).hexdigest(), + "size_bytes": len(content), + "retained": False, + }, + ) + + +def _run_workflow(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + """Invoke exactly one real Launch Kit workflow command.""" + executable = _resolve_executable(args.executable) + arguments = _parse_json_value(args.arguments_json, "--arguments-json", list) + if not all(isinstance(value, str) for value in arguments): + raise ValueError("--arguments-json must contain only strings") + environment = _environment(args.environment_json) + working_dir = Path(args.working_dir).expanduser().resolve() + working_dir.mkdir(parents=True, exist_ok=True) + artifact_dir = Path(args.artifact_dir).expanduser().resolve() + staged_user_config: Path | None = None + user_config_metadata_path: Path | None = None + try: + if args.user_config: + if args.command != "discover": + raise ValueError("--user-config is supported only with the discover workflow command") + arguments, staged_user_config, user_config_metadata = _stage_user_config( + args.user_config, + working_dir, + arguments, + ) + user_config_metadata_path = artifact_dir / "inputs" / "user-config.json" + _write_json(user_config_metadata_path, user_config_metadata) + arguments = _with_json_output(arguments) + argv = [str(executable), args.command, *arguments] + result = _run_process(argv, cwd=working_dir, env=environment) + finally: + if staged_user_config is not None: + staged_user_config.unlink(missing_ok=True) + artifacts = _record_process(artifact_dir / "commands" / args.command, argv, result) + if user_config_metadata_path is not None: + artifacts["user_config"] = str(user_config_metadata_path) + + parse_error: str | None = None + try: + documents = _parse_json_stream(str(result["stdout"]), f"l8k {args.command} stdout") + except ValueError as exc: + documents = [] + parse_error = str(exc) + + success = result["exit_code"] == 0 and parse_error is None + error = parse_error or _structured_error(documents) + if not success and error is None: + error = f"l8k {args.command} exited with code {result['exit_code']}" + if excerpt := _stderr_excerpt(str(result["stderr"])): + error = f"{error}: {excerpt}" + envelope: dict[str, Any] = { + "success": success, + "platform": "kubernetes", + "operation": args.command, + "executable": str(executable), + "argv": argv, + "working_directory": str(working_dir), + "exit_code": result["exit_code"], + "duration_seconds": result["duration_seconds"], + "documents": documents, + "artifacts": artifacts, + } + if error: + envelope["error"] = error + excerpt = _stderr_excerpt(str(result["stderr"])) + if excerpt: + envelope["stderr_excerpt"] = excerpt + exit_code = int(result["exit_code"]) + return envelope, exit_code if exit_code > 0 else (0 if success else 1) + + +def _verify_executable( + executable: Path, + artifact_dir: Path, + expected_version: str = "", + environment: dict[str, str] | None = None, +) -> tuple[dict[str, Any], bool, str | None]: + """Run Launch Kit version and schema commands and preserve both responses.""" + env = environment.copy() if environment is not None else os.environ.copy() + checks: dict[str, Any] = {} + artifacts: dict[str, dict[str, str]] = {} + errors: list[str] = [] + for name, command_args in (("version", ["version", "--output", "json"]), ("schema", ["schema"])): + argv = [str(executable), *command_args] + result = _run_process(argv, cwd=Path.cwd(), env=env) + artifacts[name] = _record_process(artifact_dir / name, argv, result) + try: + documents = _parse_json_stream(str(result["stdout"]), f"l8k {name} stdout") + except ValueError as exc: + documents = [] + errors.append(str(exc)) + if result["exit_code"] != 0: + errors.append( + _stderr_excerpt(str(result["stderr"])) or f"l8k {name} exited with code {result['exit_code']}" + ) + elif len(documents) != 1: + errors.append(f"l8k {name} must emit exactly one JSON object, got {len(documents)}") + passed = result["exit_code"] == 0 and len(documents) == 1 + if name == "schema" and passed: + commands = documents[0].get("commands") + advertised = set(commands) if isinstance(commands, dict) else set() + missing_commands = set(_WORKFLOW_COMMANDS) - advertised + if missing_commands: + passed = False + errors.append( + "l8k schema does not advertise required command(s): " + ", ".join(sorted(missing_commands)) + ) + if name == "version" and passed and expected_version: + actual_version = documents[0].get("version") + if actual_version != expected_version: + passed = False + errors.append(f"l8k version mismatch: expected {expected_version!r}, got {actual_version!r}") + checks[name] = { + "passed": passed, + "documents": documents, + "exit_code": result["exit_code"], + "artifacts": artifacts[name], + } + return {"checks": checks, "artifacts": artifacts}, not errors, "; ".join(errors) or None + + +def _download_installer(installer_ref: str, expected_sha256: str, artifact_dir: Path) -> tuple[Path, str]: + """Download an immutable installer and verify its trusted SHA-256 digest.""" + if not re.fullmatch(r"[0-9a-fA-F]{40}", installer_ref): + raise ValueError("Launch Kit installer_ref must be a full 40-character Git commit SHA") + if not re.fullmatch(r"[0-9a-fA-F]{64}", expected_sha256): + raise ValueError("Launch Kit installer_sha256 must be a 64-character SHA-256 digest") + expected_sha256 = expected_sha256.lower() + url = _INSTALLER_URL.format(ref=installer_ref.lower()) + installer = artifact_dir / "installer.sh" + installer.parent.mkdir(parents=True, exist_ok=True) + installer.unlink(missing_ok=True) + request = urllib.request.Request(url, headers={"User-Agent": "ai-cloud-validation"}) + with urllib.request.urlopen(request, timeout=30) as response: + content = response.read() + digest = hashlib.sha256(content).hexdigest() + verified = digest == expected_sha256 + _write_json( + artifact_dir / "installer-download.json", + { + "url": url, + "ref": installer_ref.lower(), + "expected_sha256": expected_sha256, + "sha256": digest, + "verified": verified, + }, + ) + if not verified: + raise ValueError(f"Launch Kit installer SHA-256 mismatch: expected {expected_sha256}, got {digest}") + installer.write_bytes(content) + return installer, url + + +def _installed_executable(prefix: str) -> Path: + """Resolve the executable installed by the official Launch Kit installer.""" + install_prefix = Path(prefix).expanduser() if prefix else Path("/usr/local") + return _resolve_executable(str(install_prefix / "bin" / "l8k")) + + +def _prepare(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + """Optionally install Launch Kit, then verify version and schema.""" + artifact_dir = Path(args.artifact_dir).expanduser().resolve() / "prepare" + environment = _environment(args.environment_json) + installed = False + install_details: dict[str, Any] | None = None + if args.mode == "install": + installer, url = _download_installer(args.installer_ref, args.installer_sha256, artifact_dir) + installer_argv = ["/bin/sh", str(installer)] + if args.prefix: + installer_argv.extend(["-d", args.prefix]) + env = environment.copy() + if args.version: + env["L8K_VERSION"] = args.version + result = _run_process(installer_argv, cwd=Path.cwd(), env=env) + install_artifacts = _record_process(artifact_dir / "install", installer_argv, result) + install_details = { + "url": url, + "exit_code": result["exit_code"], + "installer": str(installer.resolve()), + "download_metadata": str((artifact_dir / "installer-download.json").resolve()), + "artifacts": install_artifacts, + } + if result["exit_code"] != 0: + error = _stderr_excerpt(str(result["stderr"])) or "Launch Kit installer failed" + return { + "success": False, + "platform": "kubernetes", + "operation": "prepare", + "installed": False, + "install": install_details, + "error": error, + }, 1 + executable = _installed_executable(args.prefix) + installed = True + else: + executable = _resolve_executable(args.executable) + + verification, success, error = _verify_executable( + executable, + artifact_dir / "verify", + args.version, + environment, + ) + envelope: dict[str, Any] = { + "success": success, + "platform": "kubernetes", + "operation": "prepare", + "installed": installed, + "executable": str(executable), + **verification, + } + if install_details is not None: + envelope["install"] = install_details + envelope["artifacts"]["installer"] = install_details["installer"] + envelope["artifacts"]["installer_download"] = install_details["download_metadata"] + envelope["artifacts"]["install"] = install_details["artifacts"] + if error: + envelope["error"] = error + return envelope, 0 if success else 1 + + +def _verify(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + """Verify an existing Launch Kit executable without installing it.""" + executable = _resolve_executable(args.executable) + environment = _environment(args.environment_json) + verification, success, error = _verify_executable( + executable, + Path(args.artifact_dir).expanduser().resolve() / "verify-test", + args.expected_version, + environment, + ) + envelope: dict[str, Any] = { + "success": success, + "platform": "kubernetes", + "operation": "verify", + "executable": str(executable), + **verification, + } + if error: + envelope["error"] = error + return envelope, 0 if success else 1 + + +def _kubeconfig_from_workflow(raw: str) -> str | None: + """Extract one consistent explicit kubeconfig from the real workflow arguments.""" + workflow = _parse_json_value(raw, "--workflow-arguments-json", dict) + unsupported = set(workflow) - set(_WORKFLOW_COMMANDS) + if not workflow or unsupported: + supported = ", ".join(_WORKFLOW_COMMANDS) + raise ValueError(f"--workflow-arguments-json must contain a non-empty subset of: {supported}") + found: set[str] = set() + for command, values in workflow.items(): + if command not in _WORKFLOW_COMMANDS or not isinstance(values, list): + raise ValueError("--workflow-arguments-json must map Launch Kit workflow commands to argument lists") + if not all(isinstance(value, str) for value in values): + raise ValueError(f"workflow arguments for {command} must contain only strings") + index = 0 + while index < len(values): + token = values[index] + if token == "--kubeconfig": + if index + 1 >= len(values): + raise ValueError(f"{command} --kubeconfig requires a value") + value = values[index + 1] + if not value: + raise ValueError(f"{command} --kubeconfig requires a non-empty value") + found.add(value) + index += 2 + continue + if token.startswith("--kubeconfig="): + value = token.partition("=")[2] + if not value: + raise ValueError(f"{command} --kubeconfig requires a non-empty value") + found.add(value) + index += 1 + if len(found) > 1: + raise ValueError(f"Launch Kit workflow commands select different kubeconfigs: {sorted(found)}") + return next(iter(found), None) + + +def _kubectl_prefix(raw: str, environment: dict[str, str]) -> list[str]: + """Resolve the configured kubectl-compatible invocation.""" + supplied = _parse_json_value(raw, "--kubectl-command-json", list) + if supplied: + if not all(isinstance(value, str) and value for value in supplied): + raise ValueError("--kubectl-command-json must contain non-empty strings") + invocation_dir = Path.cwd() + return [ + str((invocation_dir / value).resolve()) + if not Path(value).is_absolute() and "/" in value and (invocation_dir / value).exists() + else value + for value in supplied + ] + override = environment.get("KUBECTL", "").strip() + return shlex.split(override) if override else ["kubectl"] + + +def _preflight_check( + name: str, + argv: list[str], + *, + cwd: Path, + artifact_dir: Path, + environment: dict[str, str], +) -> tuple[dict[str, Any], dict[str, Any]]: + """Run and record one Kubernetes prerequisite command.""" + result = _run_process(argv, cwd=cwd, env=environment) + artifacts = _record_process(artifact_dir / name, argv, result) + passed = result["exit_code"] == 0 + message = "command succeeded" if passed else (_stderr_excerpt(str(result["stderr"])) or "command failed") + return { + "name": name, + "passed": passed, + "message": message, + "exit_code": result["exit_code"], + "artifacts": artifacts, + }, result + + +def _preflight(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + """Prove that the Kubernetes API and at least one Ready node are available.""" + working_dir = Path(args.working_dir).expanduser().resolve() + working_dir.mkdir(parents=True, exist_ok=True) + artifact_dir = Path(args.artifact_dir).expanduser().resolve() / "kubernetes-preflight" + kubeconfig = _kubeconfig_from_workflow(args.workflow_arguments_json) + environment = _environment(args.environment_json) + prefix = _kubectl_prefix(args.kubectl_command_json, environment) + kubeconfig_args = ["--kubeconfig", kubeconfig] if kubeconfig else [] + + version_check, version_result = _preflight_check( + "api-version", + [*prefix, *kubeconfig_args, "version", "-o", "json"], + cwd=working_dir, + artifact_dir=artifact_dir, + environment=environment, + ) + nodes_check, nodes_result = _preflight_check( + "nodes", + [*prefix, *kubeconfig_args, "get", "nodes", "-o", "json"], + cwd=working_dir, + artifact_dir=artifact_dir, + environment=environment, + ) + + server_version: str | None = None + if version_check["passed"]: + try: + version_payload = json.loads(str(version_result["stdout"])) + server = version_payload.get("serverVersion") if isinstance(version_payload, dict) else None + if isinstance(server, dict) and isinstance(server.get("gitVersion"), str): + server_version = server["gitVersion"] + else: + version_check["passed"] = False + version_check["message"] = "kubectl output has no serverVersion.gitVersion" + except json.JSONDecodeError as exc: + version_check["passed"] = False + version_check["message"] = f"kubectl version output is invalid JSON: {exc}" + + total_nodes = 0 + ready_nodes = 0 + if nodes_check["passed"]: + try: + nodes_payload = json.loads(str(nodes_result["stdout"])) + items = nodes_payload.get("items") if isinstance(nodes_payload, dict) else None + if not isinstance(items, list): + raise ValueError("kubectl node output has no items list") + total_nodes = len(items) + ready_nodes = sum( + any( + isinstance(condition, dict) + and condition.get("type") == "Ready" + and condition.get("status") == "True" + for condition in (node.get("status", {}).get("conditions", []) if isinstance(node, dict) else []) + ) + for node in items + ) + except (json.JSONDecodeError, ValueError) as exc: + nodes_check["passed"] = False + nodes_check["message"] = str(exc) + + inventory_check = { + "name": "non-empty-cluster", + "passed": total_nodes > 0, + "message": f"found {total_nodes} node(s)" if total_nodes else "cluster contains no nodes", + } + readiness_check = { + "name": "ready-node", + "passed": ready_nodes > 0, + "message": f"found {ready_nodes}/{total_nodes} Ready node(s)" if ready_nodes else "cluster has no Ready nodes", + } + checks = [version_check, nodes_check, inventory_check, readiness_check] + success = all(check["passed"] is True for check in checks) + envelope: dict[str, Any] = { + "success": success, + "platform": "kubernetes", + "operation": "kubernetes-preflight", + "kubeconfig_source": "workflow arguments" if kubeconfig else "kubectl environment/default resolution", + "server_version": server_version, + "node_count": total_nodes, + "ready_node_count": ready_nodes, + "checks": checks, + "artifacts": { + "api_version": version_check["artifacts"], + "nodes": nodes_check["artifacts"], + }, + } + if not success: + failures = [f"{check['name']}: {check['message']}" for check in checks if check["passed"] is not True] + envelope["error"] = "Kubernetes prerequisite failed: " + "; ".join(failures) + envelope["remediation"] = "Select a reachable cluster and verify Kubernetes API and Ready-node access" + return envelope, 0 if success else 1 + + +def _parser() -> argparse.ArgumentParser: + """Build the provider command-line parser.""" + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="action", required=True) + + prepare = subparsers.add_parser("prepare", help="Install when requested, then verify l8k") + prepare.add_argument("--mode", choices=("verify", "install"), required=True) + prepare.add_argument("--executable", required=True) + prepare.add_argument("--version", default="") + prepare.add_argument("--installer-ref", default="") + prepare.add_argument("--installer-sha256", default="") + prepare.add_argument("--prefix", default="") + prepare.add_argument("--environment-json", default="{}") + prepare.add_argument("--artifact-dir", required=True) + + verify = subparsers.add_parser("verify", help="Verify l8k version and schema") + verify.add_argument("--executable", required=True) + verify.add_argument("--expected-version", default="") + verify.add_argument("--environment-json", default="{}") + verify.add_argument("--artifact-dir", required=True) + + preflight = subparsers.add_parser("preflight", help="Verify Kubernetes API and Ready-node access") + preflight.add_argument("--kubectl-command-json", default="[]") + preflight.add_argument("--workflow-arguments-json", required=True) + preflight.add_argument("--environment-json", default="{}") + preflight.add_argument("--working-dir", required=True) + preflight.add_argument("--artifact-dir", required=True) + + run = subparsers.add_parser("run", help="Run one real Launch Kit workflow command") + run.add_argument("--executable", required=True) + run.add_argument("--command", choices=_WORKFLOW_COMMANDS, required=True) + run.add_argument("--arguments-json", required=True) + run.add_argument("--user-config", default="") + run.add_argument("--environment-json", default="{}") + run.add_argument("--working-dir", required=True) + run.add_argument("--artifact-dir", required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + """Execute one provider operation and emit a single JSON envelope.""" + args = _parser().parse_args(argv) + try: + if args.action == "prepare": + envelope, exit_code = _prepare(args) + elif args.action == "verify": + envelope, exit_code = _verify(args) + elif args.action == "preflight": + envelope, exit_code = _preflight(args) + else: + envelope, exit_code = _run_workflow(args) + except (FileNotFoundError, OSError, TypeError, ValueError, urllib.error.URLError) as exc: + envelope = { + "success": False, + "platform": "kubernetes", + "operation": args.action, + "error": str(exc), + } + exit_code = 1 + print(json.dumps(envelope)) + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index 7b727cdbd..b74b64e7d 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -75,7 +75,11 @@ Suites: [`slurm`](slurm.yaml), [`control-plane`](control-plane.yaml), [`image-registry`](image-registry.yaml), -[`security`](security.yaml). +[`security`](security.yaml), +[`network-operator`](k8s-launch-kit/network-operator.yaml). +The Network Operator Launch Kit integration is unreleased; see the +[Launch Kit integration guide](../../../docs/guides/k8s-launch-kit/network-operator.md) +before running its cluster-mutating workflows. For the domain / script-count / AWS-reference overview see the [my-isv scaffold README](../providers/my-isv/scripts/README.md#domains). @@ -109,6 +113,11 @@ part that broke, and every member runs even after an earlier one fails. A member that needs parameters takes them inline (`- CheckName: {...}`); one that does not stays a single line. +If a member reports its own subtests, the composite forwards them as +`MemberName/probe-name`. Successful parents are summarized automatically by +subtest count in `isvctl` output; failures retain their complete diagnostic +message. This is shared renderer behavior, not a suite option. + Because a composite has no validation class to borrow from, it declares its own `description` (the catalog uses it) and its name must not shadow a class name. A check that wires one purpose-built class — `SerialConsoleCheck`, @@ -207,6 +216,67 @@ its plan item is not platform-scoped. | `switch_syslogs` | test | `providers/my-isv/scripts/observability/log_availability_test.py` | `tests.*.probes.switches_checked`, `log_source`, `entry_count`, `latest_timestamp` | | `switch_kernel_logs` | test | `providers/my-isv/scripts/observability/log_availability_test.py` | `tests.*.probes.switches_checked`, `log_source`, `entry_count`, `latest_timestamp` | +### Network Operator (`k8s-launch-kit/network-operator.yaml`, `k8s-launch-kit/network-operator-use-cases.yaml`) + +Plain suite for Kubernetes Launch Kit Network Operator self-validation. The +generic provider in `providers/k8s-launch-kit/config/provider.yaml` mirrors the real CLI as +separate verify, prerequisite, discover, generate, deploy, and validate steps. +It forwards user-supplied argument arrays and does not own Network Operator, +profile, topology, resource, or validation defaults. The suite binds fifteen +checks (one prerequisite plus fourteen currently supported PRD areas) directly +to the command output that proves them. + +GPUDirect RDMA is registered from Launch Kit's `gpudirect_dmabuf` result family; +the check skips when that family is disabled or not selected and fails on +emitted GPU topology or bandwidth errors. State restoration remains deferred +until Launch Kit provides the required snapshot/restore/verify workflow. + +`k8s-launch-kit/network-operator-use-cases.yaml` reuses those global check classes in six +separate composite tests: RoCE and InfiniBand across SR-IOV, RDMA Shared, and +host-device deployment modes. Each composite includes only checks applicable to +that use case, so unrelated fabric/deployment checks do not appear as skips in +the middle of a run. The Ethernet/RoCE composites carry `ethernet` and `roce`; +the InfiniBand composites carry `infiniband`. All six also carry `gpudirect` +because Launch Kit discovery decides whether the GPUDirect family is applicable. + +`providers/k8s-launch-kit/config/network-operator.yaml` is the production +entrypoint. It uses `l8k` and `kubectl` from `PATH` by default. In one invocation +it executes the six use-case phases sequentially, each with its own preflight, +discover, generate, deploy, validate, and evidence directories. The phases are +independent, so a failed case records a failed overall run but does not prevent +later cases from producing results. Mock executables exist only under +`isvctl/tests/providers/k8s_launch_kit/fixtures/` and are injected by tests. + +```bash +ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl test run \ + -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ + --capability kubernetes --no-upload -- -v +``` + +Add `--label ethernet` or `--label infiniband` before `--no-upload` to run only +that fabric's three workflows. Their steps use +`requires_selected_validations`, so the other fabric's mutating commands are +pruned before execution. Use `--label sriov`, `--label rdma_shared`, or +`--label host_device` to run the matching two-fabric deployment mode. Labels +compose, so `--label ethernet --label sriov` selects one use case. Omitting +labels runs all six. + +| Step | Phase | Script | Key JSON Fields | +|------|-------|--------|-----------------| +| `launch_kit_prepare` | setup | `providers/k8s-launch-kit/scripts/adapter.py prepare` | `installed`, `executable`, `checks.{version,schema}`, `artifacts` | +| `launch_kit_verify` | test | `providers/k8s-launch-kit/scripts/adapter.py verify` | `executable`, `checks.{version,schema}`, `artifacts` | +| `launch_kit_kubernetes_preflight` | test | `providers/k8s-launch-kit/scripts/adapter.py preflight` | `server_version`, `node_count`, `ready_node_count`, `checks`, `artifacts` | +| `launch_kit_discover` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k discover` | raw `documents`, `argv`, `exit_code`, `artifacts` | +| `launch_kit_generate` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k generate` | raw `documents`, `argv`, `exit_code`, `artifacts` | +| `launch_kit_deploy` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k deploy` | raw `documents` (currently empty on success), `argv`, `exit_code`, `artifacts` | +| `launch_kit_validate` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k validate` | raw static, connectivity, and report-path `documents`, `argv`, `exit_code`, `artifacts` | + +Those are the generic provider's single-workflow names. The grouped production configuration performs +prepare and verify in `setup`, then repeats the remaining five operations under +each custom use-case phase with names such as +`launch_kit_roce_sriov_preflight` through +`launch_kit_roce_sriov_validate`. + ### VM (`vm.yaml`) | Step | Phase | Script | Key JSON Fields | diff --git a/isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml b/isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml new file mode 100644 index 000000000..dce4bfff0 --- /dev/null +++ b/isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml @@ -0,0 +1,181 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Grouped Network Operator Launch Kit use cases for validation-only execution. +# +# The individual PRD checks remain catalogued in the adjacent +# network-operator.yaml. This +# suite composes those shared check implementations into six concrete profile +# tests. Providers bind each test to that profile's own validate step and pass +# the other real workflow outputs through the standard step context. + +version: "1.0" + +tests: + description: "Network Operator Kubernetes Launch Kit validation use cases" + + settings: + show_skipped_tests: false + + validations: + network_operator_roce_sriov: + step: launch_kit_roce_sriov_validate + checks: + EastWestNetworkRoceSriovCheck: + test_id: "K8S42-16" + labels: ["era", "ethernet", "gpudirect", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "roce", "slow", "sriov"] + requires: [kubernetes] + description: "Ethernet/RoCE with SR-IOV Network RDMA" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_roce_sriov_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_roce_sriov_discover | tojson }}" + generate_output: "{{ steps.launch_kit_roce_sriov_generate | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitRoceCheck + - LaunchKitSriovReadinessCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_infiniband_sriov: + step: launch_kit_infiniband_sriov_validate + checks: + EastWestNetworkInfiniBandSriovCheck: + test_id: "K8S42-17" + labels: ["era", "gpudirect", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "slow", "sriov"] + requires: [kubernetes] + description: "InfiniBand with SR-IOV Network RDMA" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_infiniband_sriov_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_infiniband_sriov_discover | tojson }}" + generate_output: "{{ steps.launch_kit_infiniband_sriov_generate | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitInfiniBandCheck + - LaunchKitSriovReadinessCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_roce_rdma_shared: + step: launch_kit_roce_rdma_shared_validate + checks: + EastWestNetworkRoceRdmaSharedCheck: + test_id: "K8S42-18" + labels: ["era", "ethernet", "gpudirect", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "rdma_shared", "roce", "slow"] + requires: [kubernetes] + description: "Ethernet/RoCE with the RDMA Shared Device Plugin" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_roce_rdma_shared_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_roce_rdma_shared_discover | tojson }}" + generate_output: "{{ steps.launch_kit_roce_rdma_shared_generate | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitRoceCheck + - LaunchKitRdmaSharedCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_infiniband_rdma_shared: + step: launch_kit_infiniband_rdma_shared_validate + checks: + EastWestNetworkInfiniBandRdmaSharedCheck: + test_id: "K8S42-19" + labels: ["era", "gpudirect", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "rdma_shared", "slow"] + requires: [kubernetes] + description: "InfiniBand/IPoIB with the RDMA Shared Device Plugin" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_infiniband_rdma_shared_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_infiniband_rdma_shared_discover | tojson }}" + generate_output: "{{ steps.launch_kit_infiniband_rdma_shared_generate | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitInfiniBandCheck + - LaunchKitRdmaSharedCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_roce_host_device: + step: launch_kit_roce_host_device_validate + checks: + EastWestNetworkRoceHostDeviceCheck: + test_id: "K8S42-20" + labels: ["era", "ethernet", "gpudirect", "host_device", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "roce", "slow"] + requires: [kubernetes] + description: "Ethernet/RoCE host-device networking for worker VMs" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_roce_host_device_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_roce_host_device_discover | tojson }}" + generate_output: "{{ steps.launch_kit_roce_host_device_generate | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitRoceCheck + - LaunchKitHostDeviceCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_infiniband_host_device: + step: launch_kit_infiniband_host_device_validate + checks: + EastWestNetworkInfiniBandHostDeviceCheck: + test_id: "K8S42-21" + labels: ["era", "gpudirect", "host_device", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "slow"] + requires: [kubernetes] + description: "InfiniBand host-device networking for worker VMs" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_infiniband_host_device_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_infiniband_host_device_discover | tojson }}" + generate_output: "{{ steps.launch_kit_infiniband_host_device_generate | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitInfiniBandCheck + - LaunchKitHostDeviceCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck diff --git a/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml b/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml new file mode 100644 index 000000000..9aa055f92 --- /dev/null +++ b/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Network Operator validation through the Kubernetes Launch Kit CLI. +# +# This suite contains catalog and result interpretation only. Providers own the +# command sequence and bind these checks to actual l8k command output. The +# generic provider is ../../providers/k8s-launch-kit/config/provider.yaml. + +version: "1.0" + +tests: + description: "Network Operator self-validation through Kubernetes Launch Kit" + + settings: + show_skipped_tests: false + + validations: + network_operator: + checks: + LaunchKitKubernetesPrerequisiteCheck: + step: launch_kit_kubernetes_preflight + test_id: "N/A" + labels: ["era", "kubernetes", "ncp", "network_operator", "prerequisite", "slow"] + requires: [kubernetes] + + LaunchKitDeploymentHealthCheck: + step: launch_kit_validate + test_id: "K8S42-01" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + + LaunchKitSriovReadinessCheck: + step: launch_kit_validate + test_id: "K8S42-02" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + discover_output: "{{ steps.launch_kit_discover | tojson }}" + + LaunchKitRdmaConnectivityCheck: + step: launch_kit_validate + test_id: "K8S42-03" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + + LaunchKitRoceCheck: + step: launch_kit_validate + test_id: "K8S42-04" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + discover_output: "{{ steps.launch_kit_discover | tojson }}" + + LaunchKitInfiniBandCheck: + step: launch_kit_validate + test_id: "K8S42-05" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + discover_output: "{{ steps.launch_kit_discover | tojson }}" + + LaunchKitHostDeviceCheck: + step: launch_kit_validate + test_id: "K8S42-06" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + discover_output: "{{ steps.launch_kit_discover | tojson }}" + + LaunchKitGpuDirectRdmaCheck: + step: launch_kit_validate + test_id: "K8S42-07" + labels: ["era", "gpudirect", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + + LaunchKitTopologyDiscoveryCheck: + step: launch_kit_discover + test_id: "K8S42-08" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + + LaunchKitSecondaryNetworkCheck: + step: launch_kit_validate + test_id: "K8S42-09" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + discover_output: "{{ steps.launch_kit_discover | tojson }}" + + LaunchKitRdmaSharedCheck: + step: launch_kit_validate + test_id: "K8S42-10" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + discover_output: "{{ steps.launch_kit_discover | tojson }}" + + LaunchKitIcmpConnectivityCheck: + step: launch_kit_validate + test_id: "K8S42-11" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + + LaunchKitRdmaBandwidthCheck: + step: launch_kit_validate + test_id: "K8S42-12" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + + LaunchKitMultirailCheck: + step: launch_kit_validate + test_id: "K8S42-13" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + discover_output: "{{ steps.launch_kit_discover | tojson }}" + + LaunchKitEvidenceCaptureCheck: + step: launch_kit_validate + test_id: "K8S42-15" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_kubernetes_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_discover | tojson }}" + generate_output: "{{ steps.launch_kit_generate | tojson }}" + deploy_output: "{{ steps.launch_kit_deploy | tojson }}" diff --git a/isvctl/src/isvctl/cli/test.py b/isvctl/src/isvctl/cli/test.py index 37a8fb3f8..91393df37 100644 --- a/isvctl/src/isvctl/cli/test.py +++ b/isvctl/src/isvctl/cli/test.py @@ -164,6 +164,23 @@ def _reported_capability(config: RunConfig, capability_context: str | None) -> s return capability_context +def _validation_result_detail(validation: dict[str, Any], reason: str | None = None) -> str: + """Return concise success text while preserving skip and failure diagnostics.""" + message = str(validation.get("message", "")) + summary = validation.get("subtest_summary") + is_success = validation.get("passed", False) and not validation.get("skipped") + if validation.get("state") != "error" and is_success and isinstance(summary, dict): + passed = int(summary.get("passed", 0) or 0) + failed = int(summary.get("failed", 0) or 0) + skipped = int(summary.get("skipped", 0) or 0) + total = int(summary.get("total", passed + failed + skipped) or 0) + if total > 0: + if passed == total: + return f"{total} subtests passed" + return f"{total} subtests: {passed} passed, {failed} failed, {skipped} skipped" + return f"{reason}: {message}" if reason and message else str(reason or message) + + def _human_readable_dry_run( config: RunConfig, capability: str | None, @@ -667,19 +684,28 @@ def run( typer.echo("ORCHESTRATION RESULTS") typer.echo("=" * 60) + show_skipped_tests = bool(config.tests and config.tests.settings.get("show_skipped_tests", False)) for phase_result in result.phases: + phase_details = phase_result.details or {} + displayed_validations = phase_details.get("validations", []) + if not show_skipped_tests: + displayed_validations = [ + validation for validation in displayed_validations if not validation.get("skipped") + ] + if phase_details.get("validations") and not displayed_validations and not phase_details.get("steps"): + continue if phase_result.message.startswith("SKIPPED:"): status = typer.style("[SKIP]", fg=typer.colors.YELLOW) elif phase_result.success: status = typer.style("[PASS]", fg=typer.colors.GREEN) else: status = typer.style("[FAIL]", fg=typer.colors.RED) - phase_name = phase_result.phase.value.upper().ljust(8) + phase_name = (phase_result.name or phase_result.phase.value).upper().ljust(24) typer.echo(f"{status} {phase_name}: {phase_result.message}") # Display step details (schema validation, errors) - if phase_result.details and "steps" in phase_result.details: - for step in phase_result.details["steps"]: + if "steps" in phase_details: + for step in phase_details["steps"]: step_name = step.get("name", "unknown") step_success = step.get("success", False) schema_valid = step.get("schema_valid", True) @@ -709,31 +735,28 @@ def run( typer.echo(f" Output: {json.dumps(output, indent=2)[:500]}") # Display centralized validation results - if phase_result.details and "validations" in phase_result.details: - validations = phase_result.details["validations"] - if validations: - for vr in validations: - vr_name = vr.get("name", "unknown") - # Handle case where name might be a dict (extract class name) - if isinstance(vr_name, dict): - vr_name = next(iter(vr_name.keys()), "unknown") - vr_message = vr.get("message", "") - vr_category = vr.get("category", "") - category_prefix = f"[{vr_category}] " if vr_category else "" - if vr.get("state") == "error": - vr_status = typer.style("ERROR", fg=typer.colors.RED) - reason = vr.get("error_reason") - elif vr.get("skipped"): - vr_status = typer.style("SKIPPED", fg=typer.colors.YELLOW) - reason = vr.get("skip_reason") - elif vr.get("passed", False): - vr_status = typer.style("PASSED", fg=typer.colors.GREEN) - reason = None - else: - vr_status = typer.style("FAILED", fg=typer.colors.RED) - reason = None - detail = f"{reason}: {vr_message}" if reason and vr_message else (reason or vr_message) - typer.echo(f" {category_prefix}{vr_name}: {vr_status} - {detail}") + if displayed_validations: + for vr in displayed_validations: + vr_name = vr.get("name", "unknown") + # Handle case where name might be a dict (extract class name) + if isinstance(vr_name, dict): + vr_name = next(iter(vr_name.keys()), "unknown") + vr_category = vr.get("category", "") + category_prefix = f"[{vr_category}] " if vr_category else "" + if vr.get("state") == "error": + vr_status = typer.style("ERROR", fg=typer.colors.RED) + reason = vr.get("error_reason") + elif vr.get("skipped"): + vr_status = typer.style("SKIPPED", fg=typer.colors.YELLOW) + reason = vr.get("skip_reason") + elif vr.get("passed", False): + vr_status = typer.style("PASSED", fg=typer.colors.GREEN) + reason = None + else: + vr_status = typer.style("FAILED", fg=typer.colors.RED) + reason = None + detail = _validation_result_detail(vr, reason) + typer.echo(f" {category_prefix}{vr_name}: {vr_status} - {detail}") typer.echo("-" * 60) if result.success: diff --git a/isvctl/src/isvctl/config/output_schemas.py b/isvctl/src/isvctl/config/output_schemas.py index 6a0e24525..99d72f324 100644 --- a/isvctl/src/isvctl/config/output_schemas.py +++ b/isvctl/src/isvctl/config/output_schemas.py @@ -1015,6 +1015,50 @@ "additionalProperties": True, "description": "Generic schema for unrecognized step names", }, + "k8s_launch_kit": { + "type": "object", + "required": ["success", "platform", "operation"], + "properties": { + **COMMON_PROPERTIES, + "operation": { + "type": "string", + "enum": [ + "prepare", + "verify", + "kubernetes-preflight", + "discover", + "generate", + "deploy", + "validate", + "clean", + ], + "description": "The actual Launch Kit or provider prerequisite operation", + }, + "executable": {"type": "string"}, + "argv": {"type": "array", "items": {"type": "string"}}, + "working_directory": { + "type": "string", + "description": "Absolute working directory used for the Launch Kit command", + }, + "exit_code": {"type": "integer"}, + "documents": { + "type": "array", + "items": {"type": "object"}, + "description": "Unmodified JSON documents emitted by l8k", + }, + "checks": { + "oneOf": [ + {"type": "object"}, + {"type": "array", "items": {"type": "object"}}, + ] + }, + "artifacts": {"type": "object"}, + "error": {"type": "string"}, + "remediation": {"type": "string"}, + }, + "additionalProperties": True, + "description": "Transport envelope around an unmodified Kubernetes Launch Kit CLI operation", + }, # ========================================================================= # Multi-cluster schemas # ========================================================================= diff --git a/isvctl/src/isvctl/config/schema.py b/isvctl/src/isvctl/config/schema.py index 7d4ad1ef1..83149fa8b 100644 --- a/isvctl/src/isvctl/config/schema.py +++ b/isvctl/src/isvctl/config/schema.py @@ -83,7 +83,10 @@ class StepConfig(BaseModel): default_factory=list, description="Command arguments (supports Jinja2 templating with {{ steps.prev_step.field }})", ) - timeout: int = Field(default=300, description="Timeout in seconds") + timeout: int | None = Field( + default=300, + description="Timeout in seconds; null disables the orchestration watchdog", + ) env: dict[str, str] = Field(default_factory=dict, description="Additional environment variables") working_dir: str | None = Field(default=None, description="Working directory for command execution") skip: bool = Field(default=False, description="Skip this step") @@ -93,7 +96,30 @@ class StepConfig(BaseModel): "Capability contexts allowed to run this step. Empty delegates capability gating to bound validations." ), ) + requires_available_validations: list[str] = Field( + default_factory=list, + description=( + "Validation names that must be available after release filtering for this step to run. " + "Unreleased validations are available only when ISVTEST_INCLUDE_UNRELEASED=1." + ), + ) + requires_selected_validations: list[str] = Field( + default_factory=list, + description=( + "Configured validation names that must be selected after release, capability, label, and suite " + "exclusion filtering for this step to run. A failed step is also reported as an error on these " + "owning validations." + ), + ) continue_on_failure: bool = Field(default=False, description="Continue to next step even if this step fails") + finalizer_for: str | None = Field( + default=None, + min_length=1, + description=( + "Step whose attempted execution activates this finalizer. A finalizer may be in the target phase " + "or in the teardown phase; it runs immediately after the target phase validations." + ), + ) phase: str = Field( default="setup", description="Phase this step belongs to: 'setup', 'test', or 'teardown'", @@ -153,11 +179,76 @@ class PlatformCommands(BaseModel): default_factory=lambda: ["setup", "teardown"], description="Ordered list of phases to execute. Steps are grouped by phase and run in this order.", ) + continue_after_failure: list[str] = Field( + default_factory=list, + description=( + "Phases whose failure must not prevent later phases from running. " + "Use this for independent test cases, never for prerequisite/setup phases." + ), + ) steps: list[StepConfig] = Field( default_factory=list, description="Sequential command steps grouped by phase", ) + @model_validator(mode="after") + def validate_continuation_phases(self) -> "PlatformCommands": + """Reject invalid continuation and linked-finalizer declarations.""" + if len(self.phases) != len(set(self.phases)): + raise ValueError("phases must not contain duplicate names") + unknown = [phase for phase in self.continue_after_failure if phase not in self.phases] + if unknown: + raise ValueError(f"continue_after_failure contains phases not listed in phases: {unknown}") + unsafe = [phase for phase in self.continue_after_failure if phase in {"setup", "teardown"}] + if unsafe: + raise ValueError(f"continue_after_failure cannot contain lifecycle phases: {unsafe}") + if len(self.continue_after_failure) != len(set(self.continue_after_failure)): + raise ValueError("continue_after_failure must not contain duplicate phase names") + + for finalizer in (step for step in self.steps if step.finalizer_for is not None): + targets = [step for step in self.steps if step.name == finalizer.finalizer_for] + if len(targets) != 1: + raise ValueError( + f"step '{finalizer.name}' finalizer_for must name exactly one configured step: " + f"{finalizer.finalizer_for!r}" + ) + target = targets[0] + target_phase = target.phase.lower() + finalizer_phase = finalizer.phase.lower() + if target_phase != finalizer_phase and finalizer_phase != "teardown": + raise ValueError( + f"step '{finalizer.name}' finalizer_for target '{target.name}' must be in the same phase " + "or the finalizer must use phase 'teardown'" + ) + if finalizer_phase == "teardown" and target_phase != finalizer_phase: + normalized_phases = [phase.lower() for phase in self.phases] + if "teardown" not in normalized_phases: + raise ValueError(f"step '{finalizer.name}' uses phase 'teardown', which is not listed in phases") + if target_phase not in normalized_phases: + raise ValueError( + f"step '{finalizer.name}' finalizer_for target '{target.name}' has an unknown phase" + ) + if normalized_phases.index(target_phase) >= normalized_phases.index("teardown"): + raise ValueError(f"step '{finalizer.name}' teardown must be ordered after target '{target.name}'") + if target.finalizer_for is not None: + raise ValueError(f"step '{finalizer.name}' cannot finalize finalizer step '{target.name}'") + gate_fields = ( + "requires", + "requires_available_validations", + "requires_selected_validations", + ) + mismatched_gates = [ + field_name + for field_name in gate_fields + if getattr(finalizer, field_name) != getattr(target, field_name) + ] + if mismatched_gates: + raise ValueError( + f"step '{finalizer.name}' must use the same gates as target '{target.name}': " + + ", ".join(mismatched_gates) + ) + return self + class KubernetesNodeOutput(BaseModel): """Schema for a single Kubernetes node in command output.""" diff --git a/isvctl/src/isvctl/config/suite_resolution.py b/isvctl/src/isvctl/config/suite_resolution.py index d0cfbbd75..8e9fdcf9f 100644 --- a/isvctl/src/isvctl/config/suite_resolution.py +++ b/isvctl/src/isvctl/config/suite_resolution.py @@ -40,7 +40,7 @@ def platform_vocabulary(configs_root: Path) -> frozenset[str]: several entry points ask for the vocabulary two or three times per run. """ platforms: set[str] = set() - for path in (configs_root / "suites").glob("*.yaml"): + for path in (configs_root / "suites").rglob("*.yaml"): try: data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} except (OSError, yaml.YAMLError): @@ -55,7 +55,7 @@ def platform_vocabulary(configs_root: Path) -> frozenset[str]: def suite_vocabulary(configs_root: Path) -> frozenset[str]: """Return plain suite names declared by canonical suite YAML.""" declarable = platform_vocabulary(configs_root) - names = {_normalize_name(path.stem) for path in (configs_root / "suites").glob("*.yaml")} + names = {_normalize_name(path.stem) for path in (configs_root / "suites").rglob("*.yaml")} return frozenset(names - declarable) @@ -162,7 +162,8 @@ def resolve_suite(provider: str | None, suite: str, *, configs_root: Path) -> Re # Best-effort per file: one malformed config must not fail `--suite` for the # whole provider. Matches `resolve_suite_name`, which already skips them. classified = [] - for path in sorted(config_dir.glob("*.yaml")): + pattern = "*.yaml" if provider is not None else "**/*.yaml" + for path in sorted(config_dir.glob(pattern)): try: classified.append((path, *_suite_name(path, declarable))) except SuiteResolutionError: diff --git a/isvctl/src/isvctl/doctor/checks/config.py b/isvctl/src/isvctl/doctor/checks/config.py index 386555691..f81f4e85d 100644 --- a/isvctl/src/isvctl/doctor/checks/config.py +++ b/isvctl/src/isvctl/doctor/checks/config.py @@ -46,7 +46,7 @@ def _check_repo_layout(root: Path) -> list[CheckResult]: results: list[CheckResult] = [] suites_dir = root / "isvctl" / "configs" / "suites" - suite_yamls = sorted(suites_dir.glob("*.yaml")) if suites_dir.is_dir() else [] + suite_yamls = sorted(suites_dir.rglob("*.yaml")) if suites_dir.is_dir() else [] if suite_yamls: results.append( CheckResult( diff --git a/isvctl/src/isvctl/orchestrator/commands.py b/isvctl/src/isvctl/orchestrator/commands.py index 373541343..b03177b54 100644 --- a/isvctl/src/isvctl/orchestrator/commands.py +++ b/isvctl/src/isvctl/orchestrator/commands.py @@ -30,6 +30,7 @@ from isvctl.config.schema import CommandConfig, CommandOutput from isvctl.orchestrator.context import _create_jinja_env +from isvctl.orchestrator.process import run_command_process from isvctl.redaction import mask_sensitive_args logger = logging.getLogger(__name__) @@ -132,12 +133,10 @@ def execute( logger.debug(f"Working directory: {cwd}") try: - result = subprocess.run( + result = run_command_process( cmd_parts, cwd=cwd, env=env, - capture_output=True, - text=True, timeout=config.timeout, ) diff --git a/isvctl/src/isvctl/orchestrator/loop.py b/isvctl/src/isvctl/orchestrator/loop.py index 6467b8f8a..242a49384 100644 --- a/isvctl/src/isvctl/orchestrator/loop.py +++ b/isvctl/src/isvctl/orchestrator/loop.py @@ -40,13 +40,14 @@ parse_validations, requirements_satisfied, resolve_entries, + resolve_entry_selection, ) from isvtest.main import run_validations_via_pytest from isvctl.config.schema import RunConfig, StepConfig from isvctl.orchestrator.commands import CommandExecutor from isvctl.orchestrator.context import Context -from isvctl.orchestrator.step_executor import StepExecutor, StepResults +from isvctl.orchestrator.step_executor import StepExecutor, StepResult, StepResults from isvctl.redaction import redact_dict, redact_junit_xml_tree logger = logging.getLogger(__name__) @@ -80,6 +81,7 @@ class PhaseResult: success: bool message: str details: dict[str, Any] | None = None + name: str | None = None @dataclass @@ -279,9 +281,74 @@ def _resolved_entry_to_result_dict(entry: ResolvedEntry) -> dict[str, Any]: "state": entry.state.value if entry.state else None, "skip_reason": entry.skip_reason.value if entry.skip_reason else None, "error_reason": entry.error_reason.value if entry.error_reason else None, + "subtest_summary": { + "total": entry.subtest_summary.total, + "passed": entry.subtest_summary.passed, + "failed": entry.subtest_summary.failed, + "skipped": entry.subtest_summary.skipped, + }, } +def _step_failure_message(result: StepResult) -> str: + """Return an operator-facing diagnostic for one failed workflow step.""" + detail = result.error + if not detail and result.schema_errors: + detail = f"output schema validation failed: {'; '.join(result.schema_errors)}" + if not detail: + detail = f"command exited with code {result.exit_code}" + return f"workflow step '{result.name}' failed: {detail}" + + +def _apply_owned_step_failures( + entries: list[ResolvedEntry], + phase_steps: list[StepConfig], + step_results: StepResults, +) -> list[ResolvedEntry]: + """Turn failed lifecycle steps into errors on their owning validations. + + ``requires_selected_validations`` is both the command-selection gate and + the explicit ownership edge between a workflow step and its selectable + tests. Without this propagation, an early step failure prevents the bound + validation step from producing output and JUnit incorrectly records a + harmless ``step_no_output`` skip. + """ + configs_by_name = {step.name: step for step in phase_steps} + errors_by_validation: dict[str, list[tuple[str, str]]] = {} + + for result in step_results.steps: + if result.success: + continue + step = configs_by_name.get(result.name) + if step is None: + continue + message = _step_failure_message(result) + for validation_name in step.requires_selected_validations: + errors_by_validation.setdefault(validation_name, []).append((step.name, message)) + + propagated: list[ResolvedEntry] = [] + for entry in entries: + step_errors = errors_by_validation.get(entry.entry.name, []) + if entry.is_ready: + # A failed validation-producing step can still return structured + # output that the validation interprets into failures/subtests. + # Only earlier owned lifecycle failures should suppress that run. + step_errors = [(name, message) for name, message in step_errors if name != entry.entry.step] + may_override = entry.is_ready or entry.skip_reason == SkipReason.STEP_NO_OUTPUT + if not step_errors or not may_override: + propagated.append(entry) + continue + propagated.append( + ResolvedEntry( + entry=entry.entry, + state=State.ERROR, + error_reason=ErrorReason.STEP_FAILED, + message="; ".join(message for _, message in step_errors), + ) + ) + return propagated + + def _resolved_entry_success(entry: ResolvedEntry) -> bool: """Return whether a resolved validation outcome should keep the phase successful.""" return entry.state in {State.PASSED, State.SKIPPED} @@ -301,8 +368,7 @@ def _requested_config_phases(config_phases: list[str], requested_phases: list[Ph if Phase.ALL in requested_phases: return config_phases - requested_phase_names = {phase.value for phase in requested_phases} - return [phase for phase in config_phases if phase in requested_phase_names] + return [phase for phase in config_phases if _phase_enum_for_name(phase) in requested_phases] def _has_explicit_pytest_selection(extra_pytest_args: list[str] | None) -> bool: @@ -314,6 +380,48 @@ def _has_explicit_pytest_selection(extra_pytest_args: list[str] | None) -> bool: ) +def _apply_selected_validation_gates( + steps: list[Any], + validation_entries: list[ValidationEntry], + *, + include_labels: set[str], + exclude_labels: set[str], + exclude_tests: set[str], + capability: str | None, +) -> list[Any]: + """Skip lifecycle steps whose required validations are not selected.""" + entries_by_name = {entry.name: entry for entry in validation_entries} + gated_steps: list[Any] = [] + for step in steps: + required_validations = getattr(step, "requires_selected_validations", []) + unselected: list[str] = [] + for validation_name in required_validations: + entry = entries_by_name.get(validation_name) + if entry is None: + unselected.append(f"{validation_name} (not configured)") + continue + result = resolve_entry_selection( + entry, + include_labels=include_labels, + exclude_labels=exclude_labels, + exclude_tests=exclude_tests, + capability=capability, + ) + if result is not None: + unselected.append(f"{validation_name} ({result.message})") + if not unselected: + gated_steps.append(step) + continue + skipped_step = step.model_copy(update={"skip": True}) + logger.info( + "Skipping step '%s' because required validation(s) are not selected: %s", + skipped_step.name, + "; ".join(unselected), + ) + gated_steps.append(skipped_step) + return gated_steps + + def _apply_capability_step_gates( steps: list[Any], validation_entries: list[ValidationEntry], @@ -473,6 +581,7 @@ def _run_steps_mode( phase=_phase_enum_for_name(phase_name), success=True, message=f"SKIPPED: platform '{platform}' is skipped by configuration", + name=phase_name, ) for phase_name in skipped_phases ], @@ -492,6 +601,9 @@ def _run_steps_mode( ], ) + continuation_phases = ( + set(self.config.commands[platform].continue_after_failure) if self.config.commands else set() + ) all_validations = {} if self.config.tests and self.config.tests.validations: all_validations = self.config.tests.validations @@ -510,6 +622,32 @@ def _run_steps_mode( ) ], ) + exclude_labels: list[str] = [] + exclude_tests: list[str] = [] + if self.config.tests and self.config.tests.exclude: + exclude_labels = self.config.tests.exclude.get("labels", []) + exclude_tests = self.config.tests.exclude.get("tests", []) + skip_config_label_exclusions = bool(self._include_labels) or _has_explicit_pytest_selection( + self._extra_pytest_args + ) + resolution_exclude_labels = set(self._exclude_labels) + if not skip_config_label_exclusions: + resolution_exclude_labels.update(exclude_labels) + + steps_before_selection = steps + steps = _apply_selected_validation_gates( + steps, + validation_entries, + include_labels=set(self._include_labels), + exclude_labels=resolution_exclude_labels, + exclude_tests=set(exclude_tests), + capability=self._capability, + ) + selection_skipped_steps = { + selected.name + for original, selected in zip(steps_before_selection, steps, strict=True) + if not original.skip and selected.skip + } steps = _apply_capability_step_gates(steps, validation_entries, self._capability) logger.info(f"Configured phases: {config_phases}") @@ -530,6 +668,8 @@ def _run_steps_mode( steps_by_phase: dict[str, list] = {phase: [] for phase in config_phases} for step in steps: + if step.name in selection_skipped_steps: + continue step_phase = (step.phase or "setup").lower() steps_by_phase[step_phase].append(step) @@ -544,25 +684,30 @@ def _run_steps_mode( step_phase = (step.phase or "setup").lower() self.context.set_step_phase(step.name, step_phase) - resolved_validations_by_index: dict[int, ResolvedEntry] = {} + configured_steps_by_name = {step.name: step for step in steps} + active_steps = [step for phase_steps in steps_by_phase.values() for step in phase_steps] + finalizers_by_target_phase: dict[str, list[StepConfig]] = {} + for finalizer in (step for step in active_steps if step.finalizer_for is not None): + target = configured_steps_by_name[finalizer.finalizer_for] + target_phase = (target.phase or "setup").lower() + finalizers_by_target_phase.setdefault(target_phase, []).append(finalizer) - exclude_labels: list[str] = [] - exclude_tests: list[str] = [] - if self.config.tests and self.config.tests.exclude: - exclude_labels = self.config.tests.exclude.get("labels", []) - exclude_tests = self.config.tests.exclude.get("tests", []) - skip_config_label_exclusions = bool(self._include_labels) or _has_explicit_pytest_selection( - self._extra_pytest_args - ) - resolution_exclude_labels = set(self._exclude_labels) - if not skip_config_label_exclusions: - resolution_exclude_labels.update(exclude_labels) + resolved_validations_by_index: dict[int, ResolvedEntry] = {} phase_results: list[PhaseResult] = [] overall_success = True + block_following_phases = False setup_steps_ran = False requested_phase_names = {p.value for p in requested_phases} + selected_config_phases = _requested_config_phases(config_phases, requested_phases) + selected_config_phase_names = set(selected_config_phases) + selected_finalizer_target_phases = selected_config_phase_names.intersection(finalizers_by_target_phase) + run_finalizers_as_teardown_recovery = ( + "teardown" in selected_config_phase_names and not selected_finalizer_target_phases + ) + attempted_step_names: set[str] = set() + executed_finalizer_names: set[str] = set() # Per-phase JUnit XML files merge at the end so later phases don't # overwrite earlier ones. @@ -574,15 +719,24 @@ def _run_steps_mode( junit_tmpdir = tempfile.mkdtemp(prefix="junit-phases-") for phase_name in config_phases: - if phase_name not in requested_phase_names and Phase.ALL not in requested_phases: + if phase_name not in selected_config_phase_names: continue - phase_steps = steps_by_phase.get(phase_name, []) + configured_phase_steps = steps_by_phase.get(phase_name, []) + phase_steps = [step for step in configured_phase_steps if step.finalizer_for is None] + declared_phase_finalizers = [step for step in configured_phase_steps if step.finalizer_for is not None] + if phase_name == "teardown" and run_finalizers_as_teardown_recovery: + phase_steps.extend(declared_phase_finalizers) + phase_finalizers = [ + step + for step in finalizers_by_target_phase.get(phase_name, []) + if step.name not in executed_finalizer_names + ] phase_enum = _phase_enum_for_name(phase_name) is_teardown = phase_name == "teardown" skip_reason: str | None = None - if not overall_success and not is_teardown: + if block_following_phases and not is_teardown: skip_reason = "previous phase failed" # Teardown gating depends on whether setup was part of this run: @@ -605,6 +759,7 @@ def _run_steps_mode( phase=phase_enum, success=True, message=f"SKIPPED: {skip_reason}", + name=phase_name, ) ) continue @@ -613,6 +768,7 @@ def _run_steps_mode( step_results = self.step_executor.execute_steps(phase_steps, self.context, best_effort=is_teardown) else: step_results = StepResults() + attempted_step_names.update(result.name for result in step_results.steps if result.attempted) # ``step_results.steps`` includes placeholder records for skip:true # steps; require at least one step that wasn't skipped before letting @@ -635,11 +791,16 @@ def _run_steps_mode( phase_entries = [validation_entries[index] for index in phase_entry_indexes] resolved_phase_entries = self._resolve_validation_entries( phase_entries, - requested_phase_names if Phase.ALL not in requested_phases else set(config_phases), + selected_config_phase_names, set(self._include_labels), resolution_exclude_labels, set(exclude_tests), ) + resolved_phase_entries = _apply_owned_step_failures( + resolved_phase_entries, + phase_steps, + step_results, + ) ready_entries = [entry for entry in resolved_phase_entries if entry.is_ready] terminal_before_pytest = [entry for entry in resolved_phase_entries if not entry.is_ready] @@ -686,14 +847,54 @@ def _run_steps_mode( phase_validations = [_resolved_entry_to_result_dict(entry) for entry in terminal_phase_entries] - if phase_steps or phase_validations: + if step_results.steps or phase_validations: phase_results.append( self._create_phase_result(phase_enum, step_results, phase_validations, phase_name) ) + eligible_finalizers = [step for step in phase_finalizers if step.finalizer_for in attempted_step_names] + for finalizer in phase_finalizers: + if finalizer not in eligible_finalizers: + logger.info( + "Skipping finalizer '%s': target step '%s' was not attempted", + finalizer.name, + finalizer.finalizer_for, + ) + finalizer_results = self.step_executor.execute_steps( + eligible_finalizers, + self.context, + best_effort=True, + ) + executed_finalizer_names.update(finalizer.name for finalizer in eligible_finalizers) + if eligible_finalizers: + phase_results.append( + self._create_phase_result( + Phase.TEARDOWN, + finalizer_results, + [], + f"{phase_name}-teardown", + ) + ) + elif phase_finalizers: + target_names = ", ".join(finalizer.finalizer_for or "unknown" for finalizer in phase_finalizers) + phase_results.append( + PhaseResult( + phase=Phase.TEARDOWN, + success=True, + message=f"SKIPPED: target step(s) were not attempted: {target_names}", + details={"steps": [], "validations": []}, + name=f"{phase_name}-teardown", + ) + ) + phase_success = step_results.success and all(v.get("passed", False) for v in phase_validations) if not phase_success: overall_success = False + if phase_name not in continuation_phases: + block_following_phases = True + if not finalizer_results.success: + overall_success = False + block_following_phases = True remaining_entries = [ (index, entry) @@ -703,7 +904,7 @@ def _run_steps_mode( if remaining_entries: terminal_remaining = self._resolve_remaining_validation_entries( remaining_entries, - requested_phase_names if Phase.ALL not in requested_phases else set(config_phases), + selected_config_phase_names, set(self._include_labels), resolution_exclude_labels, set(exclude_tests), @@ -796,6 +997,7 @@ def _create_phase_result( { "name": s.name, "success": s.success, + "attempted": s.attempted, "error": s.error, "output": redact_dict(s.output), "schema_name": s.schema_name, @@ -806,6 +1008,7 @@ def _create_phase_result( ], "validations": validation_results, }, + name=display_name, ) def _resolve_validation_entries( @@ -889,6 +1092,7 @@ def _append_resolution_only_phase_results( "steps": [], "validations": [_resolved_entry_to_result_dict(entry) for entry in resolved_entries], }, + name=phase_name, ) ) diff --git a/isvctl/src/isvctl/orchestrator/process.py b/isvctl/src/isvctl/orchestrator/process.py new file mode 100644 index 000000000..ab39123ac --- /dev/null +++ b/isvctl/src/isvctl/orchestrator/process.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Subprocess execution shared by orchestration command models.""" + +import os +import signal +import subprocess +from collections.abc import Mapping, Sequence +from pathlib import Path + +_TERMINATION_GRACE_SECONDS = 2.0 + + +def run_command_process( + args: Sequence[str], + *, + cwd: str | Path, + env: Mapping[str, str] | None, + timeout: float | None, +) -> subprocess.CompletedProcess[str]: + """Run an orchestration command and terminate its process group on timeout. + + Orchestration steps commonly invoke wrappers which then start a provider + CLI. Killing only the wrapper can leave that CLI running after the step is + reported as timed out. On POSIX, every command therefore starts in a new + session and timeout handling signals the whole process group. Other + platforms fall back to terminating the direct child process. + + Args: + args: Command and arguments to execute without a shell. + cwd: Working directory for the command. + env: Complete process environment, or ``None`` to inherit it. + timeout: Maximum execution time in seconds, or ``None`` for no limit. + + Returns: + The completed process with captured text stdout and stderr. + + Raises: + subprocess.TimeoutExpired: The command exceeded ``timeout``. Captured + stdout and stderr are attached after the process tree is stopped. + OSError: The command could not be started. + """ + command = list(args) + process = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=os.name == "posix", + ) + + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + _terminate_process_tree(process) + try: + stdout, stderr = process.communicate(timeout=_TERMINATION_GRACE_SECONDS) + except subprocess.TimeoutExpired: + _kill_process_tree(process) + stdout, stderr = process.communicate() + else: + # The direct child may exit while a descendant that closed the + # inherited pipes remains alive. Ensure the process group is gone. + _kill_process_tree(process) + + raise subprocess.TimeoutExpired(command, timeout, output=stdout, stderr=stderr) from None + + return subprocess.CompletedProcess(command, process.returncode, stdout, stderr) + + +def _terminate_process_tree(process: subprocess.Popen[str]) -> None: + """Request graceful termination of a process group or direct process.""" + try: + if os.name == "posix": + os.killpg(process.pid, signal.SIGTERM) + else: + process.terminate() + except ProcessLookupError: + pass + + +def _kill_process_tree(process: subprocess.Popen[str]) -> None: + """Force termination of a process group or direct process.""" + try: + if os.name == "posix": + os.killpg(process.pid, signal.SIGKILL) + else: + process.kill() + except ProcessLookupError: + pass diff --git a/isvctl/src/isvctl/orchestrator/step_executor.py b/isvctl/src/isvctl/orchestrator/step_executor.py index 9f8945e44..d4feacc48 100644 --- a/isvctl/src/isvctl/orchestrator/step_executor.py +++ b/isvctl/src/isvctl/orchestrator/step_executor.py @@ -60,6 +60,7 @@ from isvctl.config.output_schemas import get_schema_for_step, validate_output from isvctl.config.schema import StepConfig from isvctl.orchestrator.context import Context, _create_jinja_env +from isvctl.orchestrator.process import run_command_process from isvctl.redaction import mask_sensitive_args, redact_text logger = logging.getLogger(__name__) @@ -199,6 +200,7 @@ class StepResult: schema_errors: Schema validation error messages validation_results: Results from bound validations error: Error message if step failed + attempted: Whether the command process was actually started """ name: str @@ -212,6 +214,7 @@ class StepResult: schema_errors: list[str] = field(default_factory=list) validation_results: list[dict[str, Any]] = field(default_factory=list) error: str | None = None + attempted: bool = True @dataclass @@ -288,6 +291,7 @@ def execute_steps( stdout="", stderr="", error="Step skipped", + attempted=False, ) ) continue @@ -335,6 +339,7 @@ def _execute_step(self, step: StepConfig, context: Context) -> StepResult: stdout="", stderr="", error=f"Skipped: missing step reference steps.{e.missing_path}", + attempted=False, ) # Normalize command - replace python/python3 with current interpreter @@ -378,12 +383,10 @@ def _execute_step(self, step: StepConfig, context: Context) -> StepResult: logger.debug(f"Working directory: {cwd}") try: - result = subprocess.run( + result = run_command_process( cmd_parts, cwd=cwd, env=env, - capture_output=True, - text=True, timeout=step.timeout, ) @@ -438,6 +441,7 @@ def _execute_step(self, step: StepConfig, context: Context) -> StepResult: stdout="", stderr="", error=f"Command not found: {step.command}", + attempted=False, ) except Exception as e: return StepResult( diff --git a/isvctl/tests/providers/k8s_launch_kit/__init__.py b/isvctl/tests/providers/k8s_launch_kit/__init__.py new file mode 100644 index 000000000..77ba303a9 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the Kubernetes Launch Kit provider.""" diff --git a/isvctl/tests/providers/k8s_launch_kit/fixtures/launch_kit_scenarios.json b/isvctl/tests/providers/k8s_launch_kit/fixtures/launch_kit_scenarios.json new file mode 100644 index 000000000..c1fe18205 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/fixtures/launch_kit_scenarios.json @@ -0,0 +1,104 @@ +{ + "_copyright": "Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.", + "_license": "Apache-2.0", + "contract": { + "source_commit": "db32e4b98170", + "source_files": [ + "pkg/config/config.go", + "pkg/ui/json_output.go", + "pkg/cmd/clean.go", + "pkg/cmd/deploy.go", + "pkg/cmd/discover.go", + "pkg/cmd/generate.go", + "pkg/cmd/validate.go", + "pkg/networkoperatorplugin/clean.go", + "pkg/networkoperatorplugin/validate.go", + "pkg/networkoperatorplugin/connectivity/connectivity.go", + "pkg/networkoperatorplugin/connectivity/daemonset.go", + "pkg/networkoperatorplugin/connectivity/matrix.go", + "pkg/networkoperatorplugin/connectivity/result.go", + "pkg/networkoperatorplugin/connectivity/rdma.go", + "pkg/networkoperatorplugin/discovery/discover.go", + "profiles/host-device-rdma/40-example-daemonset.yaml", + "profiles/ipoib-rdma-shared/40-example-daemonset.yaml", + "profiles/macvlan-rdma-shared/40-example-daemonset.yaml", + "profiles/sriov-ethernet-rdma/60-example-daemonset.yaml", + "profiles/sriov-ib-rdma/60-example-daemonset.yaml" + ], + "supported_validation_checks": [ + "icmp", + "rping", + "ib_write_bw" + ], + "result_families": [ + "icmp", + "rping", + "ib_write_bw", + "gpudirect_dmabuf" + ], + "notes": [ + "discover and generate emit one ui.JSONResult object", + "successful standalone deploy emits no stdout object", + "validate emits manifest, connectivity, and reportPath JSON documents", + "connectivity DaemonSets declare test-container for RDMA and netshoot for ICMP", + "enabled GPUDirect validation follows ib_write_bw and emits a distinct gpudirect_dmabuf result family", + "clean emits one ui.JSONResult object with a cleanup summary" + ] + }, + "scenarios": { + "roce-sriov": { + "fabric": "ethernet", + "deployment": "sriov", + "profile_name": "SR-IOV Ethernet RDMA", + "network_kind": "SriovNetwork", + "network_api_version": "sriovnetwork.openshift.io/v1", + "requires_sriov": true, + "requires_ib": false + }, + "infiniband-sriov": { + "fabric": "infiniband", + "deployment": "sriov", + "profile_name": "SR-IOV Infiniband RDMA", + "network_kind": "SriovIBNetwork", + "network_api_version": "sriovnetwork.openshift.io/v1", + "requires_sriov": true, + "requires_ib": true + }, + "roce-rdma-shared": { + "fabric": "ethernet", + "deployment": "rdma_shared", + "profile_name": "Macvlan with RDMA shared device", + "network_kind": "MacvlanNetwork", + "network_api_version": "mellanox.com/v1alpha1", + "requires_sriov": false, + "requires_ib": false + }, + "infiniband-rdma-shared": { + "fabric": "infiniband", + "deployment": "rdma_shared", + "profile_name": "IP over Infiniband with RDMA shared device", + "network_kind": "IPoIBNetwork", + "network_api_version": "mellanox.com/v1alpha1", + "requires_sriov": false, + "requires_ib": true + }, + "roce-host-device": { + "fabric": "ethernet", + "deployment": "host_device", + "profile_name": "Host device RDMA", + "network_kind": "HostDeviceNetwork", + "network_api_version": "mellanox.com/v1alpha1", + "requires_sriov": false, + "requires_ib": false + }, + "infiniband-host-device": { + "fabric": "infiniband", + "deployment": "host_device", + "profile_name": "Host device RDMA", + "network_kind": "HostDeviceNetwork", + "network_api_version": "mellanox.com/v1alpha1", + "requires_sriov": false, + "requires_ib": true + } + } +} diff --git a/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_kubectl.py b/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_kubectl.py new file mode 100755 index 000000000..737863593 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_kubectl.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Small kubectl-compatible test double for Launch Kit prerequisite checks.""" + +from __future__ import annotations + +import json +import os +import sys + + +def main(argv: list[str] | None = None) -> int: + """Return Kubernetes version or Ready-node JSON for the requested command.""" + args = list(sys.argv[1:] if argv is None else argv) + if os.environ.get("L8K_MOCK_KUBERNETES_FAIL") == "1": + print("Unable to connect to the server: connection refused", file=sys.stderr) + return 1 + expected_kubeconfig = os.environ.get("L8K_MOCK_EXPECT_KUBECONFIG") + if expected_kubeconfig and os.environ.get("KUBECONFIG") != expected_kubeconfig: + print( + f"expected KUBECONFIG={expected_kubeconfig!r}, got {os.environ.get('KUBECONFIG')!r}", + file=sys.stderr, + ) + return 1 + if "version" in args: + print( + json.dumps( + { + "clientVersion": {"gitVersion": "v1.34.1"}, + "serverVersion": {"gitVersion": "v1.34.1"}, + } + ) + ) + return 0 + if "get" in args and "nodes" in args: + print( + json.dumps( + { + "apiVersion": "v1", + "items": [ + { + "metadata": {"name": "worker-a"}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + }, + { + "metadata": {"name": "worker-b"}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + }, + ], + } + ) + ) + return 0 + print(f"mock kubectl does not support: {' '.join(args)}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py b/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py new file mode 100755 index 000000000..dcf5a41a4 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py @@ -0,0 +1,706 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Executable Launch Kit test double for provider unit tests. + +Values in this file are fixed mock output, not AI Cloud Validation defaults. +The provider passes only real l8k arguments and receives the same distinct +stdout forms used by version, schema, discover, generate, deploy, validate, and clean. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path +from typing import Any + +import yaml + +_TIMESTAMP = "2026-08-05T08:27:43Z" +_SOURCE_COMMIT = "db32e4b98170" +_FIXTURE = Path(__file__).with_name("launch_kit_scenarios.json") +_VALUE_FLAGS: dict[str, set[str]] = { + "version": {"--output"}, + "discover": { + "--kubeconfig", + "--save-cluster-config", + "--user-config", + "--fabric", + "--deployment-type", + "--multirail", + "--node-selector", + "--network-operator-release", + "--output", + }, + "generate": { + "--user-config", + "--save-deployment-files", + "--network-operator-namespace", + "--output", + }, + "deploy": { + "--kubeconfig", + "--user-config", + "--deployment-files", + "--network-operator-namespace", + "--deploy-timeout", + "--output", + }, + "validate": { + "--kubeconfig", + "--user-config", + "--deployment-files", + "--network-operator-namespace", + "--connectivity", + "--connectivity-timeout", + "--validation-mode", + "--validation-checks", + "--rdma-rping-iterations", + "--rdma-ib-write-size", + "--rdma-ib-write-min-bandwidth-gbps", + "--wait", + "--report-path", + "--output", + }, + "clean": { + "--kubeconfig", + "--user-config", + "--network-operator-namespace", + "--keep-helm-chart", + "--output", + }, +} +_BOOLEAN_FLAGS: dict[str, set[str]] = { + "clean": {"--keep-helm-chart"}, +} + + +def _load_fixture() -> dict[str, Any]: + """Load the pinned mock contract and scenario definitions.""" + value = json.loads(_FIXTURE.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("Launch Kit fixture must contain an object") + return value + + +def _parse_flags(command: str, argv: list[str]) -> dict[str, str]: + """Parse the real flag subset exercised by the provider tests.""" + supported = _VALUE_FLAGS[command] + values: dict[str, str] = {} + index = 0 + while index < len(argv): + token = argv[index] + if not token.startswith("--"): + raise ValueError(f"unexpected positional argument: {token}") + if "=" in token: + flag, value = token.split("=", 1) + else: + flag = token + if flag in _BOOLEAN_FLAGS.get(command, set()): + value = "true" + else: + index += 1 + if index >= len(argv): + raise ValueError(f"missing value for {flag}") + value = argv[index] + if flag not in supported: + raise ValueError(f"unknown flag for l8k {command}: {flag}") + values[flag] = value + index += 1 + return values + + +def _emit(value: dict[str, Any], *, pretty: bool = False) -> None: + """Write one JSON document to stdout.""" + print(json.dumps(value, indent=2 if pretty else None)) + + +def _message(level: str, message: str) -> dict[str, str]: + """Build one ui.LogEntry-compatible record.""" + return {"level": level, "message": message, "timestamp": _TIMESTAMP} + + +def _profile(scenario: dict[str, Any]) -> dict[str, str]: + """Return the profile fields emitted by Launch Kit.""" + return { + "deployment": str(scenario["deployment"]), + "fabric": str(scenario["fabric"]), + "ignoreARP": "false", + "multirail": "true", + "routing": "destination-based", + } + + +def _json_result( + phase: str, + *, + profile: dict[str, str] | None = None, + generated_files: list[str] | None = None, +) -> dict[str, Any]: + """Build a successful ui.JSONResult-compatible object.""" + value: dict[str, Any] = { + "success": True, + "phase": phase, + "deployed": False, + "messages": [ + _message("info", f"Running {phase}"), + _message("success", "Workflow completed successfully"), + ], + } + if profile is not None: + value["profile"] = profile + if generated_files: + value["generatedFiles"] = generated_files + return value + + +def _structured_error(command: str, message: str) -> tuple[dict[str, Any], int]: + """Build the JSON error emitted by a failed Launch Kit command.""" + category = "cluster" if command == "discover" else "deployment" if command == "deploy" else "validation" + exit_code = 3 if category == "cluster" else 4 if category == "deployment" else 2 + return { + "success": False, + "phase": "", + "deployed": False, + "error": { + "code": f"{category.upper()}_ERROR", + "message": message, + "category": category, + "transient": category == "cluster", + "suggestion": "Inspect the preserved Launch Kit logs and correct the reported condition", + }, + "messages": None, + }, exit_code + + +def _scenario_for_profile(fabric: str, deployment: str) -> tuple[str, dict[str, Any]]: + """Find fixture data for one explicit profile.""" + scenarios = _load_fixture().get("scenarios") + if not isinstance(scenarios, dict): + raise ValueError("Launch Kit fixture has no scenarios map") + for name, scenario in scenarios.items(): + if isinstance(scenario, dict) and scenario.get("fabric") == fabric and scenario.get("deployment") == deployment: + return str(name), scenario + raise ValueError(f"unsupported mock profile: fabric={fabric!r}, deployment={deployment!r}") + + +def _load_cluster_config(path: Path) -> dict[str, Any]: + """Load a cluster configuration produced by the mock discovery command.""" + value = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path} must contain a YAML object") + return value + + +def _scenario_from_config(flags: dict[str, str]) -> tuple[str, dict[str, Any]]: + """Resolve a scenario from the persisted Launch Kit profile.""" + raw = flags.get("--user-config") + if not raw: + deployment = Path(flags.get("--deployment-files", "deployment")) + raw = str(deployment.parent / "cluster-config.yaml") + config = _load_cluster_config(Path(raw)) + profile = config.get("profile") + if not isinstance(profile, dict): + raise ValueError("cluster config has no profile") + return _scenario_for_profile(str(profile.get("fabric", "")), str(profile.get("deployment", ""))) + + +def _cluster_config_yaml(scenario: dict[str, Any]) -> str: + """Render representative output from l8k discovery.""" + return f"""# Generated by `l8k discover`. +networkOperator: + selectedRelease: "26.4" + version: v26.4.1 + namespace: nvidia-network-operator +validation: + gpuDirect: + enabled: true + gpuResourceType: nvidia.com/gpu + connectivity: true + mode: strict + checks: [icmp, rping, ib_write_bw] + rdma: + rpingIterations: 5 + ibWriteSize: 65536 + ibWriteMinBandwidthGbps: 100 +profile: + fabric: {scenario["fabric"]} + deployment: {scenario["deployment"]} + multirail: true + routing: destination-based +clusterConfig: + - identifier: mock-group + machineType: mock-vm + gpuType: NVIDIA-H100-80GB-HBM3 + workerNodes: [worker-a, worker-b] + nodeSelector: + feature.node.kubernetes.io/pci-15b3.present: "true" + pfs: + - networkInterface: ens5f0np0 + pciAddress: "0000:17:00.0" + rdmaDevice: mlx5_0 + traffic: east-west + rail: 0 + connectedGPU: GPU0 + connectedGPUPCIAddress: "0000:41:00.0" + - networkInterface: ens6f0np0 + pciAddress: "0000:31:00.0" + rdmaDevice: mlx5_1 + traffic: east-west + rail: 1 + connectedGPU: GPU1 + connectedGPUPCIAddress: "0000:71:00.0" +""" + + +def _resource_name(kind: str, rail: int) -> str: + """Return a deterministic resource name for generated mock manifests.""" + prefix = { + "IPPool": "nv-ipam-pool", + "SriovNetwork": "sriov-network", + "SriovIBNetwork": "sriov-ib-network", + "MacvlanNetwork": "macvlan-network", + "IPoIBNetwork": "ipoib-network", + "HostDeviceNetwork": "hostdev-network", + "SriovNetworkNodePolicy": "sriov-policy", + "NicNodePolicy": "nic-node-policy", + }.get(kind, kind.lower()) + return f"{prefix}-rail-{rail}-mock-group" + + +def _manifest_specs(scenario: dict[str, Any]) -> list[tuple[str, str, str, int]]: + """Return generated API/kind/file/count tuples for one profile.""" + specs = [ + ("mellanox.com/v1alpha1", "NicClusterPolicy", "10-nic-cluster-policy", 1), + ("configuration.net.nvidia.com/v1alpha1", "NicNodePolicy", "20-nic-node-policy", 1), + ("nv-ipam.nvidia.com/v1alpha1", "IPPool", "30-ip-pool", 2), + ] + if scenario.get("requires_sriov"): + specs.append(("sriovnetwork.openshift.io/v1", "SriovNetworkNodePolicy", "40-sriov-policy", 2)) + specs.append( + ( + str(scenario["network_api_version"]), + str(scenario["network_kind"]), + "50-secondary-network", + 2, + ) + ) + return specs + + +def _manifest_yaml(api_version: str, kind: str, count: int) -> str: + """Render a valid multi-document mock manifest.""" + documents: list[str] = [] + for rail in range(count): + name = "nic-cluster-policy" if kind == "NicClusterPolicy" else _resource_name(kind, rail) + namespace = "" if kind in {"NicClusterPolicy", "NicNodePolicy"} else " namespace: default\n" + documents.append( + f"apiVersion: {api_version}\nkind: {kind}\nmetadata:\n name: {name}\n{namespace}spec:\n mock: true\n" + ) + return "---\n".join(documents) + + +def _write_generated_files(root: Path, scenario: dict[str, Any]) -> list[str]: + """Materialize profile manifests and return their absolute paths.""" + manifest_dir = root / "network-operator" + manifest_dir.mkdir(parents=True, exist_ok=True) + generated: list[Path] = [] + values = manifest_dir / "values.yaml" + values.write_text("operator:\n namespace: nvidia-network-operator\n", encoding="utf-8") + generated.append(values) + for api_version, kind, stem, count in _manifest_specs(scenario): + path = manifest_dir / f"{stem}.yaml" + path.write_text(_manifest_yaml(api_version, kind, count), encoding="utf-8") + generated.append(path) + example = manifest_dir / "60-example-daemonset-mock-group.yaml" + example.write_text( + "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: l8k-network-test\n namespace: default\n" + "spec:\n" + " selector:\n" + " matchLabels:\n" + " app: l8k-network-test\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: l8k-network-test\n" + " spec:\n" + " containers:\n" + " - name: test-container\n" + " image: nvcr.io/nvidia/doca/doca:3.3.0-full-rt-host\n" + " command: [/bin/bash, -c, sleep infinity]\n" + " resources:\n" + " requests:\n" + " nvidia.com/gpu: '2'\n" + " limits:\n" + " nvidia.com/gpu: '2'\n" + " - name: netshoot\n" + " image: nicolaka/netshoot:latest\n" + " command: [/bin/bash, -c, sleep infinity]\n", + encoding="utf-8", + ) + generated.append(example) + return [str(path.resolve()) for path in generated] + + +def _manifest_results(scenario: dict[str, Any]) -> list[dict[str, Any]]: + """Build the exported Launch Kit manifest-validation results.""" + results: list[dict[str, Any]] = [] + for api_version, kind, stem, count in _manifest_specs(scenario): + for rail in range(count): + name = "nic-cluster-policy" if kind == "NicClusterPolicy" else _resource_name(kind, rail) + reason = "resource exists and is Ready" + results.append( + { + "Kind": kind, + "APIVersion": api_version, + "Name": name, + "Namespace": "" if kind in {"NicClusterPolicy", "NicNodePolicy"} else "default", + "SourceFile": f"{stem}.yaml", + "State": "success", + "Reason": reason, + "Details": {}, + "Found": True, + "Missing": False, + "Detail": reason, + } + ) + return results + + +def _ping_result( + kind: int, + src_node: str, + dst_node: str, + src_rail: str, + dst_rail: str, + fail_family: str | None, +) -> dict[str, Any]: + """Build one exported connectivity matrix row.""" + family = "icmp" if kind < 2 else "rping" if kind < 4 else "ib_write_bw" if kind < 6 else "gpudirect_dmabuf" + bandwidth_family = family in {"ib_write_bw", "gpudirect_dmabuf"} + cross_rail = src_rail != dst_rail + expectation = "forbidden" if cross_rail else "required" + observed_ok = not cross_rail + ok = True + stderr = "" + stdout = "" + bandwidth = 0.0 + if family == "icmp": + stdout = "1 packets transmitted, 1 received" if observed_ok else "" + stderr = "Network is unreachable" if cross_rail else "" + elif family == "rping": + stdout = "client DISCONNECT EVENT" if observed_ok else "" + stderr = "rping: connection timed out" if cross_rail else "" + elif observed_ok: + bandwidth = 191.25 if family == "gpudirect_dmabuf" else 187.6 + stdout = f"65536 5000 {bandwidth:.2f} {bandwidth:.2f} 0.3578" + else: + stderr = "ib_write_bw: failed to connect" + + if fail_family == family and not cross_rail and src_node == "worker-a" and src_rail == "rail-0": + ok = False + observed_ok = False + if bandwidth_family: + bandwidth = 42.5 + stderr = "observed bandwidth 42.5 Gbps below minimum 100 Gbps" + else: + stderr = f"{family}: connection refused" + + test = { + "Kind": kind, + "SrcPod": f"network-test-{src_node}", + "DstPod": f"network-test-{dst_node}", + "SrcNode": src_node, + "DstNode": dst_node, + "Rail": src_rail if not cross_rail else f"{src_rail}→{dst_rail}", + "SrcIP": "192.168.128.10" if src_node == "worker-a" else "192.168.128.11", + "DstIP": "192.168.128.11" if dst_node == "worker-b" else "192.168.128.10", + "SrcRail": src_rail, + "DstRail": dst_rail, + "SrcIface": "net1" if src_rail == "rail-0" else "net2", + "DstIface": "net1" if dst_rail == "rail-0" else "net2", + "SrcRDMADev": "mlx5_0" if src_rail == "rail-0" else "mlx5_1", + "DstRDMADev": "mlx5_0" if dst_rail == "rail-0" else "mlx5_1", + "Expectation": expectation, + } + if family == "gpudirect_dmabuf": + test.update( + { + "SrcGPUIndex": 0 if src_rail == "rail-0" else 1, + "DstGPUIndex": 0 if dst_rail == "rail-0" else 1, + "SrcGPUPCIAddress": "0000:41:00.0" if src_rail == "rail-0" else "0000:71:00.0", + "DstGPUPCIAddress": "0000:41:00.0" if dst_rail == "rail-0" else "0000:71:00.0", + } + ) + return { + "Test": test, + "Family": family, + "OK": ok, + "ObservedOK": observed_ok, + "Expectation": expectation, + "Route": {"OK": not cross_rail}, + "BandwidthGbps": bandwidth, + "MsgRateMpps": 0.3578 if bandwidth else 0.0, + "MinBandwidthGbps": 100.0 if bandwidth_family else 0.0, + "Stdout": stdout, + "Stderr": stderr, + **({"Error": stderr} if not ok else {}), + } + + +def _connectivity_result(scenario_name: str, fail_family: str | None) -> dict[str, Any]: + """Build a strict two-node, two-rail matrix.""" + rails = ["rail-0", "rail-1"] + rows: list[dict[str, Any]] = [] + for kind in range(8): + for src_node, dst_node in (("worker-a", "worker-b"), ("worker-b", "worker-a")): + pairs = [(rail, rail) for rail in rails] if kind % 2 == 0 else [(rails[0], rails[1]), (rails[1], rails[0])] + for src_rail, dst_rail in pairs: + rows.append(_ping_result(kind, src_node, dst_node, src_rail, dst_rail, fail_family)) + failed = sum(row["OK"] is not True for row in rows) + return { + "DaemonSets": [ + { + "Ref": { + "Namespace": "default", + "Name": f"l8k-network-test-{scenario_name}", + "Container": "test-container", + "RDMAContainer": "test-container", + "ICMPContainer": "netshoot", + "SourceFile": "60-example-daemonset-mock-group.yaml", + }, + "Rollout": {"Desired": 2, "Updated": 2, "Available": 2, "Ready": 2, "NotReady": 0}, + "PodCount": 2, + } + ], + "PingResults": rows, + "Skipped": None, + "Summary": {"TotalTests": len(rows), "Passed": len(rows) - failed, "Failed": failed}, + } + + +def _failure(command: str) -> tuple[bool, str | None]: + """Resolve optional failure injection as ``command[:family]``.""" + parts = os.environ.get("L8K_MOCK_FAIL", "").split(":") + if not parts or parts[0] != command: + return False, None + return len(parts) == 1, parts[1] if len(parts) > 1 else None + + +def _run_version(flags: dict[str, str]) -> int: + """Mock ``l8k version``.""" + if flags.get("--output") == "json": + _emit({"version": "v0.1.0-mock", "gitCommit": _SOURCE_COMMIT, "buildDate": _TIMESTAMP}, pretty=True) + else: + print("l8k v0.1.0-mock") + return 0 + + +def _run_schema() -> int: + """Mock ``l8k schema`` with Launch Kit-owned capabilities and defaults.""" + fixture = _load_fixture() + _emit( + { + "version": "v0.1.0-mock", + "description": "CLI tool for deploying NVIDIA cloud-native networking solutions on Kubernetes", + "commands": { + command: {"description": f"Mock l8k {command}", "example": f"l8k {command}"} + for command in ("discover", "generate", "deploy", "validate", "clean", "schema") + }, + "phases": ["discover", "generate", "deploy"], + "fabrics": sorted({str(value["fabric"]) for value in fixture["scenarios"].values()}), + "deploymentTypes": sorted({str(value["deployment"]) for value in fixture["scenarios"].values()}), + "outputFormats": ["text", "json"], + "supportedNetworkOperatorReleases": ["26.4"], + "exitCodes": {"0": "success", "2": "validation_error", "3": "cluster_error", "4": "deployment_error"}, + "flags": { + "--node-selector": { + "type": "string", + "default": "feature.node.kubernetes.io/pci-15b3.present=true", + "description": "Node selector", + }, + "--validation-mode": { + "type": "string", + "default": "inherit from validation.mode", + "description": "Validation mode", + }, + "--validation-checks": { + "type": "[]string", + "default": "inherit from validation.checks", + "description": ( + "Comma-separated checks: icmp, rping, ib_write_bw. " + "Enabled GPUDirect DMA-BUF validation follows ib_write_bw." + ), + }, + }, + }, + pretty=True, + ) + return 0 + + +def _run_discover(flags: dict[str, str]) -> int: + """Mock ``l8k discover``.""" + fail, _ = _failure("discover") + if fail: + result, exit_code = _structured_error("discover", "cluster discovery failed") + _emit(result, pretty=True) + print("Error: cluster discovery failed", file=sys.stderr) + return exit_code + _, scenario = _scenario_for_profile(flags.get("--fabric", ""), flags.get("--deployment-type", "")) + path = Path(flags.get("--save-cluster-config", "cluster-config.yaml")) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(_cluster_config_yaml(scenario), encoding="utf-8") + _emit(_json_result("discover", profile=_profile(scenario)), pretty=True) + print(f"[success] Configuration saved: {path}", file=sys.stderr) + return 0 + + +def _run_generate(flags: dict[str, str]) -> int: + """Mock ``l8k generate``.""" + fail, _ = _failure("generate") + if fail: + result, exit_code = _structured_error("generate", "deployment file generation failed") + _emit(result, pretty=True) + return exit_code + _, scenario = _scenario_from_config(flags) + root = Path(flags.get("--save-deployment-files", "deployment")) + files = _write_generated_files(root, scenario) + _emit(_json_result("generate", profile=_profile(scenario), generated_files=files), pretty=True) + print(f"[success] Generated {len(files)} files", file=sys.stderr) + return 0 + + +def _run_deploy(flags: dict[str, str]) -> int: + """Mock standalone ``l8k deploy`` including empty success stdout.""" + fail, _ = _failure("deploy") + if fail: + result, exit_code = _structured_error("deploy", "deployment failed") + _emit(result, pretty=True) + print("Error: deployment failed", file=sys.stderr) + return exit_code + _scenario_from_config(flags) + print(f"[success] Deployment completed from {flags.get('--deployment-files', 'deployment')}", file=sys.stderr) + return 0 + + +def _run_validate(flags: dict[str, str]) -> int: + """Mock the current three-document ``l8k validate`` JSON stream.""" + fail, family = _failure("validate") + if fail: + result, exit_code = _structured_error("validate", "failed to create Kubernetes client") + _emit(result) + print("Error: failed to create Kubernetes client", file=sys.stderr) + return exit_code + scenario_name, scenario = _scenario_from_config(flags) + manifests = _manifest_results(scenario) + static = { + "versionCheck": { + "Skipped": False, + "Reason": "", + "SelectedRelease": "26.4", + "ExpectedVersion": "v26.4.1", + "DeployedRelease": { + "Name": "network-operator", + "Namespace": "nvidia-network-operator", + "ChartName": "network-operator", + "ChartVersion": "26.4.1", + "AppVersion": "v26.4.1", + "Revision": 1, + "Status": "deployed", + }, + "Match": True, + }, + "manifests": manifests, + "presetDeviations": [], + "summary": { + "totalManifests": len(manifests), + "successManifests": len(manifests), + "inProgress": 0, + "errorManifests": 0, + "missingManifests": 0, + "versionMatch": True, + "deviationGroups": 0, + "success": True, + }, + } + connectivity = _connectivity_result(scenario_name, family) + deployment = Path(flags.get("--deployment-files", "deployment")) + report = Path(flags.get("--report-path", str(deployment / "k8s-launch-kit-validation-report.html"))).resolve() + report.parent.mkdir(parents=True, exist_ok=True) + verdict = "FAILED" if connectivity["Summary"]["Failed"] else "PASSED" + report.write_text(f"

VALIDATION {verdict}

\n", encoding="utf-8") + _emit(static) + _emit({"connectivity": connectivity}) + _emit({"reportPath": str(report)}) + print(f"HTML report written to {report}", file=sys.stderr) + return 4 if connectivity["Summary"]["Failed"] else 0 + + +def _run_clean(flags: dict[str, str]) -> int: + """Mock the current one-document ``l8k clean`` JSON result.""" + fail, _ = _failure("clean") + if fail: + result, exit_code = _structured_error("clean", "Network Operator cleanup failed") + _emit(result, pretty=True) + print("Error: Network Operator cleanup failed", file=sys.stderr) + return exit_code + keep_helm = flags.get("--keep-helm-chart", "false").lower() == "true" + _emit( + { + "success": True, + "phase": "clean", + "deployed": False, + "cleanup": { + "namespace": flags.get("--network-operator-namespace", "nvidia-network-operator"), + "customResourcesDeleted": 12, + "helmReleaseRemoved": not keep_helm, + "keepHelmChart": keep_helm, + }, + "messages": [], + }, + pretty=True, + ) + return 0 + + +def main(argv: list[str] | None = None) -> int: + """Execute one mocked Launch Kit command.""" + args = list(sys.argv[1:] if argv is None else argv) + if not args: + print("mock l8k expects a command", file=sys.stderr) + return 2 + command = args[0] + try: + if command == "schema": + if len(args) != 1: + raise ValueError("l8k schema accepts no arguments") + return _run_schema() + if command not in _VALUE_FLAGS: + raise ValueError(f"unknown l8k command: {command}") + flags = _parse_flags(command, args[1:]) + if command in {"discover", "generate", "deploy", "validate", "clean"} and flags.get("--output") != "json": + raise ValueError("mock workflow commands require --output json") + if command == "version": + return _run_version(flags) + if command == "discover": + return _run_discover(flags) + if command == "generate": + return _run_generate(flags) + if command == "deploy": + return _run_deploy(flags) + if command == "validate": + return _run_validate(flags) + return _run_clean(flags) + except (KeyError, OSError, TypeError, ValueError, yaml.YAMLError) as exc: + result, exit_code = _structured_error(command, str(exc)) + _emit(result, pretty=command != "validate") + print(f"Error: {exc}", file=sys.stderr) + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/tests/providers/k8s_launch_kit/test_provider.py b/isvctl/tests/providers/k8s_launch_kit/test_provider.py new file mode 100644 index 000000000..1104271e1 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/test_provider.py @@ -0,0 +1,1164 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Contract and framework tests for the generic Kubernetes Launch Kit provider.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest +import yaml +from isvtest.core.resolution import ErrorReason, State + +from isvctl.config.merger import merge_yaml_files +from isvctl.config.output_schemas import validate_output +from isvctl.config.schema import RunConfig +from isvctl.orchestrator.loop import Orchestrator, Phase + +_ISVCTL_ROOT = Path(__file__).resolve().parents[3] +_PROVIDERS = _ISVCTL_ROOT / "configs" / "providers" +_LAUNCH_KIT_PROVIDER = _PROVIDERS / "k8s-launch-kit" +_PROVIDER = _LAUNCH_KIT_PROVIDER / "scripts" / "adapter.py" +_FIXTURES = Path(__file__).resolve().parent / "fixtures" +_MOCK_L8K = _FIXTURES / "mock_l8k.py" +_MOCK_KUBECTL = _FIXTURES / "mock_kubectl.py" +_GENERIC_CONFIG = _LAUNCH_KIT_PROVIDER / "config" / "provider.yaml" +_NETWORK_OPERATOR_CONFIG = _LAUNCH_KIT_PROVIDER / "config" / "network-operator.yaml" + + +def _load_provider_module() -> ModuleType: + """Load the provider script for isolated installer tests.""" + spec = importlib.util.spec_from_file_location("k8s_launch_kit_provider", _PROVIDER) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot import {_PROVIDER}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _run_provider( + *arguments: str, + env: dict[str, str] | None = None, +) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: + """Run one provider operation from the same directory used by isvctl.""" + completed = subprocess.run( + [sys.executable, str(_PROVIDER), *arguments], + cwd=_PROVIDERS, + env=env, + check=False, + capture_output=True, + text=True, + ) + output = json.loads(completed.stdout) + assert isinstance(output, dict) + return completed, output + + +def _run_workflow( + command: str, + arguments: list[str], + *, + working_dir: Path, + artifact_dir: Path, + user_config: Path | None = None, + env: dict[str, str] | None = None, +) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: + """Run a mocked l8k workflow command through the generic transport.""" + provider_arguments = [ + "run", + "--executable", + str(_MOCK_L8K), + "--command", + command, + "--arguments-json", + json.dumps(arguments), + "--environment-json", + "{}", + "--working-dir", + str(working_dir), + "--artifact-dir", + str(artifact_dir), + ] + if user_config is not None: + provider_arguments.extend(["--user-config", str(user_config)]) + return _run_provider(*provider_arguments, env=env) + + +def _mocked_network_operator_config(tmp_path: Path) -> RunConfig: + """Load production wiring, then inject test-owned executables and paths.""" + merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) + context = merged["context"]["k8s_launch_kit"] + context["executable"] = str(_MOCK_L8K) + context["kubectl_command"] = [sys.executable, str(_MOCK_KUBECTL)] + context["shared_artifact_dir"] = str(tmp_path / "shared-evidence") + for name, use_case in context["use_cases"].items(): + use_case["working_dir"] = str(tmp_path / "use-cases" / name / "work") + use_case["artifact_dir"] = str(tmp_path / "use-cases" / name / "evidence") + return RunConfig.model_validate(merged) + + +def test_generic_provider_has_no_launch_kit_domain_defaults() -> None: + """AI Cloud Validation exposes raw argv while Launch Kit owns domain defaults.""" + merged = merge_yaml_files([_GENERIC_CONFIG]) + config = RunConfig.model_validate(merged) + context = merged["context"]["k8s_launch_kit"] + + assert set(context) == { + "executable", + "installation", + "user_config", + "kubectl_command", + "working_dir", + "artifact_dir", + "environment", + "discover", + "generate", + "deploy", + "validate", + "clean", + } + assert context["user_config"] == "" + assert context["installation"] == { + "mode": "verify", + "version": "", + "installer_ref": "", + "installer_sha256": "", + "prefix": "", + } + assert all( + context[command]["arguments"] == [] for command in ("discover", "generate", "deploy", "validate", "clean") + ) + assert [step.name for step in config.commands["network_operator"].steps] == [ + "launch_kit_prepare", + "launch_kit_verify", + "launch_kit_kubernetes_preflight", + "launch_kit_discover", + "launch_kit_generate", + "launch_kit_deploy", + "launch_kit_validate", + "launch_kit_clean", + ] + assert config.commands["network_operator"].phases == ["setup", "test", "teardown"] + discover_step = next( + step for step in config.commands["network_operator"].steps if step.name == "launch_kit_discover" + ) + prepare_step = next(step for step in config.commands["network_operator"].steps if step.name == "launch_kit_prepare") + assert "--installer-ref={{ context.k8s_launch_kit.installation.installer_ref }}" in prepare_step.args + assert "--installer-sha256={{ context.k8s_launch_kit.installation.installer_sha256 }}" in prepare_step.args + assert "--user-config={{ context.k8s_launch_kit.user_config }}" in discover_step.args + assert config.commands["network_operator"].steps[-1].phase == "teardown" + assert config.commands["network_operator"].steps[-1].finalizer_for == "launch_kit_deploy" + forbidden = { + "namespace", + "node_selector", + "expected_network_operator_version", + "driver_mode", + "rail_names", + "sriov_resource_names", + "ip_pool_names", + "gpu_count", + "validation_mode", + "validation_checks", + "rdma_rping_iterations", + "rdma_ib_write_size", + "rdma_min_bandwidth_gbps", + "timeout_seconds", + } + assert forbidden.isdisjoint(context) + + +def test_network_operator_provider_defaults_to_real_cli_tools() -> None: + """The shipped use-case provider cannot select repository test doubles.""" + merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) + config = RunConfig.model_validate(merged) + context = merged["context"]["k8s_launch_kit"] + + assert context["executable"] == "l8k" + assert context["installation"]["installer_ref"] == "" + assert context["installation"]["installer_sha256"] == "" + assert context["user_config"] == "" + assert context["kubectl_command"] == [] + assert "mock" not in json.dumps(merged).lower() + assert "poc" not in json.dumps(merged).lower() + assert len(config.commands["network_operator"].steps) == 26 + assert config.commands["network_operator"].phases[-1] == "infiniband-host-device" + discover_steps = [step for step in config.commands["network_operator"].steps if step.name.endswith("_discover")] + assert len(discover_steps) == 6 + assert all("--user-config={{ context.k8s_launch_kit.user_config }}" in step.args for step in discover_steps) + prepare_step = next(step for step in config.commands["network_operator"].steps if step.name == "launch_kit_prepare") + assert "--installer-ref={{ context.k8s_launch_kit.installation.installer_ref }}" in prepare_step.args + assert "--installer-sha256={{ context.k8s_launch_kit.installation.installer_sha256 }}" in prepare_step.args + use_case_steps = config.commands["network_operator"].steps[2:] + assert all(not step.name.endswith(("_deploy", "_clean")) for step in use_case_steps) + assert all(step.finalizer_for is None for step in use_case_steps) + + +def test_network_operator_workflows_use_launch_kit_default_paths() -> None: + """Grouped use cases leave config and deployment paths to Launch Kit.""" + merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) + use_cases = merged["context"]["k8s_launch_kit"]["use_cases"] + default_path_flags = { + "--user-config", + "--deployment-files", + "--save-cluster-config", + "--save-deployment-files", + } + + for use_case in use_cases.values(): + all_arguments = { + argument for phase in ("discover", "generate", "validate") for argument in use_case[phase]["arguments"] + } + assert default_path_flags.isdisjoint(all_arguments) + assert set(use_case) == {"working_dir", "artifact_dir", "discover", "generate", "validate"} + assert use_case["discover"]["arguments"][0] == "--fabric" + assert use_case["generate"]["arguments"] == [] + assert use_case["validate"]["arguments"] == [] + + +def test_kubectl_defaults_to_the_real_binary() -> None: + """An empty provider override resolves to kubectl from PATH.""" + module = _load_provider_module() + + assert module._kubectl_prefix("[]", {}) == ["kubectl"] + + +def test_launch_kit_executable_resolves_from_path(tmp_path: Path, monkeypatch: Any) -> None: + """The production `l8k` setting is resolved as a normal executable.""" + module = _load_provider_module() + executable = tmp_path / "l8k" + executable.write_text("test executable", encoding="utf-8") + monkeypatch.setattr(module.shutil, "which", lambda value: str(executable) if value == "l8k" else None) + + assert module._resolve_executable("l8k") == executable.resolve() + + +def test_prepare_verifies_version_and_schema(tmp_path: Path) -> None: + """Verify mode proves the executable and captures Launch Kit's schema.""" + completed, output = _run_provider( + "prepare", + "--mode", + "verify", + "--executable", + str(_MOCK_L8K), + "--artifact-dir", + str(tmp_path), + ) + + assert completed.returncode == 0 + assert output["success"] is True + assert output["operation"] == "prepare" + assert output["installed"] is False + assert set(output["checks"]) == {"version", "schema"} + assert all(check["passed"] is True for check in output["checks"].values()) + assert validate_output(output, "k8s_launch_kit") == (True, []) + + +def test_verification_rejects_an_unexpected_launch_kit_version(tmp_path: Path) -> None: + """A pinned installation cannot silently verify a different binary on PATH.""" + module = _load_provider_module() + + verification, success, error = module._verify_executable( + _MOCK_L8K, + tmp_path, + "v9.9.9", + ) + + assert success is False + assert verification["checks"]["version"]["passed"] is False + assert error == "l8k version mismatch: expected 'v9.9.9', got 'v0.1.0-mock'" + + +def test_verification_requires_the_launch_kit_clean_command(tmp_path: Path) -> None: + """A pre-clean Launch Kit binary is rejected before deployment begins.""" + module = _load_provider_module() + executable = tmp_path / "l8k" + executable.write_text( + "#!/bin/sh\n" + 'if [ "$1" = version ]; then\n' + ' echo \'{"version": "v0.1.0"}\'\n' + "else\n" + ' echo \'{"commands": {"discover": {}, "generate": {}, "deploy": {}, "validate": {}}}\'\n' + "fi\n", + encoding="utf-8", + ) + executable.chmod(0o755) + + verification, success, error = module._verify_executable(executable, tmp_path) + + assert success is False + assert verification["checks"]["schema"]["passed"] is False + assert error == "l8k schema does not advertise required command(s): clean" + + +def test_installed_executable_is_resolved_from_the_installer_prefix(tmp_path: Path) -> None: + """Install mode verifies the binary written by the installer, not a stale PATH entry.""" + module = _load_provider_module() + executable = tmp_path / "bin" / "l8k" + executable.parent.mkdir(parents=True) + executable.write_text("mock", encoding="utf-8") + + assert module._installed_executable(str(tmp_path)) == executable.resolve() + + +def test_installer_download_verifies_expected_digest(tmp_path: Path, monkeypatch: Any) -> None: + """Install mode verifies an immutable official installer before writing it.""" + module = _load_provider_module() + content = b"#!/bin/sh\nset -eu\n" + installer_ref = "a" * 40 + expected_sha256 = hashlib.sha256(content).hexdigest() + + class Response: + """Minimal context-managed urllib response.""" + + def __enter__(self) -> Response: + return self + + def __exit__(self, *_args: Any) -> None: + return None + + def read(self) -> bytes: + return content + + monkeypatch.setattr(module.urllib.request, "urlopen", lambda *_args, **_kwargs: Response()) + + installer, url = module._download_installer(installer_ref, expected_sha256, tmp_path) + metadata = json.loads((tmp_path / "installer-download.json").read_text(encoding="utf-8")) + + assert installer.read_bytes() == content + assert url.endswith(f"/{installer_ref}/scripts/install.sh") + assert metadata == { + "url": url, + "ref": installer_ref, + "expected_sha256": expected_sha256, + "sha256": expected_sha256, + "verified": True, + } + + +def test_installer_download_rejects_a_digest_mismatch(tmp_path: Path, monkeypatch: Any) -> None: + """A downloaded installer is never persisted when its trusted digest differs.""" + module = _load_provider_module() + content = b"#!/bin/sh\nexit 0\n" + + class Response: + """Minimal context-managed urllib response.""" + + def __enter__(self) -> Response: + return self + + def __exit__(self, *_args: Any) -> None: + return None + + def read(self) -> bytes: + return content + + monkeypatch.setattr(module.urllib.request, "urlopen", lambda *_args, **_kwargs: Response()) + + with pytest.raises(ValueError, match="installer SHA-256 mismatch"): + module._download_installer("b" * 40, "0" * 64, tmp_path) + + metadata = json.loads((tmp_path / "installer-download.json").read_text(encoding="utf-8")) + assert metadata["verified"] is False + assert metadata["sha256"] == hashlib.sha256(content).hexdigest() + assert not (tmp_path / "installer.sh").exists() + + +@pytest.mark.parametrize("installer_ref", ["", "main", "v0.1.0", "a" * 39]) +def test_installer_download_requires_an_immutable_commit_ref(tmp_path: Path, installer_ref: str) -> None: + """Install mode rejects mutable or abbreviated installer references before download.""" + module = _load_provider_module() + + with pytest.raises(ValueError, match="full 40-character Git commit SHA"): + module._download_installer(installer_ref, "0" * 64, tmp_path) + + +def test_install_mode_delegates_to_the_upstream_installer(tmp_path: Path, monkeypatch: Any) -> None: + """The provider does not reimplement Launch Kit archive or checksum logic.""" + module = _load_provider_module() + installer = tmp_path / "installer.sh" + installer.write_text("#!/bin/sh\n", encoding="utf-8") + calls: list[tuple[list[str], dict[str, str]]] = [] + + monkeypatch.setattr( + module, + "_download_installer", + lambda _ref, _sha256, _artifact_dir: (installer, "https://example.invalid/installer.sh"), + ) + monkeypatch.setattr(module, "_installed_executable", lambda _prefix: tmp_path / "bin" / "l8k") + monkeypatch.setattr( + module, + "_verify_executable", + lambda _executable, _artifact_dir, _expected_version, _environment: ( + {"checks": {}, "artifacts": {}}, + True, + None, + ), + ) + + def fake_run(argv: list[str], *, cwd: Path, env: dict[str, str]) -> dict[str, Any]: + del cwd + calls.append((argv, env)) + return {"exit_code": 0, "stdout": "", "stderr": "", "duration_seconds": 0.1} + + monkeypatch.setattr(module, "_run_process", fake_run) + monkeypatch.setattr(module, "_record_process", lambda *_args, **_kwargs: {}) + args = argparse.Namespace( + mode="install", + executable="l8k", + version="v0.1.0", + installer_ref="a" * 40, + installer_sha256="0" * 64, + prefix=str(tmp_path), + environment_json=json.dumps({"HTTPS_PROXY": "http://proxy.example.test"}), + artifact_dir=str(tmp_path / "evidence"), + ) + + output, exit_code = module._prepare(args) + + assert exit_code == 0 + assert output["installed"] is True + assert calls[0][0] == ["/bin/sh", str(installer), "-d", str(tmp_path)] + assert calls[0][1]["L8K_VERSION"] == "v0.1.0" + assert calls[0][1]["HTTPS_PROXY"] == "http://proxy.example.test" + + +def test_provider_runs_the_real_launch_kit_workflow_shape(tmp_path: Path) -> None: + """The transport runs the full Launch Kit lifecycle with raw argv.""" + working_dir = tmp_path / "work" + artifact_dir = tmp_path / "evidence" + kubeconfig = "kubeconfig" + discover_args = [ + "--kubeconfig", + kubeconfig, + "--fabric", + "ethernet", + "--deployment-type", + "sriov", + ] + commands = [ + ("discover", discover_args), + ("generate", []), + ("deploy", ["--kubeconfig", kubeconfig]), + ("validate", ["--kubeconfig", kubeconfig]), + ("clean", ["--kubeconfig", kubeconfig]), + ] + outputs: dict[str, dict[str, Any]] = {} + + for command, arguments in commands: + completed, output = _run_workflow( + command, + arguments, + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + assert completed.returncode == 0 + assert output["success"] is True + assert output["operation"] == command + assert output["working_directory"] == str(working_dir.resolve()) + command_index = output["argv"].index(command) + assert output["argv"][command_index + 1 :] == [*arguments, "--output", "json"] + assert validate_output(output, "k8s_launch_kit") == (True, []) + assert all(Path(path).is_file() for path in output["artifacts"].values()) + outputs[command] = output + + assert len(outputs["discover"]["documents"]) == 1 + assert len(outputs["generate"]["documents"]) == 1 + generated_files = [Path(path) for path in outputs["generate"]["documents"][0]["generatedFiles"]] + daemonset_path = next(path for path in generated_files if "example-daemonset" in path.name) + daemonset = yaml.safe_load(daemonset_path.read_text(encoding="utf-8")) + assert [container["name"] for container in daemonset["spec"]["template"]["spec"]["containers"]] == [ + "test-container", + "netshoot", + ] + test_container = daemonset["spec"]["template"]["spec"]["containers"][0] + assert test_container["resources"]["requests"]["nvidia.com/gpu"] == "2" + assert test_container["resources"]["limits"]["nvidia.com/gpu"] == "2" + assert outputs["deploy"]["documents"] == [] + assert len(outputs["validate"]["documents"]) == 3 + families = {row["Family"] for row in outputs["validate"]["documents"][1]["connectivity"]["PingResults"]} + assert families == {"icmp", "rping", "ib_write_bw", "gpudirect_dmabuf"} + assert outputs["clean"]["documents"][0]["cleanup"] == { + "namespace": "nvidia-network-operator", + "customResourcesDeleted": 12, + "helmReleaseRemoved": True, + "keepHelmChart": False, + } + assert (working_dir / "cluster-config.yaml").is_file() + assert (working_dir / "deployment" / "k8s-launch-kit-validation-report.html").is_file() + + +def test_discover_stages_user_config_transiently_without_retaining_secrets(tmp_path: Path) -> None: + """Discovery uses a private staged config but retains only safe input provenance.""" + source = tmp_path / "customer-cluster-config.yaml" + secret_values = ("customer-api-token", "registry-password", "embedded-kubeconfig") + source_contents = """networkOperator: + selectedRelease: "26.4" +profile: + fabric: ethernet + deployment: sriov +clusterConfig: [] +credentials: + token: customer-api-token + registryPassword: registry-password + kubeconfig: embedded-kubeconfig +""" + source.write_text(source_contents, encoding="utf-8") + working_dir = tmp_path / "work" + artifact_dir = tmp_path / "evidence" + + completed, output = _run_workflow( + "discover", + ["--fabric", "ethernet", "--deployment-type", "sriov"], + working_dir=working_dir, + artifact_dir=artifact_dir, + user_config=source, + ) + + staged = working_dir / "user-config.yaml" + discovered = working_dir / "cluster-config.yaml" + assert completed.returncode == 0 + assert output["success"] is True + assert source.read_text(encoding="utf-8") == source_contents + assert not staged.exists() + assert discovered.is_file() + metadata_path = artifact_dir / "inputs" / "user-config.json" + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + assert metadata == { + "source_path": str(source.resolve()), + "staged_path": str(staged.resolve()), + "sha256": hashlib.sha256(source_contents.encode()).hexdigest(), + "size_bytes": len(source_contents.encode()), + "retained": False, + } + assert output["artifacts"]["user_config"] == str(metadata_path.resolve()) + assert output["argv"][-6:] == [ + "--user-config", + str(staged.resolve()), + "--save-cluster-config", + str(discovered.resolve()), + "--output", + "json", + ] + retained_text = "\n".join( + path.read_text(encoding="utf-8", errors="replace") + for root in (working_dir, artifact_dir) + for path in root.rglob("*") + if path.is_file() + ) + assert all(secret not in retained_text for secret in secret_values) + + +def test_user_config_must_not_be_inside_the_retained_working_directory(tmp_path: Path) -> None: + """A source inside the retained output tree is rejected before it can leak as evidence.""" + working_dir = tmp_path / "work" + working_dir.mkdir() + source = working_dir / "customer-cluster-config.yaml" + source.write_text("profile: {}\ncredentials: customer-api-token\n", encoding="utf-8") + + completed, output = _run_workflow( + "discover", + [], + working_dir=working_dir, + artifact_dir=tmp_path / "evidence", + user_config=source, + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert "must be outside the retained provider working directory" in output["error"] + assert not (working_dir / "user-config.yaml").exists() + + +def test_staged_user_config_is_removed_when_discovery_fails(tmp_path: Path) -> None: + """A failed l8k discovery cannot leave the sensitive staged input behind.""" + source = tmp_path / "customer-cluster-config.yaml" + source_contents = "profile: {fabric: ethernet, deployment: sriov}\ncredentials: customer-api-token\n" + source.write_text(source_contents, encoding="utf-8") + working_dir = tmp_path / "work" + + completed, output = _run_workflow( + "discover", + [], + working_dir=working_dir, + artifact_dir=tmp_path / "evidence", + user_config=source, + env={**os.environ, "L8K_MOCK_FAIL": "discover"}, + ) + + assert completed.returncode != 0 + assert output["success"] is False + assert source.read_text(encoding="utf-8") == source_contents + assert not (working_dir / "user-config.yaml").exists() + + +@pytest.mark.parametrize("flag", ["--user-config", "--save-cluster-config"]) +def test_staged_user_config_rejects_conflicting_raw_discovery_paths(tmp_path: Path, flag: str) -> None: + """The first-class input owns both discovery config paths.""" + source = tmp_path / "customer-cluster-config.yaml" + source.write_text("profile: {}\n", encoding="utf-8") + + completed, output = _run_workflow( + "discover", + [flag, str(tmp_path / "raw.yaml")], + working_dir=tmp_path / "work", + artifact_dir=tmp_path / "evidence", + user_config=source, + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert f"cannot be combined with raw discovery flag(s): {flag}" in output["error"] + + +def test_staged_user_config_must_exist(tmp_path: Path) -> None: + """A missing first-class user config fails before l8k starts.""" + working_dir = tmp_path / "work" + + completed, output = _run_workflow( + "discover", + [], + working_dir=working_dir, + artifact_dir=tmp_path / "evidence", + user_config=tmp_path / "missing.yaml", + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert "Launch Kit user config not found" in output["error"] + assert not (working_dir / "user-config.yaml").exists() + + +def test_clean_forwards_launch_kit_boolean_flags_unchanged(tmp_path: Path) -> None: + """The transport accepts Launch Kit's native bare boolean flag syntax.""" + working_dir = tmp_path / "work" + artifact_dir = tmp_path / "evidence" + completed, _ = _run_workflow( + "discover", + ["--fabric", "ethernet", "--deployment-type", "sriov"], + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + assert completed.returncode == 0 + + completed, output = _run_workflow( + "clean", + ["--keep-helm-chart"], + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + + assert completed.returncode == 0 + assert output["argv"][-3:] == ["--keep-helm-chart", "--output", "json"] + assert output["documents"][0]["cleanup"] == { + "namespace": "nvidia-network-operator", + "customResourcesDeleted": 12, + "helmReleaseRemoved": False, + "keepHelmChart": True, + } + + +@pytest.mark.parametrize( + ("fabric", "deployment", "network_kind"), + [ + ("ethernet", "sriov", "SriovNetwork"), + ("infiniband", "sriov", "SriovIBNetwork"), + ("ethernet", "rdma_shared", "MacvlanNetwork"), + ("infiniband", "rdma_shared", "IPoIBNetwork"), + ("ethernet", "host_device", "HostDeviceNetwork"), + ("infiniband", "host_device", "HostDeviceNetwork"), + ], +) +def test_mock_supports_each_launch_kit_profile( + tmp_path: Path, + fabric: str, + deployment: str, + network_kind: str, +) -> None: + """Every pinned profile can traverse the same real command sequence.""" + working_dir = tmp_path / f"{fabric}-{deployment}" + artifact_dir = working_dir / "evidence" + commands = [ + ( + "discover", + [ + "--fabric", + fabric, + "--deployment-type", + deployment, + ], + ), + ("generate", []), + ("deploy", []), + ("validate", []), + ("clean", []), + ] + outputs: dict[str, dict[str, Any]] = {} + + for command, arguments in commands: + completed, output = _run_workflow( + command, + arguments, + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + assert completed.returncode == 0 + outputs[command] = output + + manifest_kinds = {row["Kind"] for row in outputs["validate"]["documents"][0]["manifests"]} + assert network_kind in manifest_kinds + assert outputs["clean"]["documents"][0]["phase"] == "clean" + + +def test_preflight_uses_the_workflow_kubeconfig(tmp_path: Path) -> None: + """kubectl probes target the same explicit kubeconfig supplied to l8k.""" + workflow = { + command: ["--kubeconfig", "partner.kubeconfig"] if command != "generate" else [] + for command in ("discover", "generate", "deploy", "validate", "clean") + } + completed, output = _run_provider( + "preflight", + "--kubectl-command-json", + json.dumps([sys.executable, str(_MOCK_KUBECTL)]), + "--workflow-arguments-json", + json.dumps(workflow), + "--working-dir", + str(tmp_path / "work"), + "--artifact-dir", + str(tmp_path / "evidence"), + ) + + assert completed.returncode == 0 + assert output["success"] is True + assert output["kubeconfig_source"] == "workflow arguments" + assert output["node_count"] == 2 + assert output["ready_node_count"] == 2 + command_file = Path(output["artifacts"]["api_version"]["command"]) + argv = json.loads(command_file.read_text(encoding="utf-8"))["argv"] + kubeconfig_index = argv.index("--kubeconfig") + assert argv[kubeconfig_index : kubeconfig_index + 2] == ["--kubeconfig", "partner.kubeconfig"] + + +def test_preflight_forwards_the_launch_kit_environment(tmp_path: Path) -> None: + """The safety probes use the same environment that the provider gives l8k.""" + workflow = {command: [] for command in ("discover", "generate", "deploy", "validate", "clean")} + completed, output = _run_provider( + "preflight", + "--kubectl-command-json", + json.dumps([sys.executable, str(_MOCK_KUBECTL)]), + "--workflow-arguments-json", + json.dumps(workflow), + "--environment-json", + json.dumps( + { + "KUBECONFIG": "environment.kubeconfig", + "L8K_MOCK_EXPECT_KUBECONFIG": "environment.kubeconfig", + } + ), + "--working-dir", + str(tmp_path / "work"), + "--artifact-dir", + str(tmp_path / "evidence"), + ) + + assert completed.returncode == 0 + assert output["success"] is True + + +def test_preflight_accepts_a_validation_only_workflow(tmp_path: Path) -> None: + """The prerequisite gate follows the caller's actual Launch Kit command subset.""" + workflow = {command: [] for command in ("discover", "generate", "validate")} + completed, output = _run_provider( + "preflight", + "--kubectl-command-json", + json.dumps([sys.executable, str(_MOCK_KUBECTL)]), + "--workflow-arguments-json", + json.dumps(workflow), + "--working-dir", + str(tmp_path / "work"), + "--artifact-dir", + str(tmp_path / "evidence"), + ) + + assert completed.returncode == 0 + assert output["success"] is True + + +def test_preflight_rejects_conflicting_workflow_kubeconfigs(tmp_path: Path) -> None: + """The safety gate fails closed when l8k commands would target different clusters.""" + workflow = { + "discover": ["--kubeconfig", "cluster-a"], + "generate": [], + "deploy": ["--kubeconfig=cluster-b"], + "validate": [], + "clean": [], + } + completed, output = _run_provider( + "preflight", + "--kubectl-command-json", + "[]", + "--workflow-arguments-json", + json.dumps(workflow), + "--working-dir", + str(tmp_path / "work"), + "--artifact-dir", + str(tmp_path / "evidence"), + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert "different kubeconfigs" in output["error"] + + +def test_network_operator_provider_runs_end_to_end(tmp_path: Path, monkeypatch: Any) -> None: + """The production configuration executes all six named use cases in order.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.SETUP, Phase.TEST], + capability="kubernetes", + ) + + assert result.success is True + expected_use_cases = [ + "roce_sriov", + "infiniband_sriov", + "roce_rdma_shared", + "infiniband_rdma_shared", + "roce_host_device", + "infiniband_host_device", + ] + expected_steps = ["launch_kit_prepare", "launch_kit_verify"] + for use_case in expected_use_cases: + expected_steps.extend( + f"launch_kit_{use_case}_{operation}" for operation in ("preflight", "discover", "generate", "validate") + ) + assert list(result.inventory) == expected_steps + expected_phase_names = ["setup", "launch-kit-verification"] + [ + use_case.replace("_", "-") for use_case in expected_use_cases + ] + assert [phase.name for phase in result.phases] == expected_phase_names + for use_case in expected_use_cases: + phase_name = use_case.replace("_", "-") + test_phase = next(phase for phase in result.phases if phase.name == phase_name) + assert test_phase.phase is Phase.TEST + assert [step["name"].rsplit("_", 1)[-1] for step in test_phase.details["steps"]] == [ + "preflight", + "discover", + "generate", + "validate", + ] + states = {entry.entry.name: entry.state for entry in result.validations} + assert states == { + "EastWestNetworkRoceSriovCheck": State.PASSED, + "EastWestNetworkInfiniBandSriovCheck": State.PASSED, + "EastWestNetworkRoceRdmaSharedCheck": State.PASSED, + "EastWestNetworkInfiniBandRdmaSharedCheck": State.PASSED, + "EastWestNetworkRoceHostDeviceCheck": State.PASSED, + "EastWestNetworkInfiniBandHostDeviceCheck": State.PASSED, + } + expected_subtest_counts = { + "EastWestNetworkRoceSriovCheck": 121, + "EastWestNetworkInfiniBandSriovCheck": 121, + "EastWestNetworkRoceRdmaSharedCheck": 116, + "EastWestNetworkInfiniBandRdmaSharedCheck": 116, + "EastWestNetworkRoceHostDeviceCheck": 116, + "EastWestNetworkInfiniBandHostDeviceCheck": 116, + } + for entry in result.validations: + assert entry.subtest_summary.passed == expected_subtest_counts[entry.entry.name] + assert entry.subtest_summary.failed == 0 + assert entry.subtest_summary.skipped == 0 + for use_case in expected_use_cases: + assert (tmp_path / "use-cases" / use_case / "work" / "cluster-config.yaml").is_file() + + +@pytest.mark.parametrize( + ("label", "selected_use_cases", "excluded_use_cases"), + [ + ( + "ethernet", + ["roce_sriov", "roce_rdma_shared", "roce_host_device"], + ["infiniband_sriov", "infiniband_rdma_shared", "infiniband_host_device"], + ), + ( + "infiniband", + ["infiniband_sriov", "infiniband_rdma_shared", "infiniband_host_device"], + ["roce_sriov", "roce_rdma_shared", "roce_host_device"], + ), + ( + "sriov", + ["roce_sriov", "infiniband_sriov"], + ["roce_rdma_shared", "infiniband_rdma_shared", "roce_host_device", "infiniband_host_device"], + ), + ( + "rdma_shared", + ["roce_rdma_shared", "infiniband_rdma_shared"], + ["roce_sriov", "infiniband_sriov", "roce_host_device", "infiniband_host_device"], + ), + ( + "host_device", + ["roce_host_device", "infiniband_host_device"], + ["roce_sriov", "infiniband_sriov", "roce_rdma_shared", "infiniband_rdma_shared"], + ), + ( + "gpudirect", + [ + "roce_sriov", + "infiniband_sriov", + "roce_rdma_shared", + "infiniband_rdma_shared", + "roce_host_device", + "infiniband_host_device", + ], + [], + ), + ( + ["ethernet", "sriov"], + ["roce_sriov"], + [ + "infiniband_sriov", + "roce_rdma_shared", + "infiniband_rdma_shared", + "roce_host_device", + "infiniband_host_device", + ], + ), + ], +) +def test_network_operator_provider_grouping_label_prunes_unselected_workflows( + tmp_path: Path, + monkeypatch: Any, + label: str | list[str], + selected_use_cases: list[str], + excluded_use_cases: list[str], +) -> None: + """A grouping label runs only the matching validation workflows.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.SETUP, Phase.TEST], + include_labels=[label] if isinstance(label, str) else label, + capability="kubernetes", + ) + + assert result.success is True + inventory_names = set(result.inventory) + for use_case in selected_use_cases: + assert f"launch_kit_{use_case}_validate" in inventory_names + assert f"launch_kit_{use_case}_deploy" not in inventory_names + assert f"launch_kit_{use_case}_clean" not in inventory_names + assert (tmp_path / "use-cases" / use_case / "work" / "cluster-config.yaml").is_file() + for use_case in excluded_use_cases: + assert not any(name.startswith(f"launch_kit_{use_case}_") for name in inventory_names) + assert not (tmp_path / "use-cases" / use_case / "work" / "cluster-config.yaml").exists() + + +def test_network_operator_stages_user_config_only_for_selected_use_cases( + tmp_path: Path, + monkeypatch: Any, +) -> None: + """Each selected use case receives and removes an isolated user-config copy.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + source = tmp_path / "customer-cluster-config.yaml" + source_contents = """networkOperator: + selectedRelease: "26.4" +profile: + fabric: ethernet + deployment: sriov +clusterConfig: [] +""" + source.write_text(source_contents, encoding="utf-8") + config = _mocked_network_operator_config(tmp_path) + config.context["k8s_launch_kit"]["user_config"] = str(source) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + include_labels=["ethernet", "sriov"], + capability="kubernetes", + ) + + selected_work = tmp_path / "use-cases" / "roce_sriov" / "work" + selected_evidence = tmp_path / "use-cases" / "roce_sriov" / "evidence" + assert result.success is True + assert source.read_text(encoding="utf-8") == source_contents + assert not (selected_work / "user-config.yaml").exists() + assert (selected_work / "cluster-config.yaml").is_file() + metadata = json.loads((selected_evidence / "inputs" / "user-config.json").read_text(encoding="utf-8")) + assert metadata["sha256"] == hashlib.sha256(source_contents.encode()).hexdigest() + assert metadata["retained"] is False + discover = result.inventory["launch_kit_roce_sriov_discover"] + assert discover["argv"][discover["argv"].index("--user-config") + 1] == str( + (selected_work / "user-config.yaml").resolve() + ) + for use_case in ( + "infiniband_sriov", + "roce_rdma_shared", + "infiniband_rdma_shared", + "roce_host_device", + "infiniband_host_device", + ): + assert not (tmp_path / "use-cases" / use_case / "work" / "user-config.yaml").exists() + + +def test_network_operator_provider_test_phase_verifies_without_setup(tmp_path: Path, monkeypatch: Any) -> None: + """A test-only run verifies the configured binary instead of requiring setup output.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + capability="kubernetes", + ) + + assert result.success is True + assert "launch_kit_prepare" not in result.inventory + assert result.inventory["launch_kit_verify"]["success"] is True + assert all(entry.state is State.PASSED for entry in result.validations) + + +def test_network_operator_workflow_never_invokes_deploy_or_clean(tmp_path: Path, monkeypatch: Any) -> None: + """The validation suite must not mutate or delete the ISV-managed deployment.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + monkeypatch.setenv("L8K_MOCK_FAIL", "deploy") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + include_labels=["ethernet", "sriov"], + capability="kubernetes", + ) + + assert result.success is True + assert list(result.inventory) == [ + "launch_kit_verify", + "launch_kit_roce_sriov_preflight", + "launch_kit_roce_sriov_discover", + "launch_kit_roce_sriov_generate", + "launch_kit_roce_sriov_validate", + ] + assert not any(name.endswith(("_deploy", "_clean")) for name in result.inventory) + assert all(phase.phase is not Phase.TEARDOWN for phase in result.phases) + + +def test_kubernetes_preflight_failure_stops_before_discovery(tmp_path: Path, monkeypatch: Any) -> None: + """An unreachable cluster blocks each use case before discovery without hiding later cases.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + monkeypatch.setenv("L8K_MOCK_KUBERNETES_FAIL", "1") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run(capability="kubernetes") + + assert result.success is False + assert list(result.inventory) == [ + "launch_kit_prepare", + "launch_kit_verify", + "launch_kit_roce_sriov_preflight", + "launch_kit_infiniband_sriov_preflight", + "launch_kit_roce_rdma_shared_preflight", + "launch_kit_infiniband_rdma_shared_preflight", + "launch_kit_roce_host_device_preflight", + "launch_kit_infiniband_host_device_preflight", + ] + assert all(entry.state is State.ERROR for entry in result.validations) + assert all(entry.error_reason is ErrorReason.STEP_FAILED for entry in result.validations) + assert all("preflight" in entry.message for entry in result.validations) + assert not list((tmp_path / "use-cases").glob("*/work/cluster-config.yaml")) + assert not list((tmp_path / "use-cases").glob("*/evidence/commands/discover")) + + +def test_failed_validate_is_reported_without_cluster_cleanup(tmp_path: Path, monkeypatch: Any) -> None: + """A validation failure is reported while the ISV-managed deployment remains untouched.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + monkeypatch.setenv("L8K_MOCK_FAIL", "validate:ib_write_bw") + config = _mocked_network_operator_config(tmp_path) + junit_path = tmp_path / "junit.xml" + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + include_labels=["ethernet", "sriov"], + capability="kubernetes", + junitxml=str(junit_path), + ) + + assert result.success is False + assert list(result.inventory)[-1] == "launch_kit_roce_sriov_validate" + assert not any(name.endswith(("_deploy", "_clean")) for name in result.inventory) + assert result.validations[0].state is State.FAILED + case = next( + case + for case in ET.parse(junit_path).getroot().iter("testcase") + if case.get("name") == "EastWestNetworkRoceSriovCheck" + ) + assert case.find("failure") is not None + assert case.find("error") is None + assert case.find("skipped") is None + + +def test_failed_use_case_continues_to_next_selected_validation(tmp_path: Path, monkeypatch: Any) -> None: + """Independent pre-provisioned use cases continue after an earlier validation fails.""" + monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") + monkeypatch.setenv("L8K_MOCK_FAIL", "validate:ib_write_bw") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + include_labels=["sriov"], + capability="kubernetes", + ) + + assert result.success is False + assert "launch_kit_roce_sriov_validate" in result.inventory + assert "launch_kit_infiniband_sriov_validate" in result.inventory + assert not any(name.endswith(("_deploy", "_clean")) for name in result.inventory) + + +def test_failed_validate_preserves_documents_and_process_error(tmp_path: Path) -> None: + """A non-zero l8k result retains every JSON document and a clear exit diagnostic.""" + working_dir = tmp_path / "work" + artifact_dir = tmp_path / "evidence" + _run_workflow( + "discover", + [ + "--fabric", + "ethernet", + "--deployment-type", + "sriov", + ], + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + _run_workflow( + "generate", + [], + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + env = os.environ.copy() + env["L8K_MOCK_FAIL"] = "validate:ib_write_bw" + + completed, output = _run_workflow( + "validate", + [], + working_dir=working_dir, + artifact_dir=artifact_dir, + env=env, + ) + + assert completed.returncode == 4 + assert output["success"] is False + assert len(output["documents"]) == 3 + assert "l8k validate exited with code 4" in output["error"] + assert Path(output["artifacts"]["stdout"]).read_text(encoding="utf-8") diff --git a/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py b/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py new file mode 100644 index 000000000..92123a892 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Regression tests for Launch Kit timeout ownership.""" + +from pathlib import Path +from typing import Any + +import yaml + +PROVIDER_CONFIG_DIR = Path(__file__).parents[3] / "configs" / "providers" / "k8s-launch-kit" / "config" + + +def _steps(config_name: str) -> list[dict[str, Any]]: + """Return Network Operator command steps from a provider configuration.""" + config = yaml.safe_load((PROVIDER_CONFIG_DIR / config_name).read_text()) + return config["commands"]["network_operator"]["steps"] + + +def test_generic_validate_delegates_timeout_to_launch_kit() -> None: + """The generic validate workflow must not preempt l8k's matrix budget.""" + validate_steps = [step for step in _steps("provider.yaml") if step["name"] == "launch_kit_validate"] + + assert len(validate_steps) == 1 + assert validate_steps[0]["timeout"] is None + + +def test_network_operator_validates_delegate_timeout_to_launch_kit() -> None: + """Every grouped use case must leave its validation deadline to l8k.""" + validate_steps = [step for step in _steps("network-operator.yaml") if step["name"].endswith("_validate")] + + assert len(validate_steps) == 6 + assert all(step["timeout"] is None for step in validate_steps) diff --git a/isvctl/tests/test_orchestrator_loop.py b/isvctl/tests/test_orchestrator_loop.py index 6cd083eff..0015265fc 100644 --- a/isvctl/tests/test_orchestrator_loop.py +++ b/isvctl/tests/test_orchestrator_loop.py @@ -35,6 +35,7 @@ Orchestrator, Phase, _apply_capability_step_gates, + _apply_selected_validation_gates, _entries_missing_from_junit, _merge_junit_xmls, _write_terminal_junit_xml, @@ -79,6 +80,60 @@ def test_explicit_step_requires_gate_unbound_lifecycle_steps() -> None: assert all(not step.skip for step in kubernetes_steps) +def test_selected_validation_gate_prunes_unselected_lifecycle_steps() -> None: + """Label selection prevents commands owned by another test group from running.""" + steps = [ + StepConfig( + name="run_ethernet", + command="ethernet", + phase="test", + requires_selected_validations=["EthernetCheck"], + ), + StepConfig( + name="run_infiniband", + command="infiniband", + phase="test", + requires_selected_validations=["InfiniBandCheck"], + ), + ] + entries = [ + ValidationEntry( + name="EthernetCheck", + category="network", + params_template={}, + labels=("ethernet",), + ), + ValidationEntry( + name="InfiniBandCheck", + category="network", + params_template={}, + labels=("infiniband",), + ), + ] + + all_steps = _apply_selected_validation_gates( + steps, + entries, + include_labels=set(), + exclude_labels=set(), + exclude_tests=set(), + released_tests=None, + capability=None, + ) + ethernet_steps = _apply_selected_validation_gates( + steps, + entries, + include_labels={"ethernet"}, + exclude_labels=set(), + exclude_tests=set(), + released_tests=None, + capability=None, + ) + + assert all(not step.skip for step in all_steps) + assert [step.skip for step in ethernet_steps] == [False, True] + + def test_python_script_path_falls_back_to_current_working_directory( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -371,6 +426,268 @@ def test_run_all_phases_with_failure(self) -> None: assert len(teardown_phases) == 1 assert teardown_phases[0].success + def test_independent_custom_phase_runs_after_an_allowed_failure(self) -> None: + """An opted-in failed use case does not hide results from later independent cases.""" + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig(name="case_one", command="false", phase="case-one"), + StepConfig( + name="case_two", + command="echo", + args=['{"success": true, "platform": "kubernetes"}'], + phase="case-two", + output_schema="generic", + ), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert [(phase.name, phase.success) for phase in result.phases] == [ + ("case-one", False), + ("case-two", True), + ] + + def test_phase_finalizer_runs_after_its_target_fails(self, tmp_path: Path) -> None: + """An attempted mutating step activates cleanup even when the step fails.""" + marker = tmp_path / "cleaned" + cleanup = _write_script( + tmp_path, + "cleanup.sh", + f"#!/bin/sh\ntouch {marker}\necho '{{\"success\": true}}'\n", + ) + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig(name="deploy", command="false", phase="case-one"), + StepConfig( + name="cleanup", + command=cleanup, + phase="case-one", + finalizer_for="deploy", + ), + StepConfig(name="case_two", command="true", phase="case-two"), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert marker.is_file() + assert [step["name"] for step in result.phases[0].details["steps"]] == ["deploy"] + assert [step["name"] for step in result.phases[1].details["steps"]] == ["cleanup"] + assert result.phases[1].phase is Phase.TEARDOWN + assert result.phases[1].name == "case-one-teardown" + assert result.phases[2].name == "case-two" + assert result.phases[2].success is True + + def test_phase_finalizer_skips_when_target_was_not_attempted(self, tmp_path: Path) -> None: + """A prerequisite failure cannot activate destructive cleanup before deployment.""" + marker = tmp_path / "cleaned" + cleanup = _write_script(tmp_path, "cleanup.sh", f"#!/bin/sh\ntouch {marker}\n") + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig(name="preflight", command="false", phase="case-one"), + StepConfig(name="deploy", command="true", phase="case-one"), + StepConfig( + name="cleanup", + command=cleanup, + phase="case-one", + finalizer_for="deploy", + ), + StepConfig(name="case_two", command="true", phase="case-two"), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert not marker.exists() + assert [step["name"] for step in result.phases[0].details["steps"]] == ["preflight"] + assert result.phases[1].name == "case-one-teardown" + assert result.phases[1].message.startswith("SKIPPED: target step(s) were not attempted") + assert result.phases[2].success is True + + def test_phase_finalizer_skips_when_target_process_never_started(self, tmp_path: Path) -> None: + """A command-resolution failure cannot imply that a cluster mutation occurred.""" + marker = tmp_path / "cleaned" + cleanup = _write_script(tmp_path, "cleanup.sh", f"#!/bin/sh\ntouch {marker}\n") + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig( + name="deploy", + command=str(tmp_path / "does-not-exist"), + phase="case-one", + ), + StepConfig( + name="cleanup", + command=cleanup, + phase="case-one", + finalizer_for="deploy", + ), + StepConfig(name="case_two", command="true", phase="case-two"), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert not marker.exists() + assert result.phases[0].details["steps"][0]["attempted"] is False + assert result.phases[1].message.startswith("SKIPPED: target step(s) were not attempted") + assert result.phases[2].success is True + + def test_failed_phase_finalizer_blocks_later_independent_phases(self) -> None: + """A failed cleanup leaves unsafe state and overrides continuation policy.""" + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig(name="deploy", command="true", phase="case-one"), + StepConfig( + name="cleanup", + command="false", + phase="case-one", + finalizer_for="deploy", + ), + StepConfig(name="case_two", command="true", phase="case-two"), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert [(phase.name, phase.message) for phase in result.phases] == [ + ("case-one", "deploy: passed"), + ("case-one-teardown", "cleanup: failed"), + ("case-two", "SKIPPED: previous phase failed"), + ] + + def test_teardown_finalizer_runs_between_test_phases(self, tmp_path: Path) -> None: + """A step declared in teardown executes directly after its target test phase.""" + marker = tmp_path / "cleaned" + cleanup = _write_script(tmp_path, "cleanup.sh", f"#!/bin/sh\ntouch {marker}\n") + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two", "teardown"], + steps=[ + StepConfig(name="deploy", command="true", phase="case-one"), + StepConfig(name="case_two", command="true", phase="case-two"), + StepConfig( + name="cleanup", + command=cleanup, + phase="teardown", + finalizer_for="deploy", + ), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is True + assert marker.is_file() + assert [(phase.name, phase.phase) for phase in result.phases] == [ + ("case-one", Phase.TEST), + ("case-one-teardown", Phase.TEARDOWN), + ("case-two", Phase.TEST), + ] + + def test_teardown_only_runs_linked_finalizer_as_recovery(self, tmp_path: Path) -> None: + """An explicit teardown-only run does not require an in-memory target attempt.""" + marker = tmp_path / "cleaned" + cleanup = _write_script(tmp_path, "cleanup.sh", f"#!/bin/sh\ntouch {marker}\n") + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["test", "teardown"], + steps=[ + StepConfig(name="deploy", command="true", phase="test"), + StepConfig( + name="cleanup", + command=cleanup, + phase="teardown", + finalizer_for="deploy", + ), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEARDOWN]) + + assert result.success is True + assert marker.is_file() + assert [(phase.name, phase.phase) for phase in result.phases] == [ + ("teardown", Phase.TEARDOWN), + ] + + def test_custom_phase_failure_blocks_later_phases_by_default(self) -> None: + """Without an opt-in, the existing stop-on-failure behavior is unchanged.""" + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + steps=[ + StepConfig(name="case_one", command="false", phase="case-one"), + StepConfig( + name="case_two", + command="echo", + args=['{"success": true, "platform": "kubernetes"}'], + phase="case-two", + output_schema="generic", + ), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert [(phase.name, phase.message) for phase in result.phases] == [ + ("case-one", "case_one: failed"), + ("case-two", "SKIPPED: previous phase failed"), + ] + def test_platform_detection_missing(self) -> None: """Test error when platform cannot be detected. @@ -415,6 +732,29 @@ def test_config_without_commands_runs_live_validations_only(self, monkeypatch: p assert [entry.entry.name for entry in result.validations] == ["K8sCsiStorageTypesCheck"] assert result.validations[0].state is State.PASSED + def test_config_without_commands_reports_failed_live_validation(self, monkeypatch: pytest.MonkeyPatch) -> None: + """A failed commandless validation returns a failed result instead of reading command policy.""" + monkeypatch.setattr("isvctl.orchestrator.loop.load_released_test_filter", lambda: None) + config = RunConfig( + tests=ValidationConfig( + validations={ + "live_checks": { + "checks": { + "ExistingSystemFieldCheck": { + "compose": [{"FieldExistsCheck": {"field": "missing"}}], + } + }, + } + }, + ), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST], capability="kubernetes") + + assert result.success is False + assert result.validations[0].state is State.FAILED + assert "Missing fields: missing" in result.validations[0].message + def test_config_without_commands_or_validations_is_not_a_pass(self) -> None: """Validations are all a commandless run has, so wiring none asserts nothing.""" orchestrator = Orchestrator(RunConfig(tests=ValidationConfig(validations={}))) @@ -565,9 +905,80 @@ def test_validation_without_step_output_is_reported_as_skipped(self, monkeypatch "state": "skipped", "skip_reason": "step_no_output", "error_reason": None, + "subtest_summary": {"total": 0, "passed": 0, "failed": 0, "skipped": 0}, } ] + def test_failed_owned_step_is_reported_as_validation_error( + self, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """An early workflow failure cannot become a harmless missing-output skip.""" + monkeypatch.setattr("isvctl.orchestrator.loop.load_released_test_filter", lambda: None) + failing_step = _write_script( + tmp_path, + "deploy.sh", + "#!/bin/sh\necho 'driver image not found' >&2\nexit 4\n", + ) + junit_path = tmp_path / "junit.xml" + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["use-case"], + steps=[ + StepConfig( + name="deploy_fixture", + command=failing_step, + phase="use-case", + requires_selected_validations=["ProbeSucceededCheck"], + ), + StepConfig( + name="validate_fixture", + command="true", + phase="use-case", + requires_selected_validations=["ProbeSucceededCheck"], + ), + ], + ) + }, + tests=ValidationConfig( + capability="kubernetes", + validations={ + "probe_checks": { + "step": "validate_fixture", + "checks": {"ProbeSucceededCheck": {"compose": ["StepSuccessCheck"]}}, + }, + }, + ), + ) + + result = Orchestrator(config).run( + phases=[Phase.TEST], + capability="kubernetes", + junitxml=str(junit_path), + ) + + assert result.success is False + validation = result.validations[0] + assert validation.state is State.ERROR + assert validation.error_reason is ErrorReason.STEP_FAILED + assert validation.message == ( + "workflow step 'deploy_fixture' failed: Command exited with code 4: driver image not found" + ) + + suite = ET.parse(junit_path).getroot().find("testsuite") + assert suite is not None + assert suite.get("errors") == "1" + assert suite.get("skipped") == "0" + case = suite.find("testcase") + assert case is not None + assert case.get("name") == "ProbeSucceededCheck" + error = case.find("error") + assert error is not None + assert error.get("type") == ErrorReason.STEP_FAILED.value + assert case.find("skipped") is None + def test_validation_template_error_is_reported_as_error( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/isvctl/tests/test_orchestrator_process.py b/isvctl/tests/test_orchestrator_process.py new file mode 100644 index 000000000..803d646e7 --- /dev/null +++ b/isvctl/tests/test_orchestrator_process.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for orchestration subprocess lifecycle handling.""" + +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from isvctl.orchestrator.process import run_command_process + + +def test_run_command_process_captures_output(tmp_path: Path) -> None: + """Successful commands return captured text output.""" + completed = run_command_process( + [sys.executable, "-c", "print('ready')"], + cwd=tmp_path, + env=None, + timeout=5, + ) + + assert completed.returncode == 0 + assert completed.stdout == "ready\n" + assert completed.stderr == "" + + +def test_run_command_process_accepts_no_timeout(tmp_path: Path) -> None: + """A null step timeout waits for a command that owns its deadline.""" + completed = run_command_process( + [sys.executable, "-c", "print('tool-owned-timeout')"], + cwd=tmp_path, + env=None, + timeout=None, + ) + + assert completed.returncode == 0 + assert completed.stdout == "tool-owned-timeout\n" + assert completed.stderr == "" + + +@pytest.mark.skipif(os.name != "posix", reason="process-group behavior is POSIX-specific") +def test_timeout_terminates_descendant_process(tmp_path: Path) -> None: + """A timed-out wrapper must not leave its provider CLI child running.""" + child_pid_path = tmp_path / "child.pid" + wrapper = """ +import subprocess +import sys +import time +from pathlib import Path + +child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) +Path(sys.argv[1]).write_text(str(child.pid)) +print("wrapper-ready", flush=True) +time.sleep(60) +""" + child_pid: int | None = None + + try: + with pytest.raises(subprocess.TimeoutExpired) as exc_info: + run_command_process( + [sys.executable, "-c", wrapper, str(child_pid_path)], + cwd=tmp_path, + env=None, + timeout=0.5, + ) + + assert "wrapper-ready" in (exc_info.value.stdout or "") + child_pid = int(child_pid_path.read_text()) + + deadline = time.monotonic() + 2 + while _process_exists(child_pid) and time.monotonic() < deadline: + time.sleep(0.05) + assert not _process_exists(child_pid) + finally: + if child_pid is not None and _process_exists(child_pid): + os.kill(child_pid, signal.SIGKILL) + + +def _process_exists(pid: int) -> bool: + """Return whether a process currently exists.""" + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + return True diff --git a/isvctl/tests/test_schema.py b/isvctl/tests/test_schema.py index d28e9326b..ad16a27dc 100644 --- a/isvctl/tests/test_schema.py +++ b/isvctl/tests/test_schema.py @@ -76,6 +76,8 @@ def test_minimal_step(self) -> None: assert step.phase == "setup" assert step.skip is False assert step.requires == [] + assert step.requires_available_validations == [] + assert step.requires_selected_validations == [] def test_full_step(self) -> None: """Test creating a fully specified step config.""" @@ -89,6 +91,8 @@ def test_full_step(self) -> None: phase="setup", skip=False, requires=["vm", "bare_metal"], + requires_available_validations=["NewCheck"], + requires_selected_validations=["SelectedCheck"], continue_on_failure=True, output_schema="vpc", ) @@ -99,9 +103,17 @@ def test_full_step(self) -> None: assert step.env == {"AWS_REGION": "us-west-2"} assert step.phase == "setup" assert step.requires == ["vm", "bare_metal"] + assert step.requires_available_validations == ["NewCheck"] + assert step.requires_selected_validations == ["SelectedCheck"] assert step.continue_on_failure is True assert step.output_schema == "vpc" + def test_null_timeout_disables_watchdog(self) -> None: + """A provider may delegate timeout ownership to the invoked tool.""" + step = StepConfig(name="validate", command="l8k", timeout=None) + + assert step.timeout is None + def test_step_rejects_unknown_or_duplicate_requires(self) -> None: """Step requirements use the declarable capability vocabulary.""" with pytest.raises(ValidationError, match="requires must be a list containing only"): @@ -110,6 +122,100 @@ def test_step_rejects_unknown_or_duplicate_requires(self) -> None: StepConfig(name="setup", command="echo", requires=["vm", "vm"]) +class TestPlatformCommands: + """Tests for ordered command phases.""" + + def test_phases_reject_duplicates(self) -> None: + """A duplicate phase would otherwise execute its steps more than once.""" + with pytest.raises(ValidationError, match="phases must not contain duplicate"): + PlatformCommands(phases=["setup", "test", "test"]) + + def test_continue_after_failure_rejects_unknown_phase(self) -> None: + """A typo must not silently restore stop-on-failure behavior.""" + with pytest.raises(ValidationError, match="phases not listed in phases"): + PlatformCommands( + phases=["setup", "use-case"], + continue_after_failure=["use-csae"], + ) + + @pytest.mark.parametrize("phase", ["setup", "teardown"]) + def test_continue_after_failure_rejects_lifecycle_phases(self, phase: str) -> None: + """Setup and teardown are never independent test-case phases.""" + with pytest.raises(ValidationError, match="cannot contain lifecycle phases"): + PlatformCommands( + phases=["setup", "use-case", "teardown"], + continue_after_failure=[phase], + ) + + def test_continue_after_failure_rejects_duplicates(self) -> None: + """Duplicate continuation entries are configuration errors, not useful policy.""" + with pytest.raises(ValidationError, match="must not contain duplicate"): + PlatformCommands( + phases=["setup", "use-case"], + continue_after_failure=["use-case", "use-case"], + ) + + def test_finalizer_requires_one_target_in_its_phase_or_teardown(self) -> None: + """A finalizer target must resolve unambiguously in a supported lifecycle position.""" + with pytest.raises(ValidationError, match="must name exactly one configured step"): + PlatformCommands( + phases=["test"], + steps=[ + StepConfig(name="cleanup", command="clean", phase="test", finalizer_for="deploy"), + ], + ) + + with pytest.raises(ValidationError, match="same phase or the finalizer must use phase 'teardown'"): + PlatformCommands( + phases=["setup", "test", "cleanup"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test"), + StepConfig(name="cleanup", command="clean", phase="cleanup", finalizer_for="deploy"), + ], + ) + + config = PlatformCommands( + phases=["test", "teardown"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test"), + StepConfig(name="cleanup", command="clean", phase="teardown", finalizer_for="deploy"), + ], + ) + assert config.steps[1].finalizer_for == "deploy" + + def test_teardown_finalizer_must_follow_target_and_match_its_gates(self) -> None: + """Linked teardown cannot precede its mutation or be filtered independently.""" + with pytest.raises(ValidationError, match="teardown must be ordered after target"): + PlatformCommands( + phases=["teardown", "test"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test"), + StepConfig(name="cleanup", command="clean", phase="teardown", finalizer_for="deploy"), + ], + ) + + with pytest.raises(ValidationError, match="must use the same gates as target"): + PlatformCommands( + phases=["test", "teardown"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test", requires=["kubernetes"]), + StepConfig(name="cleanup", command="clean", phase="teardown", finalizer_for="deploy"), + ], + ) + + def test_finalizer_cannot_target_another_finalizer(self) -> None: + """Finalizer chains have ambiguous activation and cleanup ordering.""" + with pytest.raises(ValidationError, match="cannot finalize finalizer step"): + PlatformCommands( + phases=["test"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test"), + StepConfig(name="cleanup", command="clean", phase="test", finalizer_for="deploy"), + StepConfig(name="verify_cleanup", command="verify", phase="test", finalizer_for="cleanup"), + ], + ) + + class TestCommandOutput: """Tests for CommandOutput model (setup command JSON output).""" diff --git a/isvctl/tests/test_stub_contracts.py b/isvctl/tests/test_stub_contracts.py index 686b29088..3c291c5a5 100644 --- a/isvctl/tests/test_stub_contracts.py +++ b/isvctl/tests/test_stub_contracts.py @@ -156,7 +156,7 @@ def _collect_yaml_checks() -> list[StepArgCheck]: checks: list[StepArgCheck] = [] yaml_paths = sorted( [ - *CONFIGS_DIR.glob("suites/*.yaml"), + *CONFIGS_DIR.glob("suites/**/*.yaml"), *CONFIGS_DIR.glob("providers/*.yaml"), # k3s.yaml, microk8s.yaml, minikube.yaml *CONFIGS_DIR.glob("providers/*/config/*.yaml"), # aws/config/*.yaml, my-isv/config/*.yaml ] diff --git a/isvctl/tests/test_suite_resolution.py b/isvctl/tests/test_suite_resolution.py index 86e253c10..0ba72c76e 100644 --- a/isvctl/tests/test_suite_resolution.py +++ b/isvctl/tests/test_suite_resolution.py @@ -52,6 +52,21 @@ def test_one_suite_flag_resolves_canonical_and_provider_suites(tmp_path: Path) - assert canonical_plain.platform is None +def test_canonical_suite_resolution_discovers_nested_suite_yaml(tmp_path: Path) -> None: + """Domain folders under suites remain selectable through the generic resolver.""" + _write_catalog(tmp_path) + domain = tmp_path / "suites" / "launch-kit" + domain.mkdir() + nested = domain / "network-operator.yaml" + nested.write_text("tests:\n validations: {}\n") + + resolved = resolve_suite(None, "network-operator", configs_root=tmp_path) + + assert resolved.config_path == nested + assert resolved.name == "network_operator" + assert resolved.platform is None + + def test_capability_uses_catalog_vocabulary(tmp_path: Path) -> None: """An unknown capability is rejected while omitted context disables filtering.""" _write_catalog(tmp_path) diff --git a/isvctl/tests/test_test_cli_labels.py b/isvctl/tests/test_test_cli_labels.py index f89107f6d..6d65e7ced 100644 --- a/isvctl/tests/test_test_cli_labels.py +++ b/isvctl/tests/test_test_cli_labels.py @@ -52,6 +52,66 @@ def _write_config(tmp_path: Path) -> Path: return config +def test_validation_result_detail_compacts_successful_subtests() -> None: + """Successful validations with probes render an aggregate instead of their long message.""" + detail = test_cli._validation_result_detail( + { + "passed": True, + "skipped": False, + "state": "passed", + "message": "long; member; output", + "subtest_summary": {"total": 6, "passed": 6, "failed": 0, "skipped": 0}, + } + ) + + assert detail == "6 subtests passed" + + +def test_validation_result_detail_preserves_failures() -> None: + """Failed validations keep their actionable message even when they report probes.""" + detail = test_cli._validation_result_detail( + { + "passed": False, + "skipped": False, + "state": "failed", + "message": "RdmaCheck: worker-a -> worker-b timed out", + "subtest_summary": {"total": 6, "passed": 5, "failed": 1, "skipped": 0}, + } + ) + + assert detail == "RdmaCheck: worker-a -> worker-b timed out" + + +def test_validation_result_detail_counts_successful_runs_with_skips() -> None: + """An allowed skipped probe remains visible in the concise success summary.""" + detail = test_cli._validation_result_detail( + { + "passed": True, + "skipped": False, + "state": "passed", + "message": "long output", + "subtest_summary": {"total": 6, "passed": 5, "failed": 0, "skipped": 1}, + } + ) + + assert detail == "6 subtests: 5 passed, 0 failed, 1 skipped" + + +def test_validation_result_detail_derives_total_for_legacy_summaries() -> None: + """Older result payloads remain concise when they omit the explicit total.""" + detail = test_cli._validation_result_detail( + { + "passed": True, + "skipped": False, + "state": "passed", + "message": "long output", + "subtest_summary": {"passed": 5, "failed": 0, "skipped": 1}, + } + ) + + assert detail == "6 subtests: 5 passed, 0 failed, 1 skipped" + + def _write_provider_config(root: Path, provider: str, name: str, suite: str, platform: str) -> Path: """Write a minimal provider config importing one suite.""" config_path = root / "providers" / provider / "config" / name @@ -129,6 +189,49 @@ def test_test_run_forwards_label_filters(monkeypatch: pytest.MonkeyPatch, tmp_pa assert _FakeOrchestrator.captured["include_labels"] == ["gpu", "slow"] +def test_orchestration_summary_hides_filtered_validations_by_default( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The default summary omits phases containing only selection-filtered checks.""" + config = _write_config(tmp_path) + + class FilteredOrchestrator(_FakeOrchestrator): + """Return one resolution-only phase for a filtered validation.""" + + def run(self, **kwargs: Any) -> OrchestratorResult: + """Return the synthetic filtered result.""" + return OrchestratorResult( + success=True, + phases=[ + PhaseResult( + phase=Phase.TEST, + success=True, + message="test phase validations resolved without execution", + details={ + "validations": [ + { + "name": "InfiniBandCheck", + "skipped": True, + "state": "skipped", + "skip_reason": "test_excluded", + "message": "does not match selected label ethernet", + } + ] + }, + ) + ], + ) + + monkeypatch.setattr(test_cli, "Orchestrator", FilteredOrchestrator) + + result = runner.invoke(test_cli.app, ["run", "-f", str(config), "--label", "ethernet", "--no-upload"]) + + assert result.exit_code == 0, result.output + assert "InfiniBandCheck" not in result.output + assert "TEST : test phase validations resolved" not in result.output + + def test_test_run_reports_and_saves_the_complete_catalog_identity( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/isvtest/src/isvtest/catalog.py b/isvtest/src/isvtest/catalog.py index 0ea7994ff..1308cddcb 100644 --- a/isvtest/src/isvtest/catalog.py +++ b/isvtest/src/isvtest/catalog.py @@ -20,7 +20,7 @@ The catalog is version-keyed by the installed isvtest package version. Suite placement and capability requirements come only from canonical -``isvctl/configs/suites/*.yaml`` wiring. +``isvctl/configs/suites/**/*.yaml`` wiring. """ import hashlib @@ -210,7 +210,7 @@ def _iter_suite_docs() -> Iterator[tuple[Path, dict[str, Any]]]: if not configs_dir: logger.warning("Could not locate isvctl/configs/ directory") return - for config_path in sorted((configs_dir / "suites").glob("*.yaml")): + for config_path in sorted((configs_dir / "suites").rglob("*.yaml")): yield config_path, yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} diff --git a/isvtest/src/isvtest/core/composite.py b/isvtest/src/isvtest/core/composite.py index 833fd1774..0b08e4d26 100644 --- a/isvtest/src/isvtest/core/composite.py +++ b/isvtest/src/isvtest/core/composite.py @@ -42,6 +42,8 @@ from typing import Any, ClassVar +import pytest + from isvtest.core.validation import BaseValidation, get_validation_class COMPOSE_KEY = "compose" @@ -86,7 +88,7 @@ class CompositeCheck(BaseValidation): _exclude_from_discovery: ClassVar[bool] = True def run(self) -> None: - """Run every configured member and fail the composite on invalid or failed members.""" + """Run every member, retaining skips and failing on invalid or failed members.""" raw = self.config.get(COMPOSE_KEY) members = composed_members(raw) if not members: @@ -115,9 +117,23 @@ def run(self) -> None: member = member_class(runner=self.runner, config={**shared, **member_params}) member.name = member_name - result = member.execute() + try: + result = member.execute() + except pytest.skip.Exception as exc: + reason = str(exc) + self.report_subtest(member_name, False, reason, skipped=True) + outputs.append(f"{member_name}: skipped - {reason}") + continue message = result["output"] if result["passed"] else result["error"] self.report_subtest(member_name, result["passed"], message, duration=result["duration"]) + for nested in result.get("subtests", []): + self.report_subtest( + f"{member_name}/{nested['name']}", + bool(nested.get("passed")), + str(nested.get("message", "")), + skipped=bool(nested.get("skipped")), + duration=nested.get("duration"), + ) if result["passed"]: outputs.append(f"{member_name}: {message}" if message else member_name) else: diff --git a/isvtest/src/isvtest/core/resolution.py b/isvtest/src/isvtest/core/resolution.py index c3401294f..492d0619e 100644 --- a/isvtest/src/isvtest/core/resolution.py +++ b/isvtest/src/isvtest/core/resolution.py @@ -20,7 +20,7 @@ import logging from collections.abc import Iterable, Mapping from collections.abc import Set as AbstractSet -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import StrEnum from functools import cache from typing import Any @@ -92,6 +92,7 @@ class ErrorReason(StrEnum): INVALID_CONFIG = "invalid_config" RUNTIME_EXCEPTION = "runtime_exception" + STEP_FAILED = "step_failed" TEMPLATE_RENDER_FAILED = "template_render_failed" @@ -108,6 +109,20 @@ class ValidationEntry: requires: tuple[str, ...] = () +@dataclass(frozen=True) +class SubtestSummary: + """Aggregate counts for the subtests reported by one validation.""" + + passed: int = 0 + failed: int = 0 + skipped: int = 0 + + @property + def total(self) -> int: + """Return the total number of reported subtests.""" + return self.passed + self.failed + self.skipped + + @dataclass class ResolvedEntry: """Lifecycle record for a single validation entry.""" @@ -119,6 +134,7 @@ class ResolvedEntry: error_reason: ErrorReason | None = None message: str = "" duration_seconds: float = 0.0 + subtest_summary: SubtestSummary = field(default_factory=SubtestSummary) @property def is_ready(self) -> bool: @@ -239,6 +255,56 @@ def parse_validations(raw_config: Mapping[str, Any]) -> list[ValidationEntry]: return entries +def resolve_entry_selection( + entry: ValidationEntry, + *, + include_labels: AbstractSet[str], + exclude_labels: AbstractSet[str], + exclude_tests: AbstractSet[str], + capability: str | None = None, +) -> ResolvedEntry | None: + """Return a terminal result when selection excludes an entry, otherwise ``None``. + + This is the provider-neutral selection boundary shared by validation + execution and lifecycle steps gated with ``requires_selected_validations``. + It deliberately stops before phase, step-output, and template resolution. + """ + config_error = _validate_entry_shape(entry) + if config_error: + return _error(entry, ErrorReason.INVALID_CONFIG, config_error) + + if entry.name in exclude_tests: + return _skip(entry, SkipReason.EXCLUDED, f"validation '{entry.name}' is excluded by name") + + if capability is not None and not requirements_satisfied(entry.requires, capability): + requirement_list = ", ".join(entry.requires) or "(none)" + return _skip( + entry, + SkipReason.CAPABILITY_REQUIREMENT, + f"requires {requirement_list} (context: {capability})", + ) + + missing_include_labels = sorted(set(include_labels).difference(entry.labels)) + if missing_include_labels: + label_list = ", ".join(sorted(include_labels)) + return _skip( + entry, + SkipReason.EXCLUDED, + f"validation '{entry.name}' does not match all selected labels: {label_list}", + ) + + label_matches = sorted(set(entry.labels).intersection(exclude_labels)) + if label_matches: + label_list = ", ".join(label_matches) + return _skip( + entry, + SkipReason.EXCLUDED, + f"validation '{entry.name}' is excluded by label: {label_list}", + ) + + return None + + def resolve_entries( entries: list[ValidationEntry], *, @@ -275,49 +341,15 @@ def resolve_entries( env = _create_jinja_env() for entry in entries: - config_error = _validate_entry_shape(entry) - if config_error: - resolved.append(_error(entry, ErrorReason.INVALID_CONFIG, config_error)) - continue - - if entry.name in exclude_tests: - resolved.append(_skip(entry, SkipReason.EXCLUDED, f"validation '{entry.name}' is excluded by name")) - continue - - if capability is not None and not requirements_satisfied(entry.requires, capability): - requirement_list = ", ".join(entry.requires) or "(none)" - context_list = capability - resolved.append( - _skip( - entry, - SkipReason.CAPABILITY_REQUIREMENT, - f"requires {requirement_list} (context: {context_list})", - ) - ) - continue - - missing_include_labels = sorted(set(include_labels).difference(entry.labels)) - if missing_include_labels: - label_list = ", ".join(sorted(include_labels)) - resolved.append( - _skip( - entry, - SkipReason.EXCLUDED, - f"validation '{entry.name}' does not match all selected labels: {label_list}", - ) - ) - continue - - label_matches = sorted(set(entry.labels).intersection(exclude_labels)) - if label_matches: - label_list = ", ".join(label_matches) - resolved.append( - _skip( - entry, - SkipReason.EXCLUDED, - f"validation '{entry.name}' is excluded by label: {label_list}", - ) - ) + selection_result = resolve_entry_selection( + entry, + include_labels=include_labels, + exclude_labels=exclude_labels, + exclude_tests=exclude_tests, + capability=capability, + ) + if selection_result is not None: + resolved.append(selection_result) continue if entry.step and entry.step in skipped_steps: diff --git a/isvtest/src/isvtest/main.py b/isvtest/src/isvtest/main.py index 44806cadd..f2b723946 100644 --- a/isvtest/src/isvtest/main.py +++ b/isvtest/src/isvtest/main.py @@ -36,7 +36,7 @@ from isvtest.config.loader import ConfigLoader from isvtest.core import runners as reframe_runner from isvtest.core.logger import setup_logger -from isvtest.core.resolution import ErrorReason, ResolvedEntry, SkipReason, State +from isvtest.core.resolution import ErrorReason, ResolvedEntry, SkipReason, State, SubtestSummary from isvtest.tests.test_validations import ( clear_validation_results, get_validation_results, @@ -223,6 +223,12 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R """Convert a captured pytest validation result to a terminal resolved entry.""" message = str(result.get("message", "")) duration = float(result.get("duration", 0.0) or 0.0) + raw_subtests = result.get("subtest_summary", {}) + subtest_summary = SubtestSummary( + passed=int(raw_subtests.get("passed", 0)) if isinstance(raw_subtests, dict) else 0, + failed=int(raw_subtests.get("failed", 0)) if isinstance(raw_subtests, dict) else 0, + skipped=int(raw_subtests.get("skipped", 0)) if isinstance(raw_subtests, dict) else 0, + ) if result.get("skipped"): return ResolvedEntry( entry=entry.entry, @@ -231,6 +237,7 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R skip_reason=SkipReason.RUNTIME_SKIP, message=message, duration_seconds=duration, + subtest_summary=subtest_summary, ) if result.get("passed", False): return ResolvedEntry( @@ -239,6 +246,7 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R state=State.PASSED, message=message, duration_seconds=duration, + subtest_summary=subtest_summary, ) if result.get("error_reason") == ErrorReason.RUNTIME_EXCEPTION.value: return ResolvedEntry( @@ -248,6 +256,7 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R error_reason=ErrorReason.RUNTIME_EXCEPTION, message=message, duration_seconds=duration, + subtest_summary=subtest_summary, ) return ResolvedEntry( entry=entry.entry, @@ -255,6 +264,7 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R state=State.FAILED, message=message, duration_seconds=duration, + subtest_summary=subtest_summary, ) diff --git a/isvtest/src/isvtest/testing/subtests.py b/isvtest/src/isvtest/testing/subtests.py index 09f889fbb..e7cd5fb83 100644 --- a/isvtest/src/isvtest/testing/subtests.py +++ b/isvtest/src/isvtest/testing/subtests.py @@ -509,11 +509,6 @@ def _inject_subtests_into_junit(junit_path: Path, reports: list[SubTestReport]) name = tc.get("name", "") testcase_indices[name] = idx - # Track counts for updating testsuite attributes - added_tests = 0 - added_failures = 0 - added_skipped = 0 - # Insert subtests after their parent, in reverse order of parent index # to avoid index shifting issues insertions: list[tuple[int, list[ET.Element]]] = [] @@ -540,16 +535,12 @@ def _inject_subtests_into_junit(junit_path: Path, reports: list[SubTestReport]) testcase.set("classname", "") # Match pytest's default testcase.set("time", f"{report.duration:.3f}") - added_tests += 1 - if report.failed: - added_failures += 1 failure = ET.SubElement(testcase, "failure") failure.set("message", f"Subtest {subtest_desc} failed") if report.longrepr: failure.text = str(report.longrepr)[:2000] # Limit length elif report.skipped: - added_skipped += 1 skipped_elem = ET.SubElement(testcase, "skipped") # longrepr from both our skipped-flag path and pytest.skip() # is a (file, line, msg) tuple; surface the msg so the @@ -578,14 +569,15 @@ def _inject_subtests_into_junit(junit_path: Path, reports: list[SubTestReport]) for i, elem in enumerate(subtest_elements): testsuite.insert(insert_pos + i, elem) - # Update testsuite counts - current_tests = int(testsuite.get("tests", "0")) - current_failures = int(testsuite.get("failures", "0")) - current_skipped = int(testsuite.get("skipped", "0")) - - testsuite.set("tests", str(current_tests + added_tests)) - testsuite.set("failures", str(current_failures + added_failures)) - testsuite.set("skipped", str(current_skipped + added_skipped)) + # pytest counts subtest reports in the testsuite attributes even though + # its JUnit plugin does not serialize them as testcase elements. Adding + # our testcase nodes and incrementing those attributes would therefore + # double-count every subtest. Reconcile counters with the serialized XML. + serialized_cases = testsuite.findall("testcase") + testsuite.set("tests", str(len(serialized_cases))) + testsuite.set("failures", str(sum(case.find("failure") is not None for case in serialized_cases))) + testsuite.set("errors", str(sum(case.find("error") is not None for case in serialized_cases))) + testsuite.set("skipped", str(sum(case.find("skipped") is not None for case in serialized_cases))) # Write back tree.write(junit_path, encoding="utf-8", xml_declaration=True) diff --git a/isvtest/src/isvtest/tests/test_validations.py b/isvtest/src/isvtest/tests/test_validations.py index 425171a93..d4497c10b 100644 --- a/isvtest/src/isvtest/tests/test_validations.py +++ b/isvtest/src/isvtest/tests/test_validations.py @@ -273,6 +273,18 @@ def test_validation( "category": category, "duration": result.get("duration", 0.0), "error_reason": result.get("error_reason"), + "subtest_summary": { + "total": len(result.get("subtests", [])), + "passed": sum( + 1 for subtest in result.get("subtests", []) if subtest.get("passed") and not subtest.get("skipped") + ), + "failed": sum( + 1 + for subtest in result.get("subtests", []) + if not subtest.get("passed") and not subtest.get("skipped") + ), + "skipped": sum(1 for subtest in result.get("subtests", []) if subtest.get("skipped")), + }, } ) diff --git a/isvtest/src/isvtest/validations/k8s_launch_kit/__init__.py b/isvtest/src/isvtest/validations/k8s_launch_kit/__init__.py new file mode 100644 index 000000000..953b56014 --- /dev/null +++ b/isvtest/src/isvtest/validations/k8s_launch_kit/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Kubernetes Launch Kit validation checks.""" diff --git a/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py b/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py new file mode 100644 index 000000000..bb14d62c1 --- /dev/null +++ b/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py @@ -0,0 +1,721 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Assertions over unmodified Kubernetes Launch Kit command output. + +Cluster interaction and Launch Kit command execution stay in the provider. +These checks interpret the real discover and validate documents and expose +resource or matrix rows as pytest subtests for actionable reporting. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, ClassVar + +import pytest + +from isvtest.core.validation import BaseValidation + +_CONNECTIVITY_FAMILIES: dict[str, set[int]] = { + "icmp": {0, 1}, + "rping": {2, 3}, + "ib_write_bw": {4, 5}, + "gpudirect_dmabuf": {6, 7}, +} + +_PROFILE_NETWORK_KINDS: dict[tuple[str, str], str] = { + ("ethernet", "sriov"): "SriovNetwork", + ("infiniband", "sriov"): "SriovIBNetwork", + ("ethernet", "rdma_shared"): "MacvlanNetwork", + ("infiniband", "rdma_shared"): "IPoIBNetwork", + ("ethernet", "host_device"): "HostDeviceNetwork", + ("infiniband", "host_device"): "HostDeviceNetwork", +} + + +def _object(value: Any) -> dict[str, Any]: + """Return ``value`` as a JSON object or an empty object.""" + return value if isinstance(value, dict) else {} + + +def _list(value: Any) -> list[Any]: + """Return ``value`` as a list or an empty list.""" + return value if isinstance(value, list) else [] + + +def _profile_network_kind(profile: dict[str, Any]) -> str | None: + """Return the expected secondary-network resource for a resolved profile.""" + fabric = profile.get("fabric") + deployment = profile.get("deployment") + if not isinstance(fabric, str) or not isinstance(deployment, str): + return None + return _PROFILE_NETWORK_KINDS.get((fabric, deployment)) + + +class _LaunchKitCheck(BaseValidation): + """Shared parsing and subtest reporting for Launch Kit checks.""" + + _exclude_from_discovery: ClassVar[bool] = True + + def _step_output(self, operation: str | None = None) -> dict[str, Any] | None: + """Return the bound provider envelope and validate its operation.""" + output = self.config.get("step_output") + if not isinstance(output, dict): + self.set_failed("Missing Launch Kit step_output") + return None + if operation is not None and output.get("operation") != operation: + self.set_failed(f"Expected Launch Kit operation {operation!r}, got {output.get('operation')!r}") + return None + return output + + def _configured_output(self, key: str) -> dict[str, Any]: + """Decode another step envelope passed through validation configuration.""" + value = self.config.get(key) + if isinstance(value, dict): + return value + if not isinstance(value, str) or not value: + return {} + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + + def _documents(self, output: dict[str, Any]) -> list[dict[str, Any]]: + """Return only object documents from a provider envelope.""" + return [document for document in _list(output.get("documents")) if isinstance(document, dict)] + + def _profile(self) -> dict[str, Any] | None: + """Return the resolved profile from the real discover JSONResult.""" + discover = self._configured_output("discover_output") + documents = self._documents(discover) + profile = documents[0].get("profile") if documents else None + if not isinstance(profile, dict): + self.set_failed("Launch Kit discover output has no resolved profile") + return None + return profile + + def _static_document(self, output: dict[str, Any]) -> dict[str, Any] | None: + """Find the manifest/version validation document.""" + for document in self._documents(output): + if {"versionCheck", "manifests", "summary"}.issubset(document): + return document + provider_error = output.get("error") + suffix = f": {provider_error}" if isinstance(provider_error, str) and provider_error else "" + self.set_failed(f"Launch Kit validate output has no static validation document{suffix}") + return None + + def _connectivity(self, output: dict[str, Any]) -> dict[str, Any] | None: + """Find the source-bound connectivity matrix document.""" + for document in self._documents(output): + connectivity = document.get("connectivity") + if isinstance(connectivity, dict): + return connectivity + provider_error = output.get("error") + suffix = f": {provider_error}" if isinstance(provider_error, str) and provider_error else "" + self.set_failed(f"Launch Kit validate output has no connectivity matrix{suffix}") + return None + + def _finish_probes(self, title: str, probes: list[dict[str, Any]]) -> None: + """Report every probe and aggregate its failures after the final row.""" + if not probes: + if not self._error: + self.set_failed(f"{title} produced no probes") + return + failures: list[str] = [] + for index, probe in enumerate(probes, start=1): + name = str(probe.get("name") or f"probe-{index}") + passed = probe.get("passed") is True + skipped = probe.get("skipped") is True + message = str(probe.get("message") or probe.get("error") or "") + self.report_subtest(name, passed=passed, skipped=skipped, message=message) + if not passed and not skipped: + failures.append(f"{name}: {message or 'failed without a diagnostic'}") + if failures: + self.set_failed(f"{title} failed: {'; '.join(failures)}") + return + self.set_passed(f"{title} passed ({len(probes)} probes)") + + def _manifest_probes( + self, + output: dict[str, Any], + *, + required_kinds: set[str] | None = None, + ) -> list[dict[str, Any]]: + """Build probes from Launch Kit manifest validation rows.""" + static = self._static_document(output) + if static is None: + return [] + manifests = [item for item in _list(static.get("manifests")) if isinstance(item, dict)] + if required_kinds is not None: + manifests = [item for item in manifests if item.get("Kind") in required_kinds] + probes = [] + for item in manifests: + kind = str(item.get("Kind") or "unknown-kind") + namespace = str(item.get("Namespace") or "cluster") + name = str(item.get("Name") or "unknown") + passed = item.get("State") == "success" and item.get("Missing") is not True + probes.append( + { + "name": f"{kind}/{namespace}/{name}", + "passed": passed, + "message": str(item.get("Reason") or item.get("Detail") or item.get("State") or ""), + } + ) + return probes + + def _matrix_probes(self, output: dict[str, Any], families: set[str]) -> list[dict[str, Any]]: + """Build one informative subtest for every selected connectivity row.""" + connectivity = self._connectivity(output) + if connectivity is None: + return [] + probes: list[dict[str, Any]] = [] + for row in _list(connectivity.get("PingResults")): + if not isinstance(row, dict): + continue + test = _object(row.get("Test")) + kind = test.get("Kind") + explicit_family = row.get("Family") + family = explicit_family if isinstance(explicit_family, str) else None + if family not in _CONNECTIVITY_FAMILIES: + family = next( + (name for name, family_kinds in _CONNECTIVITY_FAMILIES.items() if kind in family_kinds), + None, + ) + if family not in families: + continue + source = str(test.get("SrcNode") or test.get("SrcPod") or "unknown-source") + destination = str(test.get("DstNode") or test.get("DstPod") or "unknown-destination") + source_rail = str(test.get("SrcRail") or test.get("Rail") or "unknown-rail") + destination_rail = str(test.get("DstRail") or test.get("Rail") or "unknown-rail") + expectation = str(row.get("Expectation") or test.get("Expectation") or "required") + passed = row.get("OK") is True + stderr = str(row.get("Stderr") or "").strip() + error = str(row.get("Error") or "").strip() + bandwidth = row.get("BandwidthGbps") + minimum = row.get("MinBandwidthGbps") + details = [f"expectation={expectation}", f"observedOK={row.get('ObservedOK')}"] + if family in {"ib_write_bw", "gpudirect_dmabuf"}: + details.extend([f"bandwidthGbps={bandwidth}", f"minimumGbps={minimum}"]) + if family == "gpudirect_dmabuf": + source_gpu = test.get("SrcGPUIndex") + destination_gpu = test.get("DstGPUIndex") + valid_gpu_indices = all( + isinstance(index, int) and not isinstance(index, bool) and index >= 0 + for index in (source_gpu, destination_gpu) + ) + passed = passed and valid_gpu_indices + details.append(f"gpuIndices={source_gpu}->{destination_gpu}") + source_gpu_pci = test.get("SrcGPUPCIAddress") + destination_gpu_pci = test.get("DstGPUPCIAddress") + if source_gpu_pci: + details.append(f"sourceGpuPci={source_gpu_pci}") + if destination_gpu_pci: + details.append(f"destinationGpuPci={destination_gpu_pci}") + if not valid_gpu_indices: + details.append("invalid or missing endpoint GPU index") + if stderr: + details.append(f"stderr={stderr}") + if error and error != stderr: + details.append(f"error={error}") + probes.append( + { + "name": f"{family}/{source}->{destination}/{source_rail}->{destination_rail}", + "passed": passed, + "message": ", ".join(details), + "source_rail": source_rail, + "destination_rail": destination_rail, + } + ) + return probes + + def _kind_coverage_probes( + self, + output: dict[str, Any], + expected_kinds: set[str], + ) -> list[dict[str, Any]]: + """Require every applicable manifest kind to appear in Launch Kit output.""" + static = self._static_document(output) + if static is None: + return [] + observed = { + str(item.get("Kind")) + for item in _list(static.get("manifests")) + if isinstance(item, dict) and item.get("Kind") + } + return [ + { + "name": f"kind-coverage/{kind}", + "passed": kind in observed, + "message": f"observed kinds: {', '.join(sorted(observed)) or '(none)'}", + } + for kind in sorted(expected_kinds) + ] + + +class LaunchKitKubernetesPrerequisiteCheck(_LaunchKitCheck): + """Require a reachable Kubernetes API and a non-empty Ready-node inventory.""" + + description: ClassVar[str] = "Check Kubernetes prerequisites before Launch Kit execution" + + def run(self) -> None: + """Report every provider preflight probe.""" + output = self._configured_output("preflight_output") or self._step_output("kubernetes-preflight") + if output is None: + return + probes = [probe for probe in _list(output.get("checks")) if isinstance(probe, dict)] + self._finish_probes("Kubernetes prerequisite", probes) + + +class LaunchKitTopologyDiscoveryCheck(_LaunchKitCheck): + """Validate that Launch Kit completed discovery and resolved a profile.""" + + description: ClassVar[str] = "Check cluster topology discovery with Kubernetes Launch Kit" + + def run(self) -> None: + """Check the real discover JSONResult without inventing topology fields.""" + output = self._configured_output("discover_output") or self._step_output("discover") + if output is None: + return + documents = self._documents(output) + document = documents[0] if len(documents) == 1 else {} + profile = _object(document.get("profile")) + probes = [ + { + "name": "discover-command", + "passed": output.get("success") is True and document.get("success") is True, + "message": str(output.get("error") or f"phase={document.get('phase')!r}"), + }, + { + "name": "resolved-profile", + "passed": bool(profile.get("fabric") and profile.get("deployment")), + "message": f"fabric={profile.get('fabric')}, deployment={profile.get('deployment')}", + }, + ] + self._finish_probes("Launch Kit topology discovery", probes) + + +class LaunchKitDeploymentHealthCheck(_LaunchKitCheck): + """Validate the Network Operator release and every generated resource.""" + + description: ClassVar[str] = "Check Network Operator deployment health with Kubernetes Launch Kit" + + def run(self) -> None: + """Report version and manifest readiness independently of connectivity.""" + output = self._step_output("validate") + if output is None: + return + static = self._static_document(output) + if static is None: + return + version = _object(static.get("versionCheck")) + summary = _object(static.get("summary")) + version_skipped = version.get("Skipped") is True + probes = [ + { + "name": "launch-kit-validate-command", + "passed": output.get("success") is True and output.get("exit_code") == 0, + "message": str( + output.get("error") or f"success={output.get('success')!r}, exitCode={output.get('exit_code')!r}" + ), + }, + { + "name": "network-operator-version", + "passed": not version_skipped and version.get("Match") is True, + "skipped": version_skipped, + "message": ( + str(version.get("Reason")) + if version_skipped + else ( + f"selected={version.get('SelectedRelease')}, expected={version.get('ExpectedVersion')}, " + f"deployed={_object(version.get('DeployedRelease')).get('ChartVersion')}" + ) + ), + }, + { + "name": "static-summary", + "passed": summary.get("success") is True, + "message": ( + f"success={summary.get('successManifests')}/{summary.get('totalManifests')}, " + f"errors={summary.get('errorManifests')}, missing={summary.get('missingManifests')}" + ), + }, + { + "name": "manifest-inventory", + "passed": bool(_list(static.get("manifests"))), + "message": f"rows={len(_list(static.get('manifests')))}", + }, + *self._manifest_probes(output), + ] + self._finish_probes("Network Operator deployment health", probes) + + +class LaunchKitSriovReadinessCheck(_LaunchKitCheck): + """Validate SR-IOV policies and secondary-network resources.""" + + description: ClassVar[str] = "Check SR-IOV Network RDMA readiness with Kubernetes Launch Kit" + + def run(self) -> None: + """Check applicable validated resources for an SR-IOV profile.""" + output = self._step_output("validate") + if output is None: + return + profile = self._profile() + if profile is None: + return + if profile.get("deployment") != "sriov": + pytest.skip(f"selected Launch Kit deployment is {profile.get('deployment')}, not sriov") + fabric = profile.get("fabric") + network_kind = "SriovIBNetwork" if fabric == "infiniband" else "SriovNetwork" + expected_kinds = {"SriovNetworkNodePolicy", network_kind} + probes = self._kind_coverage_probes(output, expected_kinds) + probes.extend( + self._manifest_probes( + output, + required_kinds=expected_kinds, + ) + ) + self._finish_probes("SR-IOV readiness", probes) + + +class LaunchKitRdmaConnectivityCheck(_LaunchKitCheck): + """Validate every rping matrix result.""" + + description: ClassVar[str] = "Check pod-to-pod RDMA connectivity with Kubernetes Launch Kit" + + def run(self) -> None: + """Report all same-rail and cross-rail RDMA-CM observations.""" + output = self._step_output("validate") + if output is not None: + self._finish_probes("RDMA-CM connectivity", self._matrix_probes(output, {"rping"})) + + +class LaunchKitRoceCheck(_LaunchKitCheck): + """Validate the selected Ethernet/RoCE profile resources.""" + + description: ClassVar[str] = "Check RoCE secondary networking with Kubernetes Launch Kit" + + def run(self) -> None: + """Skip non-Ethernet profiles and report the selected network resources.""" + output = self._step_output("validate") + if output is None: + return + profile = self._profile() + if profile is None: + return + if profile.get("fabric") != "ethernet": + pytest.skip(f"selected Launch Kit fabric is {profile.get('fabric')}, not ethernet") + network_kind = _profile_network_kind(profile) + if network_kind is None: + self.set_failed(f"Launch Kit returned an unsupported profile: {profile}") + return + probes = self._kind_coverage_probes(output, {network_kind}) + probes.extend(self._manifest_probes(output, required_kinds={network_kind})) + self._finish_probes("Ethernet/RoCE profile", probes) + + +class LaunchKitInfiniBandCheck(_LaunchKitCheck): + """Validate the selected InfiniBand profile resources.""" + + description: ClassVar[str] = "Check InfiniBand networking with Kubernetes Launch Kit" + + def run(self) -> None: + """Skip non-IB profiles and report IB network resources.""" + output = self._step_output("validate") + if output is None: + return + profile = self._profile() + if profile is None: + return + if profile.get("fabric") != "infiniband": + pytest.skip(f"selected Launch Kit fabric is {profile.get('fabric')}, not infiniband") + network_kind = _profile_network_kind(profile) + if network_kind is None: + self.set_failed(f"Launch Kit returned an unsupported profile: {profile}") + return + probes = self._kind_coverage_probes(output, {network_kind}) + probes.extend(self._manifest_probes(output, required_kinds={network_kind})) + self._finish_probes("InfiniBand profile", probes) + + +class LaunchKitHostDeviceCheck(_LaunchKitCheck): + """Validate an applicable host-device profile.""" + + description: ClassVar[str] = "Check host-device networking with Kubernetes Launch Kit" + + def run(self) -> None: + """Skip other deployment types and report HostDeviceNetwork rows.""" + output = self._step_output("validate") + if output is None: + return + profile = self._profile() + if profile is None: + return + if profile.get("deployment") != "host_device": + pytest.skip(f"selected Launch Kit deployment is {profile.get('deployment')}, not host_device") + probes = self._kind_coverage_probes(output, {"HostDeviceNetwork"}) + probes.extend(self._manifest_probes(output, required_kinds={"HostDeviceNetwork"})) + self._finish_probes( + "host-device networking", + probes, + ) + + +class LaunchKitSecondaryNetworkCheck(_LaunchKitCheck): + """Validate secondary-network resources and test DaemonSet readiness.""" + + description: ClassVar[str] = "Check secondary-network and IPAM readiness with Kubernetes Launch Kit" + + def run(self) -> None: + """Report network/IPPool manifests and test-pod rollout state.""" + output = self._step_output("validate") + if output is None: + return + profile = self._profile() + if profile is None: + return + network_kind = _profile_network_kind(profile) + if network_kind is None: + self.set_failed(f"Launch Kit returned an unsupported profile: {profile}") + return + network_kinds = {"SriovNetwork", "SriovIBNetwork", "MacvlanNetwork", "IPoIBNetwork", "HostDeviceNetwork"} + probes = self._kind_coverage_probes(output, {"IPPool", network_kind}) + probes.extend( + self._manifest_probes( + output, + required_kinds={"IPPool", *network_kinds}, + ) + ) + static = self._static_document(output) + if static is None: + return + observed_network_kinds = { + str(item.get("Kind")) + for item in _list(static.get("manifests")) + if isinstance(item, dict) and item.get("Kind") in network_kinds + } + probes.append( + { + "name": "kind-coverage/secondary-network", + "passed": bool(observed_network_kinds), + "message": f"observed kinds: {', '.join(sorted(observed_network_kinds)) or '(none)'}", + } + ) + connectivity = self._connectivity(output) + if connectivity is None: + return + for daemonset in _list(connectivity.get("DaemonSets")): + if not isinstance(daemonset, dict): + continue + ref = _object(daemonset.get("Ref")) + rollout = _object(daemonset.get("Rollout")) + desired = rollout.get("Desired") + ready = rollout.get("Ready") + not_ready = rollout.get("NotReady") + valid_counts = all(type(value) is int and value >= 0 for value in (desired, ready, not_ready)) + rollout_detail = f"ready={ready}/{desired}, notReady={not_ready}" + if not valid_counts: + rollout_detail += ", invalid or missing integer rollout counts" + probes.append( + { + "name": f"DaemonSet/{ref.get('Namespace')}/{ref.get('Name')}", + "passed": valid_counts and desired > 0 and ready == desired and not_ready == 0, + "message": rollout_detail, + } + ) + self._finish_probes("secondary-network readiness", probes) + + +class LaunchKitRdmaSharedCheck(_LaunchKitCheck): + """Validate an applicable RDMA Shared profile.""" + + description: ClassVar[str] = "Check RDMA Shared networking with Kubernetes Launch Kit" + + def run(self) -> None: + """Skip other profiles and report Macvlan or IPoIB resources.""" + output = self._step_output("validate") + if output is None: + return + profile = self._profile() + if profile is None: + return + if profile.get("deployment") != "rdma_shared": + pytest.skip(f"selected Launch Kit deployment is {profile.get('deployment')}, not rdma_shared") + network_kind = _profile_network_kind(profile) + if network_kind is None: + self.set_failed(f"Launch Kit returned an unsupported profile: {profile}") + return + probes = self._kind_coverage_probes(output, {network_kind}) + probes.extend(self._manifest_probes(output, required_kinds={network_kind})) + self._finish_probes( + "RDMA Shared networking", + probes, + ) + + +class LaunchKitIcmpConnectivityCheck(_LaunchKitCheck): + """Validate every source-bound ICMP matrix result.""" + + description: ClassVar[str] = "Check source-bound ICMP connectivity with Kubernetes Launch Kit" + + def run(self) -> None: + """Report all same-rail and expected-isolation ICMP observations.""" + output = self._step_output("validate") + if output is not None: + self._finish_probes("source-bound ICMP", self._matrix_probes(output, {"icmp"})) + + +class LaunchKitRdmaBandwidthCheck(_LaunchKitCheck): + """Validate every ib_write_bw matrix result and its Launch Kit threshold.""" + + description: ClassVar[str] = "Check RDMA bandwidth with Kubernetes Launch Kit" + + def run(self) -> None: + """Use the observed and minimum bandwidth emitted by Launch Kit.""" + output = self._step_output("validate") + if output is not None: + self._finish_probes("RDMA bandwidth", self._matrix_probes(output, {"ib_write_bw"})) + + +class LaunchKitGpuDirectRdmaCheck(_LaunchKitCheck): + """Validate every Launch Kit GPUDirect DMA-BUF bandwidth result.""" + + description: ClassVar[str] = "Check GPUDirect RDMA DMA-BUF bandwidth with Kubernetes Launch Kit" + + def run(self) -> None: + """Report endpoint GPU topology and Launch Kit's bandwidth verdict.""" + output = self._step_output("validate") + if output is None: + return + probes = self._matrix_probes(output, {"gpudirect_dmabuf"}) + if not probes: + if self._error: + return + pytest.skip( + "Launch Kit emitted no gpudirect_dmabuf results; validation.gpuDirect is disabled " + "or ib_write_bw is not selected" + ) + self._finish_probes("GPUDirect RDMA DMA-BUF bandwidth", probes) + + +class LaunchKitMultirailCheck(_LaunchKitCheck): + """Validate same-rail reachability and expected cross-rail isolation.""" + + description: ClassVar[str] = "Check multi-rail connectivity behavior with Kubernetes Launch Kit" + + def run(self) -> None: + """Require same-rail and cross-rail coverage when multiple rails exist.""" + output = self._step_output("validate") + if output is None: + return + profile = self._profile() + if profile is None: + return + multirail = profile.get("multirail") + if multirail not in {True, "true"}: + pytest.skip(f"selected Launch Kit profile is not multirail: {multirail!r}") + probes = self._matrix_probes(output, set(_CONNECTIVITY_FAMILIES)) + rails = {rail for probe in probes for rail in (probe["source_rail"], probe["destination_rail"])} + if len(rails) == 1 and "unknown-rail" not in rails: + pytest.skip(f"Launch Kit connectivity matrix contains only one rail: {next(iter(rails))}") + same_rail = [probe for probe in probes if probe["source_rail"] == probe["destination_rail"]] + cross_rail = [probe for probe in probes if probe not in same_rail] + probes.extend( + [ + {"name": "same-rail-coverage", "passed": bool(same_rail), "message": f"rows={len(same_rail)}"}, + {"name": "cross-rail-coverage", "passed": bool(cross_rail), "message": f"rows={len(cross_rail)}"}, + ] + ) + self._finish_probes("multi-rail behavior", probes) + + +def _artifact_paths(value: Any) -> list[Path]: + """Recursively collect paths from provider artifact mappings.""" + if isinstance(value, dict): + paths: list[Path] = [] + for child in value.values(): + paths.extend(_artifact_paths(child)) + return paths + if isinstance(value, list): + paths = [] + for child in value: + paths.extend(_artifact_paths(child)) + return paths + if isinstance(value, str) and value: + return [Path(value)] + return [] + + +def _evidence_path(value: str, output: dict[str, Any]) -> Path: + """Resolve a Launch Kit-emitted path against its command working directory.""" + path = Path(value) + if path.is_absolute(): + return path + working_directory = output.get("working_directory") + if isinstance(working_directory, str) and working_directory: + return Path(working_directory) / path + return path + + +class LaunchKitEvidenceCaptureCheck(_LaunchKitCheck): + """Validate raw command logs and the Launch Kit HTML report.""" + + description: ClassVar[str] = "Check Launch Kit evidence capture" + + def run(self) -> None: + """Require fresh evidence for every real workflow command.""" + outputs = { + "verify": self._configured_output("verify_output"), + "preflight": self._configured_output("preflight_output"), + "discover": self._configured_output("discover_output"), + "generate": self._configured_output("generate_output"), + } + deploy = self._configured_output("deploy_output") + if deploy: + outputs["deploy"] = deploy + outputs["validate"] = self._step_output("validate") or {} + prepare = self._configured_output("prepare_output") + if prepare: + outputs = {"prepare": prepare, **outputs} + probes: list[dict[str, Any]] = [] + for operation, output in outputs.items(): + paths = _artifact_paths(output.get("artifacts")) + existing = [path for path in paths if path.is_file()] + probes.append( + { + "name": f"{operation}-artifacts", + "passed": bool(paths) and len(existing) == len(paths), + "message": f"found {len(existing)}/{len(paths)} files", + } + ) + generated_paths = [ + _evidence_path(path, outputs["generate"]) + for document in self._documents(outputs["generate"]) + for path in _list(document.get("generatedFiles")) + if isinstance(path, str) + ] + probes.append( + { + "name": "generated-files", + "passed": bool(generated_paths) and all(path.is_file() for path in generated_paths), + "message": ( + f"found {sum(path.is_file() for path in generated_paths)}/{len(generated_paths)} generated files" + ), + } + ) + validate = outputs["validate"] + report_paths = [ + _evidence_path(document["reportPath"], validate) + for document in self._documents(validate) + if isinstance(document.get("reportPath"), str) + ] + probes.append( + { + "name": "launch-kit-html-report", + "passed": bool(report_paths) and all(path.is_file() for path in report_paths), + "message": ", ".join(str(path) for path in report_paths) or "no reportPath document", + } + ) + self._finish_probes("Launch Kit evidence capture", probes) diff --git a/isvtest/tests/k8s_launch_kit/test_checks.py b/isvtest/tests/k8s_launch_kit/test_checks.py new file mode 100644 index 000000000..29ca2a37c --- /dev/null +++ b/isvtest/tests/k8s_launch_kit/test_checks.py @@ -0,0 +1,530 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for Kubernetes Launch Kit result interpretation.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from isvtest.core.validation import BaseValidation +from isvtest.validations.k8s_launch_kit.checks import ( + LaunchKitDeploymentHealthCheck, + LaunchKitEvidenceCaptureCheck, + LaunchKitGpuDirectRdmaCheck, + LaunchKitHostDeviceCheck, + LaunchKitIcmpConnectivityCheck, + LaunchKitInfiniBandCheck, + LaunchKitKubernetesPrerequisiteCheck, + LaunchKitMultirailCheck, + LaunchKitRdmaBandwidthCheck, + LaunchKitRdmaConnectivityCheck, + LaunchKitRdmaSharedCheck, + LaunchKitRoceCheck, + LaunchKitSecondaryNetworkCheck, + LaunchKitSriovReadinessCheck, + LaunchKitTopologyDiscoveryCheck, +) + +pytestmark = pytest.mark.unit + +_NETWORK_KIND = { + ("ethernet", "sriov"): "SriovNetwork", + ("infiniband", "sriov"): "SriovIBNetwork", + ("ethernet", "rdma_shared"): "MacvlanNetwork", + ("infiniband", "rdma_shared"): "IPoIBNetwork", + ("ethernet", "host_device"): "HostDeviceNetwork", + ("infiniband", "host_device"): "HostDeviceNetwork", +} + + +def _discover( + fabric: str = "ethernet", + deployment: str = "sriov", + *, + multirail: bool = True, +) -> dict[str, Any]: + """Build the provider envelope around a real discover-shaped document.""" + return { + "success": True, + "platform": "kubernetes", + "operation": "discover", + "exit_code": 0, + "documents": [ + { + "success": True, + "phase": "discover", + "profile": { + "fabric": fabric, + "deployment": deployment, + "multirail": "true" if multirail else "false", + }, + "deployed": False, + "messages": [], + } + ], + "artifacts": {}, + } + + +def _manifest(kind: str, *, state: str = "success") -> dict[str, Any]: + """Build one exported manifest validation row.""" + return { + "Kind": kind, + "APIVersion": "example.nvidia.com/v1", + "Name": f"mock-{kind.lower()}", + "Namespace": "default", + "State": state, + "Reason": "resource exists and is Ready" if state == "success" else "rollout has 1 unavailable pod", + "Found": True, + "Missing": False, + } + + +def _matrix_row(kind: int, *, same_rail: bool, passed: bool = True) -> dict[str, Any]: + """Build one exported connectivity row with source and destination detail.""" + destination_rail = "rail-0" if same_rail else "rail-1" + family = "icmp" if kind < 2 else "rping" if kind < 4 else "ib_write_bw" if kind < 6 else "gpudirect_dmabuf" + bandwidth_family = family in {"ib_write_bw", "gpudirect_dmabuf"} + return { + "Test": { + "Kind": kind, + "SrcNode": "worker-a", + "DstNode": "worker-b", + "SrcRail": "rail-0", + "DstRail": destination_rail, + "Expectation": "required" if same_rail else "forbidden", + **( + { + "SrcGPUIndex": 2, + "DstGPUIndex": 5, + "SrcGPUPCIAddress": "0000:41:00.0", + "DstGPUPCIAddress": "0000:71:00.0", + } + if family == "gpudirect_dmabuf" + else {} + ), + }, + "Family": family, + "OK": passed, + "ObservedOK": same_rail if passed else False, + "Expectation": "required" if same_rail else "forbidden", + "BandwidthGbps": 187.6 if bandwidth_family and passed else 42.5, + "MinBandwidthGbps": 100.0 if bandwidth_family else 0.0, + "Stderr": "" if passed else f"{family}: connection refused on rail-0", + **({"Error": f"{family} validation failed"} if not passed else {}), + } + + +def _validate( + fabric: str = "ethernet", + deployment: str = "sriov", + *, + failed_kind: str | None = None, + failed_family: str | None = None, + include_sriov_policy: bool = True, +) -> dict[str, Any]: + """Build a validate transport envelope with static and matrix documents.""" + network_kind = _NETWORK_KIND[(fabric, deployment)] + kinds = ["NicClusterPolicy", "NicNodePolicy", "IPPool"] + if deployment == "sriov" and include_sriov_policy: + kinds.append("SriovNetworkNodePolicy") + kinds.append(network_kind) + manifests = [_manifest(kind, state="error" if kind == failed_kind else "success") for kind in kinds] + rows: list[dict[str, Any]] = [] + for family, pair in { + "icmp": (0, 1), + "rping": (2, 3), + "ib_write_bw": (4, 5), + "gpudirect_dmabuf": (6, 7), + }.items(): + rows.append(_matrix_row(pair[0], same_rail=True, passed=family != failed_family)) + rows.append(_matrix_row(pair[1], same_rail=False)) + failed_manifests = sum(item["State"] != "success" for item in manifests) + failed_rows = sum(row["OK"] is not True for row in rows) + return { + "success": failed_rows == 0, + "platform": "kubernetes", + "operation": "validate", + "exit_code": 0 if failed_rows == 0 else 4, + "documents": [ + { + "versionCheck": { + "Skipped": False, + "SelectedRelease": "26.4", + "ExpectedVersion": "v26.4.1", + "DeployedRelease": {"ChartVersion": "26.4.1"}, + "Match": True, + }, + "manifests": manifests, + "presetDeviations": [], + "summary": { + "totalManifests": len(manifests), + "successManifests": len(manifests) - failed_manifests, + "errorManifests": failed_manifests, + "missingManifests": 0, + "success": failed_manifests == 0, + }, + }, + { + "connectivity": { + "DaemonSets": [ + { + "Ref": {"Namespace": "default", "Name": "l8k-network-test"}, + "Rollout": {"Desired": 2, "Ready": 2, "NotReady": 0}, + } + ], + "PingResults": rows, + "Summary": {"TotalTests": len(rows), "Failed": failed_rows}, + } + }, + ], + "artifacts": {}, + **({"error": "one or more connectivity rows failed"} if failed_rows else {}), + } + + +def _config( + output: dict[str, Any], + *, + discover: dict[str, Any] | None = None, + **extra: Any, +) -> dict[str, Any]: + """Bind a command envelope and optional earlier step outputs to a check.""" + config = {"step_output": output, **extra} + if discover is not None: + config["discover_output"] = json.dumps(discover) + return config + + +def test_kubernetes_prerequisite_reports_every_probe() -> None: + """A failed prerequisite retains successful checks and its remediation detail.""" + output = { + "success": False, + "platform": "kubernetes", + "operation": "kubernetes-preflight", + "checks": [ + {"name": "api-version", "passed": True, "message": "server v1.34.1"}, + {"name": "nodes", "passed": False, "message": "Forbidden: cannot list nodes"}, + {"name": "non-empty-cluster", "passed": False, "message": "cluster contains no nodes"}, + ], + } + + result = LaunchKitKubernetesPrerequisiteCheck(config=_config(output)).execute() + + assert result["passed"] is False + assert [probe["name"] for probe in result["subtests"]] == ["api-version", "nodes", "non-empty-cluster"] + assert "Forbidden: cannot list nodes" in result["error"] + + +def test_topology_discovery_uses_the_real_profile_document() -> None: + """Discovery succeeds only when l8k resolves both fabric and deployment.""" + result = LaunchKitTopologyDiscoveryCheck(config=_config(_discover())).execute() + + assert result["passed"] is True + assert [probe["name"] for probe in result["subtests"]] == ["discover-command", "resolved-profile"] + + +def test_deployment_health_reports_all_resources_before_failing() -> None: + """One unhealthy manifest is named without hiding later manifest rows.""" + output = _validate(failed_kind="SriovNetworkNodePolicy") + + result = LaunchKitDeploymentHealthCheck(config=_config(output)).execute() + + assert result["passed"] is False + names = [probe["name"] for probe in result["subtests"]] + assert "SriovNetworkNodePolicy/default/mock-sriovnetworknodepolicy" in names + assert names[-1] == "SriovNetwork/default/mock-sriovnetwork" + assert "rollout has 1 unavailable pod" in result["error"] + + +def test_deployment_health_honors_the_launch_kit_exit_verdict() -> None: + """A validate-level drift failure cannot be hidden by green manifest rows.""" + output = _validate() + output["success"] = False + output["exit_code"] = 4 + output["error"] = "l8k validate exited with code 4: component versions diverge" + + result = LaunchKitDeploymentHealthCheck(config=_config(output)).execute() + + assert result["passed"] is False + assert result["subtests"][0]["name"] == "launch-kit-validate-command" + assert "component versions diverge" in result["error"] + + +def test_deployment_health_allows_an_unconfigured_version_expectation() -> None: + """An optional Launch Kit version check is reported as skipped, not failed.""" + output = _validate() + output["documents"][0]["versionCheck"] = { + "Skipped": True, + "Reason": "cluster config has no selectedRelease", + } + + result = LaunchKitDeploymentHealthCheck(config=_config(output)).execute() + + assert result["passed"] is True + assert result["subtests"][1] == { + "name": "network-operator-version", + "passed": False, + "skipped": True, + "message": "cluster config has no selectedRelease", + "duration": None, + } + + +def test_sriov_readiness_requires_policy_and_network_kinds() -> None: + """A non-vacuous SR-IOV result requires both policy and attachment resources.""" + discover = _discover("infiniband", "sriov") + output = _validate("infiniband", "sriov", include_sriov_policy=False) + + result = LaunchKitSriovReadinessCheck(config=_config(output, discover=discover)).execute() + + assert result["passed"] is False + assert "kind-coverage/SriovNetworkNodePolicy" in result["error"] + assert any(probe["name"].startswith("SriovIBNetwork/") for probe in result["subtests"]) + + +@pytest.mark.parametrize( + ("check_class", "fabric", "deployment", "expected_kind"), + [ + (LaunchKitRoceCheck, "ethernet", "sriov", "SriovNetwork"), + (LaunchKitInfiniBandCheck, "infiniband", "sriov", "SriovIBNetwork"), + (LaunchKitHostDeviceCheck, "ethernet", "host_device", "HostDeviceNetwork"), + (LaunchKitRdmaSharedCheck, "infiniband", "rdma_shared", "IPoIBNetwork"), + ], +) +def test_profile_checks_require_the_applicable_network_kind( + check_class: type[BaseValidation], + fabric: str, + deployment: str, + expected_kind: str, +) -> None: + """Profile-specific checks select the exact resource implied by discover.""" + result = check_class( + config=_config(_validate(fabric, deployment), discover=_discover(fabric, deployment)) + ).execute() + + assert result["passed"] is True + names = [probe["name"] for probe in result["subtests"]] + assert names[0] == f"kind-coverage/{expected_kind}" + assert len(names) == len(set(names)) + + +def test_non_applicable_profile_is_skipped() -> None: + """An individually selectable check is skipped when the selected profile does not apply.""" + check = LaunchKitInfiniBandCheck(config=_config(_validate(), discover=_discover())) + + with pytest.raises(pytest.skip.Exception, match="not infiniband"): + check.execute() + + +@pytest.mark.parametrize( + ("check_class", "family"), + [ + (LaunchKitIcmpConnectivityCheck, "icmp"), + (LaunchKitRdmaConnectivityCheck, "rping"), + (LaunchKitRdmaBandwidthCheck, "ib_write_bw"), + (LaunchKitGpuDirectRdmaCheck, "gpudirect_dmabuf"), + ], +) +def test_connectivity_checks_create_source_bound_subtests( + check_class: type[BaseValidation], + family: str, +) -> None: + """Each matrix row becomes an independently named report item.""" + result = check_class(config=_config(_validate())).execute() + + assert result["passed"] is True + assert [probe["name"] for probe in result["subtests"]] == [ + f"{family}/worker-a->worker-b/rail-0->rail-0", + f"{family}/worker-a->worker-b/rail-0->rail-1", + ] + + +def test_connectivity_failure_preserves_stderr_and_bandwidth() -> None: + """A bandwidth failure includes endpoints, rails, observation, and threshold.""" + result = LaunchKitRdmaBandwidthCheck(config=_config(_validate(failed_family="ib_write_bw"))).execute() + + assert result["passed"] is False + assert len(result["subtests"]) == 2 + assert "bandwidthGbps=42.5" in result["error"] + assert "minimumGbps=100.0" in result["error"] + assert "connection refused on rail-0" in result["error"] + + +def test_gpudirect_failure_preserves_endpoint_gpu_and_bandwidth_evidence() -> None: + """A DMA-BUF failure identifies both endpoint GPUs and the failed threshold.""" + result = LaunchKitGpuDirectRdmaCheck(config=_config(_validate(failed_family="gpudirect_dmabuf"))).execute() + + assert result["passed"] is False + assert "gpuIndices=2->5" in result["error"] + assert "sourceGpuPci=0000:41:00.0" in result["error"] + assert "destinationGpuPci=0000:71:00.0" in result["error"] + assert "bandwidthGbps=42.5" in result["error"] + assert "minimumGbps=100.0" in result["error"] + assert "error=gpudirect_dmabuf validation failed" in result["error"] + + +def test_gpudirect_prefers_the_exported_family_contract() -> None: + """The stable Family field selects GPUDirect even if numeric kinds evolve.""" + output = _validate() + rows = output["documents"][1]["connectivity"]["PingResults"] + gpudirect_rows = [row for row in rows if row["Family"] == "gpudirect_dmabuf"] + for row in gpudirect_rows: + row["Test"]["Kind"] = 999 + output["documents"][1]["connectivity"]["PingResults"] = gpudirect_rows + + result = LaunchKitGpuDirectRdmaCheck(config=_config(output)).execute() + + assert result["passed"] is True + assert len(result["subtests"]) == 2 + + +def test_gpudirect_rejects_missing_endpoint_gpu_indices() -> None: + """A green row without explicit endpoint GPU topology is not accepted.""" + output = _validate() + row = next( + row for row in output["documents"][1]["connectivity"]["PingResults"] if row["Family"] == "gpudirect_dmabuf" + ) + row["Test"].pop("DstGPUIndex") + + result = LaunchKitGpuDirectRdmaCheck(config=_config(output)).execute() + + assert result["passed"] is False + assert "invalid or missing endpoint GPU index" in result["error"] + + +def test_gpudirect_is_skipped_when_launch_kit_does_not_emit_the_family() -> None: + """A discovery-disabled GPUDirect family is inapplicable, not failed.""" + output = _validate() + output["documents"][1]["connectivity"]["PingResults"] = [ + row for row in output["documents"][1]["connectivity"]["PingResults"] if row["Family"] != "gpudirect_dmabuf" + ] + check = LaunchKitGpuDirectRdmaCheck(config=_config(output)) + + with pytest.raises(pytest.skip.Exception, match=r"validation\.gpuDirect is disabled"): + check.execute() + + +def test_secondary_network_requires_ipam_network_and_ready_test_pods() -> None: + """Secondary-network coverage combines static resources with workload readiness.""" + discover = _discover("ethernet", "rdma_shared") + result = LaunchKitSecondaryNetworkCheck( + config=_config(_validate("ethernet", "rdma_shared"), discover=discover) + ).execute() + + assert result["passed"] is True + names = {probe["name"] for probe in result["subtests"]} + assert {"kind-coverage/IPPool", "kind-coverage/MacvlanNetwork", "DaemonSet/default/l8k-network-test"} <= names + + +@pytest.mark.parametrize( + "rollout", + [ + {}, + {"Desired": 2, "Ready": 2}, + {"Desired": 0, "Ready": 0, "NotReady": 0}, + {"Desired": True, "Ready": True, "NotReady": 0}, + ], +) +def test_secondary_network_rejects_incomplete_or_empty_daemonset_rollout(rollout: dict[str, Any]) -> None: + """Missing, invalid, or zero-sized rollout counts cannot pass vacuously.""" + discover = _discover("ethernet", "rdma_shared") + output = _validate("ethernet", "rdma_shared") + output["documents"][1]["connectivity"]["DaemonSets"][0]["Rollout"] = rollout + + result = LaunchKitSecondaryNetworkCheck(config=_config(output, discover=discover)).execute() + + assert result["passed"] is False + rollout_probe = next(probe for probe in result["subtests"] if probe["name"].startswith("DaemonSet/")) + assert rollout_probe["passed"] is False + + +def test_multirail_requires_same_and_cross_rail_coverage() -> None: + """Multi-rail validation distinguishes same-rail reachability from isolation rows.""" + result = LaunchKitMultirailCheck(config=_config(_validate(), discover=_discover())).execute() + + assert result["passed"] is True + assert result["subtests"][-2]["name"] == "same-rail-coverage" + assert result["subtests"][-1]["name"] == "cross-rail-coverage" + + +def test_multirail_is_skipped_when_matrix_contains_one_rail() -> None: + """A single-rail topology is inapplicable rather than a coverage failure.""" + output = _validate() + connectivity = output["documents"][1]["connectivity"] + connectivity["PingResults"] = [ + row for row in connectivity["PingResults"] if row["Test"]["SrcRail"] == row["Test"]["DstRail"] + ] + check = LaunchKitMultirailCheck(config=_config(output, discover=_discover())) + + with pytest.raises(pytest.skip.Exception, match="only one rail: rail-0"): + check.execute() + + assert check._subtest_results == [] + + +def test_structured_launch_kit_error_is_actionable() -> None: + """A failed l8k invocation surfaces its structured error when documents are absent.""" + output = { + "success": False, + "platform": "kubernetes", + "operation": "validate", + "documents": [{"error": {"message": "failed to create Kubernetes client"}}], + "error": "failed to create Kubernetes client; verify kubeconfig access", + } + + result = LaunchKitDeploymentHealthCheck(config=_config(output)).execute() + + assert result["passed"] is False + assert "failed to create Kubernetes client; verify kubeconfig access" in result["error"] + + +@pytest.mark.parametrize(("include_deploy", "expected_count"), [(True, 9), (False, 8)]) +def test_evidence_check_requires_selected_command_artifacts_and_html_report( + tmp_path: Path, + include_deploy: bool, + expected_count: int, +) -> None: + """Evidence supports both lifecycle-owning and validation-only workflows.""" + outputs: dict[str, dict[str, Any]] = {} + operations = ["prepare", "verify", "preflight", "discover", "generate", "validate"] + if include_deploy: + operations.insert(-1, "deploy") + for operation in operations: + artifact = tmp_path / f"{operation}.log" + artifact.write_text(f"{operation} evidence\n", encoding="utf-8") + outputs[operation] = { + "success": True, + "platform": "kubernetes", + "operation": "kubernetes-preflight" if operation == "preflight" else operation, + "working_directory": str(tmp_path), + "artifacts": {"stderr": str(artifact)}, + "documents": [], + } + generated = tmp_path / "generated" / "network-operator.yaml" + generated.parent.mkdir() + generated.write_text("kind: NicClusterPolicy\n", encoding="utf-8") + outputs["generate"]["documents"] = [{"generatedFiles": ["generated/network-operator.yaml"]}] + report = tmp_path / "k8s-launch-kit-validation-report.html" + report.write_text("passed\n", encoding="utf-8") + outputs["validate"]["documents"] = [{"reportPath": str(report)}] + + extra_outputs = { + "prepare_output": json.dumps(outputs["prepare"]), + "verify_output": json.dumps(outputs["verify"]), + "preflight_output": json.dumps(outputs["preflight"]), + "discover_output": json.dumps(outputs["discover"]), + "generate_output": json.dumps(outputs["generate"]), + } + if include_deploy: + extra_outputs["deploy_output"] = json.dumps(outputs["deploy"]) + config = _config(outputs["validate"], **extra_outputs) + result = LaunchKitEvidenceCaptureCheck(config=config).execute() + + assert result["passed"] is True + assert len(result["subtests"]) == expected_count diff --git a/isvtest/tests/test_catalog.py b/isvtest/tests/test_catalog.py index 988340bb6..01ada1427 100644 --- a/isvtest/tests/test_catalog.py +++ b/isvtest/tests/test_catalog.py @@ -55,6 +55,8 @@ def test_derives_suite_vocabulary_from_plain_suites(self) -> None: """Plain suite YAML files are listed separately from platform suites.""" suites = build_suite_vocabulary() assert "iam" in suites + assert "network_operator" in suites + assert "network_operator_use_cases" in suites assert "storage" in suites assert "kubernetes" not in suites assert "vm" not in suites @@ -112,6 +114,7 @@ def test_entries_have_suite_contract(self) -> None: assert isinstance(entry["requires"], list) if entry["capability"]: assert entry["requires"] == [] + assert "EastWestNetworkRoceSriovCheck" in names def test_extract_checks_supports_direct_dict_category_form(self, tmp_path) -> None: """Direct dict category wiring is included in catalog config scans.""" diff --git a/isvtest/tests/test_composite.py b/isvtest/tests/test_composite.py index 0d8c16f75..283fd1684 100644 --- a/isvtest/tests/test_composite.py +++ b/isvtest/tests/test_composite.py @@ -19,9 +19,11 @@ import pytest +import isvtest.core.composite as composite_module from isvtest.core.composite import CompositeCheck, composed_members, is_composite from isvtest.core.discovery import discover_all_tests from isvtest.core.resolution import parse_validations +from isvtest.core.validation import BaseValidation from isvtest.validations.generic import ( CrudOperationsCheck, FieldExistsCheck, @@ -144,6 +146,52 @@ def test_reports_each_member_as_a_subtest(self) -> None: ("FieldExistsCheck", True), ] + def test_forwards_member_subtests_with_member_qualified_names(self, monkeypatch: pytest.MonkeyPatch) -> None: + """A composite preserves member probes for terminal and JUnit diagnostics.""" + + class NestedCheck(BaseValidation): + def run(self) -> None: + self.report_subtest("probe-a", True, "probe passed") + self.report_subtest("probe-b", False, "not applicable", skipped=True) + self.set_passed("nested check passed") + + monkeypatch.setattr(composite_module, "get_validation_class", lambda name: NestedCheck) + composite = CompositeCheck(config=_config(["NestedCheck"])) + + result = composite.execute() + + assert result["passed"] is True + assert [(sub["name"], sub["skipped"]) for sub in result["subtests"]] == [ + ("NestedCheck", False), + ("NestedCheck/probe-a", False), + ("NestedCheck/probe-b", True), + ] + + def test_skipped_member_does_not_skip_or_fail_the_composite(self, monkeypatch: pytest.MonkeyPatch) -> None: + """An inapplicable member is skipped while later members still run.""" + + class SkippedCheck(BaseValidation): + def run(self) -> None: + pytest.skip("only one rail discovered") + + class PassingCheck(BaseValidation): + def run(self) -> None: + self.set_passed("later member passed") + + classes = {"SkippedCheck": SkippedCheck, "PassingCheck": PassingCheck} + monkeypatch.setattr(composite_module, "get_validation_class", classes.get) + composite = CompositeCheck(config=_config(["SkippedCheck", "PassingCheck"])) + + result = composite.execute() + + assert result["passed"] is True + assert [(sub["name"], sub["passed"], sub["skipped"]) for sub in result["subtests"]] == [ + ("SkippedCheck", False, True), + ("PassingCheck", True, False), + ] + assert "SkippedCheck: skipped - only one rail discovered" in result["output"] + assert "PassingCheck: later member passed" in result["output"] + def test_fails_naming_the_failing_member(self) -> None: """A failing member fails the composite and is named in the error.""" result = CompositeCheck( diff --git a/isvtest/tests/test_main.py b/isvtest/tests/test_main.py index efbe688c7..4b833835d 100644 --- a/isvtest/tests/test_main.py +++ b/isvtest/tests/test_main.py @@ -22,6 +22,7 @@ from isvtest.main import ( _entries_with_pytest_names, _resolved_entries_to_pytest_validations, + _result_to_resolved_entry, run_validations_via_pytest, ) @@ -102,6 +103,27 @@ def test_run_validations_via_pytest_updates_ready_entries() -> None: assert results[1].message == "NIM was not deployed" +def test_result_to_resolved_entry_preserves_subtest_summary() -> None: + """Subtest counts remain structured until the presentation layer renders them.""" + entry = _ready("StepSuccessCheck", "setup_checks", {"step_output": {"success": True}}) + + result = _result_to_resolved_entry( + entry, + { + "passed": True, + "skipped": False, + "message": "detailed validation output", + "subtest_summary": {"passed": 4, "failed": 0, "skipped": 2}, + }, + ) + + assert result.message == "detailed validation output" + assert result.subtest_summary.total == 6 + assert result.subtest_summary.passed == 4 + assert result.subtest_summary.failed == 0 + assert result.subtest_summary.skipped == 2 + + def test_run_validations_via_pytest_skips_structured_step_skips() -> None: """A step-level structured skip should skip all dependent validations.""" step_output = {"success": True, "skipped": True, "skip_reason": "No VPCs found at site"} diff --git a/isvtest/tests/test_subtests_junit.py b/isvtest/tests/test_subtests_junit.py index 3d83d20e6..767e333f6 100644 --- a/isvtest/tests/test_subtests_junit.py +++ b/isvtest/tests/test_subtests_junit.py @@ -23,9 +23,12 @@ from isvtest.testing.subtests import SubTestReport, _inject_subtests_into_junit -def _parent_junit(tmp_path: Path, parent_name: str) -> Path: +def _parent_junit(tmp_path: Path, parent_name: str, *, reported_tests: int = 1) -> Path: """Write a minimal pytest-style JUnit with a single parent testcase.""" - suite = ET.Element("testsuite", attrib={"name": "phase", "tests": "1", "skipped": "0", "failures": "0"}) + suite = ET.Element( + "testsuite", + attrib={"name": "phase", "tests": str(reported_tests), "skipped": "0", "failures": "0"}, + ) ET.SubElement(suite, "testcase", attrib={"name": parent_name, "classname": "", "time": "0.000"}) path = tmp_path / "junit.xml" ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True) @@ -92,3 +95,18 @@ def test_skipped_subtest_falls_back_when_longrepr_is_missing(tmp_path: Path) -> skipped = subtest_case.find("skipped") assert skipped is not None assert skipped.get("message") == "Subtest noisy-subtest skipped" + + +def test_injected_subtests_reconcile_precounted_junit_totals(tmp_path: Path) -> None: + """pytest's pre-counted subtest report must not be counted again after injection.""" + junit = _parent_junit(tmp_path, "ParentCheck", reported_tests=2) + report = _stub_report("::ParentCheck", "probe", failed=True, longrepr="probe failed") + + _inject_subtests_into_junit(junit, cast(list[SubTestReport], [report])) + + suite = ET.parse(junit).getroot() + assert len(suite.findall("testcase")) == 2 + assert suite.get("tests") == "2" + assert suite.get("failures") == "1" + assert suite.get("errors") == "0" + assert suite.get("skipped") == "0" diff --git a/isvtest/tests/test_validation.py b/isvtest/tests/test_validation.py index cf549e059..6662556f6 100644 --- a/isvtest/tests/test_validation.py +++ b/isvtest/tests/test_validation.py @@ -77,6 +77,16 @@ def run(self) -> None: self.set_failed("Test failed", "Error output") +class SubtestValidation(BaseValidation): + """Validation that reports passing and skipped probes.""" + + def run(self) -> None: + """Report two probe outcomes and pass the parent validation.""" + self.report_subtest("ready", True, "ready") + self.report_subtest("optional", False, "not applicable", skipped=True) + self.set_passed("All required probes passed") + + class ExceptionValidation(BaseValidation): """Validation that raises an exception.""" @@ -1999,6 +2009,16 @@ def test_passed_validation_captured(self) -> None: assert r["skipped"] is False assert r["passed"] is True + def test_subtest_summary_captured(self) -> None: + """Probe counts cross the pytest bridge without replacing the parent message.""" + subtests = MagicMock() + + run_validation_entry_point(SubtestValidation, {"_category": "test_cat"}, "SubtestValidation", subtests) + + result = _validation_results[0] + assert result["message"] == "All required probes passed" + assert result["subtest_summary"] == {"total": 2, "passed": 1, "failed": 0, "skipped": 1} + def test_failed_validation_captured(self) -> None: """Failed validations must appear with passed=False.""" config = {"_category": "test_cat"} diff --git a/scripts/requirements_source_to_md.py b/scripts/requirements_source_to_md.py index 3c5ca3923..3653cb5f6 100644 --- a/scripts/requirements_source_to_md.py +++ b/scripts/requirements_source_to_md.py @@ -17,7 +17,8 @@ """Render a source-requirements YAML to a publishable Markdown listing. YAML is the source of record (queryable, key-clean); Markdown is the published, -Google-Docs-friendly view. Handles both the `offtake` and `reference` sources. +Google-Docs-friendly view. Handles the registered offtake, reference, storage, +and project-PRD sources. Usage: python3 scripts/requirements_source_to_md.py docs/requirements/offtake-requirements.yaml @@ -38,6 +39,7 @@ REQ_DIR / "offtake-requirements.yaml", REQ_DIR / "software-reference-requirements.yaml", REQ_DIR / "storage-acceptance-requirements.yaml", + REQ_DIR / "network-operator-readiness-requirements.yaml", ] GENERATED_BANNER = "" @@ -149,7 +151,40 @@ def render_storage(doc: dict[str, Any], src_name: str) -> str: return "\n".join(out) + "\n" -RENDERERS = {"offtake": render_offtake, "reference": render_reference, "storage": render_storage} +def render_project_prd(doc: dict[str, Any], src_name: str) -> str: + """Render a project PRD listing, grouped by section.""" + out = [ + GENERATED_BANNER.format(src=src_name), + "", + f"# {doc.get('title', 'Project Requirements')}", + "", + f"> Structured source of record: `{src_name}` (version {doc.get('version', 'n/a')}).", + f"> Owner: {doc.get('owner', 'not specified')}.", + "> Edit the YAML, not this file.", + "", + ] + section = None + for requirement in doc.get("requirements", []): + if requirement.get("section") != section: + section = requirement.get("section") + heading(out, f"## {section}") + out += [ + "| Req ID | Requirement Area | Description | Status |", + "| :----- | :--------------- | :---------- | :----- |", + ] + out.append( + f"| {cell(requirement.get('req_id'))} | {cell(requirement.get('area'))} " + f"| {cell(requirement.get('description'))} | {cell(requirement.get('status', 'active'))} |" + ) + return "\n".join(out) + "\n" + + +RENDERERS = { + "offtake": render_offtake, + "project-prd": render_project_prd, + "reference": render_reference, + "storage": render_storage, +} def render(path: Path) -> None: @@ -158,9 +193,14 @@ def render(path: Path) -> None: if not isinstance(doc, dict): raise ValueError(f"{path} must contain a single mapping document") source = doc.get("source") - renderer = RENDERERS.get(source) + if not isinstance(source, str): + raise ValueError(f"{path} must declare a string source") + document_format = doc.get("format", source) + if not isinstance(document_format, str): + raise ValueError(f"{path} format must be a string") + renderer = RENDERERS.get(document_format) if renderer is None: - raise ValueError(f"{path} has unsupported source {source!r} (expected one of {sorted(RENDERERS)})") + raise ValueError(f"{path} has unsupported format {document_format!r} (expected one of {sorted(RENDERERS)})") out_path = path.with_suffix(".md") out_path.write_text(renderer(doc, path.name), encoding="utf-8") print(f"Wrote {out_path}") diff --git a/scripts/test_plan_coverage.py b/scripts/test_plan_coverage.py index ce5fc14d1..8fab388cf 100644 --- a/scripts/test_plan_coverage.py +++ b/scripts/test_plan_coverage.py @@ -131,7 +131,7 @@ def config_test_id_map(suites_dir: Path = SUITES_DIR) -> dict[str, list[str]]: both bare_metal and vm), so values still aggregate to a set across configs. """ out: dict[str, set[str]] = defaultdict(set) - for path in sorted(suites_dir.glob("*.yaml")): + for path in sorted(suites_dir.rglob("*.yaml")): for name, params in iter_config_checks(path): tid = params.get("test_id") if isinstance(tid, str) and tid: @@ -148,7 +148,7 @@ def config_label_map(suites_dir: Path = SUITES_DIR) -> dict[str, list[str]]: union of its labels. """ out: dict[str, set[str]] = defaultdict(set) - for path in sorted(suites_dir.glob("*.yaml")): + for path in sorted(suites_dir.rglob("*.yaml")): for name, params in iter_config_checks(path): out[name].update(_normalize_labels(params.get("labels"))) return {name: sorted(labels) for name, labels in out.items()} @@ -166,7 +166,7 @@ def _normalize_labels(value: Any) -> list[str]: def config_test_label_instances(suites_dir: Path = SUITES_DIR) -> list[tuple[str, str, str, list[str]]]: """Return ``(source, check_name, test_id, labels)`` for each mapped suite check.""" instances: list[tuple[str, str, str, list[str]]] = [] - for path in sorted(suites_dir.glob("*.yaml")): + for path in sorted(suites_dir.rglob("*.yaml")): try: source = path.relative_to(REPO_ROOT).as_posix() except ValueError: diff --git a/scripts/tests/test_requirements_source_to_md.py b/scripts/tests/test_requirements_source_to_md.py new file mode 100644 index 000000000..d5d8f7093 --- /dev/null +++ b/scripts/tests/test_requirements_source_to_md.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the structured-requirements Markdown renderer.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest +import yaml + +_SCRIPT = Path(__file__).resolve().parent.parent / "requirements_source_to_md.py" +_spec = importlib.util.spec_from_file_location("requirements_source_to_md", _SCRIPT) +assert _spec and _spec.loader +requirements_source_to_md = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(requirements_source_to_md) + + +def test_project_prd_format_renders_a_uniquely_named_source() -> None: + """A project PRD keeps a unique trace source while sharing a generic renderer.""" + source_path = requirements_source_to_md.REQ_DIR / "network-operator-readiness-requirements.yaml" + document = yaml.safe_load(source_path.read_text(encoding="utf-8")) + + rendered = requirements_source_to_md.RENDERERS[document["format"]](document, source_path.name) + + assert source_path in requirements_source_to_md.DEFAULT_SOURCES + assert document["source"] == "network-operator-prd" + assert document["format"] == "project-prd" + assert "# Enterprise RA Network Operator Self-Validation Integration PRD" in rendered + assert "> Owner: NVIDIA Network Operator team." in rendered + assert "## Network Validation" in rendered + assert "| ENT-REQ-008 | GPUDirect RDMA |" in rendered + + +def test_render_rejects_a_non_string_source(tmp_path: Path) -> None: + """A malformed source fails clearly before renderer dispatch.""" + source_path = tmp_path / "malformed-requirements.yaml" + source_path.write_text("source: null\nrequirements: []\n", encoding="utf-8") + + with pytest.raises(ValueError, match="must declare a string source"): + requirements_source_to_md.render(source_path) + + +def test_render_rejects_an_unknown_document_format(tmp_path: Path) -> None: + """A unique source may select only a documented reusable renderer format.""" + source_path = tmp_path / "malformed-requirements.yaml" + source_path.write_text( + "source: team-prd\nformat: not-a-renderer\nrequirements: []\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="unsupported format 'not-a-renderer'"): + requirements_source_to_md.render(source_path) diff --git a/scripts/tests/test_validate_suite_wiring.py b/scripts/tests/test_validate_suite_wiring.py index ac6ef32b5..ee4536070 100644 --- a/scripts/tests/test_validate_suite_wiring.py +++ b/scripts/tests/test_validate_suite_wiring.py @@ -95,6 +95,19 @@ def test_wiring_errors_reports_yaml_parse_failures(tmp_path: Path) -> None: assert "failed to read/parse" in errors[0] +def test_wiring_errors_scans_nested_suite_directories(tmp_path: Path) -> None: + """Domain-organized suites receive the same metadata guardrails as root suites.""" + nested = tmp_path / "launch-kit" + nested.mkdir() + (nested / "network-operator.yaml").write_text( + "tests:\n validations:\n sample:\n checks:\n MissingMetadata: {}\n" + ) + + errors = validate_suite_wiring.wiring_errors(tmp_path) + + assert any("launch-kit/network-operator.yaml" in error and "MissingMetadata" in error for error in errors) + + def test_find_check_line_numbers_supports_list_form() -> None: """List-form wiring reports each repeated check at its own line.""" lines = """ diff --git a/scripts/validate_suite_wiring.py b/scripts/validate_suite_wiring.py index a07cc007e..2fdfa5a4b 100644 --- a/scripts/validate_suite_wiring.py +++ b/scripts/validate_suite_wiring.py @@ -16,7 +16,7 @@ """Validate suite identity and check resolution in canonical and provider YAML. -Suite configs under ``isvctl/configs/suites/`` are the source of truth for +Suite configs recursively under ``isvctl/configs/suites/`` are the source of truth for validation metadata on this branch. Each wired check must declare: * ``test_id`` - a plan id from ``docs/test-plan.yaml``, or ``"N/A"`` when the @@ -190,7 +190,7 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: # Read and parse each suite once; both the dead-requirement pre-pass and the # per-check loop below work off these parsed documents. parsed: list[tuple[Path, list[str], dict[str, Any]]] = [] - for path in sorted(suites_dir.glob("*.yaml")): + for path in sorted(suites_dir.rglob("*.yaml")): try: text = path.read_text() parsed.append((path, text.splitlines(), yaml.safe_load(text) or {})) From 343eb65bdd19e5665bc2acd496fdc7160aa68f9e Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Thu, 3 Sep 2026 18:53:26 +0200 Subject: [PATCH 2/4] fix: address Launch Kit review feedback Signed-off-by: Alexander Maslennikov --- AGENTS.md | 14 +- .../guides/k8s-launch-kit/network-operator.md | 9 +- .../config/network-operator.yaml | 9 +- .../k8s-launch-kit/scripts/adapter.py | 5 +- isvctl/configs/suites/README.md | 31 +-- .../network-operator-use-cases.yaml | 181 ------------------ .../k8s-launch-kit/network-operator.yaml | 171 ++++++++++++++++- .../src/isvctl/orchestrator/step_executor.py | 9 +- .../providers/k8s_launch_kit/test_provider.py | 28 ++- isvctl/tests/test_orchestrator_loop.py | 35 ++++ .../validations/k8s_launch_kit/checks.py | 2 +- isvtest/tests/test_catalog.py | 4 +- 12 files changed, 279 insertions(+), 219 deletions(-) delete mode 100644 isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml diff --git a/AGENTS.md b/AGENTS.md index de73dd3ec..6e19dfe3d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -198,13 +198,13 @@ forwarded env vars → optional isvreporter upload. conflicts), verifies API access, requires a non-empty node inventory, and requires at least one Ready node. A failure stops the remaining steps in that workflow/use case. -- `isvctl/configs/suites/k8s-launch-kit/network-operator.yaml` owns only catalog wiring and - interpretation for the globally selectable PRD checks. Each check binds to - the real step that produced its evidence. -- `isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml` composes those shared - check classes into six concrete validation tests: RoCE and InfiniBand across - SR-IOV, RDMA Shared, and host-device modes. Include only checks applicable to - a use case; do not run all checks and hide mismatches as interleaved skips. +- `isvctl/configs/suites/k8s-launch-kit/network-operator.yaml` is the single + frontend-visible Network Operator suite. It owns catalog wiring and + interpretation for the globally selectable PRD checks and composes those + classes into six concrete validation tests: RoCE and InfiniBand across + SR-IOV, RDMA Shared, and host-device modes. Each check binds to the real step + that produced its evidence. Include only checks applicable to a use case; do + not run all checks and hide mismatches as interleaved skips. - `isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml` is the production six-use-case configuration. It defaults to real `l8k` and `kubectl`, executes each supported fabric/deployment combination as a named diff --git a/docs/guides/k8s-launch-kit/network-operator.md b/docs/guides/k8s-launch-kit/network-operator.md index b9a69fd09..e47da1573 100644 --- a/docs/guides/k8s-launch-kit/network-operator.md +++ b/docs/guides/k8s-launch-kit/network-operator.md @@ -58,16 +58,17 @@ The main files are: | Generic provider | `isvctl/configs/providers/k8s-launch-kit/config/provider.yaml` | | Network Operator validation workflow | `isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml` | | CLI transport | `isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py` | -| Individual PRD checks | `isvctl/configs/suites/k8s-launch-kit/network-operator.yaml` | -| Concrete use cases | `isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml` | +| Individual PRD checks and concrete use cases | `isvctl/configs/suites/k8s-launch-kit/network-operator.yaml` | | Result interpretation | `isvtest/src/isvtest/validations/k8s_launch_kit/checks.py` | | Provider tests and mock CLI | `isvctl/tests/providers/k8s_launch_kit/` | | Result-check tests | `isvtest/tests/k8s_launch_kit/test_checks.py` | | PRD and traceability | `docs/requirements/` | The provider and suite files are intentionally separate. The provider owns -process execution and evidence. The suite owns catalog identity, selection, -and the mapping from step outputs to reusable validation checks. +process execution and evidence. The single frontend-visible `network_operator` +suite owns catalog identity, selection, and the mapping from step outputs to +reusable validation checks. It contains both individually selectable PRD checks +and the six grouped use cases. ## Network Operator workflow diff --git a/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml b/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml index b80872807..a0ab24485 100644 --- a/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml +++ b/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml @@ -11,10 +11,17 @@ import: - provider.yaml - - ../../../suites/k8s-launch-kit/network-operator-use-cases.yaml + - ../../../suites/k8s-launch-kit/network-operator.yaml version: "1.0" +# The shared frontend suite also catalogs the individually selectable semantic +# checks. This grouped production entrypoint reports only the six use cases; +# each composite executes the applicable semantic checks as named subtests. +tests: + validations: + network_operator: [] + context: k8s_launch_kit: executable: l8k diff --git a/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py b/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py index 3db36ed28..50567411e 100644 --- a/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py +++ b/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py @@ -223,8 +223,9 @@ def _stage_user_config( content = source.read_bytes() staged.unlink(missing_ok=True) try: - staged.write_bytes(content) - staged.chmod(0o600) + descriptor = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(content) except OSError: staged.unlink(missing_ok=True) raise diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index b74b64e7d..abc4e204d 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -216,7 +216,7 @@ its plan item is not platform-scoped. | `switch_syslogs` | test | `providers/my-isv/scripts/observability/log_availability_test.py` | `tests.*.probes.switches_checked`, `log_source`, `entry_count`, `latest_timestamp` | | `switch_kernel_logs` | test | `providers/my-isv/scripts/observability/log_availability_test.py` | `tests.*.probes.switches_checked`, `log_source`, `entry_count`, `latest_timestamp` | -### Network Operator (`k8s-launch-kit/network-operator.yaml`, `k8s-launch-kit/network-operator-use-cases.yaml`) +### Network Operator (`k8s-launch-kit/network-operator.yaml`) Plain suite for Kubernetes Launch Kit Network Operator self-validation. The generic provider in `providers/k8s-launch-kit/config/provider.yaml` mirrors the real CLI as @@ -231,18 +231,22 @@ the check skips when that family is disabled or not selected and fails on emitted GPU topology or bandwidth errors. State restoration remains deferred until Launch Kit provides the required snapshot/restore/verify workflow. -`k8s-launch-kit/network-operator-use-cases.yaml` reuses those global check classes in six -separate composite tests: RoCE and InfiniBand across SR-IOV, RDMA Shared, and -host-device deployment modes. Each composite includes only checks applicable to -that use case, so unrelated fabric/deployment checks do not appear as skips in -the middle of a run. The Ethernet/RoCE composites carry `ethernet` and `roce`; -the InfiniBand composites carry `infiniband`. All six also carry `gpudirect` -because Launch Kit discovery decides whether the GPUDirect family is applicable. +The same suite reuses those global check classes in six separate composite +tests: RoCE and InfiniBand across SR-IOV, RDMA Shared, and host-device +deployment modes. Keeping both forms in one file exposes one frontend suite, +`network_operator`, rather than a second implementation-detail suite. Each +composite includes only checks applicable to that use case, so unrelated +fabric/deployment checks do not appear as skips in the middle of a run. The +Ethernet/RoCE composites carry `ethernet` and `roce`; the InfiniBand composites +carry `infiniband`. All six also carry `gpudirect` because Launch Kit discovery +decides whether the GPUDirect family is applicable. `providers/k8s-launch-kit/config/network-operator.yaml` is the production entrypoint. It uses `l8k` and `kubectl` from `PATH` by default. In one invocation -it executes the six use-case phases sequentially, each with its own preflight, -discover, generate, deploy, validate, and evidence directories. The phases are +it runs `launch_kit_prepare` in `setup`, `launch_kit_verify` in +`launch-kit-verification`, then executes the six use-case phases sequentially, +each with its own preflight, discover, generate, validate, and evidence +directories. The phases are independent, so a failed case records a failed overall run but does not prevent later cases from producing results. Mock executables exist only under `isvctl/tests/providers/k8s_launch_kit/fixtures/` and are injected by tests. @@ -271,9 +275,10 @@ labels runs all six. | `launch_kit_deploy` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k deploy` | raw `documents` (currently empty on success), `argv`, `exit_code`, `artifacts` | | `launch_kit_validate` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k validate` | raw static, connectivity, and report-path `documents`, `argv`, `exit_code`, `artifacts` | -Those are the generic provider's single-workflow names. The grouped production configuration performs -prepare and verify in `setup`, then repeats the remaining five operations under -each custom use-case phase with names such as +Those are the generic provider's single-workflow names. The grouped production +configuration performs prepare in `setup` and verify in +`launch-kit-verification`, then repeats preflight, discover, generate, and +validate under each custom use-case phase with names such as `launch_kit_roce_sriov_preflight` through `launch_kit_roce_sriov_validate`. diff --git a/isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml b/isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml deleted file mode 100644 index dce4bfff0..000000000 --- a/isvctl/configs/suites/k8s-launch-kit/network-operator-use-cases.yaml +++ /dev/null @@ -1,181 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Grouped Network Operator Launch Kit use cases for validation-only execution. -# -# The individual PRD checks remain catalogued in the adjacent -# network-operator.yaml. This -# suite composes those shared check implementations into six concrete profile -# tests. Providers bind each test to that profile's own validate step and pass -# the other real workflow outputs through the standard step context. - -version: "1.0" - -tests: - description: "Network Operator Kubernetes Launch Kit validation use cases" - - settings: - show_skipped_tests: false - - validations: - network_operator_roce_sriov: - step: launch_kit_roce_sriov_validate - checks: - EastWestNetworkRoceSriovCheck: - test_id: "K8S42-16" - labels: ["era", "ethernet", "gpudirect", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "roce", "slow", "sriov"] - requires: [kubernetes] - description: "Ethernet/RoCE with SR-IOV Network RDMA" - prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" - verify_output: "{{ steps.launch_kit_verify | tojson }}" - preflight_output: "{{ steps.launch_kit_roce_sriov_preflight | tojson }}" - discover_output: "{{ steps.launch_kit_roce_sriov_discover | tojson }}" - generate_output: "{{ steps.launch_kit_roce_sriov_generate | tojson }}" - compose: - - LaunchKitKubernetesPrerequisiteCheck - - LaunchKitTopologyDiscoveryCheck - - LaunchKitDeploymentHealthCheck - - LaunchKitRoceCheck - - LaunchKitSriovReadinessCheck - - LaunchKitSecondaryNetworkCheck - - LaunchKitIcmpConnectivityCheck - - LaunchKitRdmaConnectivityCheck - - LaunchKitRdmaBandwidthCheck - - LaunchKitGpuDirectRdmaCheck - - LaunchKitMultirailCheck - - LaunchKitEvidenceCaptureCheck - - network_operator_infiniband_sriov: - step: launch_kit_infiniband_sriov_validate - checks: - EastWestNetworkInfiniBandSriovCheck: - test_id: "K8S42-17" - labels: ["era", "gpudirect", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "slow", "sriov"] - requires: [kubernetes] - description: "InfiniBand with SR-IOV Network RDMA" - prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" - verify_output: "{{ steps.launch_kit_verify | tojson }}" - preflight_output: "{{ steps.launch_kit_infiniband_sriov_preflight | tojson }}" - discover_output: "{{ steps.launch_kit_infiniband_sriov_discover | tojson }}" - generate_output: "{{ steps.launch_kit_infiniband_sriov_generate | tojson }}" - compose: - - LaunchKitKubernetesPrerequisiteCheck - - LaunchKitTopologyDiscoveryCheck - - LaunchKitDeploymentHealthCheck - - LaunchKitInfiniBandCheck - - LaunchKitSriovReadinessCheck - - LaunchKitSecondaryNetworkCheck - - LaunchKitIcmpConnectivityCheck - - LaunchKitRdmaConnectivityCheck - - LaunchKitRdmaBandwidthCheck - - LaunchKitGpuDirectRdmaCheck - - LaunchKitMultirailCheck - - LaunchKitEvidenceCaptureCheck - - network_operator_roce_rdma_shared: - step: launch_kit_roce_rdma_shared_validate - checks: - EastWestNetworkRoceRdmaSharedCheck: - test_id: "K8S42-18" - labels: ["era", "ethernet", "gpudirect", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "rdma_shared", "roce", "slow"] - requires: [kubernetes] - description: "Ethernet/RoCE with the RDMA Shared Device Plugin" - prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" - verify_output: "{{ steps.launch_kit_verify | tojson }}" - preflight_output: "{{ steps.launch_kit_roce_rdma_shared_preflight | tojson }}" - discover_output: "{{ steps.launch_kit_roce_rdma_shared_discover | tojson }}" - generate_output: "{{ steps.launch_kit_roce_rdma_shared_generate | tojson }}" - compose: - - LaunchKitKubernetesPrerequisiteCheck - - LaunchKitTopologyDiscoveryCheck - - LaunchKitDeploymentHealthCheck - - LaunchKitRoceCheck - - LaunchKitRdmaSharedCheck - - LaunchKitSecondaryNetworkCheck - - LaunchKitIcmpConnectivityCheck - - LaunchKitRdmaConnectivityCheck - - LaunchKitRdmaBandwidthCheck - - LaunchKitGpuDirectRdmaCheck - - LaunchKitMultirailCheck - - LaunchKitEvidenceCaptureCheck - - network_operator_infiniband_rdma_shared: - step: launch_kit_infiniband_rdma_shared_validate - checks: - EastWestNetworkInfiniBandRdmaSharedCheck: - test_id: "K8S42-19" - labels: ["era", "gpudirect", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "rdma_shared", "slow"] - requires: [kubernetes] - description: "InfiniBand/IPoIB with the RDMA Shared Device Plugin" - prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" - verify_output: "{{ steps.launch_kit_verify | tojson }}" - preflight_output: "{{ steps.launch_kit_infiniband_rdma_shared_preflight | tojson }}" - discover_output: "{{ steps.launch_kit_infiniband_rdma_shared_discover | tojson }}" - generate_output: "{{ steps.launch_kit_infiniband_rdma_shared_generate | tojson }}" - compose: - - LaunchKitKubernetesPrerequisiteCheck - - LaunchKitTopologyDiscoveryCheck - - LaunchKitDeploymentHealthCheck - - LaunchKitInfiniBandCheck - - LaunchKitRdmaSharedCheck - - LaunchKitSecondaryNetworkCheck - - LaunchKitIcmpConnectivityCheck - - LaunchKitRdmaConnectivityCheck - - LaunchKitRdmaBandwidthCheck - - LaunchKitGpuDirectRdmaCheck - - LaunchKitMultirailCheck - - LaunchKitEvidenceCaptureCheck - - network_operator_roce_host_device: - step: launch_kit_roce_host_device_validate - checks: - EastWestNetworkRoceHostDeviceCheck: - test_id: "K8S42-20" - labels: ["era", "ethernet", "gpudirect", "host_device", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "roce", "slow"] - requires: [kubernetes] - description: "Ethernet/RoCE host-device networking for worker VMs" - prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" - verify_output: "{{ steps.launch_kit_verify | tojson }}" - preflight_output: "{{ steps.launch_kit_roce_host_device_preflight | tojson }}" - discover_output: "{{ steps.launch_kit_roce_host_device_discover | tojson }}" - generate_output: "{{ steps.launch_kit_roce_host_device_generate | tojson }}" - compose: - - LaunchKitKubernetesPrerequisiteCheck - - LaunchKitTopologyDiscoveryCheck - - LaunchKitDeploymentHealthCheck - - LaunchKitRoceCheck - - LaunchKitHostDeviceCheck - - LaunchKitSecondaryNetworkCheck - - LaunchKitIcmpConnectivityCheck - - LaunchKitRdmaConnectivityCheck - - LaunchKitRdmaBandwidthCheck - - LaunchKitGpuDirectRdmaCheck - - LaunchKitMultirailCheck - - LaunchKitEvidenceCaptureCheck - - network_operator_infiniband_host_device: - step: launch_kit_infiniband_host_device_validate - checks: - EastWestNetworkInfiniBandHostDeviceCheck: - test_id: "K8S42-21" - labels: ["era", "gpudirect", "host_device", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "slow"] - requires: [kubernetes] - description: "InfiniBand host-device networking for worker VMs" - prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" - verify_output: "{{ steps.launch_kit_verify | tojson }}" - preflight_output: "{{ steps.launch_kit_infiniband_host_device_preflight | tojson }}" - discover_output: "{{ steps.launch_kit_infiniband_host_device_discover | tojson }}" - generate_output: "{{ steps.launch_kit_infiniband_host_device_generate | tojson }}" - compose: - - LaunchKitKubernetesPrerequisiteCheck - - LaunchKitTopologyDiscoveryCheck - - LaunchKitDeploymentHealthCheck - - LaunchKitInfiniBandCheck - - LaunchKitHostDeviceCheck - - LaunchKitSecondaryNetworkCheck - - LaunchKitIcmpConnectivityCheck - - LaunchKitRdmaConnectivityCheck - - LaunchKitRdmaBandwidthCheck - - LaunchKitGpuDirectRdmaCheck - - LaunchKitMultirailCheck - - LaunchKitEvidenceCaptureCheck diff --git a/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml b/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml index 9aa055f92..562a9a642 100644 --- a/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml +++ b/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml @@ -3,14 +3,15 @@ # Network Operator validation through the Kubernetes Launch Kit CLI. # -# This suite contains catalog and result interpretation only. Providers own the -# command sequence and bind these checks to actual l8k command output. The -# generic provider is ../../providers/k8s-launch-kit/config/provider.yaml. +# This single frontend-visible suite contains the individual PRD checks and six +# concrete use cases. Providers own the command sequence and bind these checks +# to actual l8k command output. The generic provider is +# ../../providers/k8s-launch-kit/config/provider.yaml. version: "1.0" tests: - description: "Network Operator self-validation through Kubernetes Launch Kit" + description: "Network Operator self-validation and use cases through Kubernetes Launch Kit" settings: show_skipped_tests: false @@ -120,3 +121,165 @@ tests: discover_output: "{{ steps.launch_kit_discover | tojson }}" generate_output: "{{ steps.launch_kit_generate | tojson }}" deploy_output: "{{ steps.launch_kit_deploy | tojson }}" + + network_operator_roce_sriov: + step: launch_kit_roce_sriov_validate + checks: + EastWestNetworkRoceSriovCheck: + test_id: "K8S42-16" + labels: ["era", "ethernet", "gpudirect", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "roce", "slow", "sriov"] + requires: [kubernetes] + description: "Ethernet/RoCE with SR-IOV Network RDMA" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_roce_sriov_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_roce_sriov_discover | tojson }}" + generate_output: "{{ steps.launch_kit_roce_sriov_generate | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitRoceCheck + - LaunchKitSriovReadinessCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_infiniband_sriov: + step: launch_kit_infiniband_sriov_validate + checks: + EastWestNetworkInfiniBandSriovCheck: + test_id: "K8S42-17" + labels: ["era", "gpudirect", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "slow", "sriov"] + requires: [kubernetes] + description: "InfiniBand with SR-IOV Network RDMA" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_infiniband_sriov_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_infiniband_sriov_discover | tojson }}" + generate_output: "{{ steps.launch_kit_infiniband_sriov_generate | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitInfiniBandCheck + - LaunchKitSriovReadinessCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_roce_rdma_shared: + step: launch_kit_roce_rdma_shared_validate + checks: + EastWestNetworkRoceRdmaSharedCheck: + test_id: "K8S42-18" + labels: ["era", "ethernet", "gpudirect", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "rdma_shared", "roce", "slow"] + requires: [kubernetes] + description: "Ethernet/RoCE with the RDMA Shared Device Plugin" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_roce_rdma_shared_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_roce_rdma_shared_discover | tojson }}" + generate_output: "{{ steps.launch_kit_roce_rdma_shared_generate | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitRoceCheck + - LaunchKitRdmaSharedCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_infiniband_rdma_shared: + step: launch_kit_infiniband_rdma_shared_validate + checks: + EastWestNetworkInfiniBandRdmaSharedCheck: + test_id: "K8S42-19" + labels: ["era", "gpudirect", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "rdma_shared", "slow"] + requires: [kubernetes] + description: "InfiniBand/IPoIB with the RDMA Shared Device Plugin" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_infiniband_rdma_shared_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_infiniband_rdma_shared_discover | tojson }}" + generate_output: "{{ steps.launch_kit_infiniband_rdma_shared_generate | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitInfiniBandCheck + - LaunchKitRdmaSharedCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_roce_host_device: + step: launch_kit_roce_host_device_validate + checks: + EastWestNetworkRoceHostDeviceCheck: + test_id: "K8S42-20" + labels: ["era", "ethernet", "gpudirect", "host_device", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "roce", "slow"] + requires: [kubernetes] + description: "Ethernet/RoCE host-device networking for worker VMs" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_roce_host_device_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_roce_host_device_discover | tojson }}" + generate_output: "{{ steps.launch_kit_roce_host_device_generate | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitRoceCheck + - LaunchKitHostDeviceCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck + + network_operator_infiniband_host_device: + step: launch_kit_infiniband_host_device_validate + checks: + EastWestNetworkInfiniBandHostDeviceCheck: + test_id: "K8S42-21" + labels: ["era", "gpudirect", "host_device", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "slow"] + requires: [kubernetes] + description: "InfiniBand host-device networking for worker VMs" + prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" + verify_output: "{{ steps.launch_kit_verify | tojson }}" + preflight_output: "{{ steps.launch_kit_infiniband_host_device_preflight | tojson }}" + discover_output: "{{ steps.launch_kit_infiniband_host_device_discover | tojson }}" + generate_output: "{{ steps.launch_kit_infiniband_host_device_generate | tojson }}" + compose: + - LaunchKitKubernetesPrerequisiteCheck + - LaunchKitTopologyDiscoveryCheck + - LaunchKitDeploymentHealthCheck + - LaunchKitInfiniBandCheck + - LaunchKitHostDeviceCheck + - LaunchKitSecondaryNetworkCheck + - LaunchKitIcmpConnectivityCheck + - LaunchKitRdmaConnectivityCheck + - LaunchKitRdmaBandwidthCheck + - LaunchKitGpuDirectRdmaCheck + - LaunchKitMultirailCheck + - LaunchKitEvidenceCaptureCheck diff --git a/isvctl/src/isvctl/orchestrator/step_executor.py b/isvctl/src/isvctl/orchestrator/step_executor.py index d4feacc48..0a3c7f9e6 100644 --- a/isvctl/src/isvctl/orchestrator/step_executor.py +++ b/isvctl/src/isvctl/orchestrator/step_executor.py @@ -433,14 +433,19 @@ def _execute_step(self, step: StepConfig, context: Context) -> StepResult: stderr=e.stderr if isinstance(e.stderr, str) else "", error=f"Command timed out after {step.timeout} seconds", ) - except FileNotFoundError: + except OSError as e: + error = ( + f"Command not found: {step.command}" + if isinstance(e, FileNotFoundError) + else f"Command could not start: {e}" + ) return StepResult( name=step.name, success=False, exit_code=-1, stdout="", stderr="", - error=f"Command not found: {step.command}", + error=error, attempted=False, ) except Exception as e: diff --git a/isvctl/tests/providers/k8s_launch_kit/test_provider.py b/isvctl/tests/providers/k8s_launch_kit/test_provider.py index 1104271e1..227da0952 100644 --- a/isvctl/tests/providers/k8s_launch_kit/test_provider.py +++ b/isvctl/tests/providers/k8s_launch_kit/test_provider.py @@ -60,7 +60,13 @@ def _run_provider( capture_output=True, text=True, ) - output = json.loads(completed.stdout) + try: + output = json.loads(completed.stdout) + except json.JSONDecodeError as error: + raise AssertionError( + f"provider emitted non-JSON stdout (exit {completed.returncode}): " + f"stdout={completed.stdout!r} stderr={completed.stderr!r}" + ) from error assert isinstance(output, dict) return completed, output @@ -602,6 +608,20 @@ def test_staged_user_config_is_removed_when_discovery_fails(tmp_path: Path) -> N assert not (working_dir / "user-config.yaml").exists() +def test_staged_user_config_is_created_with_restricted_permissions(tmp_path: Path) -> None: + """Sensitive input is private from the instant its staged file is created.""" + module = _load_provider_module() + source = tmp_path / "customer-cluster-config.yaml" + source.write_text("credentials: customer-api-token\n", encoding="utf-8") + working_dir = tmp_path / "work" + working_dir.mkdir() + + _, staged, _ = module._stage_user_config(str(source), working_dir, []) + + assert staged is not None + assert staged.stat().st_mode & 0o777 == 0o600 + + @pytest.mark.parametrize("flag", ["--user-config", "--save-cluster-config"]) def test_staged_user_config_rejects_conflicting_raw_discovery_paths(tmp_path: Path, flag: str) -> None: """The first-class input owns both discovery config paths.""" @@ -1129,7 +1149,7 @@ def test_failed_validate_preserves_documents_and_process_error(tmp_path: Path) - """A non-zero l8k result retains every JSON document and a clear exit diagnostic.""" working_dir = tmp_path / "work" artifact_dir = tmp_path / "evidence" - _run_workflow( + discover, _ = _run_workflow( "discover", [ "--fabric", @@ -1140,12 +1160,14 @@ def test_failed_validate_preserves_documents_and_process_error(tmp_path: Path) - working_dir=working_dir, artifact_dir=artifact_dir, ) - _run_workflow( + assert discover.returncode == 0 + generate, _ = _run_workflow( "generate", [], working_dir=working_dir, artifact_dir=artifact_dir, ) + assert generate.returncode == 0 env = os.environ.copy() env["L8K_MOCK_FAIL"] = "validate:ib_write_bw" diff --git a/isvctl/tests/test_orchestrator_loop.py b/isvctl/tests/test_orchestrator_loop.py index 0015265fc..08ac4fcc3 100644 --- a/isvctl/tests/test_orchestrator_loop.py +++ b/isvctl/tests/test_orchestrator_loop.py @@ -565,6 +565,41 @@ def test_phase_finalizer_skips_when_target_process_never_started(self, tmp_path: assert result.phases[1].message.startswith("SKIPPED: target step(s) were not attempted") assert result.phases[2].success is True + def test_phase_finalizer_skips_when_target_is_not_executable(self, tmp_path: Path) -> None: + """A permission failure also proves that no cluster mutation occurred.""" + marker = tmp_path / "cleaned" + cleanup = _write_script(tmp_path, "cleanup.sh", f"#!/bin/sh\ntouch {marker}\n") + target = tmp_path / "deploy.sh" + target.write_text("#!/bin/sh\nexit 0\n") + target.chmod(0o644) + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig(name="deploy", command=str(target), phase="case-one"), + StepConfig( + name="cleanup", + command=cleanup, + phase="case-one", + finalizer_for="deploy", + ), + StepConfig(name="case_two", command="true", phase="case-two"), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert not marker.exists() + assert result.phases[0].details["steps"][0]["attempted"] is False + assert result.phases[1].message.startswith("SKIPPED: target step(s) were not attempted") + assert result.phases[2].success is True + def test_failed_phase_finalizer_blocks_later_independent_phases(self) -> None: """A failed cleanup leaves unsafe state and overrides continuation policy.""" config = RunConfig( diff --git a/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py b/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py index bb14d62c1..b4a7a0fbe 100644 --- a/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py +++ b/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py @@ -621,7 +621,7 @@ def run(self) -> None: if len(rails) == 1 and "unknown-rail" not in rails: pytest.skip(f"Launch Kit connectivity matrix contains only one rail: {next(iter(rails))}") same_rail = [probe for probe in probes if probe["source_rail"] == probe["destination_rail"]] - cross_rail = [probe for probe in probes if probe not in same_rail] + cross_rail = [probe for probe in probes if probe["source_rail"] != probe["destination_rail"]] probes.extend( [ {"name": "same-rail-coverage", "passed": bool(same_rail), "message": f"rows={len(same_rail)}"}, diff --git a/isvtest/tests/test_catalog.py b/isvtest/tests/test_catalog.py index 01ada1427..b3110f007 100644 --- a/isvtest/tests/test_catalog.py +++ b/isvtest/tests/test_catalog.py @@ -56,7 +56,7 @@ def test_derives_suite_vocabulary_from_plain_suites(self) -> None: suites = build_suite_vocabulary() assert "iam" in suites assert "network_operator" in suites - assert "network_operator_use_cases" in suites + assert "network_operator_use_cases" not in suites assert "storage" in suites assert "kubernetes" not in suites assert "vm" not in suites @@ -115,6 +115,8 @@ def test_entries_have_suite_contract(self) -> None: if entry["capability"]: assert entry["requires"] == [] assert "EastWestNetworkRoceSriovCheck" in names + use_case = next(entry for entry in catalog if entry["name"] == "EastWestNetworkRoceSriovCheck") + assert use_case["suite"] == "network_operator" def test_extract_checks_supports_direct_dict_category_form(self, tmp_path) -> None: """Direct dict category wiring is included in catalog config scans.""" From 2e6524fe3f582feb6cc3c142ae2e0f4a9739bd67 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Thu, 10 Sep 2026 16:32:35 +0200 Subject: [PATCH 3/4] refactor: simplify Launch Kit network validation Run the Network Operator integration as one Launch Kit validate step against user-supplied configuration and deployment files. Treat the prepared cluster and all topology choices as prerequisites instead of modeling separate use cases or lifecycle stages. Report only the connectivity matrix as dynamic subtests, preserving Launch Kit verdicts and diagnostics without requiring disabled validation families. Update catalog metadata, traceability, documentation, and mock-backed coverage to match the smaller contract. Signed-off-by: Alexander Maslennikov --- AGENTS.md | 166 ++-- docs/guides/configuration.md | 13 +- .../guides/k8s-launch-kit/network-operator.md | 384 +++------ .../test-requirements-matrix.adoc | 490 +---------- .../test-requirements-matrix.yaml | 232 +----- docs/test-plan.adoc | 279 +------ docs/test-plan.yaml | 320 +------- .../providers/k8s-launch-kit/README.md | 140 ++-- .../config/network-operator.yaml | 689 +--------------- .../k8s-launch-kit/config/provider.yaml | 16 +- .../k8s-launch-kit/scripts/adapter.py | 55 +- isvctl/configs/suites/README.md | 78 +- .../k8s-launch-kit/network-operator.yaml | 273 +------ isvctl/src/isvctl/config/schema.py | 8 - .../providers/k8s_launch_kit/test_provider.py | 433 ++++------ .../k8s_launch_kit/test_timeout_config.py | 10 +- isvctl/tests/test_orchestrator_loop.py | 7 +- isvctl/tests/test_schema.py | 3 - .../validations/k8s_launch_kit/checks.py | 763 ++---------------- isvtest/tests/k8s_launch_kit/test_checks.py | 528 ++---------- isvtest/tests/test_catalog.py | 7 +- 21 files changed, 696 insertions(+), 4198 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6e19dfe3d..f6c86de93 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -166,122 +166,58 @@ forwarded env vars → optional isvreporter upload. ### Network Operator / Kubernetes Launch Kit - All provider-owned Launch Kit files live under - `isvctl/configs/providers/k8s-launch-kit/`: generic and Network Operator YAML - in `config/`, executable transport in `scripts/`, and provider documentation - in `README.md`. -- `isvctl/configs/providers/k8s-launch-kit/config/provider.yaml` is the generic provider. Its - public API mirrors the CLI: `prepare`, `verify`, Kubernetes preflight, - `discover`, `generate`, `deploy`, `validate`, and `clean`. Workflow settings are raw - argument arrays; do not model or duplicate Launch Kit flags/defaults here. - The single file-level input, `user_config`, points to a complete Launch Kit - configuration. Discovery copies it to `/user-config.yaml` and - writes the resolved result to `/cluster-config.yaml`, preserving - the source file and the default paths used by subsequent commands. - Its `validate` step uses `timeout: null` so l8k owns the automatically - calculated or user-supplied matrix deadline; all other workflow steps retain - finite outer isvctl watchdogs. Any provider may use a null `StepConfig` - timeout when its invoked command owns a bounded deadline. -- Launch Kit-specific transport code belongs under - `isvctl/configs/providers/k8s-launch-kit/`, not `providers/shared/` (which is - reserved for scripts reused by unrelated providers). Production code lives - in `scripts/`. Executable mocks and pinned fixtures are test-only and live in - `isvctl/tests/providers/k8s_launch_kit/fixtures/`; product configuration must - never reference them. -- `prepare` supports `verify` and explicit `install` modes. Install mode - downloads and records the official Launch Kit installer, then delegates - archive selection, checksum verification, and installation to it. Both modes - verify `l8k version --output json` and `l8k schema`. The configured string - environment is shared by install, verification, preflight, and workflows. -- The Kubernetes preflight is mandatory before each normal test use case. It - accepts the non-empty subset of Launch Kit commands selected by the caller, - derives a single explicit kubeconfig from their raw arguments (and rejects - conflicts), verifies API access, requires a non-empty node inventory, and - requires at least one Ready node. A failure stops the remaining steps in that - workflow/use case. -- `isvctl/configs/suites/k8s-launch-kit/network-operator.yaml` is the single - frontend-visible Network Operator suite. It owns catalog wiring and - interpretation for the globally selectable PRD checks and composes those - classes into six concrete validation tests: RoCE and InfiniBand across - SR-IOV, RDMA Shared, and host-device modes. Each check binds to the real step - that produced its evidence. Include only checks applicable to a use case; do - not run all checks and hide mismatches as interleaved skips. -- `isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml` is the - production six-use-case configuration. It defaults to real `l8k` and - `kubectl`, executes each supported fabric/deployment combination as a named - custom phase, and gives every use case isolated working and evidence - directories. ISVs own deployment: each use case runs preflight, discover, - generate, and validate only. It must never invoke `l8k deploy`, `l8k clean`, - or a cleanup finalizer. Its fabric/deployment arguments define test identity; - Launch Kit continues to own runtime defaults and users extend raw argv in overlays. - A global `user_config` is staged independently for each selected use case; - when it is set, raw discovery arguments cannot also select user/save config paths. - Keep default config/deployment path flags out of all grouped phase arguments; - overlays may add them only when intentionally overriding Launch Kit's paths. -- Mock-backed coverage loads that same production YAML and injects test-owned - executables only in `isvctl/tests/providers/k8s_launch_kit/test_provider.py`. - Result-check tests live under `isvtest/tests/k8s_launch_kit/`. -- Independent use-case phases are listed in `continue_after_failure` so a failed - case does not suppress later evidence. The failed phase still fails the final - run. Never use that option for shared setup or dependent phases. -- `StepConfig.finalizer_for` links cleanup to a mutating target. Provider - cleanup belongs in `phase: teardown`; the orchestrator runs it immediately - after the target phase validations and reports `-teardown`, - including for `--phase test`. It only activates when the target process - started, while `--phase teardown` runs it unconditionally as recovery. Use - this instead of unconditional cleanup when a preflight failure must not - delete pre-existing state. Schema validation requires matching capability - and validation-selection gates. -- Lifecycle steps associated with a selectable test declare - `requires_selected_validations`. The gate applies release, capability, label, - and suite exclusions before command execution. It is also the reporting - ownership edge: a failed selected step makes each named validation a - `step_failed` error in structured results and JUnit rather than allowing a - later missing-output skip. Keep - `requires_available_validations` for release-only gating; pytest `-k`/`-m` - selection remains too late to prune lifecycle commands. -- `CompositeCheck` predates the Launch Kit work and is framework machinery for - `compose:` entries. It now forwards member probes as `MemberName/probe-name`. - A member-level `pytest.skip` is reported as a skipped member while the - composite continues; skipped members neither pass nor fail the parent. - Successful validations with subtests are compacted by the shared isvctl - renderer; do not add suite-specific output flags. -- Launch Kit areas are separate validation classes in - `isvtest/validations/k8s_launch_kit/checks.py`; detailed probes use `report_subtest()` so - all manifest and connectivity rows reach JUnit output before the parent fails. -- Do not invent a `selfValidation` field in l8k output. Current `discover`, - `generate`, and `clean` emit one `ui.JSONResult`, successful standalone - `deploy` emits no stdout, and `validate` emits a JSON stream (static state, - connectivity matrix, then report path). The provider wraps these unmodified documents in a transport - envelope and keeps semantic assertions in pytest. The envelope records the - absolute command working directory so validations can resolve Launch Kit's - relative evidence paths without rewriting its output. -- Current l8k base check selection remains ICMP, `rping`, and `ib_write_bw`. - When `validation.gpuDirect.enabled` is true, GPUDirect DMA-BUF follows - `ib_write_bw` and is emitted as the distinct `gpudirect_dmabuf` result family. - Consume that family without adding an AI Cloud Validation default or a fourth - `--validation-checks` value. -- `l8k clean` remains the generic provider's only supported deletion path; do - not reproduce its CR/finalizer/Helm logic with kubectl. The Network Operator - validation suite intentionally has no deletion path because the ISV owns the - pre-existing deployment. Any future state-mutating test requires an explicit - transactional restore and verification contract before it is added. -- Use `--label ethernet` or `--label infiniband` to prune the grouped run to one - fabric's three workflows. Use `--label sriov`, `--label rdma_shared`, or - `--label host_device` for one deployment-mode pair; labels compose to select - one concrete use case. `-k`/marker selection still happens after lifecycle - commands and does not prune Launch Kit workflows. -- Use `--label gpudirect` to select the six GPU-capable use-case definitions. - The semantic member skips when Launch Kit emits no `gpudirect_dmabuf` rows; - emitted failed rows must fail the parent with endpoint GPU evidence. -- Current reporting uploads JUnit/log/catalog only. Files under - `_output/k8s-launch-kit` are local evidence until the reporter gains an - explicit, redacted attachment contract. -- Design, prerequisites, unit-test boundaries, PRD mapping, and production gaps live in - `docs/guides/k8s-launch-kit/network-operator.md`. + `isvctl/configs/providers/k8s-launch-kit/`: provider YAML in `config/`, + executable transport in `scripts/`, and implementation documentation in + `README.md`. Test doubles live only under + `isvctl/tests/providers/k8s_launch_kit/fixtures/`; product configuration + must never reference them. +- `config/provider.yaml` is the generic provider. Its public API mirrors the + Launch Kit lifecycle: prepare, verify, Kubernetes preflight, discover, + generate, deploy, validate, and clean. Workflow settings are raw argument + arrays; do not model or duplicate Launch Kit flags, schema, or defaults. + Discovery can stage a complete `user_config`. Its validate step uses + `timeout: null` so Launch Kit owns the automatically calculated or + user-supplied matrix deadline. +- `config/network-operator.yaml` is deliberately independent of the generic + lifecycle provider. It runs exactly one test step: + `l8k validate --user-config --deployment-files `. + The installed binary, reachable Kubernetes cluster, reconciled Network + Operator deployment, complete Launch Kit config, and rendered deployment + files are prerequisites. Do not add prepare, verify, preflight, discover, + generate, deploy, clean, or finalizer steps to this entrypoint. +- The Network Operator provider inputs are `executable`, `user_config`, + `deployment_files`, `working_dir`, `artifact_dir`, and a string-only + `environment` mapping. Both input paths are resolved and checked before + execution, then supplied through Launch Kit's real CLI flags. The adapter + must not copy, merge, interpret, or modify them. +- `isvctl/configs/suites/k8s-launch-kit/network-operator.yaml` contains one + catalog test, `LaunchKitConnectivityCheck`. There are no fabric, deployment, + or connectivity-family use-case tests. Those choices come from the complete + Launch Kit config and current cluster state. +- `isvtest/validations/k8s_launch_kit/checks.py` consumes only + `connectivity.PingResults`. Every emitted row becomes a subtest with its + family, endpoints, and rails. Preserve bandwidth, GPU, stderr, and error + details when present. Do not require a fixed family list: disabled families + are absent without skips, and explicit new families pass through. +- A missing or empty connectivity matrix fails rather than passing vacuously. + The provider binds its step with `requires_selected_validations` so command + failures remain owned by the catalog validation and appear in structured + reporting. +- The adapter adds only `--output json`, wraps the unmodified concatenated + JSON documents, and records argv, cwd, stdout, stderr, exit code, and timing. + Do not invent a `selfValidation` result or reinterpret Launch Kit's verdict. +- The generic provider retains installation, Kubernetes preflight, and cleanup + support for other consumers. `l8k clean` remains its only supported deletion + path; never reproduce Launch Kit cleanup with kubectl. +- Mock-backed provider coverage loads the production YAML and injects + test-owned executables in memory. Result interpretation tests live under + `isvtest/tests/k8s_launch_kit/`. - The structured PRD source is - `docs/requirements/network-operator-readiness-requirements.yaml`; keep its - `ENT-REQ-*` edges in `docs/requirements/test-requirements-matrix.yaml` and - regenerate committed views with `make plan`. + `docs/requirements/network-operator-readiness-requirements.yaml`. Keep its + traceability edges in `docs/requirements/test-requirements-matrix.yaml`, + document the prerequisite boundary in + `docs/guides/k8s-launch-kit/network-operator.md`, and regenerate committed + views with `make plan`. ## Environment Variables diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index 86d9bdbc4..fa633c0b6 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -243,8 +243,7 @@ Each step defines a command to execute: | `finalizer_for` | No | Run as linked teardown after the named step's phase when that command was attempted | | `output_schema` | No | Schema name for output validation | | `requires` | No | Capability contexts this step runs in (see [Capabilities](#capabilities-and-requires)) | -| `requires_available_validations` | No | Validation names that must be available after release filtering | -| `requires_selected_validations` | No | Configured validation names that must remain selected after release, capability, label, and suite-exclusion filtering; failed steps become errors on these owning validations | +| `requires_selected_validations` | No | Configured validation names that must remain selected after capability, label, and suite-exclusion filtering; failed steps become errors on these owning validations | The timeout is an orchestration watchdog, not a provider-specific setting. Set it to `null` only when the invoked tool owns a bounded deadline; isvctl will @@ -364,8 +363,8 @@ tests: With no label filter, the validation is selected and the step runs. With `--label ethernet`, it also runs; with `--label infiniband`, the step is skipped before execution. Every listed validation must be configured and -selected. The gate also honors the release manifest, capability requirements, -`tests.exclude.tests`, and effective label exclusions. +selected. The gate also honors capability requirements, `tests.exclude.tests`, +and effective label exclusions. The same list is the reporting ownership edge for the lifecycle step. If a selected step fails before its validation can run, each listed validation is @@ -375,10 +374,6 @@ an early deploy or setup failure from being misreported as a harmless The error message names the failed step and retains its redacted command diagnostic. -`requires_available_validations` is narrower: it only prevents a step from -running when its named checks are absent from the release manifest. Retain it -for providers that only need release gating. - Pytest `-k` and `-m` expressions are evaluated inside pytest and therefore do not drive `requires_selected_validations`. Use framework `--label` filtering for lifecycle pruning in mutating suites. @@ -767,7 +762,7 @@ not register or invoke that class directly. The YAML key creates one catalog test and runs every listed validation member. Each member is reported as a subtest. If a member reports its own probes through `report_subtest()`, those probes are retained with qualified names such as -`LaunchKitRdmaConnectivityCheck/rping/worker-a->worker-b/rail-0->rail-1`. +`ConnectivityCheck/rping/worker-a->worker-b/rail-0->rail-1`. This avoids collisions between members and keeps the full probe tree in pytest and JUnit output. diff --git a/docs/guides/k8s-launch-kit/network-operator.md b/docs/guides/k8s-launch-kit/network-operator.md index e47da1573..d71f5100f 100644 --- a/docs/guides/k8s-launch-kit/network-operator.md +++ b/docs/guides/k8s-launch-kit/network-operator.md @@ -1,317 +1,175 @@ -# Network Operator validation through Kubernetes Launch Kit +# Network Operator connectivity validation through Kubernetes Launch Kit -## Safety and ownership boundary +## Scope -The production Network Operator suite validates an installation that the ISV -has already deployed and configured. Its workflow is: +The Network Operator suite performs one operation: ```text -verify l8k - -> verify Kubernetes access and at least one Ready node - -> l8k discover - -> l8k generate - -> l8k validate - -> AI Cloud Validation checks and reports +l8k validate --user-config --deployment-files ``` -It does **not** invoke `l8k deploy` or `l8k clean`. AI Cloud Validation therefore -does not install, replace, reconfigure, or remove the ISV-managed Network -Operator deployment. A validation or orchestration failure also cannot activate -a cleanup finalizer. +It reports the connectivity matrix produced by Launch Kit. It does not install +or verify the `l8k` binary, discover topology, generate manifests, deploy +Network Operator, run a separate Kubernetes preflight, or clean cluster state. -Discovery may label nodes and validation creates Launch Kit's temporary test -workloads. Those operations remain part of Launch Kit itself. The important -ownership boundary is that this suite never applies the generated Network -Operator deployment manifests and never removes the existing installation. +Those activities are prerequisites. Before starting the suite, the ISV must +provide a reachable Kubernetes cluster, bring Network Operator and the desired +networking profile into the expected state, create a complete Launch Kit +configuration, and retain the corresponding rendered deployment files. -The generic provider still exposes `discover`, `generate`, `deploy`, `validate`, -and `clean`. That API mirrors the complete Launch Kit CLI and remains available -to other suites that explicitly own deployment lifecycle. The validation-only -behavior is defined by `config/network-operator.yaml`, not by removing features -from the generic provider. +There are no separate AI Cloud Validation tests for RoCE, InfiniBand, SR-IOV, +RDMA Shared, host-device, ICMP, rping, bandwidth, or GPUDirect. The supplied +Launch Kit configuration determines the topology and enabled validation +families. This avoids duplicating Launch Kit's configuration and applicability +model in AI Cloud Validation. ## Architecture -AI Cloud Validation separates command execution from result interpretation: - ```text -provider configuration - -> ordered isvctl phases and steps - -> adapter.py executes real l8k and kubectl - -> raw command envelopes become step outputs and evidence - -> suite configuration - -> binds one use-case check to each validate step - -> supplies discover/generate/preflight outputs as context - -> isvtest validation classes - -> interpret unmodified l8k JSON - -> report member and probe-level subtests - -> console, JUnit, retained artifacts, and optional Labs upload +Network Operator provider YAML + -> one isvctl test step + -> adapter.py + -> l8k validate --user-config ... --deployment-files ... --output json + -> retained argv, stdout, stderr, exit code, and duration + -> Network Operator suite YAML + -> LaunchKitConnectivityCheck + -> one subtest for every Launch Kit connectivity row + -> console and JUnit results ``` -The main files are: +The relevant files are: | Layer | File | |---|---| -| Generic provider | `isvctl/configs/providers/k8s-launch-kit/config/provider.yaml` | -| Network Operator validation workflow | `isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml` | +| Production entrypoint | `isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml` | | CLI transport | `isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py` | -| Individual PRD checks and concrete use cases | `isvctl/configs/suites/k8s-launch-kit/network-operator.yaml` | +| Catalog wiring | `isvctl/configs/suites/k8s-launch-kit/network-operator.yaml` | | Result interpretation | `isvtest/src/isvtest/validations/k8s_launch_kit/checks.py` | -| Provider tests and mock CLI | `isvctl/tests/providers/k8s_launch_kit/` | -| Result-check tests | `isvtest/tests/k8s_launch_kit/test_checks.py` | -| PRD and traceability | `docs/requirements/` | - -The provider and suite files are intentionally separate. The provider owns -process execution and evidence. The single frontend-visible `network_operator` -suite owns catalog identity, selection, and the mapping from step outputs to -reusable validation checks. It contains both individually selectable PRD checks -and the six grouped use cases. - -## Network Operator workflow - -For every selected use case, the production provider executes these steps in -order: - -1. `launch_kit__preflight` checks Kubernetes access; -2. `launch_kit__discover` runs `l8k discover` with that use case's - fabric and deployment type; -3. `launch_kit__generate` runs `l8k generate` to reconstruct the - expected manifests used by Launch Kit validation; -4. `launch_kit__validate` runs `l8k validate` against the already - deployed cluster state. - -Generation is intentional even though deployment is external. Launch Kit -validation compares the live installation with the desired resources generated -for the selected topology. The generated files are evidence; this suite does -not apply them. - -The six concrete use cases are: - -| Test | Fabric labels | Deployment label | -|---|---|---| -| `EastWestNetworkRoceSriovCheck` | `ethernet`, `roce` | `sriov` | -| `EastWestNetworkInfiniBandSriovCheck` | `infiniband` | `sriov` | -| `EastWestNetworkRoceRdmaSharedCheck` | `ethernet`, `roce` | `rdma_shared` | -| `EastWestNetworkInfiniBandRdmaSharedCheck` | `infiniband` | `rdma_shared` | -| `EastWestNetworkRoceHostDeviceCheck` | `ethernet`, `roce` | `host_device` | -| `EastWestNetworkInfiniBandHostDeviceCheck` | `infiniband` | `host_device` | - -All applicable use cases run by default. Labels can select a fabric, a -deployment mode, or their intersection. `continue_after_failure` lets later -independent use cases collect evidence after an earlier use case fails; the -overall run still fails. - -## Provider API - -The adapter accepts raw argument arrays instead of reproducing Launch Kit's -domain configuration: - -| Key | Meaning | -|---|---| -| `executable` | Existing `l8k` command or absolute path | -| `installation.mode` | `verify` by default, or explicit `install` | -| `installation.version` | Optional exact Launch Kit version | -| `installation.installer_ref` | Immutable installer commit used in install mode | -| `installation.installer_sha256` | Trusted installer digest used in install mode | -| `installation.prefix` | Optional installation prefix | -| `user_config` | Optional path to a complete Launch Kit configuration | -| `kubectl_command` | Optional kubectl-compatible argv prefix | -| `working_dir` | Per-workflow Launch Kit working directory | -| `artifact_dir` | Per-workflow evidence directory | -| `environment` | String environment entries forwarded to all commands | -| `.arguments` | Raw arguments for that Launch Kit command | - -AI Cloud Validation defines no defaults for namespaces, node selectors, -Network Operator versions, driver modes, rails, resource names, IP pools, GPU -counts, validation modes/checks, bandwidth thresholds, or Launch Kit timeouts. -Omitted values are resolved by the installed Launch Kit release. The adapter -adds only `--output json` and rejects a conflicting user output mode. - -The generic configuration includes argument arrays for all five Launch Kit -workflow commands. The Network Operator configuration includes only `discover`, -`generate`, and `validate`, because those are the commands it actually invokes. - -### Complete user configuration - -Set `context.k8s_launch_kit.user_config` when a cluster needs settings that are -not exposed as CLI flags. This must be a complete Launch Kit configuration; -the provider does not merge partial YAML. - -For each selected use case, the adapter: - -1. copies the source to `/user-config.yaml` with mode `0600`; -2. adds `--user-config ` and - `--save-cluster-config /cluster-config.yaml` to discovery; -3. removes the staged copy as soon as discovery exits; -4. records only source path, size, and SHA-256 provenance in evidence. - -The source file is never modified or retained as an uploaded artifact. Do not -put it inside a retained working or evidence directory, and do not repeat the -provider-owned `--user-config` or `--save-cluster-config` flags in discovery -arguments. - -Example overlay: - -```yaml -import: - - /path/to/ai-cloud-validation/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml - -context: - k8s_launch_kit: - user_config: /secure/path/cluster-config.yaml - environment: - KUBECONFIG: /secure/path/kubeconfig.yaml -``` - -The supplied configuration must describe the desired use case consistently -with the selected labels. The use-case discovery flags select the authoritative -fabric and deployment type. - -## Installation verification and Kubernetes prerequisite - -`installation.mode: verify` resolves the configured executable and captures: +| Mock-backed provider tests | `isvctl/tests/providers/k8s_launch_kit/` | +| Result-check unit tests | `isvtest/tests/k8s_launch_kit/` | -- `l8k version --output json`; -- `l8k schema --output json`. +The generic provider in +`isvctl/configs/providers/k8s-launch-kit/config/provider.yaml` still mirrors the +complete Launch Kit lifecycle for other consumers. The Network Operator +entrypoint does not import it, so none of those lifecycle steps are inherited. -The generic provider contract requires the schema to advertise all five -workflow commands, including deploy and clean. This does not cause the Network -Operator suite to call those commands. An optional exact version is checked in -both setup and test-phase verification, so `--phase test` is safe when setup is -skipped. +## Inputs -Install mode requires an immutable full Git commit for the official installer -and a caller-supplied SHA-256. The adapter verifies the downloaded installer -before executing it, then verifies the installed binary. +The Network Operator provider exposes only these settings: -The preflight helper accepts the non-empty subset of Launch Kit commands used -by its caller. It extracts `--kubeconfig` from those command arguments, -rejects inconsistent kubeconfigs, and runs kubectl probes for API access and at -least one Ready node. If no command argument selects a kubeconfig, kubectl and -l8k inherit the same forwarded `KUBECONFIG` environment or normal client -defaults. +| Key | Required | Meaning | +|---|---:|---| +| `executable` | no | `l8k` command or absolute executable path; default is `l8k` | +| `user_config` | yes | Complete Launch Kit cluster configuration | +| `deployment_files` | yes | Existing rendered deployment directory validated by Launch Kit | +| `working_dir` | no | Provider process working directory | +| `artifact_dir` | no | Directory for command evidence | +| `environment` | no | String environment entries forwarded to Launch Kit, such as `KUBECONFIG` | -## Timeouts +Paths accept `~`, but absolute paths are preferable in automation. The adapter +resolves both paths, verifies that `user_config` is a file and +`deployment_files` is a directory, and passes the resolved paths to Launch Kit. +It does not copy, merge, parse, or modify either input. -Provider step timeouts are outer isvctl watchdogs. Discovery and generation -have finite watchdogs. Network Operator validate steps use `timeout: null`, so -isvctl does not preempt a valid large connectivity matrix. Launch Kit computes -and logs its bounded validation budget by default, or uses a user-supplied -Launch Kit timeout argument. +Launch Kit owns every setting inside the complete config, including the +selected profile, validation mode, enabled checks, GPUDirect behavior, +bandwidth thresholds, per-operation timeouts, routing, IP pools, and resource +names. AI Cloud Validation stores no copies of those defaults. -Other providers may use `timeout: null` only when the child tool owns a bounded -deadline. An enclosing CI job can still impose a total job timeout. +Do not also put `--user-config` or `--deployment-files` in a raw Launch Kit +argument list. The adapter rejects duplicate path sources rather than allowing +ambiguous last-value behavior. -## Result and error reporting +## Running the suite -The adapter preserves Launch Kit JSON documents without renaming fields and -records the exact argv, cwd, stdout, stderr, exit code, and timing for every -command. A non-zero process result remains attached to its step even when the -CLI emitted partial JSON. +From the repository root: -The composite use-case check converts Launch Kit resource and connectivity rows -into pytest subtests. Names identify the member check and the individual probe, -so failures point to a concrete resource, rail, source/destination pair, or -bandwidth result. A member that is inapplicable may skip without skipping the -whole use case; failed members fail the parent. +```bash +uv run isvctl test run \ + -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ + --capability kubernetes \ + --set 'context.k8s_launch_kit.user_config=/absolute/path/cluster-config.yaml' \ + --set 'context.k8s_launch_kit.deployment_files=/absolute/path/deployment' \ + --no-upload -- -v +``` -If preflight, discovery, or generation fails before validate produces output, -the owning use-case validation is emitted as a `step_failed` error in JUnit. A -normal Launch Kit validation failure is emitted as a failed testcase with its -probe diagnostics. Neither path invokes deploy or cleanup. +To use a kubeconfig that is not selected by the normal client environment, add: -`LaunchKitEvidenceCaptureCheck` accepts both lifecycle-owning and -validation-only workflows. Verify, preflight, discover, generate, and validate -evidence is required. Deploy evidence is checked only when a suite actually -provides `deploy_output`. +```text +--set 'context.k8s_launch_kit.environment={"KUBECONFIG":"/absolute/path/kubeconfig.yaml"}' +``` -## Running the suite +Omit `--no-upload` when the run should use the configured AI Cloud Labs upload +path. -Prerequisites: +## Selecting connectivity checks -- the selected kubeconfig reaches a Kubernetes cluster with at least one Ready - worker node; -- Network Operator and the resources required for the selected profile are - already deployed and reconciled; -- the installed `l8k` release supports JSON version, schema, discovery, - generation, and validation output; -- the execution identity can discover topology, create validation workloads, - exec into them, inspect events/resources, and collect logs; -- the cluster satisfies Launch Kit prerequisites for the selected SR-IOV, - RDMA Shared, host-device, RoCE, InfiniBand, and GPUDirect checks. +Selection happens in the Launch Kit config, not with AI Cloud Validation +labels. For example, disabling Launch Kit GPUDirect validation means no +`gpudirect_dmabuf` rows are emitted. The wrapper then reports the remaining +rows only; it does not create a skipped or failed GPUDirect placeholder. -Run all use cases: +Likewise, the suite does not infer a fabric or deployment mode from labels. +Run it once for the exact cluster state described by the supplied files. To +validate another topology, provision that topology and invoke the same suite +with its config and deployment directory. -```bash -ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl test run \ - -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ - --capability kubernetes --no-upload -- -v -``` +## Timeouts -Select a subset by adding labels after `--`: +The `launch_kit_validate` step has `timeout: null`. Launch Kit calculates and +logs its connectivity-matrix budget by default, or honors the timeout configured +by the user. This prevents an independent isvctl watchdog from terminating a +valid large matrix before Launch Kit's bounded checks finish. An enclosing CI +job may still impose an overall job timeout. -```bash -# Ethernet/RoCE only -... --capability kubernetes --no-upload -- -v --label ethernet +## Results and errors -# All SR-IOV profiles -... --capability kubernetes --no-upload -- -v --label sriov +`LaunchKitConnectivityCheck` finds the `connectivity.PingResults` array in the +unmodified JSON stream. Each emitted row becomes a named subtest: -# RoCE SR-IOV only -... --capability kubernetes --no-upload -- -v \ - --label ethernet --label sriov +```text +/->/-> ``` -Do not use `--phase teardown` for this Network Operator configuration; it has -no teardown phase because deployment lifecycle belongs to the ISV. +Failure messages preserve Launch Kit's expectation, observed result, bandwidth +and minimum when present, endpoint GPU information when present, stderr, and +structured error text. Explicit future `Family` values are forwarded without +requiring an AI Cloud Validation catalog update. Older numeric `Kind` values +remain supported as a compatibility fallback. + +The check fails when any emitted row has `OK != true`, when no connectivity +matrix is present, or when the matrix contains no results. A command that fails +before producing connectivity output retains its provider error in the +validation and JUnit output. -## Evidence and upload +## Evidence -Each use case writes to separate working and evidence directories under: +The adapter writes: ```text -_output/k8s-launch-kit/network-operator/use-cases// +_output/k8s-launch-kit/network-operator/ work/ - cluster-config.yaml - deployment/ - k8s-launch-kit-validation-report.html (location may vary by l8k release) evidence/ - kubernetes-preflight/ - commands/discover/ - commands/generate/ commands/validate/ + command.json + stdout.txt + stderr.log ``` -Shared setup and verification evidence is stored under -`_output/k8s-launch-kit/network-operator/shared-evidence/`. Generated manifests, -resource status, events, connectivity and bandwidth documents, stdout/stderr, -and the HTML report are registered in provider step outputs and retained -locally. Current AI Cloud Labs upload sends JUnit, the combined run log, and -catalog metadata; it does not yet upload these evidence files as attachments. - -User configuration contents are intentionally excluded; only provenance is -retained. - -## PRD coverage and remaining work - -The suite provides selectable and grouped Network Operator checks for RoCE and -InfiniBand across SR-IOV, RDMA Shared, and host-device profiles. It reuses Launch -Kit topology, manifest readiness, ICMP, rping, RDMA bandwidth, multi-rail, and -GPUDirect validation output. Catalog metadata and requirement mappings identify -ownership, labels, dependencies, applicability, and prerequisites. - -The validation-only boundary changes the interpretation of ENT-REQ-010: this -integration does not modify the Network Operator deployment, so it has no -pre-test operator state to restore. If a future test intentionally changes -operator state, that test needs a separate, explicit Launch Kit transaction or -snapshot/restore contract before it can be added here. - -Live qualification is still required for every supported hardware/fabric/ -deployment combination. Unit fixtures prove integration and reporting behavior, -not partner certification. Additional Launch Kit improvements worth considering -are stable machine-readable result schemas, explicit artifact manifests, and a -first-class API for validating an existing deployment without regenerating -desired manifests when the site already has an authoritative complete config. +`command.json` records the resolved argv, exit code, and duration. `stdout.txt` +contains Launch Kit's complete JSON stream, including static validation, +connectivity, and report-path documents; `stderr.log` retains CLI progress and +diagnostics. The adapter also registers these paths in the provider step output. +The Launch Kit HTML report remains at the `reportPath` emitted by Launch Kit, +normally below the supplied deployment directory. + +## PRD boundary + +This integration covers reportable Launch Kit connectivity validation. It +deliberately treats topology discovery, manifest generation, installation, +deployment health preparation, profile selection, and restoration as external +prerequisites. Tests that intentionally mutate Network Operator state require a +separate transaction and restoration design before they can be added. diff --git a/docs/requirements/test-requirements-matrix.adoc b/docs/requirements/test-requirements-matrix.adoc index 06d2ca229..3b01bb04b 100644 --- a/docs/requirements/test-requirements-matrix.adoc +++ b/docs/requirements/test-requirements-matrix.adoc @@ -3507,16 +3507,7 @@ docs/requirements/test-requirements-matrix.yaml. Run `make plan` to regenerate. | [[K8S42-01]]K8S42-01 | Workload Orchestration | Managed Kubernetes Control Plane -| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools -| ENT-REQ-000 -| network-operator-prd -| partial -| - -| K8S42-01 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment | ENT-REQ-001 | network-operator-prd | full @@ -3525,520 +3516,79 @@ docs/requirements/test-requirements-matrix.yaml. Run `make plan` to regenerate. | K8S42-01 | Workload Orchestration | Managed Kubernetes Control Plane -| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools -| ENT-REQ-003 -| network-operator-prd -| partial -| - -| K8S42-01 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools -| ENT-REQ-009 -| network-operator-prd -| full -| - -| K8S42-01 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools -| ENT-REQ-011 -| network-operator-prd -| partial -| - -| [[K8S42-02]]K8S42-02 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate the SR-IOV policy and profile-specific secondary network resources reported by Launch Kit -| ENT-REQ-005 -| network-operator-prd -| partial -| - -| [[K8S42-03]]K8S42-03 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate pod-to-pod RDMA-CM connectivity across selected rails +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment | ENT-REQ-002 | network-operator-prd -| full -| - -| K8S42-03 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate pod-to-pod RDMA-CM connectivity across selected rails -| ENT-REQ-005 -| network-operator-prd | partial | -| K8S42-03 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate pod-to-pod RDMA-CM connectivity across selected rails -| ENT-REQ-006 -| network-operator-prd -| partial -| - -| [[K8S42-04]]K8S42-04 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate the exact secondary network resource selected for an Ethernet or RoCE profile -| ENT-REQ-005 -| network-operator-prd -| partial -| - -| [[K8S42-05]]K8S42-05 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate the exact secondary network resource selected for an InfiniBand profile -| ENT-REQ-006 -| network-operator-prd -| partial -| - -| [[K8S42-06]]K8S42-06 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate HostDeviceNetwork readiness for an applicable Ethernet or InfiniBand profile -| ENT-REQ-007 -| network-operator-prd -| partial -| - -| [[K8S42-07]]K8S42-07 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate GPUDirect RDMA DMA-BUF bandwidth between GPU-enabled pod endpoints -| ENT-REQ-008 -| network-operator-prd -| partial -| - -| [[K8S42-08]]K8S42-08 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate Launch Kit discovery completed and resolved a fabric and deployment profile -| ENT-REQ-002 -| network-operator-prd -| partial -| - -| K8S42-08 +| K8S42-01 | Workload Orchestration | Managed Kubernetes Control Plane -| Validate Launch Kit discovery completed and resolved a fabric and deployment profile +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment | ENT-REQ-004 | network-operator-prd | partial | -| K8S42-08 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate Launch Kit discovery completed and resolved a fabric and deployment profile -| ENT-REQ-012 -| network-operator-prd -| partial -| - -| [[K8S42-09]]K8S42-09 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness -| ENT-REQ-005 -| network-operator-prd -| partial -| - -| K8S42-09 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness -| ENT-REQ-006 -| network-operator-prd -| partial -| - -| K8S42-09 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness -| ENT-REQ-009 -| network-operator-prd -| partial -| - -| [[K8S42-10]]K8S42-10 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate Macvlan or IPoIB network readiness for an applicable RDMA Shared profile -| ENT-REQ-005 -| network-operator-prd -| partial -| - -| K8S42-10 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate Macvlan or IPoIB network readiness for an applicable RDMA Shared profile -| ENT-REQ-006 -| network-operator-prd -| partial -| - -| [[K8S42-11]]K8S42-11 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation -| ENT-REQ-002 -| network-operator-prd -| partial -| - -| K8S42-11 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation -| ENT-REQ-005 -| network-operator-prd -| partial -| - -| K8S42-11 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation -| ENT-REQ-006 -| network-operator-prd -| partial -| - -| [[K8S42-12]]K8S42-12 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth -| ENT-REQ-002 -| network-operator-prd -| partial -| - -| K8S42-12 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth -| ENT-REQ-005 -| network-operator-prd -| partial -| - -| K8S42-12 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth -| ENT-REQ-006 -| network-operator-prd -| partial -| - -| [[K8S42-13]]K8S42-13 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate same-rail and cross-rail coverage for a multi-rail Launch Kit profile -| ENT-REQ-002 -| network-operator-prd -| partial -| - -| [[K8S42-15]]K8S42-15 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Verify raw phase output, generated files, connectivity results, and Launch Kit reports were preserved -| ENT-REQ-011 -| network-operator-prd -| partial -| - -| K8S42-15 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Verify raw phase output, generated files, connectivity results, and Launch Kit reports were preserved -| ENT-REQ-013 -| network-operator-prd -| partial -| - -| [[K8S42-16]]K8S42-16 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned Ethernet or RoCE SR-IOV Network RDMA profile through Launch Kit -| ENT-REQ-001 -| network-operator-prd -| partial -| - -| K8S42-16 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned Ethernet or RoCE SR-IOV Network RDMA profile through Launch Kit -| ENT-REQ-003 -| network-operator-prd -| partial -| - -| K8S42-16 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned Ethernet or RoCE SR-IOV Network RDMA profile through Launch Kit -| ENT-REQ-005 -| network-operator-prd -| partial -| - -| K8S42-16 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned Ethernet or RoCE SR-IOV Network RDMA profile through Launch Kit -| ENT-REQ-008 -| network-operator-prd -| partial -| - -| K8S42-16 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned Ethernet or RoCE SR-IOV Network RDMA profile through Launch Kit -| ENT-REQ-013 -| network-operator-prd -| partial -| - -| [[K8S42-17]]K8S42-17 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned InfiniBand SR-IOV Network RDMA profile through Launch Kit -| ENT-REQ-001 -| network-operator-prd -| partial -| - -| K8S42-17 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned InfiniBand SR-IOV Network RDMA profile through Launch Kit -| ENT-REQ-003 -| network-operator-prd -| partial -| - -| K8S42-17 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned InfiniBand SR-IOV Network RDMA profile through Launch Kit -| ENT-REQ-006 -| network-operator-prd -| partial -| - -| K8S42-17 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned InfiniBand SR-IOV Network RDMA profile through Launch Kit -| ENT-REQ-008 -| network-operator-prd -| partial -| - -| K8S42-17 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned InfiniBand SR-IOV Network RDMA profile through Launch Kit -| ENT-REQ-013 -| network-operator-prd -| partial -| - -| [[K8S42-18]]K8S42-18 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned Ethernet or RoCE RDMA Shared profile through Launch Kit -| ENT-REQ-001 -| network-operator-prd -| partial -| - -| K8S42-18 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned Ethernet or RoCE RDMA Shared profile through Launch Kit -| ENT-REQ-003 -| network-operator-prd -| partial -| - -| K8S42-18 +| K8S42-01 | Workload Orchestration | Managed Kubernetes Control Plane -| Validate a pre-provisioned Ethernet or RoCE RDMA Shared profile through Launch Kit +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment | ENT-REQ-005 | network-operator-prd | partial | -| K8S42-18 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned Ethernet or RoCE RDMA Shared profile through Launch Kit -| ENT-REQ-008 -| network-operator-prd -| partial -| - -| K8S42-18 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned Ethernet or RoCE RDMA Shared profile through Launch Kit -| ENT-REQ-013 -| network-operator-prd -| partial -| - -| [[K8S42-19]]K8S42-19 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned InfiniBand or IPoIB RDMA Shared profile through Launch Kit -| ENT-REQ-001 -| network-operator-prd -| partial -| - -| K8S42-19 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned InfiniBand or IPoIB RDMA Shared profile through Launch Kit -| ENT-REQ-003 -| network-operator-prd -| partial -| - -| K8S42-19 +| K8S42-01 | Workload Orchestration | Managed Kubernetes Control Plane -| Validate a pre-provisioned InfiniBand or IPoIB RDMA Shared profile through Launch Kit +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment | ENT-REQ-006 | network-operator-prd | partial | -| K8S42-19 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned InfiniBand or IPoIB RDMA Shared profile through Launch Kit -| ENT-REQ-008 -| network-operator-prd -| partial -| - -| K8S42-19 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate a pre-provisioned InfiniBand or IPoIB RDMA Shared profile through Launch Kit -| ENT-REQ-013 -| network-operator-prd -| partial -| - -| [[K8S42-20]]K8S42-20 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate pre-provisioned Ethernet or RoCE host-device networking on worker VMs through Launch Kit -| ENT-REQ-001 -| network-operator-prd -| partial -| - -| K8S42-20 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate pre-provisioned Ethernet or RoCE host-device networking on worker VMs through Launch Kit -| ENT-REQ-003 -| network-operator-prd -| partial -| - -| K8S42-20 +| K8S42-01 | Workload Orchestration | Managed Kubernetes Control Plane -| Validate pre-provisioned Ethernet or RoCE host-device networking on worker VMs through Launch Kit +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment | ENT-REQ-007 | network-operator-prd | partial | -| K8S42-20 +| K8S42-01 | Workload Orchestration | Managed Kubernetes Control Plane -| Validate pre-provisioned Ethernet or RoCE host-device networking on worker VMs through Launch Kit +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment | ENT-REQ-008 | network-operator-prd | partial | -| K8S42-20 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate pre-provisioned Ethernet or RoCE host-device networking on worker VMs through Launch Kit -| ENT-REQ-013 -| network-operator-prd -| partial -| - -| [[K8S42-21]]K8S42-21 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate pre-provisioned InfiniBand host-device networking on worker VMs through Launch Kit -| ENT-REQ-001 -| network-operator-prd -| partial -| - -| K8S42-21 -| Workload Orchestration -| Managed Kubernetes Control Plane -| Validate pre-provisioned InfiniBand host-device networking on worker VMs through Launch Kit -| ENT-REQ-003 -| network-operator-prd -| partial -| - -| K8S42-21 +| K8S42-01 | Workload Orchestration | Managed Kubernetes Control Plane -| Validate pre-provisioned InfiniBand host-device networking on worker VMs through Launch Kit -| ENT-REQ-007 +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment +| ENT-REQ-011 | network-operator-prd | partial | -| K8S42-21 +| K8S42-01 | Workload Orchestration | Managed Kubernetes Control Plane -| Validate pre-provisioned InfiniBand host-device networking on worker VMs through Launch Kit -| ENT-REQ-008 +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment +| ENT-REQ-012 | network-operator-prd | partial | -| K8S42-21 +| K8S42-01 | Workload Orchestration | Managed Kubernetes Control Plane -| Validate pre-provisioned InfiniBand host-device networking on worker VMs through Launch Kit +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment | ENT-REQ-013 | network-operator-prd | partial diff --git a/docs/requirements/test-requirements-matrix.yaml b/docs/requirements/test-requirements-matrix.yaml index e6dad8103..3119a6994 100644 --- a/docs/requirements/test-requirements-matrix.yaml +++ b/docs/requirements/test-requirements-matrix.yaml @@ -2570,261 +2570,35 @@ mappings: notes: '' - test_id: K8S42-01 requirements: - - req_id: ENT-REQ-000 - source: network-operator-prd - coverage: partial - req_id: ENT-REQ-001 source: network-operator-prd coverage: full - - req_id: ENT-REQ-003 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-009 - source: network-operator-prd - coverage: full - - req_id: ENT-REQ-011 - source: network-operator-prd - coverage: partial - annotations: 'Production provider establishes the integration and catalog boundary; mock-backed unit tests exercise it; long-term ownership and typed program policy metadata are not runtime assertions.' - notes: '' - - test_id: K8S42-02 - requirements: - - req_id: ENT-REQ-005 - source: network-operator-prd - coverage: partial - annotations: 'Covers SR-IOV attachment and device readiness; connectivity and bandwidth are separate selectable checks.' - notes: '' - - test_id: K8S42-03 - requirements: - - req_id: ENT-REQ-002 - source: network-operator-prd - coverage: full - - req_id: ENT-REQ-005 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-006 - source: network-operator-prd - coverage: partial - annotations: 'Reuses the Launch Kit rping matrix for both RoCE and InfiniBand profiles.' - notes: '' - - test_id: K8S42-04 - requirements: - - req_id: ENT-REQ-005 - source: network-operator-prd - coverage: partial - annotations: 'Aggregates the Ethernet and RoCE profile results.' - notes: '' - - test_id: K8S42-05 - requirements: - - req_id: ENT-REQ-006 - source: network-operator-prd - coverage: partial - annotations: 'Checks the profile-specific InfiniBand network kind; device, attachment, and connectivity evidence is split across other checks.' - notes: '' - - test_id: K8S42-06 - requirements: - - req_id: ENT-REQ-007 - source: network-operator-prd - coverage: partial - annotations: 'Production wiring and unit fixtures cover both host-device profile contracts; live VM qualification remains required.' - notes: '' - - test_id: K8S42-07 - requirements: - - req_id: ENT-REQ-008 - source: network-operator-prd - coverage: partial - annotations: 'Consumes Launch Kit gpudirect_dmabuf matrix verdicts with endpoint GPU indices, PCI addresses, bandwidth, threshold, and errors; mock-qualified only and still requires live GPUDirect hardware qualification.' - notes: '' - - test_id: K8S42-08 - requirements: - req_id: ENT-REQ-002 source: network-operator-prd coverage: partial - req_id: ENT-REQ-004 source: network-operator-prd coverage: partial - - req_id: ENT-REQ-012 - source: network-operator-prd - coverage: partial - annotations: 'Forwards user-owned discover arguments without copying Launch Kit defaults; live prerequisites are documented separately.' - notes: '' - - test_id: K8S42-09 - requirements: - - req_id: ENT-REQ-005 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-006 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-009 - source: network-operator-prd - coverage: partial - annotations: 'Checks generated secondary-network, IP pool, Multus attachment, and pod-address evidence.' - notes: '' - - test_id: K8S42-10 - requirements: - - req_id: ENT-REQ-005 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-006 - source: network-operator-prd - coverage: partial - annotations: 'Covers Macvlan and IPoIB RDMA Shared profiles.' - notes: '' - - test_id: K8S42-11 - requirements: - - req_id: ENT-REQ-002 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-005 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-006 - source: network-operator-prd - coverage: partial - annotations: 'Reuses Launch Kit strict ICMP same-rail reachability and cross-rail isolation results.' - notes: '' - - test_id: K8S42-12 - requirements: - - req_id: ENT-REQ-002 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-005 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-006 - source: network-operator-prd - coverage: partial - annotations: 'Reuses Launch Kit ib_write_bw verdicts and reports the observed and Launch Kit-resolved minimum bandwidth.' - notes: '' - - test_id: K8S42-13 - requirements: - - req_id: ENT-REQ-002 - source: network-operator-prd - coverage: partial - annotations: 'Checks the complete two-rail strict connectivity matrix.' - notes: '' - - test_id: K8S42-15 - requirements: - - req_id: ENT-REQ-011 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-013 - source: network-operator-prd - coverage: partial - annotations: 'Local evidence and framework reporting are implemented; typed catalog ownership and Labs binary attachment upload remain gaps.' - notes: '' - - test_id: K8S42-16 - requirements: - - req_id: ENT-REQ-001 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-003 - source: network-operator-prd - coverage: partial - req_id: ENT-REQ-005 source: network-operator-prd coverage: partial - - req_id: ENT-REQ-008 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-013 - source: network-operator-prd - coverage: partial - annotations: 'Concrete validation-only RoCE SR-IOV discover-generate-validate use case for an ISV-provisioned deployment; the validation path was live-qualified on a two-node Ubuntu 24.04 single-rail cluster, with the multi-rail member skipped as inapplicable.' - notes: '' - - test_id: K8S42-17 - requirements: - - req_id: ENT-REQ-001 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-003 - source: network-operator-prd - coverage: partial - req_id: ENT-REQ-006 source: network-operator-prd coverage: partial - - req_id: ENT-REQ-008 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-013 - source: network-operator-prd - coverage: partial - annotations: 'Concrete validation-only InfiniBand SR-IOV discover-generate-validate use case for an ISV-provisioned deployment; mock-qualified only.' - notes: '' - - test_id: K8S42-18 - requirements: - - req_id: ENT-REQ-001 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-003 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-005 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-008 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-013 - source: network-operator-prd - coverage: partial - annotations: 'Concrete validation-only RoCE RDMA Shared discover-generate-validate use case for an ISV-provisioned deployment; mock-qualified only.' - notes: '' - - test_id: K8S42-19 - requirements: - - req_id: ENT-REQ-001 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-003 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-006 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-008 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-013 - source: network-operator-prd - coverage: partial - annotations: 'Concrete validation-only InfiniBand and IPoIB RDMA Shared discover-generate-validate use case for an ISV-provisioned deployment; mock-qualified only.' - notes: '' - - test_id: K8S42-20 - requirements: - - req_id: ENT-REQ-001 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-003 - source: network-operator-prd - coverage: partial - req_id: ENT-REQ-007 source: network-operator-prd coverage: partial - req_id: ENT-REQ-008 source: network-operator-prd coverage: partial - - req_id: ENT-REQ-013 - source: network-operator-prd - coverage: partial - annotations: 'Concrete validation-only RoCE host-device workflow for an ISV-provisioned deployment; mock-qualified and not yet proven on worker VMs.' - notes: '' - - test_id: K8S42-21 - requirements: - - req_id: ENT-REQ-001 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-003 - source: network-operator-prd - coverage: partial - - req_id: ENT-REQ-007 + - req_id: ENT-REQ-011 source: network-operator-prd coverage: partial - - req_id: ENT-REQ-008 + - req_id: ENT-REQ-012 source: network-operator-prd coverage: partial - req_id: ENT-REQ-013 source: network-operator-prd coverage: partial - annotations: 'Concrete validation-only InfiniBand host-device workflow for an ISV-provisioned deployment; mock-qualified and not yet proven on worker VMs.' + annotations: 'Runs one l8k validate command and reports every emitted connectivity row. Cluster provisioning, topology discovery, manifest generation, deployment health, applicability, and state restoration are prerequisites outside this suite. Catalog wiring, local command evidence, and JUnit reporting are implemented; AI Cloud Labs binary attachment upload remains a gap.' notes: '' diff --git a/docs/test-plan.adoc b/docs/test-plan.adoc index b87e9bb6d..9170c977c 100644 --- a/docs/test-plan.adoc +++ b/docs/test-plan.adoc @@ -3456,7 +3456,7 @@ a| | pending | -.78+| Workload Orchestration +.59+| Workload Orchestration | Backup and Recovery | Centralized managed service to automate and govern data backup across services | AWS backup @@ -3646,7 +3646,7 @@ a| | published | -.65+| Managed Kubernetes Control Plane +.46+| Managed Kubernetes Control Plane | Tenant-isolated Kubernetes control planes for managing k8s workloads. does placement, networking, lifecyle mgmt. | AWS EKS, GCP GKE | [[K8S05-01]]K8S05-01 @@ -4033,285 +4033,20 @@ a| https://github.com/NVIDIA/ai-cloud-validation/issues/220[#220] | pending | -.20+| Network Operator self-validation through Kubernetes Launch Kit -.20+| -| [[K8S42-01]]K8S42-01 -| K8S42 -| -| era, kubernetes, ncp, network_operator, slow -| Production provider integration and unit coverage for ENT-REQ-001, ENT-REQ-009, and ENT-REQ-011 -| P0 -a| -| LimitedEnv -| -| -| Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools -| pending -| - -| [[K8S42-02]]K8S42-02 -| K8S42 -| -| era, kubernetes, ncp, network_operator, slow -| Provider wiring and unit coverage for ENT-REQ-005 -| P0 -a| -| LimitedEnv -| -| -| Validate the SR-IOV policy and profile-specific secondary network resources reported by Launch Kit -| pending -| - -| [[K8S42-03]]K8S42-03 -| K8S42 -| -| era, kubernetes, ncp, network_operator, slow -| Provider wiring and unit coverage for ENT-REQ-002 and ENT-REQ-005 -| P0 -a| -| LimitedEnv -| -| -| Validate pod-to-pod RDMA-CM connectivity across selected rails -| pending -| - -| [[K8S42-04]]K8S42-04 -| K8S42 -| -| era, kubernetes, ncp, network_operator, slow -| Provider wiring and unit coverage for ENT-REQ-005 -| P0 -a| -| LimitedEnv -| -| -| Validate the exact secondary network resource selected for an Ethernet or RoCE profile -| pending -| - -| [[K8S42-05]]K8S42-05 -| K8S42 -| -| era, kubernetes, ncp, network_operator, slow -| Provider wiring and unit coverage for ENT-REQ-006 -| P0 -a| -| LimitedEnv -| -| -| Validate the exact secondary network resource selected for an InfiniBand profile -| pending -| - -| [[K8S42-06]]K8S42-06 -| K8S42 -| -| era, kubernetes, ncp, network_operator, slow -| Provider wiring and unit coverage for ENT-REQ-007 -| P0 -a| -| LimitedEnv -| -| -| Validate HostDeviceNetwork readiness for an applicable Ethernet or InfiniBand profile -| pending -| - -| [[K8S42-07]]K8S42-07 -| K8S42 -| -| era, gpudirect, kubernetes, ncp, network_operator, slow -| Launch Kit gpudirect_dmabuf result-family integration and mock-backed unit coverage for ENT-REQ-008; live GPU qualification remains required -| P0 -a| -| LimitedEnv -| -| -| Validate GPUDirect RDMA DMA-BUF bandwidth between GPU-enabled pod endpoints -| pending -| - -| [[K8S42-08]]K8S42-08 -| K8S42 -| -| era, kubernetes, ncp, network_operator, slow -| Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-004, and ENT-REQ-012 -| P0 -a| -| LimitedEnv -| -| -| Validate Launch Kit discovery completed and resolved a fabric and deployment profile -| pending -| - -| [[K8S42-09]]K8S42-09 -| K8S42 -| -| era, kubernetes, ncp, network_operator, slow -| Provider wiring and unit coverage for ENT-REQ-005, ENT-REQ-006, and ENT-REQ-009 -| P0 -a| -| LimitedEnv -| -| -| Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness -| pending -| - -| [[K8S42-10]]K8S42-10 -| K8S42 -| -| era, kubernetes, ncp, network_operator, slow -| Provider wiring and unit coverage for ENT-REQ-005 and ENT-REQ-006 -| P0 -a| -| LimitedEnv -| -| -| Validate Macvlan or IPoIB network readiness for an applicable RDMA Shared profile -| pending -| - -| [[K8S42-11]]K8S42-11 -| K8S42 -| -| era, kubernetes, ncp, network_operator, slow -| Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-005, and ENT-REQ-006 -| P0 -a| -| LimitedEnv -| -| -| Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation -| pending -| - -| [[K8S42-12]]K8S42-12 -| K8S42 -| -| era, kubernetes, ncp, network_operator, slow -| Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-005, and ENT-REQ-006 -| P0 -a| -| LimitedEnv -| +| Network Operator connectivity validation through Kubernetes Launch Kit | -| Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth -| pending -| - -| [[K8S42-13]]K8S42-13 -| K8S42 -| -| era, kubernetes, ncp, network_operator, slow -| Provider wiring and unit coverage for Enterprise multi-rail coverage in ENT-REQ-002 -| P0 -a| -| LimitedEnv -| -| -| Validate same-rail and cross-rail coverage for a multi-rail Launch Kit profile -| pending -| - -| [[K8S42-15]]K8S42-15 +| [[K8S42-01]]K8S42-01 | K8S42 | | era, kubernetes, ncp, network_operator, slow -| Local evidence and framework reporting cover ENT-REQ-013; Labs attachment upload remains required -| P0 -a| -| LimitedEnv -| -| -| Verify raw phase output, generated files, connectivity results, and Launch Kit reports were preserved -| pending -| - -| [[K8S42-16]]K8S42-16 -| K8S42 -| -| era, ethernet, gpudirect, kubernetes, ncp, network_operator, network_operator_use_cases, roce, slow, sriov -| Validation-only composite use case; the validation path was live-qualified on a two-node Ubuntu 24.04 single-rail cluster, with multi-rail skipped as inapplicable -| P0 -a| -| LimitedEnv -| -| -| Validate a pre-provisioned Ethernet or RoCE SR-IOV Network RDMA profile through Launch Kit -| pending -| - -| [[K8S42-17]]K8S42-17 -| K8S42 -| -| era, gpudirect, infiniband, kubernetes, ncp, network_operator, network_operator_use_cases, slow, sriov -| Validation-only composite use case: preflight, discover, generate, and validate -| P0 -a| -| LimitedEnv -| -| -| Validate a pre-provisioned InfiniBand SR-IOV Network RDMA profile through Launch Kit -| pending -| - -| [[K8S42-18]]K8S42-18 -| K8S42 -| -| era, ethernet, gpudirect, kubernetes, ncp, network_operator, network_operator_use_cases, rdma_shared, roce, slow -| Validation-only composite use case: preflight, discover, generate, and validate -| P0 -a| -| LimitedEnv -| -| -| Validate a pre-provisioned Ethernet or RoCE RDMA Shared profile through Launch Kit -| pending -| - -| [[K8S42-19]]K8S42-19 -| K8S42 -| -| era, gpudirect, infiniband, kubernetes, ncp, network_operator, network_operator_use_cases, rdma_shared, slow -| Validation-only composite use case: preflight, discover, generate, and validate -| P0 -a| -| LimitedEnv -| -| -| Validate a pre-provisioned InfiniBand or IPoIB RDMA Shared profile through Launch Kit -| pending -| - -| [[K8S42-20]]K8S42-20 -| K8S42 -| -| era, ethernet, gpudirect, host_device, kubernetes, ncp, network_operator, network_operator_use_cases, roce, slow -| Validation-only composite use case: preflight, discover, generate, and validate; live VM qualification remains required -| P0 -a| -| LimitedEnv -| -| -| Validate pre-provisioned Ethernet or RoCE host-device networking on worker VMs through Launch Kit -| pending -| - -| [[K8S42-21]]K8S42-21 -| K8S42 -| -| era, gpudirect, host_device, infiniband, kubernetes, ncp, network_operator, network_operator_use_cases, slow -| Validation-only composite use case: preflight, discover, generate, and validate; live VM qualification remains required +| operator +| One l8k validate invocation consumes a complete user config and rendered deployment directory; topology, deployment readiness, and selected connectivity families are prerequisites | P0 a| | LimitedEnv | | -| Validate pre-provisioned InfiniBand host-device networking on worker VMs through Launch Kit +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment | pending | diff --git a/docs/test-plan.yaml b/docs/test-plan.yaml index ef7b8f0cf..bed9f40a5 100644 --- a/docs/test-plan.yaml +++ b/docs/test-plan.yaml @@ -3636,9 +3636,9 @@ domains: milestone: M5 github_issues: - "#220" - - description: Network Operator self-validation through Kubernetes Launch Kit + - description: Network Operator connectivity validation through Kubernetes Launch Kit tests: - - summary: Validate Network Operator deployment health, versions, policies, Multus, secondary networks, and IP pools + - summary: Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment labels: - era - kubernetes @@ -3651,321 +3651,9 @@ domains: milestone: "" req_id: K8S42 test_id: K8S42-01 + actor: operator status: pending - notes: "Production provider integration and unit coverage for ENT-REQ-001, ENT-REQ-009, and ENT-REQ-011" - - summary: Validate the SR-IOV policy and profile-specific secondary network resources reported by Launch Kit - labels: - - era - - kubernetes - - ncp - - network_operator - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-02 - status: pending - notes: "Provider wiring and unit coverage for ENT-REQ-005" - - summary: Validate pod-to-pod RDMA-CM connectivity across selected rails - labels: - - era - - kubernetes - - ncp - - network_operator - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-03 - status: pending - notes: "Provider wiring and unit coverage for ENT-REQ-002 and ENT-REQ-005" - - summary: Validate the exact secondary network resource selected for an Ethernet or RoCE profile - labels: - - era - - kubernetes - - ncp - - network_operator - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-04 - status: pending - notes: "Provider wiring and unit coverage for ENT-REQ-005" - - summary: Validate the exact secondary network resource selected for an InfiniBand profile - labels: - - era - - kubernetes - - ncp - - network_operator - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-05 - status: pending - notes: "Provider wiring and unit coverage for ENT-REQ-006" - - summary: Validate HostDeviceNetwork readiness for an applicable Ethernet or InfiniBand profile - labels: - - era - - kubernetes - - ncp - - network_operator - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-06 - status: pending - notes: "Provider wiring and unit coverage for ENT-REQ-007" - - summary: Validate GPUDirect RDMA DMA-BUF bandwidth between GPU-enabled pod endpoints - labels: - - era - - gpudirect - - kubernetes - - ncp - - network_operator - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-07 - status: pending - notes: "Launch Kit gpudirect_dmabuf result-family integration and mock-backed unit coverage for ENT-REQ-008; live GPU qualification remains required" - - summary: Validate Launch Kit discovery completed and resolved a fabric and deployment profile - labels: - - era - - kubernetes - - ncp - - network_operator - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-08 - status: pending - notes: "Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-004, and ENT-REQ-012" - - summary: Validate IP pool and profile-specific secondary-network resources plus test DaemonSet readiness - labels: - - era - - kubernetes - - ncp - - network_operator - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-09 - status: pending - notes: "Provider wiring and unit coverage for ENT-REQ-005, ENT-REQ-006, and ENT-REQ-009" - - summary: Validate Macvlan or IPoIB network readiness for an applicable RDMA Shared profile - labels: - - era - - kubernetes - - ncp - - network_operator - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-10 - status: pending - notes: "Provider wiring and unit coverage for ENT-REQ-005 and ENT-REQ-006" - - summary: Validate source-bound same-rail ICMP connectivity and strict cross-rail isolation - labels: - - era - - kubernetes - - ncp - - network_operator - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-11 - status: pending - notes: "Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-005, and ENT-REQ-006" - - summary: Validate every ib_write_bw result and report its Launch Kit-resolved minimum bandwidth - labels: - - era - - kubernetes - - ncp - - network_operator - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-12 - status: pending - notes: "Provider wiring and unit coverage for ENT-REQ-002, ENT-REQ-005, and ENT-REQ-006" - - summary: Validate same-rail and cross-rail coverage for a multi-rail Launch Kit profile - labels: - - era - - kubernetes - - ncp - - network_operator - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-13 - status: pending - notes: "Provider wiring and unit coverage for Enterprise multi-rail coverage in ENT-REQ-002" - - summary: Verify raw phase output, generated files, connectivity results, and Launch Kit reports were preserved - labels: - - era - - kubernetes - - ncp - - network_operator - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-15 - status: pending - notes: "Local evidence and framework reporting cover ENT-REQ-013; Labs attachment upload remains required" - - summary: Validate a pre-provisioned Ethernet or RoCE SR-IOV Network RDMA profile through Launch Kit - labels: - - era - - ethernet - - gpudirect - - kubernetes - - ncp - - network_operator - - network_operator_use_cases - - roce - - slow - - sriov - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-16 - status: pending - notes: "Validation-only composite use case; the validation path was live-qualified on a two-node Ubuntu 24.04 single-rail cluster, with multi-rail skipped as inapplicable" - - summary: Validate a pre-provisioned InfiniBand SR-IOV Network RDMA profile through Launch Kit - labels: - - era - - gpudirect - - infiniband - - kubernetes - - ncp - - network_operator - - network_operator_use_cases - - slow - - sriov - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-17 - status: pending - notes: "Validation-only composite use case: preflight, discover, generate, and validate" - - summary: Validate a pre-provisioned Ethernet or RoCE RDMA Shared profile through Launch Kit - labels: - - era - - ethernet - - gpudirect - - kubernetes - - ncp - - network_operator - - network_operator_use_cases - - rdma_shared - - roce - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-18 - status: pending - notes: "Validation-only composite use case: preflight, discover, generate, and validate" - - summary: Validate a pre-provisioned InfiniBand or IPoIB RDMA Shared profile through Launch Kit - labels: - - era - - gpudirect - - infiniband - - kubernetes - - ncp - - network_operator - - network_operator_use_cases - - rdma_shared - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-19 - status: pending - notes: "Validation-only composite use case: preflight, discover, generate, and validate" - - summary: Validate pre-provisioned Ethernet or RoCE host-device networking on worker VMs through Launch Kit - labels: - - era - - ethernet - - gpudirect - - host_device - - kubernetes - - ncp - - network_operator - - network_operator_use_cases - - roce - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-20 - status: pending - notes: "Validation-only composite use case: preflight, discover, generate, and validate; live VM qualification remains required" - - summary: Validate pre-provisioned InfiniBand host-device networking on worker VMs through Launch Kit - labels: - - era - - gpudirect - - host_device - - infiniband - - kubernetes - - ncp - - network_operator - - network_operator_use_cases - - slow - priority: P0 - dependencies: - - LimitedEnv - milestone: "" - req_id: K8S42 - test_id: K8S42-21 - status: pending - notes: "Validation-only composite use case: preflight, discover, generate, and validate; live VM qualification remains required" + notes: "One l8k validate invocation consumes a complete user config and rendered deployment directory; topology, deployment readiness, and selected connectivity families are prerequisites" - description: "K8s Versioning & Compliance" tests: - summary: Verify support for the three most recent minor releases (N-2) diff --git a/isvctl/configs/providers/k8s-launch-kit/README.md b/isvctl/configs/providers/k8s-launch-kit/README.md index 7eed57ff7..48c972847 100644 --- a/isvctl/configs/providers/k8s-launch-kit/README.md +++ b/isvctl/configs/providers/k8s-launch-kit/README.md @@ -3,85 +3,69 @@ # Kubernetes Launch Kit provider internals -This directory owns the implementation behind -`config/provider.yaml`. It is provider-specific code, not a cross-provider -helper. - ## Layout | Path | Purpose | |---|---| -| `config/provider.yaml` | Generic single-workflow provider using real `l8k` and `kubectl` by default | -| `config/network-operator.yaml` | Production six-use-case Network Operator workflow | -| `scripts/adapter.py` | Transport for install/verify, Kubernetes preflight, and one `l8k` workflow command | - -Test doubles and pinned scenario data intentionally live outside the shipped -provider under `isvctl/tests/providers/k8s_launch_kit/fixtures/`. The provider -tests load the production YAML and inject those paths in memory. - -The adapter must remain thin. It accepts raw argument arrays for `discover`, -`generate`, `deploy`, `validate`, and `clean`, appends `--output json`, executes the -configured `l8k` executable, and preserves the CLI's JSON documents without -renaming or interpreting fields. The one file-level input is `user_config`, a -path to a complete Launch Kit configuration. Before discovery, the adapter -copies it to the workflow as a mode-`0600` `user-config.yaml`, explicitly writes -the discovered result to `cluster-config.yaml`, and removes the staged input as -soon as discovery exits. The original is never modified, and evidence retains -only its path, size, and SHA-256 provenance rather than its potentially -sensitive contents. Launch Kit still owns the file schema, domain flags, and -defaults. Semantic assertions belong in `isvtest.validations.k8s_launch_kit`. - -Launch Kit `validate` steps use `timeout: null` so the CLI owns its deadline. -l8k calculates and logs a bounded matrix budget by default and honors a user's -explicit `--connectivity-timeout`. The remaining workflow steps keep finite -isvctl watchdogs. Other providers may also use `timeout: null`, but only when -their child command has its own bounded timeout. - -The grouped Network Operator workflow is validation-only. ISVs install and -configure Network Operator before running it. Each selected use case executes -`preflight -> discover -> generate -> validate`; it never invokes `l8k deploy` -or `l8k clean`, so AI Cloud Validation cannot replace or delete the ISV-managed -installation. The generic `provider.yaml` deliberately retains deploy and -clean as public Launch Kit operations for other consumers. - -The grouped workflow passes only its fabric and deployment identity during -discovery. With no `user_config`, Launch Kit resolves the default -`./cluster-config.yaml` and `./deployment` paths throughout the validation -workflow. With `user_config`, every selected use case stages an independent -copy, and the adapter owns `--user-config` plus `--save-cluster-config` for -discovery. Each transient copy is deleted after its discovery command. Do not -repeat either flag in the raw discovery argument array or place the source -inside the retained provider working directory. - -Each workflow envelope records the absolute working directory while retaining -Launch Kit's JSON documents unchanged. Validations use that metadata to resolve -relative `generatedFiles` paths emitted by the CLI. - -Install mode accepts only an immutable full Git commit for the official -`scripts/install.sh` plus a caller-supplied SHA-256, verifies that digest before -writing or executing the script, delegates archive selection and checksum -handling to Launch Kit, then verifies the binary at the install prefix. -When the user pins `installation.version`, both setup and test-phase -verification require `l8k version --output json` to report that exact version. -The captured schema must advertise all five generic provider commands, -including `deploy` and `clean`. This verifies that the installation satisfies -the generic provider contract even though the Network Operator suite invokes -only discover, generate, and validate. - -The preflight accepts the non-empty subset of Launch Kit commands used by the -calling workflow. It uses their explicit kubeconfig and forwarded environment, -rejects conflicting `--kubeconfig` arguments, and requires Kubernetes API -access plus at least one Ready node before validation starts. - -The same string-only environment mapping is also passed to the installer and -version/schema verification, so proxy and executable runtime settings do not -change between setup and test phases. - -The production adapter executes `executable` directly. There is no Python-file -special case: a test double must be an executable with a valid shebang, just -like any other CLI implementation. This keeps mock behavior out of the public -provider contract. - -See the [integration guide](../../../../docs/guides/k8s-launch-kit/network-operator.md) -for configuration, use cases, evidence, prerequisites, and current production -gaps. +| `config/provider.yaml` | Generic provider mirroring the full Launch Kit workflow | +| `config/network-operator.yaml` | One-step validation of an ISV-provisioned Network Operator deployment | +| `scripts/adapter.py` | Thin process and JSON evidence transport | + +Executable mocks and pinned scenarios are test-only and live under +`isvctl/tests/providers/k8s_launch_kit/fixtures/`. Product configuration must +never reference them. + +## Generic provider + +`config/provider.yaml` exposes install/verify, Kubernetes preflight, discover, +generate, deploy, validate, and clean for consumers that own the complete +Launch Kit lifecycle. Its workflow configuration is raw argument arrays. It +does not reproduce Launch Kit's domain schema or defaults. + +Discovery optionally stages a complete `user_config`, writes the resolved +`cluster-config.yaml`, and deletes the staged copy after the command. Validate +uses `timeout: null` because Launch Kit calculates a bounded connectivity budget +or honors its user-supplied timeout. The other generic steps retain finite +outer watchdogs. + +## Network Operator provider + +`config/network-operator.yaml` intentionally does not import the generic +provider. It defines one test step and executes only: + +```text +l8k validate --user-config --deployment-files --output json +``` + +The cluster, Network Operator deployment, complete Launch Kit config, rendered +deployment directory, and installed `l8k` binary are prerequisites. There are +no AI Cloud Validation use-case workflows and no discover, generate, deploy, +clean, verification, or separate preflight steps. + +The adapter resolves and validates the two input paths without copying or +parsing them. It rejects raw `--user-config` or `--deployment-files` arguments +when the dedicated inputs are used. Launch Kit remains responsible for fabric, +deployment type, enabled checks, GPUDirect applicability, thresholds, runtime +budgets, and every other value in its config. + +The one catalog validation, `LaunchKitConnectivityCheck`, converts every +emitted `connectivity.PingResults` row into a subtest. It does not expect a +fixed family list: a disabled family is absent, while a newly emitted family is +reported automatically. + +## Adapter contract + +For every workflow invocation, `adapter.py`: + +1. resolves the configured executable; +2. adds `--output json` unless the caller already selected JSON; +3. executes exactly one Launch Kit command; +4. preserves stdout, stderr, argv, exit code, and duration; +5. parses concatenated JSON objects without renaming their fields; +6. returns one provider envelope containing the raw documents and artifact paths. + +Semantic assertions belong in +`isvtest.validations.k8s_launch_kit`, not the transport. + +See the [Network Operator integration guide](../../../../docs/guides/k8s-launch-kit/network-operator.md) +for prerequisites, invocation, output, and evidence layout. diff --git a/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml b/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml index a0ab24485..c7797847c 100644 --- a/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml +++ b/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml @@ -1,704 +1,51 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Production Network Operator workflow. It invokes the real l8k and kubectl -# executables inherited from provider.yaml. Test doubles live only under tests/. -# -# Usage: -# ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl test run \ -# -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ -# --capability kubernetes --no-upload -- -v +# Network Operator connectivity validation for an ISV-provisioned cluster. +# The complete Launch Kit configuration and rendered deployment directory are +# prerequisites. This provider invokes only: +# l8k validate --user-config --deployment-files import: - - provider.yaml - ../../../suites/k8s-launch-kit/network-operator.yaml version: "1.0" -# The shared frontend suite also catalogs the individually selectable semantic -# checks. This grouped production entrypoint reports only the six use cases; -# each composite executes the applicable semantic checks as named subtests. -tests: - validations: - network_operator: [] - context: k8s_launch_kit: executable: l8k - installation: - mode: verify - version: "" - installer_ref: "" - installer_sha256: "" - prefix: "" - # Optional complete Launch Kit configuration. Every selected use case - # stages its own copy before discovery; the source file is never modified. user_config: "" - # An empty override means the adapter invokes kubectl from PATH. Users may - # replace this with any kubectl-compatible argv list in an overlay. - kubectl_command: [] + deployment_files: "" + working_dir: ../../../../../_output/k8s-launch-kit/network-operator/work + artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/evidence environment: {} - shared_artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/shared-evidence - use_cases: - roce_sriov: - working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-sriov/work - artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-sriov/evidence - discover: - arguments: - - --fabric - - ethernet - - --deployment-type - - sriov - generate: - arguments: [] - validate: - arguments: [] - infiniband_sriov: - working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-sriov/work - artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-sriov/evidence - discover: - arguments: - - --fabric - - infiniband - - --deployment-type - - sriov - generate: - arguments: [] - validate: - arguments: [] - roce_rdma_shared: - working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-rdma-shared/work - artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-rdma-shared/evidence - discover: - arguments: - - --fabric - - ethernet - - --deployment-type - - rdma_shared - generate: - arguments: [] - validate: - arguments: [] - infiniband_rdma_shared: - working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-rdma-shared/work - artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-rdma-shared/evidence - discover: - arguments: - - --fabric - - infiniband - - --deployment-type - - rdma_shared - generate: - arguments: [] - validate: - arguments: [] - roce_host_device: - working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-host-device/work - artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/roce-host-device/evidence - discover: - arguments: - - --fabric - - ethernet - - --deployment-type - - host_device - generate: - arguments: [] - validate: - arguments: [] - infiniband_host_device: - working_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-host-device/work - artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/use-cases/infiniband-host-device/evidence - discover: - arguments: - - --fabric - - infiniband - - --deployment-type - - host_device - generate: - arguments: [] - validate: - arguments: [] commands: network_operator: - phases: - - setup - - launch-kit-verification - - roce-sriov - - infiniband-sriov - - roce-rdma-shared - - infiniband-rdma-shared - - roce-host-device - - infiniband-host-device - continue_after_failure: - - roce-sriov - - infiniband-sriov - - roce-rdma-shared - - infiniband-rdma-shared - - roce-host-device - - infiniband-host-device + phases: [test] steps: - - name: launch_kit_prepare - phase: setup - command: python3 ../scripts/adapter.py - args: - - prepare - - --mode - - "{{ context.k8s_launch_kit.installation.mode }}" - - --executable - - "{{ context.k8s_launch_kit.executable }}" - - "--version={{ context.k8s_launch_kit.installation.version }}" - - "--installer-ref={{ context.k8s_launch_kit.installation.installer_ref }}" - - "--installer-sha256={{ context.k8s_launch_kit.installation.installer_sha256 }}" - - "--prefix={{ context.k8s_launch_kit.installation.prefix }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.shared_artifact_dir }}" - timeout: 900 - output_schema: k8s_launch_kit - requires: [kubernetes] - - - name: launch_kit_verify - phase: launch-kit-verification - command: python3 ../scripts/adapter.py - args: - - verify - - --executable - - "{{ steps.launch_kit_prepare.executable | default(context.k8s_launch_kit.executable) }}" - - "--expected-version={{ context.k8s_launch_kit.installation.version }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.shared_artifact_dir }}" - timeout: 60 - output_schema: k8s_launch_kit - requires: [kubernetes] - - # roce-sriov: preflight -> discover -> generate -> validate - - name: launch_kit_roce_sriov_preflight - phase: roce-sriov - command: python3 ../scripts/adapter.py - args: - - preflight - - --kubectl-command-json - - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" - - --workflow-arguments-json - - "{{ {'discover': context.k8s_launch_kit.use_cases.roce_sriov.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.roce_sriov.generate.arguments, 'validate': context.k8s_launch_kit.use_cases.roce_sriov.validate.arguments} | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.roce_sriov.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.roce_sriov.artifact_dir }}" - timeout: 60 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkRoceSriovCheck] - - - name: launch_kit_roce_sriov_discover - phase: roce-sriov - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - discover - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.roce_sriov.discover.arguments | tojson }}" - - "--user-config={{ context.k8s_launch_kit.user_config }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.roce_sriov.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.roce_sriov.artifact_dir }}" - timeout: 1800 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkRoceSriovCheck] - - - name: launch_kit_roce_sriov_generate - phase: roce-sriov - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - generate - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.roce_sriov.generate.arguments | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.roce_sriov.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.roce_sriov.artifact_dir }}" - timeout: 600 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkRoceSriovCheck] - - - name: launch_kit_roce_sriov_validate - phase: roce-sriov - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - validate - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.roce_sriov.validate.arguments | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.roce_sriov.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.roce_sriov.artifact_dir }}" - timeout: null - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkRoceSriovCheck] - - # infiniband-sriov: preflight -> discover -> generate -> validate - - name: launch_kit_infiniband_sriov_preflight - phase: infiniband-sriov - command: python3 ../scripts/adapter.py - args: - - preflight - - --kubectl-command-json - - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" - - --workflow-arguments-json - - "{{ {'discover': context.k8s_launch_kit.use_cases.infiniband_sriov.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.infiniband_sriov.generate.arguments, 'validate': context.k8s_launch_kit.use_cases.infiniband_sriov.validate.arguments} | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.artifact_dir }}" - timeout: 60 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkInfiniBandSriovCheck] - - - name: launch_kit_infiniband_sriov_discover - phase: infiniband-sriov - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - discover - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.discover.arguments | tojson }}" - - "--user-config={{ context.k8s_launch_kit.user_config }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.artifact_dir }}" - timeout: 1800 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkInfiniBandSriovCheck] - - - name: launch_kit_infiniband_sriov_generate - phase: infiniband-sriov - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - generate - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.generate.arguments | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.artifact_dir }}" - timeout: 600 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkInfiniBandSriovCheck] - - - name: launch_kit_infiniband_sriov_validate - phase: infiniband-sriov - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - validate - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.validate.arguments | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_sriov.artifact_dir }}" - timeout: null - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkInfiniBandSriovCheck] - - # roce-rdma-shared: preflight -> discover -> generate -> validate - - name: launch_kit_roce_rdma_shared_preflight - phase: roce-rdma-shared - command: python3 ../scripts/adapter.py - args: - - preflight - - --kubectl-command-json - - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" - - --workflow-arguments-json - - "{{ {'discover': context.k8s_launch_kit.use_cases.roce_rdma_shared.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.roce_rdma_shared.generate.arguments, 'validate': context.k8s_launch_kit.use_cases.roce_rdma_shared.validate.arguments} | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.artifact_dir }}" - timeout: 60 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkRoceRdmaSharedCheck] - - - name: launch_kit_roce_rdma_shared_discover - phase: roce-rdma-shared + - name: launch_kit_validate + phase: test command: python3 ../scripts/adapter.py args: - run - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - discover - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.discover.arguments | tojson }}" - - "--user-config={{ context.k8s_launch_kit.user_config }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.artifact_dir }}" - timeout: 1800 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkRoceRdmaSharedCheck] - - - name: launch_kit_roce_rdma_shared_generate - phase: roce-rdma-shared - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - generate - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.generate.arguments | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.artifact_dir }}" - timeout: 600 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkRoceRdmaSharedCheck] - - - name: launch_kit_roce_rdma_shared_validate - phase: roce-rdma-shared - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - validate - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.validate.arguments | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.roce_rdma_shared.artifact_dir }}" - timeout: null - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkRoceRdmaSharedCheck] - - # infiniband-rdma-shared: preflight -> discover -> generate -> validate - - name: launch_kit_infiniband_rdma_shared_preflight - phase: infiniband-rdma-shared - command: python3 ../scripts/adapter.py - args: - - preflight - - --kubectl-command-json - - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" - - --workflow-arguments-json - - "{{ {'discover': context.k8s_launch_kit.use_cases.infiniband_rdma_shared.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.infiniband_rdma_shared.generate.arguments, 'validate': context.k8s_launch_kit.use_cases.infiniband_rdma_shared.validate.arguments} | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.artifact_dir }}" - timeout: 60 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkInfiniBandRdmaSharedCheck] - - - name: launch_kit_infiniband_rdma_shared_discover - phase: infiniband-rdma-shared - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - discover - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.discover.arguments | tojson }}" - - "--user-config={{ context.k8s_launch_kit.user_config }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.artifact_dir }}" - timeout: 1800 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkInfiniBandRdmaSharedCheck] - - - name: launch_kit_infiniband_rdma_shared_generate - phase: infiniband-rdma-shared - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - generate - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.generate.arguments | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.artifact_dir }}" - timeout: 600 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkInfiniBandRdmaSharedCheck] - - - name: launch_kit_infiniband_rdma_shared_validate - phase: infiniband-rdma-shared - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - validate - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.validate.arguments | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_rdma_shared.artifact_dir }}" - timeout: null - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkInfiniBandRdmaSharedCheck] - - # roce-host-device: preflight -> discover -> generate -> validate - - name: launch_kit_roce_host_device_preflight - phase: roce-host-device - command: python3 ../scripts/adapter.py - args: - - preflight - - --kubectl-command-json - - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" - - --workflow-arguments-json - - "{{ {'discover': context.k8s_launch_kit.use_cases.roce_host_device.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.roce_host_device.generate.arguments, 'validate': context.k8s_launch_kit.use_cases.roce_host_device.validate.arguments} | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.roce_host_device.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.roce_host_device.artifact_dir }}" - timeout: 60 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkRoceHostDeviceCheck] - - - name: launch_kit_roce_host_device_discover - phase: roce-host-device - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - discover - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.roce_host_device.discover.arguments | tojson }}" - - "--user-config={{ context.k8s_launch_kit.user_config }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.roce_host_device.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.roce_host_device.artifact_dir }}" - timeout: 1800 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkRoceHostDeviceCheck] - - - name: launch_kit_roce_host_device_generate - phase: roce-host-device - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - generate - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.roce_host_device.generate.arguments | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.roce_host_device.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.roce_host_device.artifact_dir }}" - timeout: 600 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkRoceHostDeviceCheck] - - - name: launch_kit_roce_host_device_validate - phase: roce-host-device - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" + - "{{ context.k8s_launch_kit.executable }}" - --command - validate - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.roce_host_device.validate.arguments | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.roce_host_device.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.roce_host_device.artifact_dir }}" - timeout: null - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkRoceHostDeviceCheck] - - # infiniband-host-device: preflight -> discover -> generate -> validate - - name: launch_kit_infiniband_host_device_preflight - phase: infiniband-host-device - command: python3 ../scripts/adapter.py - args: - - preflight - - --kubectl-command-json - - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" - - --workflow-arguments-json - - "{{ {'discover': context.k8s_launch_kit.use_cases.infiniband_host_device.discover.arguments, 'generate': context.k8s_launch_kit.use_cases.infiniband_host_device.generate.arguments, 'validate': context.k8s_launch_kit.use_cases.infiniband_host_device.validate.arguments} | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.artifact_dir }}" - timeout: 60 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkInfiniBandHostDeviceCheck] - - - name: launch_kit_infiniband_host_device_discover - phase: infiniband-host-device - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - discover - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.discover.arguments | tojson }}" + - "[]" - "--user-config={{ context.k8s_launch_kit.user_config }}" + - "--deployment-files={{ context.k8s_launch_kit.deployment_files }}" - --environment-json - "{{ context.k8s_launch_kit.environment | tojson }}" - --working-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.artifact_dir }}" - timeout: 1800 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkInfiniBandHostDeviceCheck] - - - name: launch_kit_infiniband_host_device_generate - phase: infiniband-host-device - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - generate - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.generate.arguments | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.working_dir }}" - - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.artifact_dir }}" - timeout: 600 - output_schema: k8s_launch_kit - requires: [kubernetes] - requires_selected_validations: [EastWestNetworkInfiniBandHostDeviceCheck] - - - name: launch_kit_infiniband_host_device_validate - phase: infiniband-host-device - command: python3 ../scripts/adapter.py - args: - - run - - --executable - - "{{ steps.launch_kit_verify.executable }}" - - --command - - validate - - --arguments-json - - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.validate.arguments | tojson }}" - - --environment-json - - "{{ context.k8s_launch_kit.environment | tojson }}" - - --working-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.working_dir }}" + - "{{ context.k8s_launch_kit.working_dir }}" - --artifact-dir - - "{{ context.k8s_launch_kit.use_cases.infiniband_host_device.artifact_dir }}" + - "{{ context.k8s_launch_kit.artifact_dir }}" + # Launch Kit calculates the connectivity-matrix budget or honors the + # timeout configured by the user, so it owns this command's deadline. timeout: null output_schema: k8s_launch_kit requires: [kubernetes] - requires_selected_validations: [EastWestNetworkInfiniBandHostDeviceCheck] + requires_selected_validations: [LaunchKitConnectivityCheck] diff --git a/isvctl/configs/providers/k8s-launch-kit/config/provider.yaml b/isvctl/configs/providers/k8s-launch-kit/config/provider.yaml index 716208adb..8d07e6410 100644 --- a/isvctl/configs/providers/k8s-launch-kit/config/provider.yaml +++ b/isvctl/configs/providers/k8s-launch-kit/config/provider.yaml @@ -61,7 +61,7 @@ commands: timeout: 900 output_schema: k8s_launch_kit requires: [kubernetes] - requires_available_validations: [LaunchKitKubernetesPrerequisiteCheck] + requires_selected_validations: [LaunchKitConnectivityCheck] # Test-phase verification is intentional: --phase test may bypass setup. - name: launch_kit_verify @@ -79,7 +79,7 @@ commands: timeout: 60 output_schema: k8s_launch_kit requires: [kubernetes] - requires_available_validations: [LaunchKitKubernetesPrerequisiteCheck] + requires_selected_validations: [LaunchKitConnectivityCheck] # This gate always runs in the test phase, before l8k can mutate a cluster. - name: launch_kit_kubernetes_preflight @@ -100,7 +100,7 @@ commands: timeout: 60 output_schema: k8s_launch_kit requires: [kubernetes] - requires_available_validations: [LaunchKitKubernetesPrerequisiteCheck] + requires_selected_validations: [LaunchKitConnectivityCheck] - name: launch_kit_discover phase: test @@ -123,7 +123,7 @@ commands: timeout: 1800 output_schema: k8s_launch_kit requires: [kubernetes] - requires_available_validations: [LaunchKitTopologyDiscoveryCheck] + requires_selected_validations: [LaunchKitConnectivityCheck] - name: launch_kit_generate phase: test @@ -145,7 +145,7 @@ commands: timeout: 600 output_schema: k8s_launch_kit requires: [kubernetes] - requires_available_validations: [LaunchKitDeploymentHealthCheck] + requires_selected_validations: [LaunchKitConnectivityCheck] - name: launch_kit_deploy phase: test @@ -167,7 +167,7 @@ commands: timeout: 7200 output_schema: k8s_launch_kit requires: [kubernetes] - requires_available_validations: [LaunchKitDeploymentHealthCheck] + requires_selected_validations: [LaunchKitConnectivityCheck] - name: launch_kit_validate phase: test @@ -191,7 +191,7 @@ commands: timeout: null output_schema: k8s_launch_kit requires: [kubernetes] - requires_available_validations: [LaunchKitDeploymentHealthCheck] + requires_selected_validations: [LaunchKitConnectivityCheck] # Runs after phase validations when deploy was attempted, even if deploy, # validate, or a validation check failed. @@ -215,5 +215,5 @@ commands: timeout: 7200 output_schema: k8s_launch_kit requires: [kubernetes] - requires_available_validations: [LaunchKitDeploymentHealthCheck] + requires_selected_validations: [LaunchKitConnectivityCheck] finalizer_for: launch_kit_deploy diff --git a/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py b/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py index 50567411e..d7b3a1d36 100644 --- a/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py +++ b/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py @@ -6,11 +6,10 @@ The provider deliberately exposes the real Launch Kit operations. It forwards user-supplied arguments verbatim and adds ``--output json`` so stdout can be -preserved as structured evidence. When a complete user config is supplied, the -discover operation also stages it transiently in the working directory, binds -Launch Kit's native ``--user-config`` and ``--save-cluster-config`` flags, and -removes the staged input after discovery. Launch Kit remains the owner of -command flags, configuration schema, and defaults. +preserved as structured evidence. Discovery can stage a complete user config +transiently. Validation can bind an existing complete user config and rendered +deployment directory directly. Launch Kit remains the owner of command flags, +configuration schema, and defaults. """ from __future__ import annotations @@ -250,6 +249,43 @@ def _stage_user_config( ) +def _bind_validate_inputs( + user_config_value: str, + deployment_files_value: str, + arguments: list[str], +) -> list[str]: + """Bind required, pre-existing Launch Kit validation inputs.""" + if not user_config_value: + raise ValueError("context.k8s_launch_kit.user_config is required for Network Operator validation") + if not deployment_files_value: + raise ValueError("context.k8s_launch_kit.deployment_files is required for Network Operator validation") + + conflicting_flags = [ + flag + for flag in ("--user-config", "--deployment-files") + if any(token == flag or token.startswith(f"{flag}=") for token in arguments) + ] + if conflicting_flags: + raise ValueError( + "dedicated Launch Kit validation inputs cannot be combined with raw flag(s): " + + ", ".join(conflicting_flags) + ) + + user_config = Path(user_config_value).expanduser().resolve() + if not user_config.is_file(): + raise FileNotFoundError(f"Launch Kit user config not found: {user_config}") + deployment_files = Path(deployment_files_value).expanduser().resolve() + if not deployment_files.is_dir(): + raise FileNotFoundError(f"Launch Kit deployment directory not found: {deployment_files}") + return [ + *arguments, + "--user-config", + str(user_config), + "--deployment-files", + str(deployment_files), + ] + + def _run_workflow(args: argparse.Namespace) -> tuple[dict[str, Any], int]: """Invoke exactly one real Launch Kit workflow command.""" executable = _resolve_executable(args.executable) @@ -263,9 +299,11 @@ def _run_workflow(args: argparse.Namespace) -> tuple[dict[str, Any], int]: staged_user_config: Path | None = None user_config_metadata_path: Path | None = None try: - if args.user_config: + if args.command == "validate" and args.deployment_files is not None: + arguments = _bind_validate_inputs(args.user_config, args.deployment_files, arguments) + elif args.user_config: if args.command != "discover": - raise ValueError("--user-config is supported only with the discover workflow command") + raise ValueError("--user-config requires --deployment-files for the validate workflow command") arguments, staged_user_config, user_config_metadata = _stage_user_config( args.user_config, working_dir, @@ -691,6 +729,7 @@ def _parser() -> argparse.ArgumentParser: run.add_argument("--command", choices=_WORKFLOW_COMMANDS, required=True) run.add_argument("--arguments-json", required=True) run.add_argument("--user-config", default="") + run.add_argument("--deployment-files", default=None) run.add_argument("--environment-json", default="{}") run.add_argument("--working-dir", required=True) run.add_argument("--artifact-dir", required=True) @@ -713,7 +752,7 @@ def main(argv: list[str] | None = None) -> int: envelope = { "success": False, "platform": "kubernetes", - "operation": args.action, + "operation": args.command if args.action == "run" else args.action, "error": str(exc), } exit_code = 1 diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index abc4e204d..3686ee79c 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -218,69 +218,35 @@ its plan item is not platform-scoped. ### Network Operator (`k8s-launch-kit/network-operator.yaml`) -Plain suite for Kubernetes Launch Kit Network Operator self-validation. The -generic provider in `providers/k8s-launch-kit/config/provider.yaml` mirrors the real CLI as -separate verify, prerequisite, discover, generate, deploy, and validate steps. -It forwards user-supplied argument arrays and does not own Network Operator, -profile, topology, resource, or validation defaults. The suite binds fifteen -checks (one prerequisite plus fourteen currently supported PRD areas) directly -to the command output that proves them. - -GPUDirect RDMA is registered from Launch Kit's `gpudirect_dmabuf` result family; -the check skips when that family is disabled or not selected and fails on -emitted GPU topology or bandwidth errors. State restoration remains deferred -until Launch Kit provides the required snapshot/restore/verify workflow. - -The same suite reuses those global check classes in six separate composite -tests: RoCE and InfiniBand across SR-IOV, RDMA Shared, and host-device -deployment modes. Keeping both forms in one file exposes one frontend suite, -`network_operator`, rather than a second implementation-detail suite. Each -composite includes only checks applicable to that use case, so unrelated -fabric/deployment checks do not appear as skips in the middle of a run. The -Ethernet/RoCE composites carry `ethernet` and `roce`; the InfiniBand composites -carry `infiniband`. All six also carry `gpudirect` because Launch Kit discovery -decides whether the GPUDirect family is applicable. - -`providers/k8s-launch-kit/config/network-operator.yaml` is the production -entrypoint. It uses `l8k` and `kubectl` from `PATH` by default. In one invocation -it runs `launch_kit_prepare` in `setup`, `launch_kit_verify` in -`launch-kit-verification`, then executes the six use-case phases sequentially, -each with its own preflight, discover, generate, validate, and evidence -directories. The phases are -independent, so a failed case records a failed overall run but does not prevent -later cases from producing results. Mock executables exist only under -`isvctl/tests/providers/k8s_launch_kit/fixtures/` and are injected by tests. +The Network Operator suite contains one catalog entry, +`LaunchKitConnectivityCheck`. Its production provider invokes only `l8k +validate` with a caller-supplied complete `user_config` and existing +`deployment_files` directory. The Kubernetes cluster, Network Operator +deployment, Launch Kit installation, configuration, and generated manifests +are prerequisites. + +Fabric, deployment type, enabled checks, and GPUDirect applicability are not +modeled as suite labels or separate tests. Every +`connectivity.PingResults` row emitted by Launch Kit becomes a subtest. A +disabled family is simply absent, and new explicit family values are reported +without suite changes. ```bash -ISVTEST_INCLUDE_UNRELEASED=1 uv run isvctl test run \ +uv run isvctl test run \ -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ - --capability kubernetes --no-upload -- -v + --capability kubernetes \ + --set 'context.k8s_launch_kit.user_config=/absolute/path/cluster-config.yaml' \ + --set 'context.k8s_launch_kit.deployment_files=/absolute/path/deployment' \ + --no-upload -- -v ``` -Add `--label ethernet` or `--label infiniband` before `--no-upload` to run only -that fabric's three workflows. Their steps use -`requires_selected_validations`, so the other fabric's mutating commands are -pruned before execution. Use `--label sriov`, `--label rdma_shared`, or -`--label host_device` to run the matching two-fabric deployment mode. Labels -compose, so `--label ethernet --label sriov` selects one use case. Omitting -labels runs all six. - | Step | Phase | Script | Key JSON Fields | |------|-------|--------|-----------------| -| `launch_kit_prepare` | setup | `providers/k8s-launch-kit/scripts/adapter.py prepare` | `installed`, `executable`, `checks.{version,schema}`, `artifacts` | -| `launch_kit_verify` | test | `providers/k8s-launch-kit/scripts/adapter.py verify` | `executable`, `checks.{version,schema}`, `artifacts` | -| `launch_kit_kubernetes_preflight` | test | `providers/k8s-launch-kit/scripts/adapter.py preflight` | `server_version`, `node_count`, `ready_node_count`, `checks`, `artifacts` | -| `launch_kit_discover` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k discover` | raw `documents`, `argv`, `exit_code`, `artifacts` | -| `launch_kit_generate` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k generate` | raw `documents`, `argv`, `exit_code`, `artifacts` | -| `launch_kit_deploy` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k deploy` | raw `documents` (currently empty on success), `argv`, `exit_code`, `artifacts` | -| `launch_kit_validate` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k validate` | raw static, connectivity, and report-path `documents`, `argv`, `exit_code`, `artifacts` | - -Those are the generic provider's single-workflow names. The grouped production -configuration performs prepare in `setup` and verify in -`launch-kit-verification`, then repeats preflight, discover, generate, and -validate under each custom use-case phase with names such as -`launch_kit_roce_sriov_preflight` through -`launch_kit_roce_sriov_validate`. +| `launch_kit_validate` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k validate` | raw `documents`, `argv`, `exit_code`, `duration_seconds`, `artifacts` | + +The generic `providers/k8s-launch-kit/config/provider.yaml` remains available +to consumers that need the full Launch Kit lifecycle. See the +[Launch Kit integration guide](../../../docs/guides/k8s-launch-kit/network-operator.md). ### VM (`vm.yaml`) diff --git a/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml b/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml index 562a9a642..ea874fdbe 100644 --- a/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml +++ b/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml @@ -1,17 +1,15 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Network Operator validation through the Kubernetes Launch Kit CLI. -# -# This single frontend-visible suite contains the individual PRD checks and six -# concrete use cases. Providers own the command sequence and bind these checks -# to actual l8k command output. The generic provider is -# ../../providers/k8s-launch-kit/config/provider.yaml. +# One catalog test consumes the connectivity matrix emitted by l8k validate. +# Fabric, deployment type, enabled checks, and all Launch Kit parameters belong +# to the supplied complete user config; they are prerequisites, not separate +# AI Cloud Validation use cases. version: "1.0" tests: - description: "Network Operator self-validation and use cases through Kubernetes Launch Kit" + description: "Network Operator connectivity validation through Kubernetes Launch Kit" settings: show_skipped_tests: false @@ -19,267 +17,8 @@ tests: validations: network_operator: checks: - LaunchKitKubernetesPrerequisiteCheck: - step: launch_kit_kubernetes_preflight - test_id: "N/A" - labels: ["era", "kubernetes", "ncp", "network_operator", "prerequisite", "slow"] - requires: [kubernetes] - - LaunchKitDeploymentHealthCheck: + LaunchKitConnectivityCheck: step: launch_kit_validate test_id: "K8S42-01" labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] requires: [kubernetes] - - LaunchKitSriovReadinessCheck: - step: launch_kit_validate - test_id: "K8S42-02" - labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] - requires: [kubernetes] - discover_output: "{{ steps.launch_kit_discover | tojson }}" - - LaunchKitRdmaConnectivityCheck: - step: launch_kit_validate - test_id: "K8S42-03" - labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] - requires: [kubernetes] - - LaunchKitRoceCheck: - step: launch_kit_validate - test_id: "K8S42-04" - labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] - requires: [kubernetes] - discover_output: "{{ steps.launch_kit_discover | tojson }}" - - LaunchKitInfiniBandCheck: - step: launch_kit_validate - test_id: "K8S42-05" - labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] - requires: [kubernetes] - discover_output: "{{ steps.launch_kit_discover | tojson }}" - - LaunchKitHostDeviceCheck: - step: launch_kit_validate - test_id: "K8S42-06" - labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] - requires: [kubernetes] - discover_output: "{{ steps.launch_kit_discover | tojson }}" - - LaunchKitGpuDirectRdmaCheck: - step: launch_kit_validate - test_id: "K8S42-07" - labels: ["era", "gpudirect", "kubernetes", "ncp", "network_operator", "slow"] - requires: [kubernetes] - - LaunchKitTopologyDiscoveryCheck: - step: launch_kit_discover - test_id: "K8S42-08" - labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] - requires: [kubernetes] - - LaunchKitSecondaryNetworkCheck: - step: launch_kit_validate - test_id: "K8S42-09" - labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] - requires: [kubernetes] - discover_output: "{{ steps.launch_kit_discover | tojson }}" - - LaunchKitRdmaSharedCheck: - step: launch_kit_validate - test_id: "K8S42-10" - labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] - requires: [kubernetes] - discover_output: "{{ steps.launch_kit_discover | tojson }}" - - LaunchKitIcmpConnectivityCheck: - step: launch_kit_validate - test_id: "K8S42-11" - labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] - requires: [kubernetes] - - LaunchKitRdmaBandwidthCheck: - step: launch_kit_validate - test_id: "K8S42-12" - labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] - requires: [kubernetes] - - LaunchKitMultirailCheck: - step: launch_kit_validate - test_id: "K8S42-13" - labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] - requires: [kubernetes] - discover_output: "{{ steps.launch_kit_discover | tojson }}" - - LaunchKitEvidenceCaptureCheck: - step: launch_kit_validate - test_id: "K8S42-15" - labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] - requires: [kubernetes] - prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" - verify_output: "{{ steps.launch_kit_verify | tojson }}" - preflight_output: "{{ steps.launch_kit_kubernetes_preflight | tojson }}" - discover_output: "{{ steps.launch_kit_discover | tojson }}" - generate_output: "{{ steps.launch_kit_generate | tojson }}" - deploy_output: "{{ steps.launch_kit_deploy | tojson }}" - - network_operator_roce_sriov: - step: launch_kit_roce_sriov_validate - checks: - EastWestNetworkRoceSriovCheck: - test_id: "K8S42-16" - labels: ["era", "ethernet", "gpudirect", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "roce", "slow", "sriov"] - requires: [kubernetes] - description: "Ethernet/RoCE with SR-IOV Network RDMA" - prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" - verify_output: "{{ steps.launch_kit_verify | tojson }}" - preflight_output: "{{ steps.launch_kit_roce_sriov_preflight | tojson }}" - discover_output: "{{ steps.launch_kit_roce_sriov_discover | tojson }}" - generate_output: "{{ steps.launch_kit_roce_sriov_generate | tojson }}" - compose: - - LaunchKitKubernetesPrerequisiteCheck - - LaunchKitTopologyDiscoveryCheck - - LaunchKitDeploymentHealthCheck - - LaunchKitRoceCheck - - LaunchKitSriovReadinessCheck - - LaunchKitSecondaryNetworkCheck - - LaunchKitIcmpConnectivityCheck - - LaunchKitRdmaConnectivityCheck - - LaunchKitRdmaBandwidthCheck - - LaunchKitGpuDirectRdmaCheck - - LaunchKitMultirailCheck - - LaunchKitEvidenceCaptureCheck - - network_operator_infiniband_sriov: - step: launch_kit_infiniband_sriov_validate - checks: - EastWestNetworkInfiniBandSriovCheck: - test_id: "K8S42-17" - labels: ["era", "gpudirect", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "slow", "sriov"] - requires: [kubernetes] - description: "InfiniBand with SR-IOV Network RDMA" - prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" - verify_output: "{{ steps.launch_kit_verify | tojson }}" - preflight_output: "{{ steps.launch_kit_infiniband_sriov_preflight | tojson }}" - discover_output: "{{ steps.launch_kit_infiniband_sriov_discover | tojson }}" - generate_output: "{{ steps.launch_kit_infiniband_sriov_generate | tojson }}" - compose: - - LaunchKitKubernetesPrerequisiteCheck - - LaunchKitTopologyDiscoveryCheck - - LaunchKitDeploymentHealthCheck - - LaunchKitInfiniBandCheck - - LaunchKitSriovReadinessCheck - - LaunchKitSecondaryNetworkCheck - - LaunchKitIcmpConnectivityCheck - - LaunchKitRdmaConnectivityCheck - - LaunchKitRdmaBandwidthCheck - - LaunchKitGpuDirectRdmaCheck - - LaunchKitMultirailCheck - - LaunchKitEvidenceCaptureCheck - - network_operator_roce_rdma_shared: - step: launch_kit_roce_rdma_shared_validate - checks: - EastWestNetworkRoceRdmaSharedCheck: - test_id: "K8S42-18" - labels: ["era", "ethernet", "gpudirect", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "rdma_shared", "roce", "slow"] - requires: [kubernetes] - description: "Ethernet/RoCE with the RDMA Shared Device Plugin" - prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" - verify_output: "{{ steps.launch_kit_verify | tojson }}" - preflight_output: "{{ steps.launch_kit_roce_rdma_shared_preflight | tojson }}" - discover_output: "{{ steps.launch_kit_roce_rdma_shared_discover | tojson }}" - generate_output: "{{ steps.launch_kit_roce_rdma_shared_generate | tojson }}" - compose: - - LaunchKitKubernetesPrerequisiteCheck - - LaunchKitTopologyDiscoveryCheck - - LaunchKitDeploymentHealthCheck - - LaunchKitRoceCheck - - LaunchKitRdmaSharedCheck - - LaunchKitSecondaryNetworkCheck - - LaunchKitIcmpConnectivityCheck - - LaunchKitRdmaConnectivityCheck - - LaunchKitRdmaBandwidthCheck - - LaunchKitGpuDirectRdmaCheck - - LaunchKitMultirailCheck - - LaunchKitEvidenceCaptureCheck - - network_operator_infiniband_rdma_shared: - step: launch_kit_infiniband_rdma_shared_validate - checks: - EastWestNetworkInfiniBandRdmaSharedCheck: - test_id: "K8S42-19" - labels: ["era", "gpudirect", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "rdma_shared", "slow"] - requires: [kubernetes] - description: "InfiniBand/IPoIB with the RDMA Shared Device Plugin" - prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" - verify_output: "{{ steps.launch_kit_verify | tojson }}" - preflight_output: "{{ steps.launch_kit_infiniband_rdma_shared_preflight | tojson }}" - discover_output: "{{ steps.launch_kit_infiniband_rdma_shared_discover | tojson }}" - generate_output: "{{ steps.launch_kit_infiniband_rdma_shared_generate | tojson }}" - compose: - - LaunchKitKubernetesPrerequisiteCheck - - LaunchKitTopologyDiscoveryCheck - - LaunchKitDeploymentHealthCheck - - LaunchKitInfiniBandCheck - - LaunchKitRdmaSharedCheck - - LaunchKitSecondaryNetworkCheck - - LaunchKitIcmpConnectivityCheck - - LaunchKitRdmaConnectivityCheck - - LaunchKitRdmaBandwidthCheck - - LaunchKitGpuDirectRdmaCheck - - LaunchKitMultirailCheck - - LaunchKitEvidenceCaptureCheck - - network_operator_roce_host_device: - step: launch_kit_roce_host_device_validate - checks: - EastWestNetworkRoceHostDeviceCheck: - test_id: "K8S42-20" - labels: ["era", "ethernet", "gpudirect", "host_device", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "roce", "slow"] - requires: [kubernetes] - description: "Ethernet/RoCE host-device networking for worker VMs" - prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" - verify_output: "{{ steps.launch_kit_verify | tojson }}" - preflight_output: "{{ steps.launch_kit_roce_host_device_preflight | tojson }}" - discover_output: "{{ steps.launch_kit_roce_host_device_discover | tojson }}" - generate_output: "{{ steps.launch_kit_roce_host_device_generate | tojson }}" - compose: - - LaunchKitKubernetesPrerequisiteCheck - - LaunchKitTopologyDiscoveryCheck - - LaunchKitDeploymentHealthCheck - - LaunchKitRoceCheck - - LaunchKitHostDeviceCheck - - LaunchKitSecondaryNetworkCheck - - LaunchKitIcmpConnectivityCheck - - LaunchKitRdmaConnectivityCheck - - LaunchKitRdmaBandwidthCheck - - LaunchKitGpuDirectRdmaCheck - - LaunchKitMultirailCheck - - LaunchKitEvidenceCaptureCheck - - network_operator_infiniband_host_device: - step: launch_kit_infiniband_host_device_validate - checks: - EastWestNetworkInfiniBandHostDeviceCheck: - test_id: "K8S42-21" - labels: ["era", "gpudirect", "host_device", "infiniband", "kubernetes", "ncp", "network_operator", "network_operator_use_cases", "slow"] - requires: [kubernetes] - description: "InfiniBand host-device networking for worker VMs" - prepare_output: "{{ steps.launch_kit_prepare | default({}) | tojson }}" - verify_output: "{{ steps.launch_kit_verify | tojson }}" - preflight_output: "{{ steps.launch_kit_infiniband_host_device_preflight | tojson }}" - discover_output: "{{ steps.launch_kit_infiniband_host_device_discover | tojson }}" - generate_output: "{{ steps.launch_kit_infiniband_host_device_generate | tojson }}" - compose: - - LaunchKitKubernetesPrerequisiteCheck - - LaunchKitTopologyDiscoveryCheck - - LaunchKitDeploymentHealthCheck - - LaunchKitInfiniBandCheck - - LaunchKitHostDeviceCheck - - LaunchKitSecondaryNetworkCheck - - LaunchKitIcmpConnectivityCheck - - LaunchKitRdmaConnectivityCheck - - LaunchKitRdmaBandwidthCheck - - LaunchKitGpuDirectRdmaCheck - - LaunchKitMultirailCheck - - LaunchKitEvidenceCaptureCheck diff --git a/isvctl/src/isvctl/config/schema.py b/isvctl/src/isvctl/config/schema.py index 83149fa8b..14e74effd 100644 --- a/isvctl/src/isvctl/config/schema.py +++ b/isvctl/src/isvctl/config/schema.py @@ -96,13 +96,6 @@ class StepConfig(BaseModel): "Capability contexts allowed to run this step. Empty delegates capability gating to bound validations." ), ) - requires_available_validations: list[str] = Field( - default_factory=list, - description=( - "Validation names that must be available after release filtering for this step to run. " - "Unreleased validations are available only when ISVTEST_INCLUDE_UNRELEASED=1." - ), - ) requires_selected_validations: list[str] = Field( default_factory=list, description=( @@ -234,7 +227,6 @@ def validate_continuation_phases(self) -> "PlatformCommands": raise ValueError(f"step '{finalizer.name}' cannot finalize finalizer step '{target.name}'") gate_fields = ( "requires", - "requires_available_validations", "requires_selected_validations", ) mismatched_gates = [ diff --git a/isvctl/tests/providers/k8s_launch_kit/test_provider.py b/isvctl/tests/providers/k8s_launch_kit/test_provider.py index 227da0952..7a2bb6cf1 100644 --- a/isvctl/tests/providers/k8s_launch_kit/test_provider.py +++ b/isvctl/tests/providers/k8s_launch_kit/test_provider.py @@ -19,7 +19,7 @@ import pytest import yaml -from isvtest.core.resolution import ErrorReason, State +from isvtest.core.resolution import State from isvctl.config.merger import merge_yaml_files from isvctl.config.output_schemas import validate_output @@ -78,6 +78,7 @@ def _run_workflow( working_dir: Path, artifact_dir: Path, user_config: Path | None = None, + deployment_files: Path | None = None, env: dict[str, str] | None = None, ) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: """Run a mocked l8k workflow command through the generic transport.""" @@ -98,6 +99,8 @@ def _run_workflow( ] if user_config is not None: provider_arguments.extend(["--user-config", str(user_config)]) + if deployment_files is not None: + provider_arguments.extend(["--deployment-files", str(deployment_files)]) return _run_provider(*provider_arguments, env=env) @@ -105,12 +108,24 @@ def _mocked_network_operator_config(tmp_path: Path) -> RunConfig: """Load production wiring, then inject test-owned executables and paths.""" merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) context = merged["context"]["k8s_launch_kit"] + user_config = tmp_path / "cluster-config.yaml" + user_config.write_text( + """networkOperator: + selectedRelease: "26.4" +profile: + fabric: ethernet + deployment: sriov +clusterConfig: [] +""", + encoding="utf-8", + ) + deployment_files = tmp_path / "deployment" + deployment_files.mkdir() context["executable"] = str(_MOCK_L8K) - context["kubectl_command"] = [sys.executable, str(_MOCK_KUBECTL)] - context["shared_artifact_dir"] = str(tmp_path / "shared-evidence") - for name, use_case in context["use_cases"].items(): - use_case["working_dir"] = str(tmp_path / "use-cases" / name / "work") - use_case["artifact_dir"] = str(tmp_path / "use-cases" / name / "evidence") + context["user_config"] = str(user_config) + context["deployment_files"] = str(deployment_files) + context["working_dir"] = str(tmp_path / "work") + context["artifact_dir"] = str(tmp_path / "evidence") return RunConfig.model_validate(merged) @@ -185,51 +200,38 @@ def test_generic_provider_has_no_launch_kit_domain_defaults() -> None: def test_network_operator_provider_defaults_to_real_cli_tools() -> None: - """The shipped use-case provider cannot select repository test doubles.""" + """The shipped provider contains one real, validation-only workflow.""" merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) config = RunConfig.model_validate(merged) context = merged["context"]["k8s_launch_kit"] - assert context["executable"] == "l8k" - assert context["installation"]["installer_ref"] == "" - assert context["installation"]["installer_sha256"] == "" - assert context["user_config"] == "" - assert context["kubectl_command"] == [] + assert context == { + "executable": "l8k", + "user_config": "", + "deployment_files": "", + "working_dir": "../../../../../_output/k8s-launch-kit/network-operator/work", + "artifact_dir": "../../../../../_output/k8s-launch-kit/network-operator/evidence", + "environment": {}, + } assert "mock" not in json.dumps(merged).lower() assert "poc" not in json.dumps(merged).lower() - assert len(config.commands["network_operator"].steps) == 26 - assert config.commands["network_operator"].phases[-1] == "infiniband-host-device" - discover_steps = [step for step in config.commands["network_operator"].steps if step.name.endswith("_discover")] - assert len(discover_steps) == 6 - assert all("--user-config={{ context.k8s_launch_kit.user_config }}" in step.args for step in discover_steps) - prepare_step = next(step for step in config.commands["network_operator"].steps if step.name == "launch_kit_prepare") - assert "--installer-ref={{ context.k8s_launch_kit.installation.installer_ref }}" in prepare_step.args - assert "--installer-sha256={{ context.k8s_launch_kit.installation.installer_sha256 }}" in prepare_step.args - use_case_steps = config.commands["network_operator"].steps[2:] - assert all(not step.name.endswith(("_deploy", "_clean")) for step in use_case_steps) - assert all(step.finalizer_for is None for step in use_case_steps) - - -def test_network_operator_workflows_use_launch_kit_default_paths() -> None: - """Grouped use cases leave config and deployment paths to Launch Kit.""" + command = config.commands["network_operator"] + assert command.phases == ["test"] + assert [step.name for step in command.steps] == ["launch_kit_validate"] + step = command.steps[0] + assert step.timeout is None + assert "--user-config={{ context.k8s_launch_kit.user_config }}" in step.args + assert "--deployment-files={{ context.k8s_launch_kit.deployment_files }}" in step.args + assert step.requires_selected_validations == ["LaunchKitConnectivityCheck"] + + +def test_network_operator_suite_has_one_catalog_check() -> None: + """Fabric and deployment choices are prerequisites, not test entries.""" merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) - use_cases = merged["context"]["k8s_launch_kit"]["use_cases"] - default_path_flags = { - "--user-config", - "--deployment-files", - "--save-cluster-config", - "--save-deployment-files", - } + checks = merged["tests"]["validations"]["network_operator"]["checks"] - for use_case in use_cases.values(): - all_arguments = { - argument for phase in ("discover", "generate", "validate") for argument in use_case[phase]["arguments"] - } - assert default_path_flags.isdisjoint(all_arguments) - assert set(use_case) == {"working_dir", "artifact_dir", "discover", "generate", "validate"} - assert use_case["discover"]["arguments"][0] == "--fabric" - assert use_case["generate"]["arguments"] == [] - assert use_case["validate"]["arguments"] == [] + assert list(checks) == ["LaunchKitConnectivityCheck"] + assert checks["LaunchKitConnectivityCheck"]["test_id"] == "K8S42-01" def test_kubectl_defaults_to_the_real_binary() -> None: @@ -840,309 +842,154 @@ def test_preflight_rejects_conflicting_workflow_kubeconfigs(tmp_path: Path) -> N assert "different kubeconfigs" in output["error"] -def test_network_operator_provider_runs_end_to_end(tmp_path: Path, monkeypatch: Any) -> None: - """The production configuration executes all six named use cases in order.""" - monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") +def test_network_operator_provider_runs_only_validate(tmp_path: Path) -> None: + """The production configuration invokes one validation over prerequisite inputs.""" config = _mocked_network_operator_config(tmp_path) result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( - phases=[Phase.SETUP, Phase.TEST], + phases=[Phase.TEST], capability="kubernetes", ) assert result.success is True - expected_use_cases = [ - "roce_sriov", - "infiniband_sriov", - "roce_rdma_shared", - "infiniband_rdma_shared", - "roce_host_device", - "infiniband_host_device", - ] - expected_steps = ["launch_kit_prepare", "launch_kit_verify"] - for use_case in expected_use_cases: - expected_steps.extend( - f"launch_kit_{use_case}_{operation}" for operation in ("preflight", "discover", "generate", "validate") - ) - assert list(result.inventory) == expected_steps - expected_phase_names = ["setup", "launch-kit-verification"] + [ - use_case.replace("_", "-") for use_case in expected_use_cases - ] - assert [phase.name for phase in result.phases] == expected_phase_names - for use_case in expected_use_cases: - phase_name = use_case.replace("_", "-") - test_phase = next(phase for phase in result.phases if phase.name == phase_name) - assert test_phase.phase is Phase.TEST - assert [step["name"].rsplit("_", 1)[-1] for step in test_phase.details["steps"]] == [ - "preflight", - "discover", - "generate", - "validate", - ] - states = {entry.entry.name: entry.state for entry in result.validations} - assert states == { - "EastWestNetworkRoceSriovCheck": State.PASSED, - "EastWestNetworkInfiniBandSriovCheck": State.PASSED, - "EastWestNetworkRoceRdmaSharedCheck": State.PASSED, - "EastWestNetworkInfiniBandRdmaSharedCheck": State.PASSED, - "EastWestNetworkRoceHostDeviceCheck": State.PASSED, - "EastWestNetworkInfiniBandHostDeviceCheck": State.PASSED, - } - expected_subtest_counts = { - "EastWestNetworkRoceSriovCheck": 121, - "EastWestNetworkInfiniBandSriovCheck": 121, - "EastWestNetworkRoceRdmaSharedCheck": 116, - "EastWestNetworkInfiniBandRdmaSharedCheck": 116, - "EastWestNetworkRoceHostDeviceCheck": 116, - "EastWestNetworkInfiniBandHostDeviceCheck": 116, - } - for entry in result.validations: - assert entry.subtest_summary.passed == expected_subtest_counts[entry.entry.name] - assert entry.subtest_summary.failed == 0 - assert entry.subtest_summary.skipped == 0 - for use_case in expected_use_cases: - assert (tmp_path / "use-cases" / use_case / "work" / "cluster-config.yaml").is_file() + assert list(result.inventory) == ["launch_kit_validate"] + assert [phase.name for phase in result.phases] == ["test"] + assert len(result.validations) == 1 + validation = result.validations[0] + assert validation.entry.name == "LaunchKitConnectivityCheck" + assert validation.state is State.PASSED + assert validation.subtest_summary.passed == 32 + assert validation.subtest_summary.failed == 0 + assert validation.subtest_summary.skipped == 0 + + argv = result.inventory["launch_kit_validate"]["argv"] + assert argv[1] == "validate" + assert argv[argv.index("--user-config") + 1] == str((tmp_path / "cluster-config.yaml").resolve()) + assert argv[argv.index("--deployment-files") + 1] == str((tmp_path / "deployment").resolve()) + assert argv[-2:] == ["--output", "json"] + + +def test_network_operator_provider_expands_input_paths(tmp_path: Path, monkeypatch: Any) -> None: + """Tilde inputs are resolved before they are supplied to Launch Kit.""" + monkeypatch.setenv("HOME", str(tmp_path)) + user_config = tmp_path / "l8k" / "cluster-config.yaml" + user_config.parent.mkdir() + user_config.write_text( + "profile:\n fabric: ethernet\n deployment: sriov\n", + encoding="utf-8", + ) + deployment_files = tmp_path / "l8k" / "deployment" + deployment_files.mkdir() + + completed, output = _run_workflow( + "validate", + [], + working_dir=tmp_path / "work", + artifact_dir=tmp_path / "evidence", + user_config=Path("~/l8k/cluster-config.yaml"), + deployment_files=Path("~/l8k/deployment"), + ) + + assert completed.returncode == 0 + assert output["argv"][output["argv"].index("--user-config") + 1] == str(user_config) + assert output["argv"][output["argv"].index("--deployment-files") + 1] == str(deployment_files) @pytest.mark.parametrize( - ("label", "selected_use_cases", "excluded_use_cases"), + ("user_config", "deployment_files", "expected"), [ - ( - "ethernet", - ["roce_sriov", "roce_rdma_shared", "roce_host_device"], - ["infiniband_sriov", "infiniband_rdma_shared", "infiniband_host_device"], - ), - ( - "infiniband", - ["infiniband_sriov", "infiniband_rdma_shared", "infiniband_host_device"], - ["roce_sriov", "roce_rdma_shared", "roce_host_device"], - ), - ( - "sriov", - ["roce_sriov", "infiniband_sriov"], - ["roce_rdma_shared", "infiniband_rdma_shared", "roce_host_device", "infiniband_host_device"], - ), - ( - "rdma_shared", - ["roce_rdma_shared", "infiniband_rdma_shared"], - ["roce_sriov", "infiniband_sriov", "roce_host_device", "infiniband_host_device"], - ), - ( - "host_device", - ["roce_host_device", "infiniband_host_device"], - ["roce_sriov", "infiniband_sriov", "roce_rdma_shared", "infiniband_rdma_shared"], - ), - ( - "gpudirect", - [ - "roce_sriov", - "infiniband_sriov", - "roce_rdma_shared", - "infiniband_rdma_shared", - "roce_host_device", - "infiniband_host_device", - ], - [], - ), - ( - ["ethernet", "sriov"], - ["roce_sriov"], - [ - "infiniband_sriov", - "roce_rdma_shared", - "infiniband_rdma_shared", - "roce_host_device", - "infiniband_host_device", - ], - ), + (None, "deployment", "user_config is required"), + ("cluster-config.yaml", None, "--user-config requires --deployment-files"), ], ) -def test_network_operator_provider_grouping_label_prunes_unselected_workflows( +def test_validate_requires_both_prerequisite_inputs( tmp_path: Path, - monkeypatch: Any, - label: str | list[str], - selected_use_cases: list[str], - excluded_use_cases: list[str], + user_config: str | None, + deployment_files: str | None, + expected: str, ) -> None: - """A grouping label runs only the matching validation workflows.""" - monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") - config = _mocked_network_operator_config(tmp_path) + """Partial prerequisite input fails before Launch Kit execution.""" + config_path = tmp_path / "cluster-config.yaml" + config_path.write_text("profile: {}\n", encoding="utf-8") + deployment_path = tmp_path / "deployment" + deployment_path.mkdir() - result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( - phases=[Phase.SETUP, Phase.TEST], - include_labels=[label] if isinstance(label, str) else label, - capability="kubernetes", - ) - - assert result.success is True - inventory_names = set(result.inventory) - for use_case in selected_use_cases: - assert f"launch_kit_{use_case}_validate" in inventory_names - assert f"launch_kit_{use_case}_deploy" not in inventory_names - assert f"launch_kit_{use_case}_clean" not in inventory_names - assert (tmp_path / "use-cases" / use_case / "work" / "cluster-config.yaml").is_file() - for use_case in excluded_use_cases: - assert not any(name.startswith(f"launch_kit_{use_case}_") for name in inventory_names) - assert not (tmp_path / "use-cases" / use_case / "work" / "cluster-config.yaml").exists() - - -def test_network_operator_stages_user_config_only_for_selected_use_cases( - tmp_path: Path, - monkeypatch: Any, -) -> None: - """Each selected use case receives and removes an isolated user-config copy.""" - monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") - source = tmp_path / "customer-cluster-config.yaml" - source_contents = """networkOperator: - selectedRelease: "26.4" -profile: - fabric: ethernet - deployment: sriov -clusterConfig: [] -""" - source.write_text(source_contents, encoding="utf-8") - config = _mocked_network_operator_config(tmp_path) - config.context["k8s_launch_kit"]["user_config"] = str(source) - - result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( - phases=[Phase.TEST], - include_labels=["ethernet", "sriov"], - capability="kubernetes", - ) - - selected_work = tmp_path / "use-cases" / "roce_sriov" / "work" - selected_evidence = tmp_path / "use-cases" / "roce_sriov" / "evidence" - assert result.success is True - assert source.read_text(encoding="utf-8") == source_contents - assert not (selected_work / "user-config.yaml").exists() - assert (selected_work / "cluster-config.yaml").is_file() - metadata = json.loads((selected_evidence / "inputs" / "user-config.json").read_text(encoding="utf-8")) - assert metadata["sha256"] == hashlib.sha256(source_contents.encode()).hexdigest() - assert metadata["retained"] is False - discover = result.inventory["launch_kit_roce_sriov_discover"] - assert discover["argv"][discover["argv"].index("--user-config") + 1] == str( - (selected_work / "user-config.yaml").resolve() - ) - for use_case in ( - "infiniband_sriov", - "roce_rdma_shared", - "infiniband_rdma_shared", - "roce_host_device", - "infiniband_host_device", - ): - assert not (tmp_path / "use-cases" / use_case / "work" / "user-config.yaml").exists() - - -def test_network_operator_provider_test_phase_verifies_without_setup(tmp_path: Path, monkeypatch: Any) -> None: - """A test-only run verifies the configured binary instead of requiring setup output.""" - monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") - config = _mocked_network_operator_config(tmp_path) - - result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( - phases=[Phase.TEST], - capability="kubernetes", + completed, output = _run_workflow( + "validate", + [], + working_dir=tmp_path / "work", + artifact_dir=tmp_path / "evidence", + user_config=config_path if user_config is not None else None, + deployment_files=deployment_path if deployment_files is not None else None, ) - assert result.success is True - assert "launch_kit_prepare" not in result.inventory - assert result.inventory["launch_kit_verify"]["success"] is True - assert all(entry.state is State.PASSED for entry in result.validations) + assert completed.returncode == 1 + assert output["success"] is False + assert expected in output["error"] -def test_network_operator_workflow_never_invokes_deploy_or_clean(tmp_path: Path, monkeypatch: Any) -> None: - """The validation suite must not mutate or delete the ISV-managed deployment.""" - monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") - monkeypatch.setenv("L8K_MOCK_FAIL", "deploy") - config = _mocked_network_operator_config(tmp_path) +def test_validate_rejects_duplicate_path_flags(tmp_path: Path) -> None: + """Dedicated inputs cannot silently conflict with raw Launch Kit arguments.""" + user_config = tmp_path / "cluster-config.yaml" + user_config.write_text("profile: {}\n", encoding="utf-8") + deployment_files = tmp_path / "deployment" + deployment_files.mkdir() - result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( - phases=[Phase.TEST], - include_labels=["ethernet", "sriov"], - capability="kubernetes", + completed, output = _run_workflow( + "validate", + ["--user-config", "other.yaml"], + working_dir=tmp_path / "work", + artifact_dir=tmp_path / "evidence", + user_config=user_config, + deployment_files=deployment_files, ) - assert result.success is True - assert list(result.inventory) == [ - "launch_kit_verify", - "launch_kit_roce_sriov_preflight", - "launch_kit_roce_sriov_discover", - "launch_kit_roce_sriov_generate", - "launch_kit_roce_sriov_validate", - ] - assert not any(name.endswith(("_deploy", "_clean")) for name in result.inventory) - assert all(phase.phase is not Phase.TEARDOWN for phase in result.phases) - - -def test_kubernetes_preflight_failure_stops_before_discovery(tmp_path: Path, monkeypatch: Any) -> None: - """An unreachable cluster blocks each use case before discovery without hiding later cases.""" - monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") - monkeypatch.setenv("L8K_MOCK_KUBERNETES_FAIL", "1") - config = _mocked_network_operator_config(tmp_path) - - result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run(capability="kubernetes") - - assert result.success is False - assert list(result.inventory) == [ - "launch_kit_prepare", - "launch_kit_verify", - "launch_kit_roce_sriov_preflight", - "launch_kit_infiniband_sriov_preflight", - "launch_kit_roce_rdma_shared_preflight", - "launch_kit_infiniband_rdma_shared_preflight", - "launch_kit_roce_host_device_preflight", - "launch_kit_infiniband_host_device_preflight", - ] - assert all(entry.state is State.ERROR for entry in result.validations) - assert all(entry.error_reason is ErrorReason.STEP_FAILED for entry in result.validations) - assert all("preflight" in entry.message for entry in result.validations) - assert not list((tmp_path / "use-cases").glob("*/work/cluster-config.yaml")) - assert not list((tmp_path / "use-cases").glob("*/evidence/commands/discover")) + assert completed.returncode == 1 + assert "cannot be combined with raw flag(s): --user-config" in output["error"] -def test_failed_validate_is_reported_without_cluster_cleanup(tmp_path: Path, monkeypatch: Any) -> None: - """A validation failure is reported while the ISV-managed deployment remains untouched.""" - monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") +def test_failed_connectivity_is_a_junit_failure(tmp_path: Path, monkeypatch: Any) -> None: + """A failed Launch Kit matrix row is retained as a test failure in JUnit.""" monkeypatch.setenv("L8K_MOCK_FAIL", "validate:ib_write_bw") config = _mocked_network_operator_config(tmp_path) junit_path = tmp_path / "junit.xml" result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( phases=[Phase.TEST], - include_labels=["ethernet", "sriov"], capability="kubernetes", junitxml=str(junit_path), ) assert result.success is False - assert list(result.inventory)[-1] == "launch_kit_roce_sriov_validate" - assert not any(name.endswith(("_deploy", "_clean")) for name in result.inventory) + assert list(result.inventory) == ["launch_kit_validate"] assert result.validations[0].state is State.FAILED + assert result.validations[0].subtest_summary.failed == 1 case = next( case for case in ET.parse(junit_path).getroot().iter("testcase") - if case.get("name") == "EastWestNetworkRoceSriovCheck" + if case.get("name") == "LaunchKitConnectivityCheck" ) assert case.find("failure") is not None assert case.find("error") is None assert case.find("skipped") is None -def test_failed_use_case_continues_to_next_selected_validation(tmp_path: Path, monkeypatch: Any) -> None: - """Independent pre-provisioned use cases continue after an earlier validation fails.""" - monkeypatch.setenv("ISVTEST_INCLUDE_UNRELEASED", "1") - monkeypatch.setenv("L8K_MOCK_FAIL", "validate:ib_write_bw") - config = _mocked_network_operator_config(tmp_path) +def test_missing_prerequisites_are_a_step_error(tmp_path: Path) -> None: + """An unset prerequisite produces an actionable validation error.""" + merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) + merged["context"]["k8s_launch_kit"]["executable"] = str(_MOCK_L8K) + merged["context"]["k8s_launch_kit"]["working_dir"] = str(tmp_path / "work") + merged["context"]["k8s_launch_kit"]["artifact_dir"] = str(tmp_path / "evidence") + config = RunConfig.model_validate(merged) result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( phases=[Phase.TEST], - include_labels=["sriov"], capability="kubernetes", ) assert result.success is False - assert "launch_kit_roce_sriov_validate" in result.inventory - assert "launch_kit_infiniband_sriov_validate" in result.inventory - assert not any(name.endswith(("_deploy", "_clean")) for name in result.inventory) + assert result.validations[0].state is State.FAILED + assert "user_config is required" in result.validations[0].message def test_failed_validate_preserves_documents_and_process_error(tmp_path: Path) -> None: diff --git a/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py b/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py index 92123a892..66f2eca4f 100644 --- a/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py +++ b/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py @@ -25,9 +25,9 @@ def test_generic_validate_delegates_timeout_to_launch_kit() -> None: assert validate_steps[0]["timeout"] is None -def test_network_operator_validates_delegate_timeout_to_launch_kit() -> None: - """Every grouped use case must leave its validation deadline to l8k.""" - validate_steps = [step for step in _steps("network-operator.yaml") if step["name"].endswith("_validate")] +def test_network_operator_validate_delegates_timeout_to_launch_kit() -> None: + """The one Network Operator validation leaves its deadline to l8k.""" + validate_steps = [step for step in _steps("network-operator.yaml") if step["name"] == "launch_kit_validate"] - assert len(validate_steps) == 6 - assert all(step["timeout"] is None for step in validate_steps) + assert len(validate_steps) == 1 + assert validate_steps[0]["timeout"] is None diff --git a/isvctl/tests/test_orchestrator_loop.py b/isvctl/tests/test_orchestrator_loop.py index 08ac4fcc3..5d49fa818 100644 --- a/isvctl/tests/test_orchestrator_loop.py +++ b/isvctl/tests/test_orchestrator_loop.py @@ -117,7 +117,6 @@ def test_selected_validation_gate_prunes_unselected_lifecycle_steps() -> None: include_labels=set(), exclude_labels=set(), exclude_tests=set(), - released_tests=None, capability=None, ) ethernet_steps = _apply_selected_validation_gates( @@ -126,7 +125,6 @@ def test_selected_validation_gate_prunes_unselected_lifecycle_steps() -> None: include_labels={"ethernet"}, exclude_labels=set(), exclude_tests=set(), - released_tests=None, capability=None, ) @@ -767,9 +765,8 @@ def test_config_without_commands_runs_live_validations_only(self, monkeypatch: p assert [entry.entry.name for entry in result.validations] == ["K8sCsiStorageTypesCheck"] assert result.validations[0].state is State.PASSED - def test_config_without_commands_reports_failed_live_validation(self, monkeypatch: pytest.MonkeyPatch) -> None: + def test_config_without_commands_reports_failed_live_validation(self) -> None: """A failed commandless validation returns a failed result instead of reading command policy.""" - monkeypatch.setattr("isvctl.orchestrator.loop.load_released_test_filter", lambda: None) config = RunConfig( tests=ValidationConfig( validations={ @@ -947,10 +944,8 @@ def test_validation_without_step_output_is_reported_as_skipped(self, monkeypatch def test_failed_owned_step_is_reported_as_validation_error( self, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: """An early workflow failure cannot become a harmless missing-output skip.""" - monkeypatch.setattr("isvctl.orchestrator.loop.load_released_test_filter", lambda: None) failing_step = _write_script( tmp_path, "deploy.sh", diff --git a/isvctl/tests/test_schema.py b/isvctl/tests/test_schema.py index ad16a27dc..eb943a1dd 100644 --- a/isvctl/tests/test_schema.py +++ b/isvctl/tests/test_schema.py @@ -76,7 +76,6 @@ def test_minimal_step(self) -> None: assert step.phase == "setup" assert step.skip is False assert step.requires == [] - assert step.requires_available_validations == [] assert step.requires_selected_validations == [] def test_full_step(self) -> None: @@ -91,7 +90,6 @@ def test_full_step(self) -> None: phase="setup", skip=False, requires=["vm", "bare_metal"], - requires_available_validations=["NewCheck"], requires_selected_validations=["SelectedCheck"], continue_on_failure=True, output_schema="vpc", @@ -103,7 +101,6 @@ def test_full_step(self) -> None: assert step.env == {"AWS_REGION": "us-west-2"} assert step.phase == "setup" assert step.requires == ["vm", "bare_metal"] - assert step.requires_available_validations == ["NewCheck"] assert step.requires_selected_validations == ["SelectedCheck"] assert step.continue_on_failure is True assert step.output_schema == "vpc" diff --git a/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py b/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py index b4a7a0fbe..c4d3e6678 100644 --- a/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py +++ b/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py @@ -1,116 +1,52 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Assertions over unmodified Kubernetes Launch Kit command output. - -Cluster interaction and Launch Kit command execution stay in the provider. -These checks interpret the real discover and validate documents and expose -resource or matrix rows as pytest subtests for actionable reporting. -""" +"""Connectivity assertions over unmodified Kubernetes Launch Kit output.""" from __future__ import annotations -import json -from pathlib import Path from typing import Any, ClassVar -import pytest - from isvtest.core.validation import BaseValidation -_CONNECTIVITY_FAMILIES: dict[str, set[int]] = { - "icmp": {0, 1}, - "rping": {2, 3}, - "ib_write_bw": {4, 5}, - "gpudirect_dmabuf": {6, 7}, +_FAMILY_BY_KIND = { + 0: "icmp", + 1: "icmp", + 2: "rping", + 3: "rping", + 4: "ib_write_bw", + 5: "ib_write_bw", + 6: "gpudirect_dmabuf", + 7: "gpudirect_dmabuf", } -_PROFILE_NETWORK_KINDS: dict[tuple[str, str], str] = { - ("ethernet", "sriov"): "SriovNetwork", - ("infiniband", "sriov"): "SriovIBNetwork", - ("ethernet", "rdma_shared"): "MacvlanNetwork", - ("infiniband", "rdma_shared"): "IPoIBNetwork", - ("ethernet", "host_device"): "HostDeviceNetwork", - ("infiniband", "host_device"): "HostDeviceNetwork", -} - -def _object(value: Any) -> dict[str, Any]: - """Return ``value`` as a JSON object or an empty object.""" +def _mapping(value: Any) -> dict[str, Any]: + """Return ``value`` as a mapping or an empty mapping.""" return value if isinstance(value, dict) else {} -def _list(value: Any) -> list[Any]: - """Return ``value`` as a list or an empty list.""" +def _sequence(value: Any) -> list[Any]: + """Return ``value`` as a sequence or an empty sequence.""" return value if isinstance(value, list) else [] -def _profile_network_kind(profile: dict[str, Any]) -> str | None: - """Return the expected secondary-network resource for a resolved profile.""" - fabric = profile.get("fabric") - deployment = profile.get("deployment") - if not isinstance(fabric, str) or not isinstance(deployment, str): - return None - return _PROFILE_NETWORK_KINDS.get((fabric, deployment)) +class LaunchKitConnectivityCheck(BaseValidation): + """Report every connectivity result emitted by ``l8k validate``.""" + description: ClassVar[str] = "Check the Kubernetes Launch Kit connectivity matrix" -class _LaunchKitCheck(BaseValidation): - """Shared parsing and subtest reporting for Launch Kit checks.""" - - _exclude_from_discovery: ClassVar[bool] = True - - def _step_output(self, operation: str | None = None) -> dict[str, Any] | None: - """Return the bound provider envelope and validate its operation.""" + def _connectivity(self) -> dict[str, Any] | None: + """Find the connectivity document in the bound provider output.""" output = self.config.get("step_output") if not isinstance(output, dict): self.set_failed("Missing Launch Kit step_output") return None - if operation is not None and output.get("operation") != operation: - self.set_failed(f"Expected Launch Kit operation {operation!r}, got {output.get('operation')!r}") - return None - return output - - def _configured_output(self, key: str) -> dict[str, Any]: - """Decode another step envelope passed through validation configuration.""" - value = self.config.get(key) - if isinstance(value, dict): - return value - if not isinstance(value, str) or not value: - return {} - try: - parsed = json.loads(value) - except json.JSONDecodeError: - return {} - return parsed if isinstance(parsed, dict) else {} - - def _documents(self, output: dict[str, Any]) -> list[dict[str, Any]]: - """Return only object documents from a provider envelope.""" - return [document for document in _list(output.get("documents")) if isinstance(document, dict)] - - def _profile(self) -> dict[str, Any] | None: - """Return the resolved profile from the real discover JSONResult.""" - discover = self._configured_output("discover_output") - documents = self._documents(discover) - profile = documents[0].get("profile") if documents else None - if not isinstance(profile, dict): - self.set_failed("Launch Kit discover output has no resolved profile") + if output.get("operation") != "validate": + self.set_failed(f"Expected Launch Kit operation 'validate', got {output.get('operation')!r}") return None - return profile - - def _static_document(self, output: dict[str, Any]) -> dict[str, Any] | None: - """Find the manifest/version validation document.""" - for document in self._documents(output): - if {"versionCheck", "manifests", "summary"}.issubset(document): - return document - provider_error = output.get("error") - suffix = f": {provider_error}" if isinstance(provider_error, str) and provider_error else "" - self.set_failed(f"Launch Kit validate output has no static validation document{suffix}") - return None - - def _connectivity(self, output: dict[str, Any]) -> dict[str, Any] | None: - """Find the source-bound connectivity matrix document.""" - for document in self._documents(output): - connectivity = document.get("connectivity") + for document in _sequence(output.get("documents")): + connectivity = _mapping(document).get("connectivity") if isinstance(connectivity, dict): return connectivity provider_error = output.get("error") @@ -118,604 +54,75 @@ def _connectivity(self, output: dict[str, Any]) -> dict[str, Any] | None: self.set_failed(f"Launch Kit validate output has no connectivity matrix{suffix}") return None - def _finish_probes(self, title: str, probes: list[dict[str, Any]]) -> None: - """Report every probe and aggregate its failures after the final row.""" - if not probes: - if not self._error: - self.set_failed(f"{title} produced no probes") - return - failures: list[str] = [] - for index, probe in enumerate(probes, start=1): - name = str(probe.get("name") or f"probe-{index}") - passed = probe.get("passed") is True - skipped = probe.get("skipped") is True - message = str(probe.get("message") or probe.get("error") or "") - self.report_subtest(name, passed=passed, skipped=skipped, message=message) - if not passed and not skipped: - failures.append(f"{name}: {message or 'failed without a diagnostic'}") - if failures: - self.set_failed(f"{title} failed: {'; '.join(failures)}") - return - self.set_passed(f"{title} passed ({len(probes)} probes)") - - def _manifest_probes( - self, - output: dict[str, Any], - *, - required_kinds: set[str] | None = None, - ) -> list[dict[str, Any]]: - """Build probes from Launch Kit manifest validation rows.""" - static = self._static_document(output) - if static is None: - return [] - manifests = [item for item in _list(static.get("manifests")) if isinstance(item, dict)] - if required_kinds is not None: - manifests = [item for item in manifests if item.get("Kind") in required_kinds] - probes = [] - for item in manifests: - kind = str(item.get("Kind") or "unknown-kind") - namespace = str(item.get("Namespace") or "cluster") - name = str(item.get("Name") or "unknown") - passed = item.get("State") == "success" and item.get("Missing") is not True - probes.append( - { - "name": f"{kind}/{namespace}/{name}", - "passed": passed, - "message": str(item.get("Reason") or item.get("Detail") or item.get("State") or ""), - } - ) - return probes - - def _matrix_probes(self, output: dict[str, Any], families: set[str]) -> list[dict[str, Any]]: - """Build one informative subtest for every selected connectivity row.""" - connectivity = self._connectivity(output) - if connectivity is None: - return [] - probes: list[dict[str, Any]] = [] - for row in _list(connectivity.get("PingResults")): - if not isinstance(row, dict): - continue - test = _object(row.get("Test")) - kind = test.get("Kind") - explicit_family = row.get("Family") - family = explicit_family if isinstance(explicit_family, str) else None - if family not in _CONNECTIVITY_FAMILIES: - family = next( - (name for name, family_kinds in _CONNECTIVITY_FAMILIES.items() if kind in family_kinds), - None, - ) - if family not in families: - continue - source = str(test.get("SrcNode") or test.get("SrcPod") or "unknown-source") - destination = str(test.get("DstNode") or test.get("DstPod") or "unknown-destination") - source_rail = str(test.get("SrcRail") or test.get("Rail") or "unknown-rail") - destination_rail = str(test.get("DstRail") or test.get("Rail") or "unknown-rail") - expectation = str(row.get("Expectation") or test.get("Expectation") or "required") - passed = row.get("OK") is True - stderr = str(row.get("Stderr") or "").strip() - error = str(row.get("Error") or "").strip() - bandwidth = row.get("BandwidthGbps") - minimum = row.get("MinBandwidthGbps") - details = [f"expectation={expectation}", f"observedOK={row.get('ObservedOK')}"] - if family in {"ib_write_bw", "gpudirect_dmabuf"}: - details.extend([f"bandwidthGbps={bandwidth}", f"minimumGbps={minimum}"]) - if family == "gpudirect_dmabuf": - source_gpu = test.get("SrcGPUIndex") - destination_gpu = test.get("DstGPUIndex") - valid_gpu_indices = all( - isinstance(index, int) and not isinstance(index, bool) and index >= 0 - for index in (source_gpu, destination_gpu) - ) - passed = passed and valid_gpu_indices - details.append(f"gpuIndices={source_gpu}->{destination_gpu}") - source_gpu_pci = test.get("SrcGPUPCIAddress") - destination_gpu_pci = test.get("DstGPUPCIAddress") - if source_gpu_pci: - details.append(f"sourceGpuPci={source_gpu_pci}") - if destination_gpu_pci: - details.append(f"destinationGpuPci={destination_gpu_pci}") - if not valid_gpu_indices: - details.append("invalid or missing endpoint GPU index") - if stderr: - details.append(f"stderr={stderr}") - if error and error != stderr: - details.append(f"error={error}") - probes.append( - { - "name": f"{family}/{source}->{destination}/{source_rail}->{destination_rail}", - "passed": passed, - "message": ", ".join(details), - "source_rail": source_rail, - "destination_rail": destination_rail, - } - ) - return probes - - def _kind_coverage_probes( - self, - output: dict[str, Any], - expected_kinds: set[str], - ) -> list[dict[str, Any]]: - """Require every applicable manifest kind to appear in Launch Kit output.""" - static = self._static_document(output) - if static is None: - return [] - observed = { - str(item.get("Kind")) - for item in _list(static.get("manifests")) - if isinstance(item, dict) and item.get("Kind") + @staticmethod + def _probe(row: dict[str, Any], index: int) -> dict[str, Any]: + """Convert one Launch Kit matrix row into one informative subtest.""" + test = _mapping(row.get("Test")) + explicit_family = row.get("Family") + family = explicit_family if isinstance(explicit_family, str) and explicit_family else None + if family is None: + family = _FAMILY_BY_KIND.get(test.get("Kind"), f"kind-{test.get('Kind', 'unknown')}") + + source = str(test.get("SrcNode") or test.get("SrcPod") or "unknown-source") + destination = str(test.get("DstNode") or test.get("DstPod") or "unknown-destination") + source_rail = str(test.get("SrcRail") or test.get("Rail") or "unknown-rail") + destination_rail = str(test.get("DstRail") or test.get("Rail") or "unknown-rail") + expectation = str(row.get("Expectation") or test.get("Expectation") or "required") + details = [f"expectation={expectation}", f"observedOK={row.get('ObservedOK')}"] + + bandwidth = row.get("BandwidthGbps") + minimum = row.get("MinBandwidthGbps") + if bandwidth is not None or minimum is not None: + details.extend([f"bandwidthGbps={bandwidth}", f"minimumGbps={minimum}"]) + + source_gpu = test.get("SrcGPUIndex") + destination_gpu = test.get("DstGPUIndex") + if source_gpu is not None or destination_gpu is not None: + details.append(f"gpuIndices={source_gpu}->{destination_gpu}") + source_gpu_pci = test.get("SrcGPUPCIAddress") + destination_gpu_pci = test.get("DstGPUPCIAddress") + if source_gpu_pci: + details.append(f"sourceGpuPci={source_gpu_pci}") + if destination_gpu_pci: + details.append(f"destinationGpuPci={destination_gpu_pci}") + + stderr = str(row.get("Stderr") or "").strip() + error = str(row.get("Error") or "").strip() + if stderr: + details.append(f"stderr={stderr}") + if error and error != stderr: + details.append(f"error={error}") + + return { + "name": f"{family}/{source}->{destination}/{source_rail}->{destination_rail}", + "passed": row.get("OK") is True, + "message": ", ".join(details) or f"matrix row {index}", } - return [ - { - "name": f"kind-coverage/{kind}", - "passed": kind in observed, - "message": f"observed kinds: {', '.join(sorted(observed)) or '(none)'}", - } - for kind in sorted(expected_kinds) - ] - - -class LaunchKitKubernetesPrerequisiteCheck(_LaunchKitCheck): - """Require a reachable Kubernetes API and a non-empty Ready-node inventory.""" - - description: ClassVar[str] = "Check Kubernetes prerequisites before Launch Kit execution" def run(self) -> None: - """Report every provider preflight probe.""" - output = self._configured_output("preflight_output") or self._step_output("kubernetes-preflight") - if output is None: - return - probes = [probe for probe in _list(output.get("checks")) if isinstance(probe, dict)] - self._finish_probes("Kubernetes prerequisite", probes) - - -class LaunchKitTopologyDiscoveryCheck(_LaunchKitCheck): - """Validate that Launch Kit completed discovery and resolved a profile.""" - - description: ClassVar[str] = "Check cluster topology discovery with Kubernetes Launch Kit" - - def run(self) -> None: - """Check the real discover JSONResult without inventing topology fields.""" - output = self._configured_output("discover_output") or self._step_output("discover") - if output is None: - return - documents = self._documents(output) - document = documents[0] if len(documents) == 1 else {} - profile = _object(document.get("profile")) - probes = [ - { - "name": "discover-command", - "passed": output.get("success") is True and document.get("success") is True, - "message": str(output.get("error") or f"phase={document.get('phase')!r}"), - }, - { - "name": "resolved-profile", - "passed": bool(profile.get("fabric") and profile.get("deployment")), - "message": f"fabric={profile.get('fabric')}, deployment={profile.get('deployment')}", - }, - ] - self._finish_probes("Launch Kit topology discovery", probes) - - -class LaunchKitDeploymentHealthCheck(_LaunchKitCheck): - """Validate the Network Operator release and every generated resource.""" - - description: ClassVar[str] = "Check Network Operator deployment health with Kubernetes Launch Kit" - - def run(self) -> None: - """Report version and manifest readiness independently of connectivity.""" - output = self._step_output("validate") - if output is None: - return - static = self._static_document(output) - if static is None: + """Expose the complete Launch Kit connectivity matrix as subtests.""" + connectivity = self._connectivity() + if connectivity is None: return - version = _object(static.get("versionCheck")) - summary = _object(static.get("summary")) - version_skipped = version.get("Skipped") is True probes = [ - { - "name": "launch-kit-validate-command", - "passed": output.get("success") is True and output.get("exit_code") == 0, - "message": str( - output.get("error") or f"success={output.get('success')!r}, exitCode={output.get('exit_code')!r}" - ), - }, - { - "name": "network-operator-version", - "passed": not version_skipped and version.get("Match") is True, - "skipped": version_skipped, - "message": ( - str(version.get("Reason")) - if version_skipped - else ( - f"selected={version.get('SelectedRelease')}, expected={version.get('ExpectedVersion')}, " - f"deployed={_object(version.get('DeployedRelease')).get('ChartVersion')}" - ) - ), - }, - { - "name": "static-summary", - "passed": summary.get("success") is True, - "message": ( - f"success={summary.get('successManifests')}/{summary.get('totalManifests')}, " - f"errors={summary.get('errorManifests')}, missing={summary.get('missingManifests')}" - ), - }, - { - "name": "manifest-inventory", - "passed": bool(_list(static.get("manifests"))), - "message": f"rows={len(_list(static.get('manifests')))}", - }, - *self._manifest_probes(output), + self._probe(row, index) + for index, row in enumerate(_sequence(connectivity.get("PingResults")), start=1) + if isinstance(row, dict) ] - self._finish_probes("Network Operator deployment health", probes) - - -class LaunchKitSriovReadinessCheck(_LaunchKitCheck): - """Validate SR-IOV policies and secondary-network resources.""" - - description: ClassVar[str] = "Check SR-IOV Network RDMA readiness with Kubernetes Launch Kit" - - def run(self) -> None: - """Check applicable validated resources for an SR-IOV profile.""" - output = self._step_output("validate") - if output is None: - return - profile = self._profile() - if profile is None: - return - if profile.get("deployment") != "sriov": - pytest.skip(f"selected Launch Kit deployment is {profile.get('deployment')}, not sriov") - fabric = profile.get("fabric") - network_kind = "SriovIBNetwork" if fabric == "infiniband" else "SriovNetwork" - expected_kinds = {"SriovNetworkNodePolicy", network_kind} - probes = self._kind_coverage_probes(output, expected_kinds) - probes.extend( - self._manifest_probes( - output, - required_kinds=expected_kinds, - ) - ) - self._finish_probes("SR-IOV readiness", probes) - - -class LaunchKitRdmaConnectivityCheck(_LaunchKitCheck): - """Validate every rping matrix result.""" - - description: ClassVar[str] = "Check pod-to-pod RDMA connectivity with Kubernetes Launch Kit" - - def run(self) -> None: - """Report all same-rail and cross-rail RDMA-CM observations.""" - output = self._step_output("validate") - if output is not None: - self._finish_probes("RDMA-CM connectivity", self._matrix_probes(output, {"rping"})) - - -class LaunchKitRoceCheck(_LaunchKitCheck): - """Validate the selected Ethernet/RoCE profile resources.""" - - description: ClassVar[str] = "Check RoCE secondary networking with Kubernetes Launch Kit" - - def run(self) -> None: - """Skip non-Ethernet profiles and report the selected network resources.""" - output = self._step_output("validate") - if output is None: - return - profile = self._profile() - if profile is None: - return - if profile.get("fabric") != "ethernet": - pytest.skip(f"selected Launch Kit fabric is {profile.get('fabric')}, not ethernet") - network_kind = _profile_network_kind(profile) - if network_kind is None: - self.set_failed(f"Launch Kit returned an unsupported profile: {profile}") - return - probes = self._kind_coverage_probes(output, {network_kind}) - probes.extend(self._manifest_probes(output, required_kinds={network_kind})) - self._finish_probes("Ethernet/RoCE profile", probes) - - -class LaunchKitInfiniBandCheck(_LaunchKitCheck): - """Validate the selected InfiniBand profile resources.""" - - description: ClassVar[str] = "Check InfiniBand networking with Kubernetes Launch Kit" - - def run(self) -> None: - """Skip non-IB profiles and report IB network resources.""" - output = self._step_output("validate") - if output is None: - return - profile = self._profile() - if profile is None: - return - if profile.get("fabric") != "infiniband": - pytest.skip(f"selected Launch Kit fabric is {profile.get('fabric')}, not infiniband") - network_kind = _profile_network_kind(profile) - if network_kind is None: - self.set_failed(f"Launch Kit returned an unsupported profile: {profile}") - return - probes = self._kind_coverage_probes(output, {network_kind}) - probes.extend(self._manifest_probes(output, required_kinds={network_kind})) - self._finish_probes("InfiniBand profile", probes) - - -class LaunchKitHostDeviceCheck(_LaunchKitCheck): - """Validate an applicable host-device profile.""" - - description: ClassVar[str] = "Check host-device networking with Kubernetes Launch Kit" - - def run(self) -> None: - """Skip other deployment types and report HostDeviceNetwork rows.""" - output = self._step_output("validate") - if output is None: - return - profile = self._profile() - if profile is None: - return - if profile.get("deployment") != "host_device": - pytest.skip(f"selected Launch Kit deployment is {profile.get('deployment')}, not host_device") - probes = self._kind_coverage_probes(output, {"HostDeviceNetwork"}) - probes.extend(self._manifest_probes(output, required_kinds={"HostDeviceNetwork"})) - self._finish_probes( - "host-device networking", - probes, - ) - - -class LaunchKitSecondaryNetworkCheck(_LaunchKitCheck): - """Validate secondary-network resources and test DaemonSet readiness.""" - - description: ClassVar[str] = "Check secondary-network and IPAM readiness with Kubernetes Launch Kit" - - def run(self) -> None: - """Report network/IPPool manifests and test-pod rollout state.""" - output = self._step_output("validate") - if output is None: - return - profile = self._profile() - if profile is None: - return - network_kind = _profile_network_kind(profile) - if network_kind is None: - self.set_failed(f"Launch Kit returned an unsupported profile: {profile}") - return - network_kinds = {"SriovNetwork", "SriovIBNetwork", "MacvlanNetwork", "IPoIBNetwork", "HostDeviceNetwork"} - probes = self._kind_coverage_probes(output, {"IPPool", network_kind}) - probes.extend( - self._manifest_probes( - output, - required_kinds={"IPPool", *network_kinds}, - ) - ) - static = self._static_document(output) - if static is None: - return - observed_network_kinds = { - str(item.get("Kind")) - for item in _list(static.get("manifests")) - if isinstance(item, dict) and item.get("Kind") in network_kinds - } - probes.append( - { - "name": "kind-coverage/secondary-network", - "passed": bool(observed_network_kinds), - "message": f"observed kinds: {', '.join(sorted(observed_network_kinds)) or '(none)'}", - } - ) - connectivity = self._connectivity(output) - if connectivity is None: - return - for daemonset in _list(connectivity.get("DaemonSets")): - if not isinstance(daemonset, dict): - continue - ref = _object(daemonset.get("Ref")) - rollout = _object(daemonset.get("Rollout")) - desired = rollout.get("Desired") - ready = rollout.get("Ready") - not_ready = rollout.get("NotReady") - valid_counts = all(type(value) is int and value >= 0 for value in (desired, ready, not_ready)) - rollout_detail = f"ready={ready}/{desired}, notReady={not_ready}" - if not valid_counts: - rollout_detail += ", invalid or missing integer rollout counts" - probes.append( - { - "name": f"DaemonSet/{ref.get('Namespace')}/{ref.get('Name')}", - "passed": valid_counts and desired > 0 and ready == desired and not_ready == 0, - "message": rollout_detail, - } - ) - self._finish_probes("secondary-network readiness", probes) - - -class LaunchKitRdmaSharedCheck(_LaunchKitCheck): - """Validate an applicable RDMA Shared profile.""" - - description: ClassVar[str] = "Check RDMA Shared networking with Kubernetes Launch Kit" - - def run(self) -> None: - """Skip other profiles and report Macvlan or IPoIB resources.""" - output = self._step_output("validate") - if output is None: - return - profile = self._profile() - if profile is None: - return - if profile.get("deployment") != "rdma_shared": - pytest.skip(f"selected Launch Kit deployment is {profile.get('deployment')}, not rdma_shared") - network_kind = _profile_network_kind(profile) - if network_kind is None: - self.set_failed(f"Launch Kit returned an unsupported profile: {profile}") - return - probes = self._kind_coverage_probes(output, {network_kind}) - probes.extend(self._manifest_probes(output, required_kinds={network_kind})) - self._finish_probes( - "RDMA Shared networking", - probes, - ) - - -class LaunchKitIcmpConnectivityCheck(_LaunchKitCheck): - """Validate every source-bound ICMP matrix result.""" - - description: ClassVar[str] = "Check source-bound ICMP connectivity with Kubernetes Launch Kit" - - def run(self) -> None: - """Report all same-rail and expected-isolation ICMP observations.""" - output = self._step_output("validate") - if output is not None: - self._finish_probes("source-bound ICMP", self._matrix_probes(output, {"icmp"})) - - -class LaunchKitRdmaBandwidthCheck(_LaunchKitCheck): - """Validate every ib_write_bw matrix result and its Launch Kit threshold.""" - - description: ClassVar[str] = "Check RDMA bandwidth with Kubernetes Launch Kit" - - def run(self) -> None: - """Use the observed and minimum bandwidth emitted by Launch Kit.""" - output = self._step_output("validate") - if output is not None: - self._finish_probes("RDMA bandwidth", self._matrix_probes(output, {"ib_write_bw"})) - - -class LaunchKitGpuDirectRdmaCheck(_LaunchKitCheck): - """Validate every Launch Kit GPUDirect DMA-BUF bandwidth result.""" - - description: ClassVar[str] = "Check GPUDirect RDMA DMA-BUF bandwidth with Kubernetes Launch Kit" - - def run(self) -> None: - """Report endpoint GPU topology and Launch Kit's bandwidth verdict.""" - output = self._step_output("validate") - if output is None: - return - probes = self._matrix_probes(output, {"gpudirect_dmabuf"}) if not probes: - if self._error: - return - pytest.skip( - "Launch Kit emitted no gpudirect_dmabuf results; validation.gpuDirect is disabled " - "or ib_write_bw is not selected" - ) - self._finish_probes("GPUDirect RDMA DMA-BUF bandwidth", probes) - - -class LaunchKitMultirailCheck(_LaunchKitCheck): - """Validate same-rail reachability and expected cross-rail isolation.""" - - description: ClassVar[str] = "Check multi-rail connectivity behavior with Kubernetes Launch Kit" - - def run(self) -> None: - """Require same-rail and cross-rail coverage when multiple rails exist.""" - output = self._step_output("validate") - if output is None: - return - profile = self._profile() - if profile is None: + self.set_failed("Launch Kit connectivity matrix produced no results") return - multirail = profile.get("multirail") - if multirail not in {True, "true"}: - pytest.skip(f"selected Launch Kit profile is not multirail: {multirail!r}") - probes = self._matrix_probes(output, set(_CONNECTIVITY_FAMILIES)) - rails = {rail for probe in probes for rail in (probe["source_rail"], probe["destination_rail"])} - if len(rails) == 1 and "unknown-rail" not in rails: - pytest.skip(f"Launch Kit connectivity matrix contains only one rail: {next(iter(rails))}") - same_rail = [probe for probe in probes if probe["source_rail"] == probe["destination_rail"]] - cross_rail = [probe for probe in probes if probe["source_rail"] != probe["destination_rail"]] - probes.extend( - [ - {"name": "same-rail-coverage", "passed": bool(same_rail), "message": f"rows={len(same_rail)}"}, - {"name": "cross-rail-coverage", "passed": bool(cross_rail), "message": f"rows={len(cross_rail)}"}, - ] - ) - self._finish_probes("multi-rail behavior", probes) - - -def _artifact_paths(value: Any) -> list[Path]: - """Recursively collect paths from provider artifact mappings.""" - if isinstance(value, dict): - paths: list[Path] = [] - for child in value.values(): - paths.extend(_artifact_paths(child)) - return paths - if isinstance(value, list): - paths = [] - for child in value: - paths.extend(_artifact_paths(child)) - return paths - if isinstance(value, str) and value: - return [Path(value)] - return [] - - -def _evidence_path(value: str, output: dict[str, Any]) -> Path: - """Resolve a Launch Kit-emitted path against its command working directory.""" - path = Path(value) - if path.is_absolute(): - return path - working_directory = output.get("working_directory") - if isinstance(working_directory, str) and working_directory: - return Path(working_directory) / path - return path - - -class LaunchKitEvidenceCaptureCheck(_LaunchKitCheck): - """Validate raw command logs and the Launch Kit HTML report.""" - description: ClassVar[str] = "Check Launch Kit evidence capture" - - def run(self) -> None: - """Require fresh evidence for every real workflow command.""" - outputs = { - "verify": self._configured_output("verify_output"), - "preflight": self._configured_output("preflight_output"), - "discover": self._configured_output("discover_output"), - "generate": self._configured_output("generate_output"), - } - deploy = self._configured_output("deploy_output") - if deploy: - outputs["deploy"] = deploy - outputs["validate"] = self._step_output("validate") or {} - prepare = self._configured_output("prepare_output") - if prepare: - outputs = {"prepare": prepare, **outputs} - probes: list[dict[str, Any]] = [] - for operation, output in outputs.items(): - paths = _artifact_paths(output.get("artifacts")) - existing = [path for path in paths if path.is_file()] - probes.append( - { - "name": f"{operation}-artifacts", - "passed": bool(paths) and len(existing) == len(paths), - "message": f"found {len(existing)}/{len(paths)} files", - } + failures: list[str] = [] + for probe in probes: + self.report_subtest( + probe["name"], + passed=probe["passed"], + message=probe["message"], ) - generated_paths = [ - _evidence_path(path, outputs["generate"]) - for document in self._documents(outputs["generate"]) - for path in _list(document.get("generatedFiles")) - if isinstance(path, str) - ] - probes.append( - { - "name": "generated-files", - "passed": bool(generated_paths) and all(path.is_file() for path in generated_paths), - "message": ( - f"found {sum(path.is_file() for path in generated_paths)}/{len(generated_paths)} generated files" - ), - } - ) - validate = outputs["validate"] - report_paths = [ - _evidence_path(document["reportPath"], validate) - for document in self._documents(validate) - if isinstance(document.get("reportPath"), str) - ] - probes.append( - { - "name": "launch-kit-html-report", - "passed": bool(report_paths) and all(path.is_file() for path in report_paths), - "message": ", ".join(str(path) for path in report_paths) or "no reportPath document", - } - ) - self._finish_probes("Launch Kit evidence capture", probes) + if not probe["passed"]: + failures.append(f"{probe['name']}: {probe['message']}") + if failures: + self.set_failed("Launch Kit connectivity failed: " + "; ".join(failures)) + return + self.set_passed(f"Launch Kit connectivity passed ({len(probes)} results)") diff --git a/isvtest/tests/k8s_launch_kit/test_checks.py b/isvtest/tests/k8s_launch_kit/test_checks.py index 29ca2a37c..51324e490 100644 --- a/isvtest/tests/k8s_launch_kit/test_checks.py +++ b/isvtest/tests/k8s_launch_kit/test_checks.py @@ -1,95 +1,28 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for Kubernetes Launch Kit result interpretation.""" +"""Unit tests for Kubernetes Launch Kit connectivity interpretation.""" from __future__ import annotations -import json -from pathlib import Path from typing import Any import pytest -from isvtest.core.validation import BaseValidation -from isvtest.validations.k8s_launch_kit.checks import ( - LaunchKitDeploymentHealthCheck, - LaunchKitEvidenceCaptureCheck, - LaunchKitGpuDirectRdmaCheck, - LaunchKitHostDeviceCheck, - LaunchKitIcmpConnectivityCheck, - LaunchKitInfiniBandCheck, - LaunchKitKubernetesPrerequisiteCheck, - LaunchKitMultirailCheck, - LaunchKitRdmaBandwidthCheck, - LaunchKitRdmaConnectivityCheck, - LaunchKitRdmaSharedCheck, - LaunchKitRoceCheck, - LaunchKitSecondaryNetworkCheck, - LaunchKitSriovReadinessCheck, - LaunchKitTopologyDiscoveryCheck, -) +from isvtest.validations.k8s_launch_kit.checks import LaunchKitConnectivityCheck pytestmark = pytest.mark.unit -_NETWORK_KIND = { - ("ethernet", "sriov"): "SriovNetwork", - ("infiniband", "sriov"): "SriovIBNetwork", - ("ethernet", "rdma_shared"): "MacvlanNetwork", - ("infiniband", "rdma_shared"): "IPoIBNetwork", - ("ethernet", "host_device"): "HostDeviceNetwork", - ("infiniband", "host_device"): "HostDeviceNetwork", -} - -def _discover( - fabric: str = "ethernet", - deployment: str = "sriov", +def _matrix_row( + family: str | None, + kind: int, *, - multirail: bool = True, + passed: bool = True, + destination_rail: str = "rail-0", ) -> dict[str, Any]: - """Build the provider envelope around a real discover-shaped document.""" - return { - "success": True, - "platform": "kubernetes", - "operation": "discover", - "exit_code": 0, - "documents": [ - { - "success": True, - "phase": "discover", - "profile": { - "fabric": fabric, - "deployment": deployment, - "multirail": "true" if multirail else "false", - }, - "deployed": False, - "messages": [], - } - ], - "artifacts": {}, - } - - -def _manifest(kind: str, *, state: str = "success") -> dict[str, Any]: - """Build one exported manifest validation row.""" - return { - "Kind": kind, - "APIVersion": "example.nvidia.com/v1", - "Name": f"mock-{kind.lower()}", - "Namespace": "default", - "State": state, - "Reason": "resource exists and is Ready" if state == "success" else "rollout has 1 unavailable pod", - "Found": True, - "Missing": False, - } - - -def _matrix_row(kind: int, *, same_rail: bool, passed: bool = True) -> dict[str, Any]: - """Build one exported connectivity row with source and destination detail.""" - destination_rail = "rail-0" if same_rail else "rail-1" - family = "icmp" if kind < 2 else "rping" if kind < 4 else "ib_write_bw" if kind < 6 else "gpudirect_dmabuf" - bandwidth_family = family in {"ib_write_bw", "gpudirect_dmabuf"} + """Build one Launch Kit connectivity result.""" + bandwidth = family in {"ib_write_bw", "gpudirect_dmabuf"} return { "Test": { "Kind": kind, @@ -97,7 +30,7 @@ def _matrix_row(kind: int, *, same_rail: bool, passed: bool = True) -> dict[str, "DstNode": "worker-b", "SrcRail": "rail-0", "DstRail": destination_rail, - "Expectation": "required" if same_rail else "forbidden", + "Expectation": "required", **( { "SrcGPUIndex": 2, @@ -109,422 +42,137 @@ def _matrix_row(kind: int, *, same_rail: bool, passed: bool = True) -> dict[str, else {} ), }, - "Family": family, + **({"Family": family} if family is not None else {}), "OK": passed, - "ObservedOK": same_rail if passed else False, - "Expectation": "required" if same_rail else "forbidden", - "BandwidthGbps": 187.6 if bandwidth_family and passed else 42.5, - "MinBandwidthGbps": 100.0 if bandwidth_family else 0.0, - "Stderr": "" if passed else f"{family}: connection refused on rail-0", - **({"Error": f"{family} validation failed"} if not passed else {}), + "ObservedOK": passed, + "Expectation": "required", + **( + { + "BandwidthGbps": 187.6 if passed else 42.5, + "MinBandwidthGbps": 100.0, + } + if bandwidth + else {} + ), + **( + { + "Stderr": f"{family}: connection refused on rail-0", + "Error": f"{family} validation failed", + } + if not passed + else {} + ), } -def _validate( - fabric: str = "ethernet", - deployment: str = "sriov", - *, - failed_kind: str | None = None, - failed_family: str | None = None, - include_sriov_policy: bool = True, -) -> dict[str, Any]: - """Build a validate transport envelope with static and matrix documents.""" - network_kind = _NETWORK_KIND[(fabric, deployment)] - kinds = ["NicClusterPolicy", "NicNodePolicy", "IPPool"] - if deployment == "sriov" and include_sriov_policy: - kinds.append("SriovNetworkNodePolicy") - kinds.append(network_kind) - manifests = [_manifest(kind, state="error" if kind == failed_kind else "success") for kind in kinds] - rows: list[dict[str, Any]] = [] - for family, pair in { - "icmp": (0, 1), - "rping": (2, 3), - "ib_write_bw": (4, 5), - "gpudirect_dmabuf": (6, 7), - }.items(): - rows.append(_matrix_row(pair[0], same_rail=True, passed=family != failed_family)) - rows.append(_matrix_row(pair[1], same_rail=False)) - failed_manifests = sum(item["State"] != "success" for item in manifests) - failed_rows = sum(row["OK"] is not True for row in rows) +def _validate(rows: list[dict[str, Any]]) -> dict[str, Any]: + """Wrap matrix rows in the provider contract consumed by the check.""" + failed = sum(row.get("OK") is not True for row in rows) return { - "success": failed_rows == 0, + "success": failed == 0, "platform": "kubernetes", "operation": "validate", - "exit_code": 0 if failed_rows == 0 else 4, + "exit_code": 0 if failed == 0 else 4, "documents": [ - { - "versionCheck": { - "Skipped": False, - "SelectedRelease": "26.4", - "ExpectedVersion": "v26.4.1", - "DeployedRelease": {"ChartVersion": "26.4.1"}, - "Match": True, - }, - "manifests": manifests, - "presetDeviations": [], - "summary": { - "totalManifests": len(manifests), - "successManifests": len(manifests) - failed_manifests, - "errorManifests": failed_manifests, - "missingManifests": 0, - "success": failed_manifests == 0, - }, - }, + {"versionCheck": {}, "manifests": [], "summary": {}}, { "connectivity": { - "DaemonSets": [ - { - "Ref": {"Namespace": "default", "Name": "l8k-network-test"}, - "Rollout": {"Desired": 2, "Ready": 2, "NotReady": 0}, - } - ], "PingResults": rows, - "Summary": {"TotalTests": len(rows), "Failed": failed_rows}, + "Summary": {"TotalTests": len(rows), "Failed": failed}, } }, ], - "artifacts": {}, - **({"error": "one or more connectivity rows failed"} if failed_rows else {}), - } - - -def _config( - output: dict[str, Any], - *, - discover: dict[str, Any] | None = None, - **extra: Any, -) -> dict[str, Any]: - """Bind a command envelope and optional earlier step outputs to a check.""" - config = {"step_output": output, **extra} - if discover is not None: - config["discover_output"] = json.dumps(discover) - return config - - -def test_kubernetes_prerequisite_reports_every_probe() -> None: - """A failed prerequisite retains successful checks and its remediation detail.""" - output = { - "success": False, - "platform": "kubernetes", - "operation": "kubernetes-preflight", - "checks": [ - {"name": "api-version", "passed": True, "message": "server v1.34.1"}, - {"name": "nodes", "passed": False, "message": "Forbidden: cannot list nodes"}, - {"name": "non-empty-cluster", "passed": False, "message": "cluster contains no nodes"}, - ], + **({"error": "one or more connectivity rows failed"} if failed else {}), } - result = LaunchKitKubernetesPrerequisiteCheck(config=_config(output)).execute() - - assert result["passed"] is False - assert [probe["name"] for probe in result["subtests"]] == ["api-version", "nodes", "non-empty-cluster"] - assert "Forbidden: cannot list nodes" in result["error"] - - -def test_topology_discovery_uses_the_real_profile_document() -> None: - """Discovery succeeds only when l8k resolves both fabric and deployment.""" - result = LaunchKitTopologyDiscoveryCheck(config=_config(_discover())).execute() - - assert result["passed"] is True - assert [probe["name"] for probe in result["subtests"]] == ["discover-command", "resolved-profile"] - -def test_deployment_health_reports_all_resources_before_failing() -> None: - """One unhealthy manifest is named without hiding later manifest rows.""" - output = _validate(failed_kind="SriovNetworkNodePolicy") +def _execute(output: dict[str, Any]) -> dict[str, Any]: + """Execute the catalog check against one provider envelope.""" + return LaunchKitConnectivityCheck(config={"step_output": output}).execute() - result = LaunchKitDeploymentHealthCheck(config=_config(output)).execute() - assert result["passed"] is False - names = [probe["name"] for probe in result["subtests"]] - assert "SriovNetworkNodePolicy/default/mock-sriovnetworknodepolicy" in names - assert names[-1] == "SriovNetwork/default/mock-sriovnetwork" - assert "rollout has 1 unavailable pod" in result["error"] - - -def test_deployment_health_honors_the_launch_kit_exit_verdict() -> None: - """A validate-level drift failure cannot be hidden by green manifest rows.""" - output = _validate() - output["success"] = False - output["exit_code"] = 4 - output["error"] = "l8k validate exited with code 4: component versions diverge" - - result = LaunchKitDeploymentHealthCheck(config=_config(output)).execute() - - assert result["passed"] is False - assert result["subtests"][0]["name"] == "launch-kit-validate-command" - assert "component versions diverge" in result["error"] - - -def test_deployment_health_allows_an_unconfigured_version_expectation() -> None: - """An optional Launch Kit version check is reported as skipped, not failed.""" - output = _validate() - output["documents"][0]["versionCheck"] = { - "Skipped": True, - "Reason": "cluster config has no selectedRelease", - } - - result = LaunchKitDeploymentHealthCheck(config=_config(output)).execute() - - assert result["passed"] is True - assert result["subtests"][1] == { - "name": "network-operator-version", - "passed": False, - "skipped": True, - "message": "cluster config has no selectedRelease", - "duration": None, - } - - -def test_sriov_readiness_requires_policy_and_network_kinds() -> None: - """A non-vacuous SR-IOV result requires both policy and attachment resources.""" - discover = _discover("infiniband", "sriov") - output = _validate("infiniband", "sriov", include_sriov_policy=False) - - result = LaunchKitSriovReadinessCheck(config=_config(output, discover=discover)).execute() - - assert result["passed"] is False - assert "kind-coverage/SriovNetworkNodePolicy" in result["error"] - assert any(probe["name"].startswith("SriovIBNetwork/") for probe in result["subtests"]) - - -@pytest.mark.parametrize( - ("check_class", "fabric", "deployment", "expected_kind"), - [ - (LaunchKitRoceCheck, "ethernet", "sriov", "SriovNetwork"), - (LaunchKitInfiniBandCheck, "infiniband", "sriov", "SriovIBNetwork"), - (LaunchKitHostDeviceCheck, "ethernet", "host_device", "HostDeviceNetwork"), - (LaunchKitRdmaSharedCheck, "infiniband", "rdma_shared", "IPoIBNetwork"), - ], -) -def test_profile_checks_require_the_applicable_network_kind( - check_class: type[BaseValidation], - fabric: str, - deployment: str, - expected_kind: str, -) -> None: - """Profile-specific checks select the exact resource implied by discover.""" - result = check_class( - config=_config(_validate(fabric, deployment), discover=_discover(fabric, deployment)) - ).execute() - - assert result["passed"] is True - names = [probe["name"] for probe in result["subtests"]] - assert names[0] == f"kind-coverage/{expected_kind}" - assert len(names) == len(set(names)) - - -def test_non_applicable_profile_is_skipped() -> None: - """An individually selectable check is skipped when the selected profile does not apply.""" - check = LaunchKitInfiniBandCheck(config=_config(_validate(), discover=_discover())) - - with pytest.raises(pytest.skip.Exception, match="not infiniband"): - check.execute() +def test_reports_every_emitted_connectivity_family() -> None: + """The single check exposes every Launch Kit matrix row in stream order.""" + rows = [ + _matrix_row("icmp", 0), + _matrix_row("rping", 2), + _matrix_row("ib_write_bw", 4), + _matrix_row("gpudirect_dmabuf", 6), + ] - -@pytest.mark.parametrize( - ("check_class", "family"), - [ - (LaunchKitIcmpConnectivityCheck, "icmp"), - (LaunchKitRdmaConnectivityCheck, "rping"), - (LaunchKitRdmaBandwidthCheck, "ib_write_bw"), - (LaunchKitGpuDirectRdmaCheck, "gpudirect_dmabuf"), - ], -) -def test_connectivity_checks_create_source_bound_subtests( - check_class: type[BaseValidation], - family: str, -) -> None: - """Each matrix row becomes an independently named report item.""" - result = check_class(config=_config(_validate())).execute() + result = _execute(_validate(rows)) assert result["passed"] is True - assert [probe["name"] for probe in result["subtests"]] == [ - f"{family}/worker-a->worker-b/rail-0->rail-0", - f"{family}/worker-a->worker-b/rail-0->rail-1", + assert [subtest["name"] for subtest in result["subtests"]] == [ + f"{family}/worker-a->worker-b/rail-0->rail-0" for family in ("icmp", "rping", "ib_write_bw", "gpudirect_dmabuf") ] -def test_connectivity_failure_preserves_stderr_and_bandwidth() -> None: - """A bandwidth failure includes endpoints, rails, observation, and threshold.""" - result = LaunchKitRdmaBandwidthCheck(config=_config(_validate(failed_family="ib_write_bw"))).execute() +def test_failure_preserves_endpoints_rails_bandwidth_and_stderr() -> None: + """A failed row retains the diagnostics emitted by Launch Kit.""" + result = _execute(_validate([_matrix_row("ib_write_bw", 4, passed=False)])) assert result["passed"] is False - assert len(result["subtests"]) == 2 + assert result["subtests"][0]["passed"] is False + assert "ib_write_bw/worker-a->worker-b/rail-0->rail-0" in result["error"] assert "bandwidthGbps=42.5" in result["error"] assert "minimumGbps=100.0" in result["error"] assert "connection refused on rail-0" in result["error"] -def test_gpudirect_failure_preserves_endpoint_gpu_and_bandwidth_evidence() -> None: - """A DMA-BUF failure identifies both endpoint GPUs and the failed threshold.""" - result = LaunchKitGpuDirectRdmaCheck(config=_config(_validate(failed_family="gpudirect_dmabuf"))).execute() - - assert result["passed"] is False - assert "gpuIndices=2->5" in result["error"] - assert "sourceGpuPci=0000:41:00.0" in result["error"] - assert "destinationGpuPci=0000:71:00.0" in result["error"] - assert "bandwidthGbps=42.5" in result["error"] - assert "minimumGbps=100.0" in result["error"] - assert "error=gpudirect_dmabuf validation failed" in result["error"] - - -def test_gpudirect_prefers_the_exported_family_contract() -> None: - """The stable Family field selects GPUDirect even if numeric kinds evolve.""" - output = _validate() - rows = output["documents"][1]["connectivity"]["PingResults"] - gpudirect_rows = [row for row in rows if row["Family"] == "gpudirect_dmabuf"] - for row in gpudirect_rows: - row["Test"]["Kind"] = 999 - output["documents"][1]["connectivity"]["PingResults"] = gpudirect_rows - - result = LaunchKitGpuDirectRdmaCheck(config=_config(output)).execute() +def test_gpudirect_details_are_reported_without_a_separate_expected_check() -> None: + """GPUDirect rows are ordinary connectivity rows when Launch Kit emits them.""" + result = _execute(_validate([_matrix_row("gpudirect_dmabuf", 6)])) assert result["passed"] is True - assert len(result["subtests"]) == 2 - + message = result["subtests"][0]["message"] + assert "gpuIndices=2->5" in message + assert "sourceGpuPci=0000:41:00.0" in message + assert "destinationGpuPci=0000:71:00.0" in message -def test_gpudirect_rejects_missing_endpoint_gpu_indices() -> None: - """A green row without explicit endpoint GPU topology is not accepted.""" - output = _validate() - row = next( - row for row in output["documents"][1]["connectivity"]["PingResults"] if row["Family"] == "gpudirect_dmabuf" - ) - row["Test"].pop("DstGPUIndex") - result = LaunchKitGpuDirectRdmaCheck(config=_config(output)).execute() +def test_disabled_gpudirect_requires_no_skip_or_placeholder() -> None: + """Families disabled by user config are absent instead of becoming skipped tests.""" + result = _execute(_validate([_matrix_row("icmp", 0), _matrix_row("rping", 2)])) - assert result["passed"] is False - assert "invalid or missing endpoint GPU index" in result["error"] + assert result["passed"] is True + assert all("gpudirect" not in subtest["name"] for subtest in result["subtests"]) + assert all(subtest["skipped"] is False for subtest in result["subtests"]) -def test_gpudirect_is_skipped_when_launch_kit_does_not_emit_the_family() -> None: - """A discovery-disabled GPUDirect family is inapplicable, not failed.""" - output = _validate() - output["documents"][1]["connectivity"]["PingResults"] = [ - row for row in output["documents"][1]["connectivity"]["PingResults"] if row["Family"] != "gpudirect_dmabuf" - ] - check = LaunchKitGpuDirectRdmaCheck(config=_config(output)) +def test_explicit_future_family_is_not_filtered_out() -> None: + """The wrapper forwards new Launch Kit families without a catalog update.""" + result = _execute(_validate([_matrix_row("future_connectivity", 999)])) - with pytest.raises(pytest.skip.Exception, match=r"validation\.gpuDirect is disabled"): - check.execute() + assert result["passed"] is True + assert result["subtests"][0]["name"].startswith("future_connectivity/") -def test_secondary_network_requires_ipam_network_and_ready_test_pods() -> None: - """Secondary-network coverage combines static resources with workload readiness.""" - discover = _discover("ethernet", "rdma_shared") - result = LaunchKitSecondaryNetworkCheck( - config=_config(_validate("ethernet", "rdma_shared"), discover=discover) - ).execute() +def test_legacy_numeric_kind_resolves_a_family() -> None: + """Older Launch Kit output without Family remains readable.""" + result = _execute(_validate([_matrix_row(None, 3)])) assert result["passed"] is True - names = {probe["name"] for probe in result["subtests"]} - assert {"kind-coverage/IPPool", "kind-coverage/MacvlanNetwork", "DaemonSet/default/l8k-network-test"} <= names + assert result["subtests"][0]["name"].startswith("rping/") @pytest.mark.parametrize( - "rollout", + "output", [ - {}, - {"Desired": 2, "Ready": 2}, - {"Desired": 0, "Ready": 0, "NotReady": 0}, - {"Desired": True, "Ready": True, "NotReady": 0}, + {"operation": "validate", "documents": [], "error": "Kubernetes client failed"}, + {"operation": "discover", "documents": [{"connectivity": {"PingResults": []}}]}, ], ) -def test_secondary_network_rejects_incomplete_or_empty_daemonset_rollout(rollout: dict[str, Any]) -> None: - """Missing, invalid, or zero-sized rollout counts cannot pass vacuously.""" - discover = _discover("ethernet", "rdma_shared") - output = _validate("ethernet", "rdma_shared") - output["documents"][1]["connectivity"]["DaemonSets"][0]["Rollout"] = rollout - - result = LaunchKitSecondaryNetworkCheck(config=_config(output, discover=discover)).execute() +def test_rejects_missing_or_wrong_validate_output(output: dict[str, Any]) -> None: + """Transport failures remain actionable instead of passing vacuously.""" + result = _execute(output) assert result["passed"] is False - rollout_probe = next(probe for probe in result["subtests"] if probe["name"].startswith("DaemonSet/")) - assert rollout_probe["passed"] is False - -def test_multirail_requires_same_and_cross_rail_coverage() -> None: - """Multi-rail validation distinguishes same-rail reachability from isolation rows.""" - result = LaunchKitMultirailCheck(config=_config(_validate(), discover=_discover())).execute() - - assert result["passed"] is True - assert result["subtests"][-2]["name"] == "same-rail-coverage" - assert result["subtests"][-1]["name"] == "cross-rail-coverage" - -def test_multirail_is_skipped_when_matrix_contains_one_rail() -> None: - """A single-rail topology is inapplicable rather than a coverage failure.""" - output = _validate() - connectivity = output["documents"][1]["connectivity"] - connectivity["PingResults"] = [ - row for row in connectivity["PingResults"] if row["Test"]["SrcRail"] == row["Test"]["DstRail"] - ] - check = LaunchKitMultirailCheck(config=_config(output, discover=_discover())) - - with pytest.raises(pytest.skip.Exception, match="only one rail: rail-0"): - check.execute() - - assert check._subtest_results == [] - - -def test_structured_launch_kit_error_is_actionable() -> None: - """A failed l8k invocation surfaces its structured error when documents are absent.""" - output = { - "success": False, - "platform": "kubernetes", - "operation": "validate", - "documents": [{"error": {"message": "failed to create Kubernetes client"}}], - "error": "failed to create Kubernetes client; verify kubeconfig access", - } - - result = LaunchKitDeploymentHealthCheck(config=_config(output)).execute() +def test_empty_connectivity_matrix_fails() -> None: + """A validate response with no connectivity rows is not evidence of success.""" + result = _execute(_validate([])) assert result["passed"] is False - assert "failed to create Kubernetes client; verify kubeconfig access" in result["error"] - - -@pytest.mark.parametrize(("include_deploy", "expected_count"), [(True, 9), (False, 8)]) -def test_evidence_check_requires_selected_command_artifacts_and_html_report( - tmp_path: Path, - include_deploy: bool, - expected_count: int, -) -> None: - """Evidence supports both lifecycle-owning and validation-only workflows.""" - outputs: dict[str, dict[str, Any]] = {} - operations = ["prepare", "verify", "preflight", "discover", "generate", "validate"] - if include_deploy: - operations.insert(-1, "deploy") - for operation in operations: - artifact = tmp_path / f"{operation}.log" - artifact.write_text(f"{operation} evidence\n", encoding="utf-8") - outputs[operation] = { - "success": True, - "platform": "kubernetes", - "operation": "kubernetes-preflight" if operation == "preflight" else operation, - "working_directory": str(tmp_path), - "artifacts": {"stderr": str(artifact)}, - "documents": [], - } - generated = tmp_path / "generated" / "network-operator.yaml" - generated.parent.mkdir() - generated.write_text("kind: NicClusterPolicy\n", encoding="utf-8") - outputs["generate"]["documents"] = [{"generatedFiles": ["generated/network-operator.yaml"]}] - report = tmp_path / "k8s-launch-kit-validation-report.html" - report.write_text("passed\n", encoding="utf-8") - outputs["validate"]["documents"] = [{"reportPath": str(report)}] - - extra_outputs = { - "prepare_output": json.dumps(outputs["prepare"]), - "verify_output": json.dumps(outputs["verify"]), - "preflight_output": json.dumps(outputs["preflight"]), - "discover_output": json.dumps(outputs["discover"]), - "generate_output": json.dumps(outputs["generate"]), - } - if include_deploy: - extra_outputs["deploy_output"] = json.dumps(outputs["deploy"]) - config = _config(outputs["validate"], **extra_outputs) - result = LaunchKitEvidenceCaptureCheck(config=config).execute() - - assert result["passed"] is True - assert len(result["subtests"]) == expected_count + assert result["error"] == "Launch Kit connectivity matrix produced no results" diff --git a/isvtest/tests/test_catalog.py b/isvtest/tests/test_catalog.py index b3110f007..5d530761c 100644 --- a/isvtest/tests/test_catalog.py +++ b/isvtest/tests/test_catalog.py @@ -114,9 +114,10 @@ def test_entries_have_suite_contract(self) -> None: assert isinstance(entry["requires"], list) if entry["capability"]: assert entry["requires"] == [] - assert "EastWestNetworkRoceSriovCheck" in names - use_case = next(entry for entry in catalog if entry["name"] == "EastWestNetworkRoceSriovCheck") - assert use_case["suite"] == "network_operator" + assert "LaunchKitConnectivityCheck" in names + launch_kit = next(entry for entry in catalog if entry["name"] == "LaunchKitConnectivityCheck") + assert launch_kit["suite"] == "network_operator" + assert launch_kit["test_ids"] == ["K8S42-01"] def test_extract_checks_supports_direct_dict_category_form(self, tmp_path) -> None: """Direct dict category wiring is included in catalog config scans.""" From 2257b62d10929ed7cbdc87c74744990308c22e11 Mon Sep 17 00:00:00 2001 From: Alexander Maslennikov Date: Thu, 10 Sep 2026 19:06:30 +0200 Subject: [PATCH 4/4] feat: collect Launch Kit sosreport after validation Signed-off-by: Alexander Maslennikov --- AGENTS.md | 24 +++- docs/README.md | 2 +- docs/guides/configuration.md | 2 +- .../guides/k8s-launch-kit/network-operator.md | 61 ++++++++- .../providers/k8s-launch-kit/README.md | 27 +++- .../config/network-operator.yaml | 28 +++- .../k8s-launch-kit/scripts/adapter.py | 127 +++++++++++++++-- isvctl/configs/suites/README.md | 14 +- isvctl/src/isvctl/config/output_schemas.py | 5 + .../k8s_launch_kit/fixtures/mock_l8k.py | 28 +++- .../providers/k8s_launch_kit/test_provider.py | 128 ++++++++++++++++-- .../k8s_launch_kit/test_timeout_config.py | 8 ++ 12 files changed, 399 insertions(+), 55 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f6c86de93..3d06533f0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -179,12 +179,16 @@ forwarded env vars → optional isvreporter upload. `timeout: null` so Launch Kit owns the automatically calculated or user-supplied matrix deadline. - `config/network-operator.yaml` is deliberately independent of the generic - lifecycle provider. It runs exactly one test step: - `l8k validate --user-config --deployment-files `. + lifecycle provider. It runs exactly one catalog-owning test step: + `l8k validate --user-config --deployment-files `, followed + by a linked same-phase finalizer that always invokes `l8k sosreport` after an + attempted validation. Sosreport is evidence collection, not another test. The installed binary, reachable Kubernetes cluster, reconciled Network Operator deployment, complete Launch Kit config, and rendered deployment - files are prerequisites. Do not add prepare, verify, preflight, discover, - generate, deploy, clean, or finalizer steps to this entrypoint. + files are prerequisites. The installation must also expose Launch Kit's + `kubectl-netop_sosreport` helper. Do not add prepare, verify, preflight, + discover, generate, deploy, clean, or unrelated finalizer steps to this + entrypoint. - The Network Operator provider inputs are `executable`, `user_config`, `deployment_files`, `working_dir`, `artifact_dir`, and a string-only `environment` mapping. Both input paths are resolved and checked before @@ -203,9 +207,15 @@ forwarded env vars → optional isvreporter upload. The provider binds its step with `requires_selected_validations` so command failures remain owned by the catalog validation and appear in structured reporting. -- The adapter adds only `--output json`, wraps the unmodified concatenated - JSON documents, and records argv, cwd, stdout, stderr, exit code, and timing. - Do not invent a `selfValidation` result or reinterpret Launch Kit's verdict. +- The adapter adds only `--output json` to commands that emit structured + output, wraps the unmodified concatenated JSON documents, and records argv, + cwd, stdout, stderr, exit code, and timing. For `validate`, use the emitted + `reportPath` as the authoritative HTML report source and copy it to + `/k8s-launch-kit-validation-report.html`. `l8k sosreport` is + text-streaming; default its `--output-dir` to the provider evidence directory, + retain that directory as an artifact, and wrap the command without parsing + its output. Do not invent a `selfValidation` result or reinterpret Launch + Kit's verdict. - The generic provider retains installation, Kubernetes preflight, and cleanup support for other consumers. `l8k clean` remains its only supported deletion path; never reproduce Launch Kit cleanup with kubectl. diff --git a/docs/README.md b/docs/README.md index ed828d15d..77e9f4727 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,7 +19,7 @@ Welcome to the documentation for NVIDIA AI Cloud Validation suite - a collection - [Configuration](guides/configuration.md) - Configuration file format and options - [External Validation Guide](guides/external-validation-guide.md) - Create custom validations without modifying the repo -- [Network Operator Launch Kit integration](guides/k8s-launch-kit/network-operator.md) - Production provider, Kubernetes preflight, mock-backed unit coverage, evidence, and limitations +- [Network Operator Launch Kit integration](guides/k8s-launch-kit/network-operator.md) - Connectivity validation, always-run sosreport evidence, mock-backed unit coverage, and limitations - [Remote Deployment](guides/remote-deployment.md) - Deploy and run tests on remote machines - [Local Development](guides/local-development.md) - MicroK8s setup for local testing - [Troubleshooting: Test runs stuck in STARTED](guides/troubleshooting-started-tests.md) - Why runs stay STARTED in the portal and how to fix it diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index fa633c0b6..e4cf69bb2 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -57,7 +57,7 @@ Pre-built configs are provided in `isvctl/configs/`: | `providers/aws/config/iam.yaml` | AWS IAM user lifecycle | | `providers/aws/config/eks.yaml` | AWS EKS with GPU nodes | | `providers/k8s-launch-kit/config/provider.yaml` | Generic Kubernetes Launch Kit workflow | -| `providers/k8s-launch-kit/config/network-operator.yaml` | Six Network Operator Launch Kit use cases | +| `providers/k8s-launch-kit/config/network-operator.yaml` | Launch Kit connectivity validation with post-run sosreport collection | | `suites/k8s.yaml` | Standard Kubernetes cluster | | `suites/k8s-launch-kit/*.yaml` | Launch Kit-specific Network Operator catalog wiring | | `suites/slurm.yaml` | Slurm HPC cluster | diff --git a/docs/guides/k8s-launch-kit/network-operator.md b/docs/guides/k8s-launch-kit/network-operator.md index d71f5100f..c207c91e0 100644 --- a/docs/guides/k8s-launch-kit/network-operator.md +++ b/docs/guides/k8s-launch-kit/network-operator.md @@ -5,13 +5,22 @@ ## Scope -The Network Operator suite performs one operation: +The Network Operator suite performs one validation operation: ```text l8k validate --user-config --deployment-files ``` -It reports the connectivity matrix produced by Launch Kit. It does not install +It reports the connectivity matrix produced by Launch Kit. After that command +is attempted, an always-run linked finalizer invokes: + +```text +l8k sosreport --output-dir /sosreport +``` + +The diagnostic command runs whether validation passes, returns an error, or +produces a failing connectivity matrix. It is evidence collection, not a +second catalog test. The suite does not install or verify the `l8k` binary, discover topology, generate manifests, deploy Network Operator, run a separate Kubernetes preflight, or clean cluster state. @@ -30,13 +39,17 @@ model in AI Cloud Validation. ```text Network Operator provider YAML - -> one isvctl test step + -> validation step -> adapter.py -> l8k validate --user-config ... --deployment-files ... --output json - -> retained argv, stdout, stderr, exit code, and duration + -> retained argv, stdout, stderr, exit code, duration, and HTML report -> Network Operator suite YAML -> LaunchKitConnectivityCheck -> one subtest for every Launch Kit connectivity row + -> linked finalizer, after the connectivity assertion + -> adapter.py + -> l8k sosreport --output-dir .../evidence/sosreport + -> retained diagnostic directory, stdout, stderr, exit code, and duration -> console and JUnit results ``` @@ -56,6 +69,13 @@ The generic provider in complete Launch Kit lifecycle for other consumers. The Network Operator entrypoint does not import it, so none of those lifecycle steps are inherited. +The `l8k` installation must also make the upstream +`kubectl-netop_sosreport` helper available to `l8k sosreport`. Validate this +once with a direct `l8k sosreport --output-dir ` call. If +Launch Kit reports that the script is missing, install the helper below the +same installation prefix at `share/l8k/scripts/kubectl-netop_sosreport` before +running the suite. + ## Inputs The Network Operator provider exposes only these settings: @@ -125,6 +145,12 @@ by the user. This prevents an independent isvctl watchdog from terminating a valid large matrix before Launch Kit's bounded checks finish. An enclosing CI job may still impose an overall job timeout. +The `launch_kit_sosreport` finalizer has a 30-minute orchestration watchdog. +Unlike connectivity validation, the current Launch Kit sosreport command does +not calculate its own total deadline. A timeout or sosreport command error is +reported as a separate `test-teardown` orchestration failure; it does not +replace the connectivity test result. + ## Results and errors `LaunchKitConnectivityCheck` finds the `connectivity.PingResults` array in the @@ -145,6 +171,10 @@ matrix is present, or when the matrix contains no results. A command that fails before producing connectivity output retains its provider error in the validation and JUnit output. +The sosreport finalizer runs after this assertion. If sosreport itself fails, +the connectivity result remains intact and the overall orchestration reports +the diagnostic-collection failure separately. + ## Evidence The adapter writes: @@ -153,18 +183,35 @@ The adapter writes: _output/k8s-launch-kit/network-operator/ work/ evidence/ + k8s-launch-kit-validation-report.html commands/validate/ command.json stdout.txt stderr.log + commands/sosreport/ + command.json + stdout.txt + stderr.log + sosreport/ + ... files produced by the Network Operator sosreport helper ... ``` `command.json` records the resolved argv, exit code, and duration. `stdout.txt` contains Launch Kit's complete JSON stream, including static validation, connectivity, and report-path documents; `stderr.log` retains CLI progress and -diagnostics. The adapter also registers these paths in the provider step output. -The Launch Kit HTML report remains at the `reportPath` emitted by Launch Kit, -normally below the supplied deployment directory. +diagnostics. The adapter uses the emitted `reportPath` as the authoritative +source, copies the HTML file to +`evidence/k8s-launch-kit-validation-report.html`, and registers the copied path +as the `validation_report` artifact. The original report remains at the path +written by Launch Kit, normally below the supplied deployment directory. A +report emitted for a failed connectivity matrix is copied in the same way. If +Launch Kit advertises a report that cannot be read, the provider returns an +evidence-retention error instead of silently reusing an older report. + +The sosreport command currently streams human-readable output even when the +global `--output` flag is available. The adapter therefore preserves that +stream in `commands/sosreport/stdout.txt` and emits its own normal structured +step envelope; it does not attempt to reinterpret the diagnostic contents. ## PRD boundary diff --git a/isvctl/configs/providers/k8s-launch-kit/README.md b/isvctl/configs/providers/k8s-launch-kit/README.md index 48c972847..cf6b3f533 100644 --- a/isvctl/configs/providers/k8s-launch-kit/README.md +++ b/isvctl/configs/providers/k8s-launch-kit/README.md @@ -8,7 +8,7 @@ | Path | Purpose | |---|---| | `config/provider.yaml` | Generic provider mirroring the full Launch Kit workflow | -| `config/network-operator.yaml` | One-step validation of an ISV-provisioned Network Operator deployment | +| `config/network-operator.yaml` | Connectivity validation plus always-run diagnostics for an ISV-provisioned Network Operator deployment | | `scripts/adapter.py` | Thin process and JSON evidence transport | Executable mocks and pinned scenarios are test-only and live under @@ -31,12 +31,22 @@ outer watchdogs. ## Network Operator provider `config/network-operator.yaml` intentionally does not import the generic -provider. It defines one test step and executes only: +provider. It defines one catalog-owning test step: ```text l8k validate --user-config --deployment-files --output json ``` +After validation is attempted, a linked finalizer always executes: + +```text +l8k sosreport --output-dir /sosreport +``` + +Sosreport runs after both successful and failed validation commands and after +the connectivity assertion. It is diagnostic evidence, not another catalog +test. Its failure is reported as a separate teardown result. + The cluster, Network Operator deployment, complete Launch Kit config, rendered deployment directory, and installed `l8k` binary are prerequisites. There are no AI Cloud Validation use-case workflows and no discover, generate, deploy, @@ -55,14 +65,23 @@ reported automatically. ## Adapter contract -For every workflow invocation, `adapter.py`: +For every structured workflow invocation, `adapter.py`: 1. resolves the configured executable; 2. adds `--output json` unless the caller already selected JSON; 3. executes exactly one Launch Kit command; 4. preserves stdout, stderr, argv, exit code, and duration; 5. parses concatenated JSON objects without renaming their fields; -6. returns one provider envelope containing the raw documents and artifact paths. +6. copies the HTML file advertised by a validation `reportPath` into the + provider evidence directory; +7. returns one provider envelope containing the raw documents and artifact paths. + +For `sosreport`, the adapter does not force JSON because the current Launch Kit +command streams text output. It defaults `--output-dir` to the provider evidence +directory, records that directory as an artifact, and still returns the same +structured provider envelope. The installed Launch Kit must make its +`kubectl-netop_sosreport` helper available under the Launch Kit installation +prefix. Semantic assertions belong in `isvtest.validations.k8s_launch_kit`, not the transport. diff --git a/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml b/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml index c7797847c..eb13b5595 100644 --- a/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml +++ b/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml @@ -3,8 +3,9 @@ # Network Operator connectivity validation for an ISV-provisioned cluster. # The complete Launch Kit configuration and rendered deployment directory are -# prerequisites. This provider invokes only: +# prerequisites. This provider validates them, then always collects diagnostics: # l8k validate --user-config --deployment-files +# l8k sosreport --output-dir /sosreport import: - ../../../suites/k8s-launch-kit/network-operator.yaml @@ -49,3 +50,28 @@ commands: output_schema: k8s_launch_kit requires: [kubernetes] requires_selected_validations: [LaunchKitConnectivityCheck] + + # A linked same-phase finalizer runs after the connectivity assertion even + # when validate exits nonzero. It is not a separate catalog test. + - name: launch_kit_sosreport + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ context.k8s_launch_kit.executable }}" + - --command + - sosreport + - --arguments-json + - "[]" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [LaunchKitConnectivityCheck] + finalizer_for: launch_kit_validate diff --git a/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py b/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py index d7b3a1d36..62c14e368 100644 --- a/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py +++ b/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py @@ -5,11 +5,12 @@ """Thin AI Cloud Validation transport for the Kubernetes Launch Kit CLI. The provider deliberately exposes the real Launch Kit operations. It forwards -user-supplied arguments verbatim and adds ``--output json`` so stdout can be -preserved as structured evidence. Discovery can stage a complete user config -transiently. Validation can bind an existing complete user config and rendered -deployment directory directly. Launch Kit remains the owner of command flags, -configuration schema, and defaults. +user-supplied arguments verbatim and requests structured output from commands +that implement it. Discovery can stage a complete user config transiently. +Validation can bind an existing complete user config and rendered deployment +directory directly. Its emitted HTML report and sosreport output are retained +below the provider artifact directory. Launch Kit remains the owner of command +flags, configuration schema, and defaults. """ from __future__ import annotations @@ -31,9 +32,11 @@ from typing import Any _WORKFLOW_COMMANDS = ("discover", "generate", "deploy", "validate", "clean") +_RUN_COMMANDS = (*_WORKFLOW_COMMANDS, "sosreport") _INSTALLER_URL = "https://raw.githubusercontent.com/NVIDIA/k8s-launch-kit/{ref}/scripts/install.sh" _STAGED_USER_CONFIG = "user-config.yaml" _DISCOVERED_CLUSTER_CONFIG = "cluster-config.yaml" +_VALIDATION_REPORT_NAME = "k8s-launch-kit-validation-report.html" def _parse_json_value(raw: str, source: str, expected_type: type[Any]) -> Any: @@ -105,6 +108,67 @@ def _with_json_output(arguments: list[str]) -> list[str]: return result +def _bind_sosreport_output(arguments: list[str], *, working_dir: Path, artifact_dir: Path) -> tuple[list[str], Path]: + """Resolve the sosreport output directory and default it to retained evidence.""" + result = list(arguments) + output_dir: Path | None = None + index = 0 + while index < len(result): + token = result[index] + if token == "--output-dir": + if index + 1 >= len(result) or not result[index + 1]: + raise ValueError("--output-dir requires a non-empty value") + output_dir = Path(result[index + 1]).expanduser() + index += 2 + continue + if token.startswith("--output-dir="): + value = token.partition("=")[2] + if not value: + raise ValueError("--output-dir requires a non-empty value") + output_dir = Path(value).expanduser() + index += 1 + + if output_dir is None: + output_dir = artifact_dir / "sosreport" + result.extend(["--output-dir", str(output_dir)]) + elif not output_dir.is_absolute(): + output_dir = (working_dir / output_dir).resolve() + + return result, output_dir.resolve() + + +def _retain_validation_report( + documents: list[dict[str, Any]], + *, + working_dir: Path, + artifact_dir: Path, +) -> Path | None: + """Copy the HTML report advertised by Launch Kit into retained evidence.""" + report_value = next( + ( + document["reportPath"] + for document in reversed(documents) + if isinstance(document.get("reportPath"), str) and document["reportPath"] + ), + None, + ) + if report_value is None: + return None + + source = Path(report_value).expanduser() + if not source.is_absolute(): + source = working_dir / source + source = source.resolve() + if not source.is_file(): + raise FileNotFoundError(f"Launch Kit HTML validation report not found: {source}") + + destination = (artifact_dir / _VALIDATION_REPORT_NAME).resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + if source != destination: + shutil.copy2(source, destination) + return destination + + def _structured_error(documents: list[dict[str, Any]]) -> str | None: """Extract the most actionable Launch Kit structured error.""" for document in reversed(documents): @@ -296,8 +360,12 @@ def _run_workflow(args: argparse.Namespace) -> tuple[dict[str, Any], int]: working_dir = Path(args.working_dir).expanduser().resolve() working_dir.mkdir(parents=True, exist_ok=True) artifact_dir = Path(args.artifact_dir).expanduser().resolve() + retained_validation_report = artifact_dir / _VALIDATION_REPORT_NAME + if args.command == "validate": + retained_validation_report.unlink(missing_ok=True) staged_user_config: Path | None = None user_config_metadata_path: Path | None = None + sosreport_output_dir: Path | None = None try: if args.command == "validate" and args.deployment_files is not None: arguments = _bind_validate_inputs(args.user_config, args.deployment_files, arguments) @@ -311,7 +379,14 @@ def _run_workflow(args: argparse.Namespace) -> tuple[dict[str, Any], int]: ) user_config_metadata_path = artifact_dir / "inputs" / "user-config.json" _write_json(user_config_metadata_path, user_config_metadata) - arguments = _with_json_output(arguments) + if args.command == "sosreport": + arguments, sosreport_output_dir = _bind_sosreport_output( + arguments, + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + else: + arguments = _with_json_output(arguments) argv = [str(executable), args.command, *arguments] result = _run_process(argv, cwd=working_dir, env=environment) finally: @@ -320,20 +395,44 @@ def _run_workflow(args: argparse.Namespace) -> tuple[dict[str, Any], int]: artifacts = _record_process(artifact_dir / "commands" / args.command, argv, result) if user_config_metadata_path is not None: artifacts["user_config"] = str(user_config_metadata_path) + if sosreport_output_dir is not None and sosreport_output_dir.exists(): + artifacts["sosreport"] = str(sosreport_output_dir) parse_error: str | None = None - try: - documents = _parse_json_stream(str(result["stdout"]), f"l8k {args.command} stdout") - except ValueError as exc: + if args.command == "sosreport": + # The current sosreport command accepts the global --output flag but + # streams human-readable helper output in both modes. Preserve it as a + # process artifact and let this adapter provide the structured envelope. documents = [] - parse_error = str(exc) + else: + try: + documents = _parse_json_stream(str(result["stdout"]), f"l8k {args.command} stdout") + except ValueError as exc: + documents = [] + parse_error = str(exc) + + report_retention_error: str | None = None + if args.command == "validate" and parse_error is None: + try: + retained_report = _retain_validation_report( + documents, + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + except (FileNotFoundError, OSError) as exc: + retained_report = None + report_retention_error = f"failed to retain Launch Kit HTML validation report: {exc}" + if retained_report is not None: + artifacts["validation_report"] = str(retained_report) - success = result["exit_code"] == 0 and parse_error is None + success = result["exit_code"] == 0 and parse_error is None and report_retention_error is None error = parse_error or _structured_error(documents) - if not success and error is None: + if result["exit_code"] != 0 and error is None: error = f"l8k {args.command} exited with code {result['exit_code']}" if excerpt := _stderr_excerpt(str(result["stderr"])): error = f"{error}: {excerpt}" + if report_retention_error is not None: + error = f"{error}; {report_retention_error}" if error else report_retention_error envelope: dict[str, Any] = { "success": success, "platform": "kubernetes", @@ -346,6 +445,8 @@ def _run_workflow(args: argparse.Namespace) -> tuple[dict[str, Any], int]: "documents": documents, "artifacts": artifacts, } + if sosreport_output_dir is not None: + envelope["sosreport_output_directory"] = str(sosreport_output_dir) if error: envelope["error"] = error excerpt = _stderr_excerpt(str(result["stderr"])) @@ -726,7 +827,7 @@ def _parser() -> argparse.ArgumentParser: run = subparsers.add_parser("run", help="Run one real Launch Kit workflow command") run.add_argument("--executable", required=True) - run.add_argument("--command", choices=_WORKFLOW_COMMANDS, required=True) + run.add_argument("--command", choices=_RUN_COMMANDS, required=True) run.add_argument("--arguments-json", required=True) run.add_argument("--user-config", default="") run.add_argument("--deployment-files", default=None) diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index 3686ee79c..4466d832a 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -219,11 +219,12 @@ its plan item is not platform-scoped. ### Network Operator (`k8s-launch-kit/network-operator.yaml`) The Network Operator suite contains one catalog entry, -`LaunchKitConnectivityCheck`. Its production provider invokes only `l8k -validate` with a caller-supplied complete `user_config` and existing -`deployment_files` directory. The Kubernetes cluster, Network Operator -deployment, Launch Kit installation, configuration, and generated manifests -are prerequisites. +`LaunchKitConnectivityCheck`. Its production provider invokes `l8k validate` +with a caller-supplied complete `user_config` and existing `deployment_files` +directory, then invokes `l8k sosreport` as an always-run linked finalizer. The +Kubernetes cluster, Network Operator deployment, Launch Kit installation +(including its sosreport helper), configuration, and generated manifests are +prerequisites. Fabric, deployment type, enabled checks, and GPUDirect applicability are not modeled as suite labels or separate tests. Every @@ -242,7 +243,8 @@ uv run isvctl test run \ | Step | Phase | Script | Key JSON Fields | |------|-------|--------|-----------------| -| `launch_kit_validate` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k validate` | raw `documents`, `argv`, `exit_code`, `duration_seconds`, `artifacts` | +| `launch_kit_validate` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k validate` | raw `documents`, `argv`, `exit_code`, `duration_seconds`, `artifacts.validation_report` | +| `launch_kit_sosreport` | test finalizer | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k sosreport` | `argv`, `exit_code`, `duration_seconds`, `sosreport_output_directory`, `artifacts` | The generic `providers/k8s-launch-kit/config/provider.yaml` remains available to consumers that need the full Launch Kit lifecycle. See the diff --git a/isvctl/src/isvctl/config/output_schemas.py b/isvctl/src/isvctl/config/output_schemas.py index 99d72f324..a9f68a585 100644 --- a/isvctl/src/isvctl/config/output_schemas.py +++ b/isvctl/src/isvctl/config/output_schemas.py @@ -1031,6 +1031,7 @@ "deploy", "validate", "clean", + "sosreport", ], "description": "The actual Launch Kit or provider prerequisite operation", }, @@ -1046,6 +1047,10 @@ "items": {"type": "object"}, "description": "Unmodified JSON documents emitted by l8k", }, + "sosreport_output_directory": { + "type": "string", + "description": "Absolute directory selected for l8k sosreport output", + }, "checks": { "oneOf": [ {"type": "object"}, diff --git a/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py b/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py index dcf5a41a4..d2f6ae334 100755 --- a/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py +++ b/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py @@ -6,7 +6,8 @@ Values in this file are fixed mock output, not AI Cloud Validation defaults. The provider passes only real l8k arguments and receives the same distinct -stdout forms used by version, schema, discover, generate, deploy, validate, and clean. +stdout forms used by version, schema, discover, generate, deploy, validate, +clean, and sosreport. """ from __future__ import annotations @@ -72,6 +73,11 @@ "--keep-helm-chart", "--output", }, + "sosreport": { + "--kubeconfig", + "--output-dir", + "--output", + }, } _BOOLEAN_FLAGS: dict[str, set[str]] = { "clean": {"--keep-helm-chart"}, @@ -508,7 +514,7 @@ def _run_schema() -> int: "description": "CLI tool for deploying NVIDIA cloud-native networking solutions on Kubernetes", "commands": { command: {"description": f"Mock l8k {command}", "example": f"l8k {command}"} - for command in ("discover", "generate", "deploy", "validate", "clean", "schema") + for command in ("discover", "generate", "deploy", "validate", "clean", "sosreport", "schema") }, "phases": ["discover", "generate", "deploy"], "fabrics": sorted({str(value["fabric"]) for value in fixture["scenarios"].values()}), @@ -667,6 +673,22 @@ def _run_clean(flags: dict[str, str]) -> int: return 0 +def _run_sosreport(flags: dict[str, str]) -> int: + """Mock the current text-streaming ``l8k sosreport`` command.""" + output_dir = Path(flags.get("--output-dir", "./sosreport")).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + archive = output_dir / "network-operator-sosreport.tar.gz" + archive.write_text("mock Network Operator diagnostic archive\n", encoding="utf-8") + print("Collecting sosreport from cluster...") + print(f" Output: {output_dir}") + fail, _ = _failure("sosreport") + if fail: + print("Error: sosreport collection failed", file=sys.stderr) + return 3 + print(f"\nSosreport collected: {output_dir}") + return 0 + + def main(argv: list[str] | None = None) -> int: """Execute one mocked Launch Kit command.""" args = list(sys.argv[1:] if argv is None else argv) @@ -694,6 +716,8 @@ def main(argv: list[str] | None = None) -> int: return _run_deploy(flags) if command == "validate": return _run_validate(flags) + if command == "sosreport": + return _run_sosreport(flags) return _run_clean(flags) except (KeyError, OSError, TypeError, ValueError, yaml.YAMLError) as exc: result, exit_code = _structured_error(command, str(exc)) diff --git a/isvctl/tests/providers/k8s_launch_kit/test_provider.py b/isvctl/tests/providers/k8s_launch_kit/test_provider.py index 7a2bb6cf1..bdba31a57 100644 --- a/isvctl/tests/providers/k8s_launch_kit/test_provider.py +++ b/isvctl/tests/providers/k8s_launch_kit/test_provider.py @@ -200,7 +200,7 @@ def test_generic_provider_has_no_launch_kit_domain_defaults() -> None: def test_network_operator_provider_defaults_to_real_cli_tools() -> None: - """The shipped provider contains one real, validation-only workflow.""" + """The shipped provider validates once and always collects diagnostics.""" merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) config = RunConfig.model_validate(merged) context = merged["context"]["k8s_launch_kit"] @@ -217,12 +217,17 @@ def test_network_operator_provider_defaults_to_real_cli_tools() -> None: assert "poc" not in json.dumps(merged).lower() command = config.commands["network_operator"] assert command.phases == ["test"] - assert [step.name for step in command.steps] == ["launch_kit_validate"] - step = command.steps[0] - assert step.timeout is None - assert "--user-config={{ context.k8s_launch_kit.user_config }}" in step.args - assert "--deployment-files={{ context.k8s_launch_kit.deployment_files }}" in step.args - assert step.requires_selected_validations == ["LaunchKitConnectivityCheck"] + assert [step.name for step in command.steps] == ["launch_kit_validate", "launch_kit_sosreport"] + validate_step, sosreport_step = command.steps + assert validate_step.timeout is None + assert "--user-config={{ context.k8s_launch_kit.user_config }}" in validate_step.args + assert "--deployment-files={{ context.k8s_launch_kit.deployment_files }}" in validate_step.args + assert validate_step.requires_selected_validations == ["LaunchKitConnectivityCheck"] + assert sosreport_step.timeout == 1800 + assert sosreport_step.phase == "test" + assert sosreport_step.finalizer_for == "launch_kit_validate" + assert sosreport_step.requires == validate_step.requires + assert sosreport_step.requires_selected_validations == validate_step.requires_selected_validations def test_network_operator_suite_has_one_catalog_check() -> None: @@ -494,6 +499,10 @@ def test_provider_runs_the_real_launch_kit_workflow_shape(tmp_path: Path) -> Non assert test_container["resources"]["limits"]["nvidia.com/gpu"] == "2" assert outputs["deploy"]["documents"] == [] assert len(outputs["validate"]["documents"]) == 3 + source_report = working_dir / "deployment" / "k8s-launch-kit-validation-report.html" + retained_report = artifact_dir / "k8s-launch-kit-validation-report.html" + assert outputs["validate"]["artifacts"]["validation_report"] == str(retained_report) + assert retained_report.read_bytes() == source_report.read_bytes() families = {row["Family"] for row in outputs["validate"]["documents"][1]["connectivity"]["PingResults"]} assert families == {"icmp", "rping", "ib_write_bw", "gpudirect_dmabuf"} assert outputs["clean"]["documents"][0]["cleanup"] == { @@ -503,7 +512,31 @@ def test_provider_runs_the_real_launch_kit_workflow_shape(tmp_path: Path) -> Non "keepHelmChart": False, } assert (working_dir / "cluster-config.yaml").is_file() - assert (working_dir / "deployment" / "k8s-launch-kit-validation-report.html").is_file() + assert source_report.is_file() + + +def test_sosreport_preserves_text_output_and_registers_its_directory(tmp_path: Path) -> None: + """The adapter retains the text-only sosreport contract as structured evidence.""" + working_dir = tmp_path / "work" + artifact_dir = tmp_path / "evidence" + + completed, output = _run_workflow( + "sosreport", + [], + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + + assert completed.returncode == 0 + assert output["success"] is True + assert output["operation"] == "sosreport" + assert output["documents"] == [] + assert output["argv"][1:] == ["sosreport", "--output-dir", str(artifact_dir / "sosreport")] + assert output["sosreport_output_directory"] == str(artifact_dir / "sosreport") + assert output["artifacts"]["sosreport"] == str(artifact_dir / "sosreport") + assert (artifact_dir / "sosreport" / "network-operator-sosreport.tar.gz").is_file() + assert "Sosreport collected" in Path(output["artifacts"]["stdout"]).read_text(encoding="utf-8") + assert validate_output(output, "k8s_launch_kit") == (True, []) def test_discover_stages_user_config_transiently_without_retaining_secrets(tmp_path: Path) -> None: @@ -842,8 +875,8 @@ def test_preflight_rejects_conflicting_workflow_kubeconfigs(tmp_path: Path) -> N assert "different kubeconfigs" in output["error"] -def test_network_operator_provider_runs_only_validate(tmp_path: Path) -> None: - """The production configuration invokes one validation over prerequisite inputs.""" +def test_network_operator_provider_runs_validate_then_sosreport(tmp_path: Path) -> None: + """The production configuration validates once and then collects diagnostics.""" config = _mocked_network_operator_config(tmp_path) result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( @@ -852,8 +885,8 @@ def test_network_operator_provider_runs_only_validate(tmp_path: Path) -> None: ) assert result.success is True - assert list(result.inventory) == ["launch_kit_validate"] - assert [phase.name for phase in result.phases] == ["test"] + assert list(result.inventory) == ["launch_kit_validate", "launch_kit_sosreport"] + assert [phase.name for phase in result.phases] == ["test", "test-teardown"] assert len(result.validations) == 1 validation = result.validations[0] assert validation.entry.name == "LaunchKitConnectivityCheck" @@ -867,6 +900,32 @@ def test_network_operator_provider_runs_only_validate(tmp_path: Path) -> None: assert argv[argv.index("--user-config") + 1] == str((tmp_path / "cluster-config.yaml").resolve()) assert argv[argv.index("--deployment-files") + 1] == str((tmp_path / "deployment").resolve()) assert argv[-2:] == ["--output", "json"] + report = tmp_path / "evidence" / "k8s-launch-kit-validation-report.html" + assert result.inventory["launch_kit_validate"]["artifacts"]["validation_report"] == str(report) + assert report.is_file() + sosreport = result.inventory["launch_kit_sosreport"] + assert sosreport["argv"][1:] == ["sosreport", "--output-dir", str(tmp_path / "evidence" / "sosreport")] + assert Path(sosreport["artifacts"]["sosreport"]).is_dir() + + +def test_sosreport_failure_does_not_replace_connectivity_result(tmp_path: Path, monkeypatch: Any) -> None: + """Diagnostic failure is separate while the connectivity assertion stays passed.""" + monkeypatch.setenv("L8K_MOCK_FAIL", "sosreport") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + capability="kubernetes", + ) + + assert result.success is False + assert result.validations[0].state is State.PASSED + assert [(phase.name, phase.success) for phase in result.phases] == [ + ("test", True), + ("test-teardown", False), + ] + assert result.inventory["launch_kit_sosreport"]["success"] is False + assert "sosreport collection failed" in result.inventory["launch_kit_sosreport"]["error"] def test_network_operator_provider_expands_input_paths(tmp_path: Path, monkeypatch: Any) -> None: @@ -961,7 +1020,11 @@ def test_failed_connectivity_is_a_junit_failure(tmp_path: Path, monkeypatch: Any ) assert result.success is False - assert list(result.inventory) == ["launch_kit_validate"] + assert list(result.inventory) == ["launch_kit_validate", "launch_kit_sosreport"] + report = tmp_path / "evidence" / "k8s-launch-kit-validation-report.html" + assert result.inventory["launch_kit_validate"]["artifacts"]["validation_report"] == str(report) + assert report.is_file() + assert (tmp_path / "evidence" / "sosreport" / "network-operator-sosreport.tar.gz").is_file() assert result.validations[0].state is State.FAILED assert result.validations[0].subtest_summary.failed == 1 case = next( @@ -988,6 +1051,8 @@ def test_missing_prerequisites_are_a_step_error(tmp_path: Path) -> None: ) assert result.success is False + assert list(result.inventory) == ["launch_kit_validate", "launch_kit_sosreport"] + assert (tmp_path / "evidence" / "sosreport" / "network-operator-sosreport.tar.gz").is_file() assert result.validations[0].state is State.FAILED assert "user_config is required" in result.validations[0].message @@ -1030,4 +1095,41 @@ def test_failed_validate_preserves_documents_and_process_error(tmp_path: Path) - assert output["success"] is False assert len(output["documents"]) == 3 assert "l8k validate exited with code 4" in output["error"] + assert Path(output["artifacts"]["validation_report"]).is_file() assert Path(output["artifacts"]["stdout"]).read_text(encoding="utf-8") + + +def test_missing_advertised_validation_report_is_an_evidence_error(tmp_path: Path) -> None: + """A stale report cannot satisfy a new Launch Kit reportPath document.""" + missing_report = tmp_path / "missing-validation-report.html" + executable = tmp_path / "l8k" + executable.write_text( + f"#!/bin/sh\nprintf '%s\\n' '{{\"reportPath\":\"{missing_report}\"}}'\n", + encoding="utf-8", + ) + executable.chmod(0o755) + artifact_dir = tmp_path / "evidence" + retained_report = artifact_dir / "k8s-launch-kit-validation-report.html" + retained_report.parent.mkdir(parents=True) + retained_report.write_text("stale report\n", encoding="utf-8") + + completed, output = _run_provider( + "run", + "--executable", + str(executable), + "--command", + "validate", + "--arguments-json", + "[]", + "--working-dir", + str(tmp_path / "work"), + "--artifact-dir", + str(artifact_dir), + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert "failed to retain Launch Kit HTML validation report" in output["error"] + assert str(missing_report) in output["error"] + assert "validation_report" not in output["artifacts"] + assert not retained_report.exists() diff --git a/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py b/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py index 66f2eca4f..1e235304c 100644 --- a/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py +++ b/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py @@ -31,3 +31,11 @@ def test_network_operator_validate_delegates_timeout_to_launch_kit() -> None: assert len(validate_steps) == 1 assert validate_steps[0]["timeout"] is None + + +def test_network_operator_sosreport_has_a_bounded_watchdog() -> None: + """Diagnostic collection must not hang the orchestration indefinitely.""" + sosreport_steps = [step for step in _steps("network-operator.yaml") if step["name"] == "launch_kit_sosreport"] + + assert len(sosreport_steps) == 1 + assert sosreport_steps[0]["timeout"] == 1800