diff --git a/AGENTS.md b/AGENTS.md index ee7720a00..3d06533f0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,7 +77,8 @@ Entry point: `isvctl/src/isvctl/main.py` (Typer). - `cli/` - subcommands (`test`, `deploy`, `clean`, `docs`, `report`) - `orchestrator/` - `loop.py` (phase loop), `step_executor.py` (step + validation - execution, supports `best_effort` mode), `commands.py` (timeouts), `context.py` + execution, supports `best_effort` mode), `commands.py` (legacy command model), + `process.py` (shared subprocess and process-group timeout handling), `context.py` (Jinja2 with missing-reference warnings) - `config/` - `schema.py` (Pydantic), `output_schemas.py` (per-step JSON schemas), `merger.py` (multi-file merge) @@ -139,7 +140,9 @@ forwarded env vars → optional isvreporter upload. - Workspace root `pyproject.toml` defines members; each package has its own `pyproject.toml`; all source under `src/`. -- `isvctl/configs/suites/` - provider-agnostic test contracts. +- `isvctl/configs/suites/` - provider-agnostic test contracts. Discovery is + recursive, so related domain suites may be grouped in a subdirectory; YAML + filename stems must remain globally unique. - `isvctl/configs/providers//` - one folder per provider (`aws/`, `my-isv/`, ...): - `config/` - YAML wiring (imports a suite, supplies commands) - `scripts/` - executable scripts (Python/Bash) that do the work, organized by @@ -160,6 +163,72 @@ forwarded env vars → optional isvreporter upload. `aws/scripts/common/` provides `ec2`, `errors` (with `delete_with_retry`), `ssh_utils.wait_for_ssh`, `serial_console`, `vpc`. +### Network Operator / Kubernetes Launch Kit + +- All provider-owned Launch Kit files live under + `isvctl/configs/providers/k8s-launch-kit/`: provider YAML in `config/`, + executable transport in `scripts/`, and implementation documentation in + `README.md`. Test doubles live only under + `isvctl/tests/providers/k8s_launch_kit/fixtures/`; product configuration + must never reference them. +- `config/provider.yaml` is the generic provider. Its public API mirrors the + Launch Kit lifecycle: prepare, verify, Kubernetes preflight, discover, + generate, deploy, validate, and clean. Workflow settings are raw argument + arrays; do not model or duplicate Launch Kit flags, schema, or defaults. + Discovery can stage a complete `user_config`. Its validate step uses + `timeout: null` so Launch Kit owns the automatically calculated or + user-supplied matrix deadline. +- `config/network-operator.yaml` is deliberately independent of the generic + lifecycle provider. It runs exactly one catalog-owning test step: + `l8k validate --user-config --deployment-files `, followed + by a linked same-phase finalizer that always invokes `l8k sosreport` after an + attempted validation. Sosreport is evidence collection, not another test. + The installed binary, reachable Kubernetes cluster, reconciled Network + Operator deployment, complete Launch Kit config, and rendered deployment + files are prerequisites. The installation must also expose Launch Kit's + `kubectl-netop_sosreport` helper. Do not add prepare, verify, preflight, + discover, generate, deploy, clean, or unrelated finalizer steps to this + entrypoint. +- The Network Operator provider inputs are `executable`, `user_config`, + `deployment_files`, `working_dir`, `artifact_dir`, and a string-only + `environment` mapping. Both input paths are resolved and checked before + execution, then supplied through Launch Kit's real CLI flags. The adapter + must not copy, merge, interpret, or modify them. +- `isvctl/configs/suites/k8s-launch-kit/network-operator.yaml` contains one + catalog test, `LaunchKitConnectivityCheck`. There are no fabric, deployment, + or connectivity-family use-case tests. Those choices come from the complete + Launch Kit config and current cluster state. +- `isvtest/validations/k8s_launch_kit/checks.py` consumes only + `connectivity.PingResults`. Every emitted row becomes a subtest with its + family, endpoints, and rails. Preserve bandwidth, GPU, stderr, and error + details when present. Do not require a fixed family list: disabled families + are absent without skips, and explicit new families pass through. +- A missing or empty connectivity matrix fails rather than passing vacuously. + The provider binds its step with `requires_selected_validations` so command + failures remain owned by the catalog validation and appear in structured + reporting. +- The adapter adds only `--output json` to commands that emit structured + output, wraps the unmodified concatenated JSON documents, and records argv, + cwd, stdout, stderr, exit code, and timing. For `validate`, use the emitted + `reportPath` as the authoritative HTML report source and copy it to + `/k8s-launch-kit-validation-report.html`. `l8k sosreport` is + text-streaming; default its `--output-dir` to the provider evidence directory, + retain that directory as an artifact, and wrap the command without parsing + its output. Do not invent a `selfValidation` result or reinterpret Launch + Kit's verdict. +- The generic provider retains installation, Kubernetes preflight, and cleanup + support for other consumers. `l8k clean` remains its only supported deletion + path; never reproduce Launch Kit cleanup with kubectl. +- Mock-backed provider coverage loads the production YAML and injects + test-owned executables in memory. Result interpretation tests live under + `isvtest/tests/k8s_launch_kit/`. +- The structured PRD source is + `docs/requirements/network-operator-readiness-requirements.yaml`. Keep its + traceability edges in `docs/requirements/test-requirements-matrix.yaml`, + document the prerequisite boundary in + `docs/guides/k8s-launch-kit/network-operator.md`, and regenerate committed + views with `make plan`. + ## Environment Variables | Variable | Description | Used by | diff --git a/docs/README.md b/docs/README.md index 052375a85..77e9f4727 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,6 +19,7 @@ Welcome to the documentation for NVIDIA AI Cloud Validation suite - a collection - [Configuration](guides/configuration.md) - Configuration file format and options - [External Validation Guide](guides/external-validation-guide.md) - Create custom validations without modifying the repo +- [Network Operator Launch Kit integration](guides/k8s-launch-kit/network-operator.md) - Connectivity validation, always-run sosreport evidence, mock-backed unit coverage, and limitations - [Remote Deployment](guides/remote-deployment.md) - Deploy and run tests on remote machines - [Local Development](guides/local-development.md) - MicroK8s setup for local testing - [Troubleshooting: Test runs stuck in STARTED](guides/troubleshooting-started-tests.md) - Why runs stay STARTED in the portal and how to fix it diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index 9e5547144..e4cf69bb2 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -56,7 +56,10 @@ Pre-built configs are provided in `isvctl/configs/`: | `providers/aws/config/vm.yaml` | AWS EC2 GPU instance tests | | `providers/aws/config/iam.yaml` | AWS IAM user lifecycle | | `providers/aws/config/eks.yaml` | AWS EKS with GPU nodes | +| `providers/k8s-launch-kit/config/provider.yaml` | Generic Kubernetes Launch Kit workflow | +| `providers/k8s-launch-kit/config/network-operator.yaml` | Launch Kit connectivity validation with post-run sosreport collection | | `suites/k8s.yaml` | Standard Kubernetes cluster | +| `suites/k8s-launch-kit/*.yaml` | Launch Kit-specific Network Operator catalog wiring | | `suites/slurm.yaml` | Slurm HPC cluster | ## Basic Usage @@ -155,17 +158,60 @@ Each platform defines phases and steps: commands: network: phases: ["setup", "test", "teardown"] # Execution order + continue_after_failure: [] # Optional independent test phases steps: [...] # Steps grouped by phase ``` | Field | Required | Description | | ----- | -------- | ----------- | -| `phases` | No | Ordered list of phases (default: `["setup", "test", "teardown"]`) | +| `phases` | No | Ordered list of phases (default: `["setup", "teardown"]`) | +| `continue_after_failure` | No | Phase names whose failure records a failed run but does not prevent later phases from running | | `steps` | Yes | List of step configurations | | `skip` | No | Skip this entire platform | **Important:** If a step's `phase` is not in the `phases` list, an error is raised. +Phase names are not limited to `setup`, `test`, and `teardown`. Any other name +is a custom test phase: it runs in the declared order, appears under its own +name in the orchestration summary, and is selected by `--phase test`. A +validation bound to a step runs after that step's custom phase. + +By default, a failed phase prevents later non-teardown phases from running. Use +`continue_after_failure` only when the named phases are independent test cases +and collecting every result in one invocation is more useful than stopping at +the first failure: + +```yaml +commands: + network_operator: + phases: [setup, roce-sriov, infiniband-sriov, roce-host-device] + continue_after_failure: [roce-sriov, infiniband-sriov, roce-host-device] + steps: + - name: prepare + phase: setup + command: ./prepare.sh + - name: test_roce_sriov + phase: roce-sriov + command: ./run-use-case.sh + args: [roce-sriov] + - name: test_infiniband_sriov + phase: infiniband-sriov + command: ./run-use-case.sh + args: [infiniband-sriov] + - name: test_roce_host_device + phase: roce-host-device + command: ./run-use-case.sh + args: [roce-host-device] +``` + +This setting changes continuation, not the verdict: if `roce-sriov` fails, +later listed use cases still run, but the final orchestration result remains +failed. Every continuation name must also appear once in `phases`; +configuration validation rejects unknown or duplicate names and forbids +`setup` and `teardown`. Do not list prerequisites shared by later phases or +phases that leave state on which later phases depend. Teardown retains its +existing `teardown_on_failure` behavior. + ### Step Configuration Each step defines a command to execute: @@ -180,6 +226,7 @@ Each step defines a command to execute: AWS_PROFILE: "production" skip: false continue_on_failure: false + finalizer_for: null output_schema: vpc ``` @@ -189,12 +236,83 @@ Each step defines a command to execute: | `phase` | No | Phase this step belongs to (default: `setup`) | | `command` | Yes | Script/command to execute | | `args` | No | Arguments (supports Jinja2 templates) | -| `timeout` | No | Timeout in seconds (default: 300) | +| `timeout` | No | Orchestration watchdog in seconds (default: 300); `null` disables it | | `env` | No | Environment variables | | `skip` | No | Skip this step | | `continue_on_failure` | No | Continue even if this step fails | +| `finalizer_for` | No | Run as linked teardown after the named step's phase when that command was attempted | | `output_schema` | No | Schema name for output validation | | `requires` | No | Capability contexts this step runs in (see [Capabilities](#capabilities-and-requires)) | +| `requires_selected_validations` | No | Configured validation names that must remain selected after capability, label, and suite-exclusion filtering; failed steps become errors on these owning validations | + +The timeout is an orchestration watchdog, not a provider-specific setting. Set +it to `null` only when the invoked tool owns a bounded deadline; isvctl will +then wait for the command to exit. On POSIX systems, isvctl starts each step in +a separate process group. When the +watchdog expires, it sends `SIGTERM` to the entire group, waits briefly, then +uses `SIGKILL` if needed. This prevents a wrapper's child CLI from continuing +to modify infrastructure after the wrapper step has been reported as timed +out. On non-POSIX systems, isvctl terminates the direct child process. + +#### Linked teardown finalizers + +Use `finalizer_for` when cleanup must run after the validations for one custom +test phase, including when the mutating step or a validation failed. Declare +cleanup in `phase: teardown`; the orchestrator executes it directly after its +target's test phase instead of waiting until every test case has finished: + +```yaml +commands: + network: + phases: [setup, use-case-one, use-case-two, teardown] + continue_after_failure: [use-case-one] + steps: + - name: deploy_fixture + phase: use-case-one + command: ./deploy.sh + + - name: clean_fixture + phase: teardown + command: ./clean.sh + finalizer_for: deploy_fixture +``` + +The finalizer target must resolve to one unique step, precede the configured +`teardown` phase, and cannot itself be a finalizer. The finalizer must use the +same capability and validation-selection gates as its target. Configuration +validation rejects violations of these rules. + +The orchestrator withholds linked teardown from normal phase execution, runs +the target phase validations, and then executes the eligible cleanup in +best-effort mode. The result is reported separately as +`-teardown`. This interleaving applies even to `--phase test`, so +multiple independent cases cannot leave deployments overlapping until the end +of the suite. A target activates cleanup only when its command process actually +started, whether it passed or failed. If an earlier prerequisite stopped the +phase, a template could not be rendered, or the executable could not be +started, cleanup is reported as skipped; this prevents deletion of pre-existing +state the current run never mutated. + +An explicit `--phase teardown` run executes linked teardown steps without an +in-memory target attempt. This is the standalone recovery path for resources +left by an interrupted earlier run. When target test phases and teardown are +part of the same invocation, already-linked cleanup is not run again in the +final teardown position. + +An ordinary use-case failure may still honor `continue_after_failure` after its +finalizers succeed. A failed finalizer always blocks later non-teardown phases, +because the fixture can no longer be assumed clean. Finalizer command output +and failure details are recorded in the teardown phase result. Keep finalizers +lifecycle-only rather than binding validations to their output, because target +phase validations intentionally run before cleanup. A same-phase finalizer is +still supported for compatibility, but a destructive provider cleanup should +normally be declared in `phase: teardown` so its lifecycle role and reporting +are explicit. + +Finalizers are an orchestration guarantee, not a recovery service. An abrupt +isvctl process termination, host failure, or `SIGKILL` can prevent them from +running. Provider cleanup commands should therefore be idempotent and usable as +standalone recovery commands. #### Gating a step with `requires` @@ -217,6 +335,54 @@ same one, so setup and teardown always move together. A step that survives the gate must not reference a gated-off step's output; use `default(...)` if it legitimately might be absent. +#### Gating Mutating Steps by Test Selection + +Use `requires_selected_validations` when a lifecycle step exists only to serve +specific validation entries. This applies selection before the command runs, +so `--label` and `--exclude-label` do not execute an unrelated deployment and +then discard its result: + +```yaml +commands: + network: + steps: + - name: deploy_ethernet_fixture + phase: ethernet + command: ./deploy-ethernet.sh + requires_selected_validations: [EthernetConnectivityCheck] + +tests: + validations: + network: + checks: + EthernetConnectivityCheck: + step: deploy_ethernet_fixture + labels: [ethernet] +``` + +With no label filter, the validation is selected and the step runs. With +`--label ethernet`, it also runs; with `--label infiniband`, the step is +skipped before execution. Every listed validation must be configured and +selected. The gate also honors capability requirements, `tests.exclude.tests`, +and effective label exclusions. + +The same list is the reporting ownership edge for the lifecycle step. If a +selected step fails before its validation can run, each listed validation is +reported as `error` with reason `step_failed`, including in JUnit. This prevents +an early deploy or setup failure from being misreported as a harmless +`step_no_output` skip merely because a later validation step was never reached. +The error message names the failed step and retains its redacted command +diagnostic. + +Pytest `-k` and `-m` expressions are evaluated inside pytest and therefore do +not drive `requires_selected_validations`. Use framework `--label` filtering +for lifecycle pruning in mutating suites. + +Selection-filtered validations remain in the structured result and JUnit +report. With the default `tests.settings.show_skipped_tests: false`, terminal +output omits summary phases containing only those filtered validations. Set it +to `true` when the skipped selection decisions should be visible interactively. + ### Validation Configuration Validations are centralized in `tests.validations`, grouped by category. Each group binds to a step and lists checks as a dict: @@ -351,6 +517,13 @@ Capability names and plain-suite names share one namespace, so a plain suite may not be named after a capability. `catalog_document` and `scripts/validate_suite_wiring.py` both reject the collision. +Suite discovery is recursive under `isvctl/configs/suites/`. A domain with +multiple related suites may therefore use a subdirectory such as +`suites/k8s-launch-kit/`; catalog generation, `--suite` resolution, doctor, +wiring validation, and test-plan coverage all discover the nested YAMLs. Suite +identity is still the YAML filename stem, so stems must remain unique across +the complete suite tree. + ## Import and Override Provider configs can import a canonical test suite and override command definitions while inheriting validations (unless explicitly overridden): @@ -584,6 +757,37 @@ checks: fields: ["network_id"] ``` +`CompositeCheck` is the existing framework runner behind `compose:`; authors do +not register or invoke that class directly. The YAML key creates one catalog +test and runs every listed validation member. Each member is reported as a +subtest. If a member reports its own probes through `report_subtest()`, those +probes are retained with qualified names such as +`ConnectivityCheck/rping/worker-a->worker-b/rail-0->rail-1`. +This avoids collisions between members and keeps the full probe tree in pytest +and JUnit output. + +A member may call `pytest.skip` when it is not applicable to the current +environment. `CompositeCheck` records that member as a skipped subtest and +continues with the remaining members. The skip neither passes nor fails the +member, and the parent composite passes when every non-skipped member passes. +This is different from skipping the step output or the composite itself, both +of which skip the entire parent validation. + +The orchestration summary automatically abbreviates a successful validation +that reported subtests: + +```text +MyUseCase: PASSED - 12 subtests passed +``` + +If optional probes were skipped, the summary includes passed, failed, and +skipped counts. Failed and errored validations keep their original diagnostic +message instead of being abbreviated. There is no YAML presentation flag; +this behavior applies to composites and ordinary validation classes alike. +After subtest testcase nodes are injected into JUnit, the suite's tests, +failures, errors, and skipped counters are recalculated from those serialized +nodes so reports do not double-count pytest's pre-counted subtest events. + `SchemaValidation` remains directly wireable, but is catalog-excluded because the step executor runs schema checks automatically. diff --git a/docs/guides/k8s-launch-kit/network-operator.md b/docs/guides/k8s-launch-kit/network-operator.md new file mode 100644 index 000000000..c207c91e0 --- /dev/null +++ b/docs/guides/k8s-launch-kit/network-operator.md @@ -0,0 +1,222 @@ + + + +# Network Operator connectivity validation through Kubernetes Launch Kit + +## Scope + +The Network Operator suite performs one validation operation: + +```text +l8k validate --user-config --deployment-files +``` + +It reports the connectivity matrix produced by Launch Kit. After that command +is attempted, an always-run linked finalizer invokes: + +```text +l8k sosreport --output-dir /sosreport +``` + +The diagnostic command runs whether validation passes, returns an error, or +produces a failing connectivity matrix. It is evidence collection, not a +second catalog test. The suite does not install +or verify the `l8k` binary, discover topology, generate manifests, deploy +Network Operator, run a separate Kubernetes preflight, or clean cluster state. + +Those activities are prerequisites. Before starting the suite, the ISV must +provide a reachable Kubernetes cluster, bring Network Operator and the desired +networking profile into the expected state, create a complete Launch Kit +configuration, and retain the corresponding rendered deployment files. + +There are no separate AI Cloud Validation tests for RoCE, InfiniBand, SR-IOV, +RDMA Shared, host-device, ICMP, rping, bandwidth, or GPUDirect. The supplied +Launch Kit configuration determines the topology and enabled validation +families. This avoids duplicating Launch Kit's configuration and applicability +model in AI Cloud Validation. + +## Architecture + +```text +Network Operator provider YAML + -> validation step + -> adapter.py + -> l8k validate --user-config ... --deployment-files ... --output json + -> retained argv, stdout, stderr, exit code, duration, and HTML report + -> Network Operator suite YAML + -> LaunchKitConnectivityCheck + -> one subtest for every Launch Kit connectivity row + -> linked finalizer, after the connectivity assertion + -> adapter.py + -> l8k sosreport --output-dir .../evidence/sosreport + -> retained diagnostic directory, stdout, stderr, exit code, and duration + -> console and JUnit results +``` + +The relevant files are: + +| Layer | File | +|---|---| +| Production entrypoint | `isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml` | +| CLI transport | `isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py` | +| Catalog wiring | `isvctl/configs/suites/k8s-launch-kit/network-operator.yaml` | +| Result interpretation | `isvtest/src/isvtest/validations/k8s_launch_kit/checks.py` | +| Mock-backed provider tests | `isvctl/tests/providers/k8s_launch_kit/` | +| Result-check unit tests | `isvtest/tests/k8s_launch_kit/` | + +The generic provider in +`isvctl/configs/providers/k8s-launch-kit/config/provider.yaml` still mirrors the +complete Launch Kit lifecycle for other consumers. The Network Operator +entrypoint does not import it, so none of those lifecycle steps are inherited. + +The `l8k` installation must also make the upstream +`kubectl-netop_sosreport` helper available to `l8k sosreport`. Validate this +once with a direct `l8k sosreport --output-dir ` call. If +Launch Kit reports that the script is missing, install the helper below the +same installation prefix at `share/l8k/scripts/kubectl-netop_sosreport` before +running the suite. + +## Inputs + +The Network Operator provider exposes only these settings: + +| Key | Required | Meaning | +|---|---:|---| +| `executable` | no | `l8k` command or absolute executable path; default is `l8k` | +| `user_config` | yes | Complete Launch Kit cluster configuration | +| `deployment_files` | yes | Existing rendered deployment directory validated by Launch Kit | +| `working_dir` | no | Provider process working directory | +| `artifact_dir` | no | Directory for command evidence | +| `environment` | no | String environment entries forwarded to Launch Kit, such as `KUBECONFIG` | + +Paths accept `~`, but absolute paths are preferable in automation. The adapter +resolves both paths, verifies that `user_config` is a file and +`deployment_files` is a directory, and passes the resolved paths to Launch Kit. +It does not copy, merge, parse, or modify either input. + +Launch Kit owns every setting inside the complete config, including the +selected profile, validation mode, enabled checks, GPUDirect behavior, +bandwidth thresholds, per-operation timeouts, routing, IP pools, and resource +names. AI Cloud Validation stores no copies of those defaults. + +Do not also put `--user-config` or `--deployment-files` in a raw Launch Kit +argument list. The adapter rejects duplicate path sources rather than allowing +ambiguous last-value behavior. + +## Running the suite + +From the repository root: + +```bash +uv run isvctl test run \ + -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ + --capability kubernetes \ + --set 'context.k8s_launch_kit.user_config=/absolute/path/cluster-config.yaml' \ + --set 'context.k8s_launch_kit.deployment_files=/absolute/path/deployment' \ + --no-upload -- -v +``` + +To use a kubeconfig that is not selected by the normal client environment, add: + +```text +--set 'context.k8s_launch_kit.environment={"KUBECONFIG":"/absolute/path/kubeconfig.yaml"}' +``` + +Omit `--no-upload` when the run should use the configured AI Cloud Labs upload +path. + +## Selecting connectivity checks + +Selection happens in the Launch Kit config, not with AI Cloud Validation +labels. For example, disabling Launch Kit GPUDirect validation means no +`gpudirect_dmabuf` rows are emitted. The wrapper then reports the remaining +rows only; it does not create a skipped or failed GPUDirect placeholder. + +Likewise, the suite does not infer a fabric or deployment mode from labels. +Run it once for the exact cluster state described by the supplied files. To +validate another topology, provision that topology and invoke the same suite +with its config and deployment directory. + +## Timeouts + +The `launch_kit_validate` step has `timeout: null`. Launch Kit calculates and +logs its connectivity-matrix budget by default, or honors the timeout configured +by the user. This prevents an independent isvctl watchdog from terminating a +valid large matrix before Launch Kit's bounded checks finish. An enclosing CI +job may still impose an overall job timeout. + +The `launch_kit_sosreport` finalizer has a 30-minute orchestration watchdog. +Unlike connectivity validation, the current Launch Kit sosreport command does +not calculate its own total deadline. A timeout or sosreport command error is +reported as a separate `test-teardown` orchestration failure; it does not +replace the connectivity test result. + +## Results and errors + +`LaunchKitConnectivityCheck` finds the `connectivity.PingResults` array in the +unmodified JSON stream. Each emitted row becomes a named subtest: + +```text +/->/-> +``` + +Failure messages preserve Launch Kit's expectation, observed result, bandwidth +and minimum when present, endpoint GPU information when present, stderr, and +structured error text. Explicit future `Family` values are forwarded without +requiring an AI Cloud Validation catalog update. Older numeric `Kind` values +remain supported as a compatibility fallback. + +The check fails when any emitted row has `OK != true`, when no connectivity +matrix is present, or when the matrix contains no results. A command that fails +before producing connectivity output retains its provider error in the +validation and JUnit output. + +The sosreport finalizer runs after this assertion. If sosreport itself fails, +the connectivity result remains intact and the overall orchestration reports +the diagnostic-collection failure separately. + +## Evidence + +The adapter writes: + +```text +_output/k8s-launch-kit/network-operator/ + work/ + evidence/ + k8s-launch-kit-validation-report.html + commands/validate/ + command.json + stdout.txt + stderr.log + commands/sosreport/ + command.json + stdout.txt + stderr.log + sosreport/ + ... files produced by the Network Operator sosreport helper ... +``` + +`command.json` records the resolved argv, exit code, and duration. `stdout.txt` +contains Launch Kit's complete JSON stream, including static validation, +connectivity, and report-path documents; `stderr.log` retains CLI progress and +diagnostics. The adapter uses the emitted `reportPath` as the authoritative +source, copies the HTML file to +`evidence/k8s-launch-kit-validation-report.html`, and registers the copied path +as the `validation_report` artifact. The original report remains at the path +written by Launch Kit, normally below the supplied deployment directory. A +report emitted for a failed connectivity matrix is copied in the same way. If +Launch Kit advertises a report that cannot be read, the provider returns an +evidence-retention error instead of silently reusing an older report. + +The sosreport command currently streams human-readable output even when the +global `--output` flag is available. The adapter therefore preserves that +stream in `commands/sosreport/stdout.txt` and emits its own normal structured +step envelope; it does not attempt to reinterpret the diagnostic contents. + +## PRD boundary + +This integration covers reportable Launch Kit connectivity validation. It +deliberately treats topology discovery, manifest generation, installation, +deployment health preparation, profile selection, and restoration as external +prerequisites. Tests that intentionally mutate Network Operator state require a +separate transaction and restoration design before they can be added. diff --git a/docs/packages/isvctl.md b/docs/packages/isvctl.md index 10ba7f3a4..348082ab6 100644 --- a/docs/packages/isvctl.md +++ b/docs/packages/isvctl.md @@ -133,6 +133,27 @@ isvctl test validate -f isvctl/configs/suites/k8s.yaml See [Configuration Guide](../guides/configuration.md) for full details. +All lifecycle and step commands run with captured stdout/stderr and an outer +watchdog. On POSIX, a timeout terminates the command's complete process group +(`SIGTERM`, then `SIGKILL` after a short grace period), which prevents a child +provider CLI from continuing after its wrapper step has timed out. See +[Step Configuration](../guides/configuration.md#step-configuration). + +Cleanup steps may use `phase: teardown` with +`finalizer_for: `. The linked teardown runs directly after the +target's phase validations whenever that command started, including after +target or validation failure, and is reported as `-teardown`. +An explicit teardown-only run executes it as standalone recovery. Cleanup +failure blocks later non-teardown phases. See +[Linked teardown finalizers](../guides/configuration.md#linked-teardown-finalizers) +for activation, ordering, and process-failure limitations. + +Steps gated with `requires_selected_validations` also declare which validation +owns their lifecycle result. If one of those steps fails, the named validation +is emitted as a `step_failed` error in structured results and JUnit even when a +later validation-producing step never runs. See +[Gating mutating steps by test selection](../guides/configuration.md#gating-mutating-steps-by-test-selection). + ### Unified Config Structure ```yaml diff --git a/docs/packages/isvtest.md b/docs/packages/isvtest.md index 32c65483f..8d93e2914 100644 --- a/docs/packages/isvtest.md +++ b/docs/packages/isvtest.md @@ -67,6 +67,25 @@ Utility checks that work with any step output. `SchemaValidation` remains directly wireable, but is catalog-excluded because the step executor runs schema checks automatically. +### Composite checks and nested results + +`CompositeCheck` is existing internal framework machinery used when a suite +declares `compose:`. It is excluded from discovery and the catalog; the named +YAML entry is the test identity. Every member runs, even after an earlier member +fails, and is reported as a parent subtest. Subtests reported by a member are +also forwarded with `MemberName/probe-name` names, so their detailed messages +and timing remain available in pytest and JUnit. If a member calls +`pytest.skip`, `CompositeCheck` records that member as skipped and continues +with the remaining members. A skipped member neither passes nor fails the +composite; the composite passes when every non-skipped member passes. + +The isvctl orchestration summary uses the structured subtest counts to render a +concise line for successful parents. Failure and error messages are never +replaced by that summary. JUnit suite counters are reconciled with the emitted +parent and subtest testcase nodes after injection. See the +[configuration guide](../guides/configuration.md#available-validations) for +YAML and output examples. + | Validation | Platforms | Description | | ---------- | --------- | ----------- | | `StepSuccessCheck` | all | Compose-only: check step completed successfully | diff --git a/docs/requirements/README.md b/docs/requirements/README.md index d7e3de1b6..d81fbbabb 100644 --- a/docs/requirements/README.md +++ b/docs/requirements/README.md @@ -15,6 +15,8 @@ reconciles all of these different goals. | `software-reference-requirements.md` | Generated rendering of the reference YAML (`make plan`); one contributing requirements doc among several. | | `storage-acceptance-requirements.yaml` | **Source of record** for the DGXC Storage Acceptance Test requirements (PRD-ref namespace). | | `storage-acceptance-requirements.md` | Generated rendering of the storage YAML (`make plan`). | +| `network-operator-readiness-requirements.yaml` | **Source of record** for the Enterprise RA Network Operator self-validation integration PRD. | +| `network-operator-readiness-requirements.md` | Generated rendering of the Network Operator PRD YAML (`make plan`). | | `test-requirements-matrix.yaml` | The **traceability matrix (index)**: which requirement(s) each test relates to, across documents (`source`). | | `test-requirements-matrix.adoc` | Generated and committed traceability matrix, viewable in github (or renderable to html) | | `../../scripts/reqtrace.py` | Integrity checks (`reqtrace validate`; `make reqcheck`). | @@ -96,6 +98,7 @@ record. | `BFX` (04+) | reference | break-fix health (continues offtake `BFX`) | | `BENCH` | reference | exemplar benchmarking | | `N-*` | storage | Storage Acceptance test IDs | +| `ENT-REQ-*` | network-operator-prd | Enterprise Network Operator self-validation integration | ## 3. `legacy_ids` @@ -146,16 +149,22 @@ belonging to a given requirements document*. We keep the data flexible for this: When a new team's requirements document is blessed, reconcile it here: -1. **Register prefix(es)** for the new doc in the registry (sec. 2). Resolve +1. **Add a structured requirements source.** Give it a globally unique + top-level `source`, because that value is the matrix join key. For a project + PRD, set `format: project-prd` to reuse the generic section/area renderer; + do not add a source-specific renderer branch. Add the file to + `DEFAULT_SOURCES` in `requirements_source_to_md.py` when `make plan` should + render it by default. +2. **Register prefix(es)** for the new doc in the registry (sec. 2). Resolve any overload before proceeding (see the `CP` lesson). -2. **Assign IDs.** Prefer mirroring the upstream requirement IDs. On collision +3. **Assign IDs.** Prefer mirroring the upstream requirement IDs. On collision with an existing prefix, apply the collision policy (sec. 2): continue the number space, or (selectively) choose another resolution and record why. -3. **Add/adjust tests** in `test-plan.yaml` (the canonical truth). Use +4. **Add/adjust tests** in `test-plan.yaml` (the canonical truth). Use `legacy_ids` for any renames. -4. **Update the matrix** (`test-requirements-matrix.yaml`): add each +5. **Update the matrix** (`test-requirements-matrix.yaml`): add each test->requirement edge with the new `source`, plus `annotations`/`notes`. -5. **Validate**: `make reqcheck` must pass; **regenerate**: `make plan`. +6. **Validate**: `make reqcheck` must pass; **regenerate**: `make plan`. > Kept as a subsection for now; promote to its own `ONBOARDING.md` if it grows. diff --git a/docs/requirements/network-operator-readiness-requirements.md b/docs/requirements/network-operator-readiness-requirements.md new file mode 100644 index 000000000..12e0a8485 --- /dev/null +++ b/docs/requirements/network-operator-readiness-requirements.md @@ -0,0 +1,51 @@ + + +# Enterprise RA Network Operator Self-Validation Integration PRD + +> Structured source of record: `network-operator-readiness-requirements.yaml` (version prd-snapshot-2026-08-04). +> Owner: NVIDIA Network Operator team. +> Edit the YAML, not this file. + +## Ownership + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-000 | Integration ownership | The Network Operator team owns and maintains the integration solution, including compatibility updates for the underlying tests. | active | + +## Framework Integration + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-001 | Standard validation workflow | Integrate Network Operator self-validation tests into AI Cloud Validation so Enterprise and AI Cloud Ready users can run them through the standard workflow. | active | +| ENT-REQ-002 | Launch Kit reuse | Reuse applicable Kubernetes Launch Kit validation components, including topology discovery, manifest readiness, RDMA connectivity, and RDMA bandwidth. | active | +| ENT-REQ-003 | Selection and program profiles | Support individual and grouped Network Operator validation, with Enterprise and AI Cloud Ready profiles able to mark checks required or optional. | active | +| ENT-REQ-004 | Runtime parameters | Expose applicable runtime parameters such as namespace, node selector, network and driver modes, rail and network names, resource and IP pool names, GPU count, and timeout. | active | + +## Network Validation + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-005 | Ethernet and RoCE | Validate SR-IOV Network RDMA and RDMA Shared scenarios, including secondary network attachment, RDMA device availability, pod-to-pod RDMA or RoCE connectivity, and basic bandwidth. | active | +| ENT-REQ-006 | InfiniBand | Validate InfiniBand SR-IOV and RDMA Shared with IPoIB scenarios, including IB device availability, pod network attachment, and pod-to-pod InfiniBand connectivity. | active | +| ENT-REQ-007 | Host-device networking | Validate host-device networking for Kubernetes workers running in virtual machines, covering both Ethernet or RoCE and InfiniBand. | active | +| ENT-REQ-008 | GPUDirect RDMA | Validate GPUDirect RDMA peer-to-peer connectivity between GPU-enabled pods across supported worker nodes. | active | +| ENT-REQ-009 | Deployment health | Validate Network Operator deployment health and required resources for the selected mode, including policies, secondary networks, IP pools, Multus and CNI components, and drivers. | active | + +## Lifecycle Safety + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-010 | State restoration | For tests that modify Network Operator or cluster state, capture the pre-test configuration and restore the original Network Operator state after success or failure. | active | + +## Catalog and Documentation + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-011 | Catalog metadata | Add catalog entries for all tests with owner, labels, dependencies, descriptions, and required YAML updates while following repository contribution standards. | active | +| ENT-REQ-012 | Prerequisites | Document the required cluster prerequisites for each validation area. | active | + +## Reporting + +| Req ID | Requirement Area | Description | Status | +| :----- | :--------------- | :---------- | :----- | +| ENT-REQ-013 | Results and evidence | Integrate pass or fail status, logs, Launch Kit reports, generated manifests, Kubernetes state, connectivity results, and bandwidth results into AI Cloud Validation reporting, catalog, and AI Cloud Labs artifacts. | active | diff --git a/docs/requirements/network-operator-readiness-requirements.yaml b/docs/requirements/network-operator-readiness-requirements.yaml new file mode 100644 index 000000000..5dc2aabef --- /dev/null +++ b/docs/requirements/network-operator-readiness-requirements.yaml @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Enterprise RA Network Operator self-validation integration requirements. +# Structured from the PRD supplied for the Kubernetes Launch Kit integration. +# Render the publishable Markdown view with `make plan`. + +source: network-operator-prd +format: project-prd +title: Enterprise RA Network Operator Self-Validation Integration PRD +version: prd-snapshot-2026-08-04 +owner: NVIDIA Network Operator team +requirements: + - req_id: ENT-REQ-000 + section: Ownership + area: Integration ownership + description: The Network Operator team owns and maintains the integration solution, including compatibility updates for the underlying tests. + status: active + - req_id: ENT-REQ-001 + section: Framework Integration + area: Standard validation workflow + description: Integrate Network Operator self-validation tests into AI Cloud Validation so Enterprise and AI Cloud Ready users can run them through the standard workflow. + status: active + - req_id: ENT-REQ-002 + section: Framework Integration + area: Launch Kit reuse + description: Reuse applicable Kubernetes Launch Kit validation components, including topology discovery, manifest readiness, RDMA connectivity, and RDMA bandwidth. + status: active + - req_id: ENT-REQ-003 + section: Framework Integration + area: Selection and program profiles + description: Support individual and grouped Network Operator validation, with Enterprise and AI Cloud Ready profiles able to mark checks required or optional. + status: active + - req_id: ENT-REQ-004 + section: Framework Integration + area: Runtime parameters + description: Expose applicable runtime parameters such as namespace, node selector, network and driver modes, rail and network names, resource and IP pool names, GPU count, and timeout. + status: active + - req_id: ENT-REQ-005 + section: Network Validation + area: Ethernet and RoCE + description: Validate SR-IOV Network RDMA and RDMA Shared scenarios, including secondary network attachment, RDMA device availability, pod-to-pod RDMA or RoCE connectivity, and basic bandwidth. + status: active + - req_id: ENT-REQ-006 + section: Network Validation + area: InfiniBand + description: Validate InfiniBand SR-IOV and RDMA Shared with IPoIB scenarios, including IB device availability, pod network attachment, and pod-to-pod InfiniBand connectivity. + status: active + - req_id: ENT-REQ-007 + section: Network Validation + area: Host-device networking + description: Validate host-device networking for Kubernetes workers running in virtual machines, covering both Ethernet or RoCE and InfiniBand. + status: active + - req_id: ENT-REQ-008 + section: Network Validation + area: GPUDirect RDMA + description: Validate GPUDirect RDMA peer-to-peer connectivity between GPU-enabled pods across supported worker nodes. + status: active + - req_id: ENT-REQ-009 + section: Network Validation + area: Deployment health + description: Validate Network Operator deployment health and required resources for the selected mode, including policies, secondary networks, IP pools, Multus and CNI components, and drivers. + status: active + - req_id: ENT-REQ-010 + section: Lifecycle Safety + area: State restoration + description: For tests that modify Network Operator or cluster state, capture the pre-test configuration and restore the original Network Operator state after success or failure. + status: active + - req_id: ENT-REQ-011 + section: Catalog and Documentation + area: Catalog metadata + description: Add catalog entries for all tests with owner, labels, dependencies, descriptions, and required YAML updates while following repository contribution standards. + status: active + - req_id: ENT-REQ-012 + section: Catalog and Documentation + area: Prerequisites + description: Document the required cluster prerequisites for each validation area. + status: active + - req_id: ENT-REQ-013 + section: Reporting + area: Results and evidence + description: Integrate pass or fail status, logs, Launch Kit reports, generated manifests, Kubernetes state, connectivity results, and bandwidth results into AI Cloud Validation reporting, catalog, and AI Cloud Labs artifacts. + status: active diff --git a/docs/requirements/test-requirements-matrix.adoc b/docs/requirements/test-requirements-matrix.adoc index 49745c250..3b01bb04b 100644 --- a/docs/requirements/test-requirements-matrix.adoc +++ b/docs/requirements/test-requirements-matrix.adoc @@ -3504,4 +3504,94 @@ docs/requirements/test-requirements-matrix.yaml. Run `make plan` to regenerate. | full | +| [[K8S42-01]]K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment +| ENT-REQ-001 +| network-operator-prd +| full +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment +| ENT-REQ-002 +| network-operator-prd +| partial +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment +| ENT-REQ-004 +| network-operator-prd +| partial +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment +| ENT-REQ-005 +| network-operator-prd +| partial +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment +| ENT-REQ-006 +| network-operator-prd +| partial +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment +| ENT-REQ-007 +| network-operator-prd +| partial +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment +| ENT-REQ-008 +| network-operator-prd +| partial +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment +| ENT-REQ-011 +| network-operator-prd +| partial +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment +| ENT-REQ-012 +| network-operator-prd +| partial +| + +| K8S42-01 +| Workload Orchestration +| Managed Kubernetes Control Plane +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment +| ENT-REQ-013 +| network-operator-prd +| partial +| + |=== diff --git a/docs/requirements/test-requirements-matrix.yaml b/docs/requirements/test-requirements-matrix.yaml index 64b889c7c..3119a6994 100644 --- a/docs/requirements/test-requirements-matrix.yaml +++ b/docs/requirements/test-requirements-matrix.yaml @@ -8,7 +8,7 @@ # # Per mapping: # test_id - matches docs/test-plan.yaml -# requirements - list of { req_id, source: offtake|reference, coverage: full|partial } +# requirements - list of { req_id, source: , coverage: full|partial } # annotations - free-form (e.g. how this relationship was decided) # notes - free-form scratch # @@ -2568,3 +2568,37 @@ mappings: coverage: full annotations: '' notes: '' + - test_id: K8S42-01 + requirements: + - req_id: ENT-REQ-001 + source: network-operator-prd + coverage: full + - req_id: ENT-REQ-002 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-004 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-005 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-006 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-007 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-008 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-011 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-012 + source: network-operator-prd + coverage: partial + - req_id: ENT-REQ-013 + source: network-operator-prd + coverage: partial + annotations: 'Runs one l8k validate command and reports every emitted connectivity row. Cluster provisioning, topology discovery, manifest generation, deployment health, applicability, and state restoration are prerequisites outside this suite. Catalog wiring, local command evidence, and JUnit reporting are implemented; AI Cloud Labs binary attachment upload remains a gap.' + notes: '' diff --git a/docs/test-plan.adoc b/docs/test-plan.adoc index fabd52576..9170c977c 100644 --- a/docs/test-plan.adoc +++ b/docs/test-plan.adoc @@ -3456,7 +3456,7 @@ a| | pending | -.58+| Workload Orchestration +.59+| Workload Orchestration | Backup and Recovery | Centralized managed service to automate and govern data backup across services | AWS backup @@ -3646,7 +3646,7 @@ a| | published | -.45+| Managed Kubernetes Control Plane +.46+| Managed Kubernetes Control Plane | Tenant-isolated Kubernetes control planes for managing k8s workloads. does placement, networking, lifecyle mgmt. | AWS EKS, GCP GKE | [[K8S05-01]]K8S05-01 @@ -4033,6 +4033,23 @@ a| https://github.com/NVIDIA/ai-cloud-validation/issues/220[#220] | pending | +| Network Operator connectivity validation through Kubernetes Launch Kit +| +| [[K8S42-01]]K8S42-01 +| K8S42 +| +| era, kubernetes, ncp, network_operator, slow +| operator +| One l8k validate invocation consumes a complete user config and rendered deployment directory; topology, deployment readiness, and selected connectivity families are prerequisites +| P0 +a| +| LimitedEnv +| +| +| Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment +| pending +| + .2+| K8s Versioning & Compliance .2+| | [[K8S02-01]]K8S02-01 diff --git a/docs/test-plan.yaml b/docs/test-plan.yaml index bf209c912..bed9f40a5 100644 --- a/docs/test-plan.yaml +++ b/docs/test-plan.yaml @@ -3636,6 +3636,24 @@ domains: milestone: M5 github_issues: - "#220" + - description: Network Operator connectivity validation through Kubernetes Launch Kit + tests: + - summary: Run the Launch Kit connectivity matrix against an ISV-provisioned Network Operator deployment + labels: + - era + - kubernetes + - ncp + - network_operator + - slow + priority: P0 + dependencies: + - LimitedEnv + milestone: "" + req_id: K8S42 + test_id: K8S42-01 + actor: operator + status: pending + notes: "One l8k validate invocation consumes a complete user config and rendered deployment directory; topology, deployment readiness, and selected connectivity families are prerequisites" - description: "K8s Versioning & Compliance" tests: - summary: Verify support for the three most recent minor releases (N-2) diff --git a/isvctl/configs/providers/k8s-launch-kit/README.md b/isvctl/configs/providers/k8s-launch-kit/README.md new file mode 100644 index 000000000..cf6b3f533 --- /dev/null +++ b/isvctl/configs/providers/k8s-launch-kit/README.md @@ -0,0 +1,90 @@ + + + +# Kubernetes Launch Kit provider internals + +## Layout + +| Path | Purpose | +|---|---| +| `config/provider.yaml` | Generic provider mirroring the full Launch Kit workflow | +| `config/network-operator.yaml` | Connectivity validation plus always-run diagnostics for an ISV-provisioned Network Operator deployment | +| `scripts/adapter.py` | Thin process and JSON evidence transport | + +Executable mocks and pinned scenarios are test-only and live under +`isvctl/tests/providers/k8s_launch_kit/fixtures/`. Product configuration must +never reference them. + +## Generic provider + +`config/provider.yaml` exposes install/verify, Kubernetes preflight, discover, +generate, deploy, validate, and clean for consumers that own the complete +Launch Kit lifecycle. Its workflow configuration is raw argument arrays. It +does not reproduce Launch Kit's domain schema or defaults. + +Discovery optionally stages a complete `user_config`, writes the resolved +`cluster-config.yaml`, and deletes the staged copy after the command. Validate +uses `timeout: null` because Launch Kit calculates a bounded connectivity budget +or honors its user-supplied timeout. The other generic steps retain finite +outer watchdogs. + +## Network Operator provider + +`config/network-operator.yaml` intentionally does not import the generic +provider. It defines one catalog-owning test step: + +```text +l8k validate --user-config --deployment-files --output json +``` + +After validation is attempted, a linked finalizer always executes: + +```text +l8k sosreport --output-dir /sosreport +``` + +Sosreport runs after both successful and failed validation commands and after +the connectivity assertion. It is diagnostic evidence, not another catalog +test. Its failure is reported as a separate teardown result. + +The cluster, Network Operator deployment, complete Launch Kit config, rendered +deployment directory, and installed `l8k` binary are prerequisites. There are +no AI Cloud Validation use-case workflows and no discover, generate, deploy, +clean, verification, or separate preflight steps. + +The adapter resolves and validates the two input paths without copying or +parsing them. It rejects raw `--user-config` or `--deployment-files` arguments +when the dedicated inputs are used. Launch Kit remains responsible for fabric, +deployment type, enabled checks, GPUDirect applicability, thresholds, runtime +budgets, and every other value in its config. + +The one catalog validation, `LaunchKitConnectivityCheck`, converts every +emitted `connectivity.PingResults` row into a subtest. It does not expect a +fixed family list: a disabled family is absent, while a newly emitted family is +reported automatically. + +## Adapter contract + +For every structured workflow invocation, `adapter.py`: + +1. resolves the configured executable; +2. adds `--output json` unless the caller already selected JSON; +3. executes exactly one Launch Kit command; +4. preserves stdout, stderr, argv, exit code, and duration; +5. parses concatenated JSON objects without renaming their fields; +6. copies the HTML file advertised by a validation `reportPath` into the + provider evidence directory; +7. returns one provider envelope containing the raw documents and artifact paths. + +For `sosreport`, the adapter does not force JSON because the current Launch Kit +command streams text output. It defaults `--output-dir` to the provider evidence +directory, records that directory as an artifact, and still returns the same +structured provider envelope. The installed Launch Kit must make its +`kubectl-netop_sosreport` helper available under the Launch Kit installation +prefix. + +Semantic assertions belong in +`isvtest.validations.k8s_launch_kit`, not the transport. + +See the [Network Operator integration guide](../../../../docs/guides/k8s-launch-kit/network-operator.md) +for prerequisites, invocation, output, and evidence layout. diff --git a/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml b/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml new file mode 100644 index 000000000..eb13b5595 --- /dev/null +++ b/isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Network Operator connectivity validation for an ISV-provisioned cluster. +# The complete Launch Kit configuration and rendered deployment directory are +# prerequisites. This provider validates them, then always collects diagnostics: +# l8k validate --user-config --deployment-files +# l8k sosreport --output-dir /sosreport + +import: + - ../../../suites/k8s-launch-kit/network-operator.yaml + +version: "1.0" + +context: + k8s_launch_kit: + executable: l8k + user_config: "" + deployment_files: "" + working_dir: ../../../../../_output/k8s-launch-kit/network-operator/work + artifact_dir: ../../../../../_output/k8s-launch-kit/network-operator/evidence + environment: {} + +commands: + network_operator: + phases: [test] + steps: + - name: launch_kit_validate + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ context.k8s_launch_kit.executable }}" + - --command + - validate + - --arguments-json + - "[]" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - "--deployment-files={{ context.k8s_launch_kit.deployment_files }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + # Launch Kit calculates the connectivity-matrix budget or honors the + # timeout configured by the user, so it owns this command's deadline. + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [LaunchKitConnectivityCheck] + + # A linked same-phase finalizer runs after the connectivity assertion even + # when validate exits nonzero. It is not a separate catalog test. + - name: launch_kit_sosreport + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ context.k8s_launch_kit.executable }}" + - --command + - sosreport + - --arguments-json + - "[]" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [LaunchKitConnectivityCheck] + finalizer_for: launch_kit_validate diff --git a/isvctl/configs/providers/k8s-launch-kit/config/provider.yaml b/isvctl/configs/providers/k8s-launch-kit/config/provider.yaml new file mode 100644 index 000000000..8d07e6410 --- /dev/null +++ b/isvctl/configs/providers/k8s-launch-kit/config/provider.yaml @@ -0,0 +1,219 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Generic Kubernetes Launch Kit provider. This production configuration invokes +# l8k and kubectl from PATH unless a user overlay explicitly replaces them. +# +# This file intentionally contains no Network Operator, topology, resource, +# validation, or l8k timeout defaults. The arguments for every command are +# passed directly to l8k. Omitted values are resolved by Launch Kit itself. + +version: "1.0" + +context: + k8s_launch_kit: + executable: l8k + installation: + mode: verify + version: "" + installer_ref: "" + installer_sha256: "" + prefix: "" + # Optional complete Launch Kit configuration. Discovery stages a copy as + # ./user-config.yaml and writes its resolved output to ./cluster-config.yaml. + user_config: "" + kubectl_command: [] + working_dir: ../../../../../_output/k8s-launch-kit/work + artifact_dir: ../../../../../_output/k8s-launch-kit/evidence + environment: {} + discover: + arguments: [] + generate: + arguments: [] + deploy: + arguments: [] + validate: + arguments: [] + clean: + arguments: [] + +commands: + network_operator: + phases: [setup, test, teardown] + steps: + - name: launch_kit_prepare + phase: setup + command: python3 ../scripts/adapter.py + args: + - prepare + - --mode + - "{{ context.k8s_launch_kit.installation.mode }}" + - --executable + - "{{ context.k8s_launch_kit.executable }}" + - "--version={{ context.k8s_launch_kit.installation.version }}" + - "--installer-ref={{ context.k8s_launch_kit.installation.installer_ref }}" + - "--installer-sha256={{ context.k8s_launch_kit.installation.installer_sha256 }}" + - "--prefix={{ context.k8s_launch_kit.installation.prefix }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 900 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [LaunchKitConnectivityCheck] + + # Test-phase verification is intentional: --phase test may bypass setup. + - name: launch_kit_verify + phase: test + command: python3 ../scripts/adapter.py + args: + - verify + - --executable + - "{{ steps.launch_kit_prepare.executable | default(context.k8s_launch_kit.executable) }}" + - "--expected-version={{ context.k8s_launch_kit.installation.version }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [LaunchKitConnectivityCheck] + + # This gate always runs in the test phase, before l8k can mutate a cluster. + - name: launch_kit_kubernetes_preflight + phase: test + command: python3 ../scripts/adapter.py + args: + - preflight + - --kubectl-command-json + - "{{ context.k8s_launch_kit.kubectl_command | tojson }}" + - --workflow-arguments-json + - "{{ {'discover': context.k8s_launch_kit.discover.arguments, 'generate': context.k8s_launch_kit.generate.arguments, 'deploy': context.k8s_launch_kit.deploy.arguments, 'validate': context.k8s_launch_kit.validate.arguments, 'clean': context.k8s_launch_kit.clean.arguments} | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 60 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [LaunchKitConnectivityCheck] + + - name: launch_kit_discover + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - discover + - --arguments-json + - "{{ context.k8s_launch_kit.discover.arguments | tojson }}" + - "--user-config={{ context.k8s_launch_kit.user_config }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 1800 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [LaunchKitConnectivityCheck] + + - name: launch_kit_generate + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - generate + - --arguments-json + - "{{ context.k8s_launch_kit.generate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 600 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [LaunchKitConnectivityCheck] + + - name: launch_kit_deploy + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - deploy + - --arguments-json + - "{{ context.k8s_launch_kit.deploy.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [LaunchKitConnectivityCheck] + + - name: launch_kit_validate + phase: test + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable }}" + - --command + - validate + - --arguments-json + - "{{ context.k8s_launch_kit.validate.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + # l8k calculates a bounded matrix budget or honors the user's explicit + # connectivity timeout, so it owns the deadline for this command. + timeout: null + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [LaunchKitConnectivityCheck] + + # Runs after phase validations when deploy was attempted, even if deploy, + # validate, or a validation check failed. + - name: launch_kit_clean + phase: teardown + command: python3 ../scripts/adapter.py + args: + - run + - --executable + - "{{ steps.launch_kit_verify.executable | default(context.k8s_launch_kit.executable) }}" + - --command + - clean + - --arguments-json + - "{{ context.k8s_launch_kit.clean.arguments | tojson }}" + - --environment-json + - "{{ context.k8s_launch_kit.environment | tojson }}" + - --working-dir + - "{{ context.k8s_launch_kit.working_dir }}" + - --artifact-dir + - "{{ context.k8s_launch_kit.artifact_dir }}" + timeout: 7200 + output_schema: k8s_launch_kit + requires: [kubernetes] + requires_selected_validations: [LaunchKitConnectivityCheck] + finalizer_for: launch_kit_deploy diff --git a/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py b/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py new file mode 100644 index 000000000..62c14e368 --- /dev/null +++ b/isvctl/configs/providers/k8s-launch-kit/scripts/adapter.py @@ -0,0 +1,865 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Thin AI Cloud Validation transport for the Kubernetes Launch Kit CLI. + +The provider deliberately exposes the real Launch Kit operations. It forwards +user-supplied arguments verbatim and requests structured output from commands +that implement it. Discovery can stage a complete user config transiently. +Validation can bind an existing complete user config and rendered deployment +directory directly. Its emitted HTML report and sosreport output are retained +below the provider artifact directory. Launch Kit remains the owner of command +flags, configuration schema, and defaults. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.request +from json import JSONDecoder +from pathlib import Path +from typing import Any + +_WORKFLOW_COMMANDS = ("discover", "generate", "deploy", "validate", "clean") +_RUN_COMMANDS = (*_WORKFLOW_COMMANDS, "sosreport") +_INSTALLER_URL = "https://raw.githubusercontent.com/NVIDIA/k8s-launch-kit/{ref}/scripts/install.sh" +_STAGED_USER_CONFIG = "user-config.yaml" +_DISCOVERED_CLUSTER_CONFIG = "cluster-config.yaml" +_VALIDATION_REPORT_NAME = "k8s-launch-kit-validation-report.html" + + +def _parse_json_value(raw: str, source: str, expected_type: type[Any]) -> Any: + """Parse a JSON CLI value and enforce its root type.""" + try: + value = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"{source} is not valid JSON: {exc}") from exc + if not isinstance(value, expected_type): + raise ValueError(f"{source} must contain a {expected_type.__name__}") + return value + + +def _parse_json_stream(raw: str, source: str) -> list[dict[str, Any]]: + """Parse zero or more concatenated JSON objects from ``raw``.""" + decoder = JSONDecoder() + documents: list[dict[str, Any]] = [] + offset = 0 + while offset < len(raw): + while offset < len(raw) and raw[offset].isspace(): + offset += 1 + if offset >= len(raw): + break + try: + value, offset = decoder.raw_decode(raw, offset) + except json.JSONDecodeError as exc: + raise ValueError(f"{source} contains invalid JSON at byte {exc.pos}: {exc.msg}") from exc + if not isinstance(value, dict): + raise ValueError(f"{source} document #{len(documents) + 1} is not an object") + documents.append(value) + return documents + + +def _write_json(path: Path, value: Any) -> None: + """Write deterministic structured evidence.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2) + "\n", encoding="utf-8") + + +def _resolve_executable(value: str) -> Path: + """Resolve an explicit path or a command available on ``PATH``.""" + candidate = Path(value).expanduser() + if candidate.is_absolute() or candidate.parent != Path("."): + resolved = candidate.resolve() + if not resolved.is_file(): + raise FileNotFoundError(f"Launch Kit executable not found: {resolved}") + return resolved + found = shutil.which(value) + if found is None: + raise FileNotFoundError(f"Launch Kit executable not found on PATH: {value}") + return Path(found).resolve() + + +def _with_json_output(arguments: list[str]) -> list[str]: + """Return workflow arguments that request Launch Kit's automation output.""" + result = list(arguments) + for index, token in enumerate(result): + if token == "--output": + if index + 1 >= len(result): + raise ValueError("--output requires a value") + if result[index + 1] != "json": + raise ValueError("the Launch Kit provider requires --output json") + return result + if token.startswith("--output="): + if token.partition("=")[2] != "json": + raise ValueError("the Launch Kit provider requires --output json") + return result + result.extend(["--output", "json"]) + return result + + +def _bind_sosreport_output(arguments: list[str], *, working_dir: Path, artifact_dir: Path) -> tuple[list[str], Path]: + """Resolve the sosreport output directory and default it to retained evidence.""" + result = list(arguments) + output_dir: Path | None = None + index = 0 + while index < len(result): + token = result[index] + if token == "--output-dir": + if index + 1 >= len(result) or not result[index + 1]: + raise ValueError("--output-dir requires a non-empty value") + output_dir = Path(result[index + 1]).expanduser() + index += 2 + continue + if token.startswith("--output-dir="): + value = token.partition("=")[2] + if not value: + raise ValueError("--output-dir requires a non-empty value") + output_dir = Path(value).expanduser() + index += 1 + + if output_dir is None: + output_dir = artifact_dir / "sosreport" + result.extend(["--output-dir", str(output_dir)]) + elif not output_dir.is_absolute(): + output_dir = (working_dir / output_dir).resolve() + + return result, output_dir.resolve() + + +def _retain_validation_report( + documents: list[dict[str, Any]], + *, + working_dir: Path, + artifact_dir: Path, +) -> Path | None: + """Copy the HTML report advertised by Launch Kit into retained evidence.""" + report_value = next( + ( + document["reportPath"] + for document in reversed(documents) + if isinstance(document.get("reportPath"), str) and document["reportPath"] + ), + None, + ) + if report_value is None: + return None + + source = Path(report_value).expanduser() + if not source.is_absolute(): + source = working_dir / source + source = source.resolve() + if not source.is_file(): + raise FileNotFoundError(f"Launch Kit HTML validation report not found: {source}") + + destination = (artifact_dir / _VALIDATION_REPORT_NAME).resolve() + destination.parent.mkdir(parents=True, exist_ok=True) + if source != destination: + shutil.copy2(source, destination) + return destination + + +def _structured_error(documents: list[dict[str, Any]]) -> str | None: + """Extract the most actionable Launch Kit structured error.""" + for document in reversed(documents): + error = document.get("error") + if not isinstance(error, dict): + continue + message = error.get("message") + if not isinstance(message, str) or not message: + continue + suggestion = error.get("suggestion") + if isinstance(suggestion, str) and suggestion: + return f"{message}; {suggestion}" + return message + return None + + +def _stderr_excerpt(stderr: str) -> str | None: + """Return the last non-empty stderr line without flooding the envelope.""" + lines = [line.strip() for line in stderr.splitlines() if line.strip()] + return lines[-1] if lines else None + + +def _run_process(argv: list[str], *, cwd: Path, env: dict[str, str]) -> dict[str, Any]: + """Execute a child process and retain both output streams.""" + started = time.monotonic() + try: + completed = subprocess.run( + argv, + cwd=cwd, + env=env, + check=False, + capture_output=True, + text=True, + ) + return { + "exit_code": completed.returncode, + "stdout": completed.stdout, + "stderr": completed.stderr, + "duration_seconds": time.monotonic() - started, + } + except OSError as exc: + return { + "exit_code": -1, + "stdout": "", + "stderr": str(exc), + "duration_seconds": time.monotonic() - started, + } + + +def _record_process(directory: Path, argv: list[str], result: dict[str, Any]) -> dict[str, str]: + """Persist one command, stdout, and stderr as evidence.""" + directory.mkdir(parents=True, exist_ok=True) + stdout_path = directory / "stdout.txt" + stderr_path = directory / "stderr.log" + command_path = directory / "command.json" + stdout_path.write_text(str(result["stdout"]), encoding="utf-8") + stderr_path.write_text(str(result["stderr"]), encoding="utf-8") + _write_json( + command_path, + { + "argv": argv, + "exit_code": result["exit_code"], + "duration_seconds": result["duration_seconds"], + }, + ) + return { + "stdout": str(stdout_path.resolve()), + "stderr": str(stderr_path.resolve()), + "command": str(command_path.resolve()), + } + + +def _environment(raw: str) -> dict[str, str]: + """Merge user-supplied string environment entries with the process environment.""" + supplied = _parse_json_value(raw, "--environment-json", dict) + invalid = [str(key) for key, value in supplied.items() if not isinstance(key, str) or not isinstance(value, str)] + if invalid: + raise ValueError("--environment-json keys and values must be strings") + env = os.environ.copy() + env.update(supplied) + return env + + +def _stage_user_config( + source_value: str, + working_dir: Path, + arguments: list[str], +) -> tuple[list[str], Path | None, dict[str, Any] | None]: + """Stage a complete user config for discovery and return safe provenance.""" + if not source_value: + return arguments, None, None + + conflicting_flags = [ + flag + for flag in ("--user-config", "--save-cluster-config") + if any(token == flag or token.startswith(f"{flag}=") for token in arguments) + ] + if conflicting_flags: + raise ValueError( + "context.k8s_launch_kit.user_config cannot be combined with raw discovery flag(s): " + + ", ".join(conflicting_flags) + ) + + source = Path(source_value).expanduser().resolve() + if not source.is_file(): + raise FileNotFoundError(f"Launch Kit user config not found: {source}") + try: + source.relative_to(working_dir) + except ValueError: + pass + else: + raise ValueError("Launch Kit user config must be outside the retained provider working directory") + + staged = working_dir / _STAGED_USER_CONFIG + content = source.read_bytes() + staged.unlink(missing_ok=True) + try: + descriptor = os.open(staged, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as stream: + stream.write(content) + except OSError: + staged.unlink(missing_ok=True) + raise + + discovered = working_dir / _DISCOVERED_CLUSTER_CONFIG + return ( + [ + *arguments, + "--user-config", + str(staged), + "--save-cluster-config", + str(discovered), + ], + staged, + { + "source_path": str(source), + "staged_path": str(staged), + "sha256": hashlib.sha256(content).hexdigest(), + "size_bytes": len(content), + "retained": False, + }, + ) + + +def _bind_validate_inputs( + user_config_value: str, + deployment_files_value: str, + arguments: list[str], +) -> list[str]: + """Bind required, pre-existing Launch Kit validation inputs.""" + if not user_config_value: + raise ValueError("context.k8s_launch_kit.user_config is required for Network Operator validation") + if not deployment_files_value: + raise ValueError("context.k8s_launch_kit.deployment_files is required for Network Operator validation") + + conflicting_flags = [ + flag + for flag in ("--user-config", "--deployment-files") + if any(token == flag or token.startswith(f"{flag}=") for token in arguments) + ] + if conflicting_flags: + raise ValueError( + "dedicated Launch Kit validation inputs cannot be combined with raw flag(s): " + + ", ".join(conflicting_flags) + ) + + user_config = Path(user_config_value).expanduser().resolve() + if not user_config.is_file(): + raise FileNotFoundError(f"Launch Kit user config not found: {user_config}") + deployment_files = Path(deployment_files_value).expanduser().resolve() + if not deployment_files.is_dir(): + raise FileNotFoundError(f"Launch Kit deployment directory not found: {deployment_files}") + return [ + *arguments, + "--user-config", + str(user_config), + "--deployment-files", + str(deployment_files), + ] + + +def _run_workflow(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + """Invoke exactly one real Launch Kit workflow command.""" + executable = _resolve_executable(args.executable) + arguments = _parse_json_value(args.arguments_json, "--arguments-json", list) + if not all(isinstance(value, str) for value in arguments): + raise ValueError("--arguments-json must contain only strings") + environment = _environment(args.environment_json) + working_dir = Path(args.working_dir).expanduser().resolve() + working_dir.mkdir(parents=True, exist_ok=True) + artifact_dir = Path(args.artifact_dir).expanduser().resolve() + retained_validation_report = artifact_dir / _VALIDATION_REPORT_NAME + if args.command == "validate": + retained_validation_report.unlink(missing_ok=True) + staged_user_config: Path | None = None + user_config_metadata_path: Path | None = None + sosreport_output_dir: Path | None = None + try: + if args.command == "validate" and args.deployment_files is not None: + arguments = _bind_validate_inputs(args.user_config, args.deployment_files, arguments) + elif args.user_config: + if args.command != "discover": + raise ValueError("--user-config requires --deployment-files for the validate workflow command") + arguments, staged_user_config, user_config_metadata = _stage_user_config( + args.user_config, + working_dir, + arguments, + ) + user_config_metadata_path = artifact_dir / "inputs" / "user-config.json" + _write_json(user_config_metadata_path, user_config_metadata) + if args.command == "sosreport": + arguments, sosreport_output_dir = _bind_sosreport_output( + arguments, + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + else: + arguments = _with_json_output(arguments) + argv = [str(executable), args.command, *arguments] + result = _run_process(argv, cwd=working_dir, env=environment) + finally: + if staged_user_config is not None: + staged_user_config.unlink(missing_ok=True) + artifacts = _record_process(artifact_dir / "commands" / args.command, argv, result) + if user_config_metadata_path is not None: + artifacts["user_config"] = str(user_config_metadata_path) + if sosreport_output_dir is not None and sosreport_output_dir.exists(): + artifacts["sosreport"] = str(sosreport_output_dir) + + parse_error: str | None = None + if args.command == "sosreport": + # The current sosreport command accepts the global --output flag but + # streams human-readable helper output in both modes. Preserve it as a + # process artifact and let this adapter provide the structured envelope. + documents = [] + else: + try: + documents = _parse_json_stream(str(result["stdout"]), f"l8k {args.command} stdout") + except ValueError as exc: + documents = [] + parse_error = str(exc) + + report_retention_error: str | None = None + if args.command == "validate" and parse_error is None: + try: + retained_report = _retain_validation_report( + documents, + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + except (FileNotFoundError, OSError) as exc: + retained_report = None + report_retention_error = f"failed to retain Launch Kit HTML validation report: {exc}" + if retained_report is not None: + artifacts["validation_report"] = str(retained_report) + + success = result["exit_code"] == 0 and parse_error is None and report_retention_error is None + error = parse_error or _structured_error(documents) + if result["exit_code"] != 0 and error is None: + error = f"l8k {args.command} exited with code {result['exit_code']}" + if excerpt := _stderr_excerpt(str(result["stderr"])): + error = f"{error}: {excerpt}" + if report_retention_error is not None: + error = f"{error}; {report_retention_error}" if error else report_retention_error + envelope: dict[str, Any] = { + "success": success, + "platform": "kubernetes", + "operation": args.command, + "executable": str(executable), + "argv": argv, + "working_directory": str(working_dir), + "exit_code": result["exit_code"], + "duration_seconds": result["duration_seconds"], + "documents": documents, + "artifacts": artifacts, + } + if sosreport_output_dir is not None: + envelope["sosreport_output_directory"] = str(sosreport_output_dir) + if error: + envelope["error"] = error + excerpt = _stderr_excerpt(str(result["stderr"])) + if excerpt: + envelope["stderr_excerpt"] = excerpt + exit_code = int(result["exit_code"]) + return envelope, exit_code if exit_code > 0 else (0 if success else 1) + + +def _verify_executable( + executable: Path, + artifact_dir: Path, + expected_version: str = "", + environment: dict[str, str] | None = None, +) -> tuple[dict[str, Any], bool, str | None]: + """Run Launch Kit version and schema commands and preserve both responses.""" + env = environment.copy() if environment is not None else os.environ.copy() + checks: dict[str, Any] = {} + artifacts: dict[str, dict[str, str]] = {} + errors: list[str] = [] + for name, command_args in (("version", ["version", "--output", "json"]), ("schema", ["schema"])): + argv = [str(executable), *command_args] + result = _run_process(argv, cwd=Path.cwd(), env=env) + artifacts[name] = _record_process(artifact_dir / name, argv, result) + try: + documents = _parse_json_stream(str(result["stdout"]), f"l8k {name} stdout") + except ValueError as exc: + documents = [] + errors.append(str(exc)) + if result["exit_code"] != 0: + errors.append( + _stderr_excerpt(str(result["stderr"])) or f"l8k {name} exited with code {result['exit_code']}" + ) + elif len(documents) != 1: + errors.append(f"l8k {name} must emit exactly one JSON object, got {len(documents)}") + passed = result["exit_code"] == 0 and len(documents) == 1 + if name == "schema" and passed: + commands = documents[0].get("commands") + advertised = set(commands) if isinstance(commands, dict) else set() + missing_commands = set(_WORKFLOW_COMMANDS) - advertised + if missing_commands: + passed = False + errors.append( + "l8k schema does not advertise required command(s): " + ", ".join(sorted(missing_commands)) + ) + if name == "version" and passed and expected_version: + actual_version = documents[0].get("version") + if actual_version != expected_version: + passed = False + errors.append(f"l8k version mismatch: expected {expected_version!r}, got {actual_version!r}") + checks[name] = { + "passed": passed, + "documents": documents, + "exit_code": result["exit_code"], + "artifacts": artifacts[name], + } + return {"checks": checks, "artifacts": artifacts}, not errors, "; ".join(errors) or None + + +def _download_installer(installer_ref: str, expected_sha256: str, artifact_dir: Path) -> tuple[Path, str]: + """Download an immutable installer and verify its trusted SHA-256 digest.""" + if not re.fullmatch(r"[0-9a-fA-F]{40}", installer_ref): + raise ValueError("Launch Kit installer_ref must be a full 40-character Git commit SHA") + if not re.fullmatch(r"[0-9a-fA-F]{64}", expected_sha256): + raise ValueError("Launch Kit installer_sha256 must be a 64-character SHA-256 digest") + expected_sha256 = expected_sha256.lower() + url = _INSTALLER_URL.format(ref=installer_ref.lower()) + installer = artifact_dir / "installer.sh" + installer.parent.mkdir(parents=True, exist_ok=True) + installer.unlink(missing_ok=True) + request = urllib.request.Request(url, headers={"User-Agent": "ai-cloud-validation"}) + with urllib.request.urlopen(request, timeout=30) as response: + content = response.read() + digest = hashlib.sha256(content).hexdigest() + verified = digest == expected_sha256 + _write_json( + artifact_dir / "installer-download.json", + { + "url": url, + "ref": installer_ref.lower(), + "expected_sha256": expected_sha256, + "sha256": digest, + "verified": verified, + }, + ) + if not verified: + raise ValueError(f"Launch Kit installer SHA-256 mismatch: expected {expected_sha256}, got {digest}") + installer.write_bytes(content) + return installer, url + + +def _installed_executable(prefix: str) -> Path: + """Resolve the executable installed by the official Launch Kit installer.""" + install_prefix = Path(prefix).expanduser() if prefix else Path("/usr/local") + return _resolve_executable(str(install_prefix / "bin" / "l8k")) + + +def _prepare(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + """Optionally install Launch Kit, then verify version and schema.""" + artifact_dir = Path(args.artifact_dir).expanduser().resolve() / "prepare" + environment = _environment(args.environment_json) + installed = False + install_details: dict[str, Any] | None = None + if args.mode == "install": + installer, url = _download_installer(args.installer_ref, args.installer_sha256, artifact_dir) + installer_argv = ["/bin/sh", str(installer)] + if args.prefix: + installer_argv.extend(["-d", args.prefix]) + env = environment.copy() + if args.version: + env["L8K_VERSION"] = args.version + result = _run_process(installer_argv, cwd=Path.cwd(), env=env) + install_artifacts = _record_process(artifact_dir / "install", installer_argv, result) + install_details = { + "url": url, + "exit_code": result["exit_code"], + "installer": str(installer.resolve()), + "download_metadata": str((artifact_dir / "installer-download.json").resolve()), + "artifacts": install_artifacts, + } + if result["exit_code"] != 0: + error = _stderr_excerpt(str(result["stderr"])) or "Launch Kit installer failed" + return { + "success": False, + "platform": "kubernetes", + "operation": "prepare", + "installed": False, + "install": install_details, + "error": error, + }, 1 + executable = _installed_executable(args.prefix) + installed = True + else: + executable = _resolve_executable(args.executable) + + verification, success, error = _verify_executable( + executable, + artifact_dir / "verify", + args.version, + environment, + ) + envelope: dict[str, Any] = { + "success": success, + "platform": "kubernetes", + "operation": "prepare", + "installed": installed, + "executable": str(executable), + **verification, + } + if install_details is not None: + envelope["install"] = install_details + envelope["artifacts"]["installer"] = install_details["installer"] + envelope["artifacts"]["installer_download"] = install_details["download_metadata"] + envelope["artifacts"]["install"] = install_details["artifacts"] + if error: + envelope["error"] = error + return envelope, 0 if success else 1 + + +def _verify(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + """Verify an existing Launch Kit executable without installing it.""" + executable = _resolve_executable(args.executable) + environment = _environment(args.environment_json) + verification, success, error = _verify_executable( + executable, + Path(args.artifact_dir).expanduser().resolve() / "verify-test", + args.expected_version, + environment, + ) + envelope: dict[str, Any] = { + "success": success, + "platform": "kubernetes", + "operation": "verify", + "executable": str(executable), + **verification, + } + if error: + envelope["error"] = error + return envelope, 0 if success else 1 + + +def _kubeconfig_from_workflow(raw: str) -> str | None: + """Extract one consistent explicit kubeconfig from the real workflow arguments.""" + workflow = _parse_json_value(raw, "--workflow-arguments-json", dict) + unsupported = set(workflow) - set(_WORKFLOW_COMMANDS) + if not workflow or unsupported: + supported = ", ".join(_WORKFLOW_COMMANDS) + raise ValueError(f"--workflow-arguments-json must contain a non-empty subset of: {supported}") + found: set[str] = set() + for command, values in workflow.items(): + if command not in _WORKFLOW_COMMANDS or not isinstance(values, list): + raise ValueError("--workflow-arguments-json must map Launch Kit workflow commands to argument lists") + if not all(isinstance(value, str) for value in values): + raise ValueError(f"workflow arguments for {command} must contain only strings") + index = 0 + while index < len(values): + token = values[index] + if token == "--kubeconfig": + if index + 1 >= len(values): + raise ValueError(f"{command} --kubeconfig requires a value") + value = values[index + 1] + if not value: + raise ValueError(f"{command} --kubeconfig requires a non-empty value") + found.add(value) + index += 2 + continue + if token.startswith("--kubeconfig="): + value = token.partition("=")[2] + if not value: + raise ValueError(f"{command} --kubeconfig requires a non-empty value") + found.add(value) + index += 1 + if len(found) > 1: + raise ValueError(f"Launch Kit workflow commands select different kubeconfigs: {sorted(found)}") + return next(iter(found), None) + + +def _kubectl_prefix(raw: str, environment: dict[str, str]) -> list[str]: + """Resolve the configured kubectl-compatible invocation.""" + supplied = _parse_json_value(raw, "--kubectl-command-json", list) + if supplied: + if not all(isinstance(value, str) and value for value in supplied): + raise ValueError("--kubectl-command-json must contain non-empty strings") + invocation_dir = Path.cwd() + return [ + str((invocation_dir / value).resolve()) + if not Path(value).is_absolute() and "/" in value and (invocation_dir / value).exists() + else value + for value in supplied + ] + override = environment.get("KUBECTL", "").strip() + return shlex.split(override) if override else ["kubectl"] + + +def _preflight_check( + name: str, + argv: list[str], + *, + cwd: Path, + artifact_dir: Path, + environment: dict[str, str], +) -> tuple[dict[str, Any], dict[str, Any]]: + """Run and record one Kubernetes prerequisite command.""" + result = _run_process(argv, cwd=cwd, env=environment) + artifacts = _record_process(artifact_dir / name, argv, result) + passed = result["exit_code"] == 0 + message = "command succeeded" if passed else (_stderr_excerpt(str(result["stderr"])) or "command failed") + return { + "name": name, + "passed": passed, + "message": message, + "exit_code": result["exit_code"], + "artifacts": artifacts, + }, result + + +def _preflight(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + """Prove that the Kubernetes API and at least one Ready node are available.""" + working_dir = Path(args.working_dir).expanduser().resolve() + working_dir.mkdir(parents=True, exist_ok=True) + artifact_dir = Path(args.artifact_dir).expanduser().resolve() / "kubernetes-preflight" + kubeconfig = _kubeconfig_from_workflow(args.workflow_arguments_json) + environment = _environment(args.environment_json) + prefix = _kubectl_prefix(args.kubectl_command_json, environment) + kubeconfig_args = ["--kubeconfig", kubeconfig] if kubeconfig else [] + + version_check, version_result = _preflight_check( + "api-version", + [*prefix, *kubeconfig_args, "version", "-o", "json"], + cwd=working_dir, + artifact_dir=artifact_dir, + environment=environment, + ) + nodes_check, nodes_result = _preflight_check( + "nodes", + [*prefix, *kubeconfig_args, "get", "nodes", "-o", "json"], + cwd=working_dir, + artifact_dir=artifact_dir, + environment=environment, + ) + + server_version: str | None = None + if version_check["passed"]: + try: + version_payload = json.loads(str(version_result["stdout"])) + server = version_payload.get("serverVersion") if isinstance(version_payload, dict) else None + if isinstance(server, dict) and isinstance(server.get("gitVersion"), str): + server_version = server["gitVersion"] + else: + version_check["passed"] = False + version_check["message"] = "kubectl output has no serverVersion.gitVersion" + except json.JSONDecodeError as exc: + version_check["passed"] = False + version_check["message"] = f"kubectl version output is invalid JSON: {exc}" + + total_nodes = 0 + ready_nodes = 0 + if nodes_check["passed"]: + try: + nodes_payload = json.loads(str(nodes_result["stdout"])) + items = nodes_payload.get("items") if isinstance(nodes_payload, dict) else None + if not isinstance(items, list): + raise ValueError("kubectl node output has no items list") + total_nodes = len(items) + ready_nodes = sum( + any( + isinstance(condition, dict) + and condition.get("type") == "Ready" + and condition.get("status") == "True" + for condition in (node.get("status", {}).get("conditions", []) if isinstance(node, dict) else []) + ) + for node in items + ) + except (json.JSONDecodeError, ValueError) as exc: + nodes_check["passed"] = False + nodes_check["message"] = str(exc) + + inventory_check = { + "name": "non-empty-cluster", + "passed": total_nodes > 0, + "message": f"found {total_nodes} node(s)" if total_nodes else "cluster contains no nodes", + } + readiness_check = { + "name": "ready-node", + "passed": ready_nodes > 0, + "message": f"found {ready_nodes}/{total_nodes} Ready node(s)" if ready_nodes else "cluster has no Ready nodes", + } + checks = [version_check, nodes_check, inventory_check, readiness_check] + success = all(check["passed"] is True for check in checks) + envelope: dict[str, Any] = { + "success": success, + "platform": "kubernetes", + "operation": "kubernetes-preflight", + "kubeconfig_source": "workflow arguments" if kubeconfig else "kubectl environment/default resolution", + "server_version": server_version, + "node_count": total_nodes, + "ready_node_count": ready_nodes, + "checks": checks, + "artifacts": { + "api_version": version_check["artifacts"], + "nodes": nodes_check["artifacts"], + }, + } + if not success: + failures = [f"{check['name']}: {check['message']}" for check in checks if check["passed"] is not True] + envelope["error"] = "Kubernetes prerequisite failed: " + "; ".join(failures) + envelope["remediation"] = "Select a reachable cluster and verify Kubernetes API and Ready-node access" + return envelope, 0 if success else 1 + + +def _parser() -> argparse.ArgumentParser: + """Build the provider command-line parser.""" + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="action", required=True) + + prepare = subparsers.add_parser("prepare", help="Install when requested, then verify l8k") + prepare.add_argument("--mode", choices=("verify", "install"), required=True) + prepare.add_argument("--executable", required=True) + prepare.add_argument("--version", default="") + prepare.add_argument("--installer-ref", default="") + prepare.add_argument("--installer-sha256", default="") + prepare.add_argument("--prefix", default="") + prepare.add_argument("--environment-json", default="{}") + prepare.add_argument("--artifact-dir", required=True) + + verify = subparsers.add_parser("verify", help="Verify l8k version and schema") + verify.add_argument("--executable", required=True) + verify.add_argument("--expected-version", default="") + verify.add_argument("--environment-json", default="{}") + verify.add_argument("--artifact-dir", required=True) + + preflight = subparsers.add_parser("preflight", help="Verify Kubernetes API and Ready-node access") + preflight.add_argument("--kubectl-command-json", default="[]") + preflight.add_argument("--workflow-arguments-json", required=True) + preflight.add_argument("--environment-json", default="{}") + preflight.add_argument("--working-dir", required=True) + preflight.add_argument("--artifact-dir", required=True) + + run = subparsers.add_parser("run", help="Run one real Launch Kit workflow command") + run.add_argument("--executable", required=True) + run.add_argument("--command", choices=_RUN_COMMANDS, required=True) + run.add_argument("--arguments-json", required=True) + run.add_argument("--user-config", default="") + run.add_argument("--deployment-files", default=None) + run.add_argument("--environment-json", default="{}") + run.add_argument("--working-dir", required=True) + run.add_argument("--artifact-dir", required=True) + return parser + + +def main(argv: list[str] | None = None) -> int: + """Execute one provider operation and emit a single JSON envelope.""" + args = _parser().parse_args(argv) + try: + if args.action == "prepare": + envelope, exit_code = _prepare(args) + elif args.action == "verify": + envelope, exit_code = _verify(args) + elif args.action == "preflight": + envelope, exit_code = _preflight(args) + else: + envelope, exit_code = _run_workflow(args) + except (FileNotFoundError, OSError, TypeError, ValueError, urllib.error.URLError) as exc: + envelope = { + "success": False, + "platform": "kubernetes", + "operation": args.command if args.action == "run" else args.action, + "error": str(exc), + } + exit_code = 1 + print(json.dumps(envelope)) + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index 7b727cdbd..4466d832a 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -75,7 +75,11 @@ Suites: [`slurm`](slurm.yaml), [`control-plane`](control-plane.yaml), [`image-registry`](image-registry.yaml), -[`security`](security.yaml). +[`security`](security.yaml), +[`network-operator`](k8s-launch-kit/network-operator.yaml). +The Network Operator Launch Kit integration is unreleased; see the +[Launch Kit integration guide](../../../docs/guides/k8s-launch-kit/network-operator.md) +before running its cluster-mutating workflows. For the domain / script-count / AWS-reference overview see the [my-isv scaffold README](../providers/my-isv/scripts/README.md#domains). @@ -109,6 +113,11 @@ part that broke, and every member runs even after an earlier one fails. A member that needs parameters takes them inline (`- CheckName: {...}`); one that does not stays a single line. +If a member reports its own subtests, the composite forwards them as +`MemberName/probe-name`. Successful parents are summarized automatically by +subtest count in `isvctl` output; failures retain their complete diagnostic +message. This is shared renderer behavior, not a suite option. + Because a composite has no validation class to borrow from, it declares its own `description` (the catalog uses it) and its name must not shadow a class name. A check that wires one purpose-built class — `SerialConsoleCheck`, @@ -207,6 +216,40 @@ its plan item is not platform-scoped. | `switch_syslogs` | test | `providers/my-isv/scripts/observability/log_availability_test.py` | `tests.*.probes.switches_checked`, `log_source`, `entry_count`, `latest_timestamp` | | `switch_kernel_logs` | test | `providers/my-isv/scripts/observability/log_availability_test.py` | `tests.*.probes.switches_checked`, `log_source`, `entry_count`, `latest_timestamp` | +### Network Operator (`k8s-launch-kit/network-operator.yaml`) + +The Network Operator suite contains one catalog entry, +`LaunchKitConnectivityCheck`. Its production provider invokes `l8k validate` +with a caller-supplied complete `user_config` and existing `deployment_files` +directory, then invokes `l8k sosreport` as an always-run linked finalizer. The +Kubernetes cluster, Network Operator deployment, Launch Kit installation +(including its sosreport helper), configuration, and generated manifests are +prerequisites. + +Fabric, deployment type, enabled checks, and GPUDirect applicability are not +modeled as suite labels or separate tests. Every +`connectivity.PingResults` row emitted by Launch Kit becomes a subtest. A +disabled family is simply absent, and new explicit family values are reported +without suite changes. + +```bash +uv run isvctl test run \ + -f isvctl/configs/providers/k8s-launch-kit/config/network-operator.yaml \ + --capability kubernetes \ + --set 'context.k8s_launch_kit.user_config=/absolute/path/cluster-config.yaml' \ + --set 'context.k8s_launch_kit.deployment_files=/absolute/path/deployment' \ + --no-upload -- -v +``` + +| Step | Phase | Script | Key JSON Fields | +|------|-------|--------|-----------------| +| `launch_kit_validate` | test | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k validate` | raw `documents`, `argv`, `exit_code`, `duration_seconds`, `artifacts.validation_report` | +| `launch_kit_sosreport` | test finalizer | `providers/k8s-launch-kit/scripts/adapter.py run` -> `l8k sosreport` | `argv`, `exit_code`, `duration_seconds`, `sosreport_output_directory`, `artifacts` | + +The generic `providers/k8s-launch-kit/config/provider.yaml` remains available +to consumers that need the full Launch Kit lifecycle. See the +[Launch Kit integration guide](../../../docs/guides/k8s-launch-kit/network-operator.md). + ### VM (`vm.yaml`) | Step | Phase | Script | Key JSON Fields | diff --git a/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml b/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml new file mode 100644 index 000000000..ea874fdbe --- /dev/null +++ b/isvctl/configs/suites/k8s-launch-kit/network-operator.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# One catalog test consumes the connectivity matrix emitted by l8k validate. +# Fabric, deployment type, enabled checks, and all Launch Kit parameters belong +# to the supplied complete user config; they are prerequisites, not separate +# AI Cloud Validation use cases. + +version: "1.0" + +tests: + description: "Network Operator connectivity validation through Kubernetes Launch Kit" + + settings: + show_skipped_tests: false + + validations: + network_operator: + checks: + LaunchKitConnectivityCheck: + step: launch_kit_validate + test_id: "K8S42-01" + labels: ["era", "kubernetes", "ncp", "network_operator", "slow"] + requires: [kubernetes] diff --git a/isvctl/src/isvctl/cli/test.py b/isvctl/src/isvctl/cli/test.py index 37a8fb3f8..91393df37 100644 --- a/isvctl/src/isvctl/cli/test.py +++ b/isvctl/src/isvctl/cli/test.py @@ -164,6 +164,23 @@ def _reported_capability(config: RunConfig, capability_context: str | None) -> s return capability_context +def _validation_result_detail(validation: dict[str, Any], reason: str | None = None) -> str: + """Return concise success text while preserving skip and failure diagnostics.""" + message = str(validation.get("message", "")) + summary = validation.get("subtest_summary") + is_success = validation.get("passed", False) and not validation.get("skipped") + if validation.get("state") != "error" and is_success and isinstance(summary, dict): + passed = int(summary.get("passed", 0) or 0) + failed = int(summary.get("failed", 0) or 0) + skipped = int(summary.get("skipped", 0) or 0) + total = int(summary.get("total", passed + failed + skipped) or 0) + if total > 0: + if passed == total: + return f"{total} subtests passed" + return f"{total} subtests: {passed} passed, {failed} failed, {skipped} skipped" + return f"{reason}: {message}" if reason and message else str(reason or message) + + def _human_readable_dry_run( config: RunConfig, capability: str | None, @@ -667,19 +684,28 @@ def run( typer.echo("ORCHESTRATION RESULTS") typer.echo("=" * 60) + show_skipped_tests = bool(config.tests and config.tests.settings.get("show_skipped_tests", False)) for phase_result in result.phases: + phase_details = phase_result.details or {} + displayed_validations = phase_details.get("validations", []) + if not show_skipped_tests: + displayed_validations = [ + validation for validation in displayed_validations if not validation.get("skipped") + ] + if phase_details.get("validations") and not displayed_validations and not phase_details.get("steps"): + continue if phase_result.message.startswith("SKIPPED:"): status = typer.style("[SKIP]", fg=typer.colors.YELLOW) elif phase_result.success: status = typer.style("[PASS]", fg=typer.colors.GREEN) else: status = typer.style("[FAIL]", fg=typer.colors.RED) - phase_name = phase_result.phase.value.upper().ljust(8) + phase_name = (phase_result.name or phase_result.phase.value).upper().ljust(24) typer.echo(f"{status} {phase_name}: {phase_result.message}") # Display step details (schema validation, errors) - if phase_result.details and "steps" in phase_result.details: - for step in phase_result.details["steps"]: + if "steps" in phase_details: + for step in phase_details["steps"]: step_name = step.get("name", "unknown") step_success = step.get("success", False) schema_valid = step.get("schema_valid", True) @@ -709,31 +735,28 @@ def run( typer.echo(f" Output: {json.dumps(output, indent=2)[:500]}") # Display centralized validation results - if phase_result.details and "validations" in phase_result.details: - validations = phase_result.details["validations"] - if validations: - for vr in validations: - vr_name = vr.get("name", "unknown") - # Handle case where name might be a dict (extract class name) - if isinstance(vr_name, dict): - vr_name = next(iter(vr_name.keys()), "unknown") - vr_message = vr.get("message", "") - vr_category = vr.get("category", "") - category_prefix = f"[{vr_category}] " if vr_category else "" - if vr.get("state") == "error": - vr_status = typer.style("ERROR", fg=typer.colors.RED) - reason = vr.get("error_reason") - elif vr.get("skipped"): - vr_status = typer.style("SKIPPED", fg=typer.colors.YELLOW) - reason = vr.get("skip_reason") - elif vr.get("passed", False): - vr_status = typer.style("PASSED", fg=typer.colors.GREEN) - reason = None - else: - vr_status = typer.style("FAILED", fg=typer.colors.RED) - reason = None - detail = f"{reason}: {vr_message}" if reason and vr_message else (reason or vr_message) - typer.echo(f" {category_prefix}{vr_name}: {vr_status} - {detail}") + if displayed_validations: + for vr in displayed_validations: + vr_name = vr.get("name", "unknown") + # Handle case where name might be a dict (extract class name) + if isinstance(vr_name, dict): + vr_name = next(iter(vr_name.keys()), "unknown") + vr_category = vr.get("category", "") + category_prefix = f"[{vr_category}] " if vr_category else "" + if vr.get("state") == "error": + vr_status = typer.style("ERROR", fg=typer.colors.RED) + reason = vr.get("error_reason") + elif vr.get("skipped"): + vr_status = typer.style("SKIPPED", fg=typer.colors.YELLOW) + reason = vr.get("skip_reason") + elif vr.get("passed", False): + vr_status = typer.style("PASSED", fg=typer.colors.GREEN) + reason = None + else: + vr_status = typer.style("FAILED", fg=typer.colors.RED) + reason = None + detail = _validation_result_detail(vr, reason) + typer.echo(f" {category_prefix}{vr_name}: {vr_status} - {detail}") typer.echo("-" * 60) if result.success: diff --git a/isvctl/src/isvctl/config/output_schemas.py b/isvctl/src/isvctl/config/output_schemas.py index 6a0e24525..a9f68a585 100644 --- a/isvctl/src/isvctl/config/output_schemas.py +++ b/isvctl/src/isvctl/config/output_schemas.py @@ -1015,6 +1015,55 @@ "additionalProperties": True, "description": "Generic schema for unrecognized step names", }, + "k8s_launch_kit": { + "type": "object", + "required": ["success", "platform", "operation"], + "properties": { + **COMMON_PROPERTIES, + "operation": { + "type": "string", + "enum": [ + "prepare", + "verify", + "kubernetes-preflight", + "discover", + "generate", + "deploy", + "validate", + "clean", + "sosreport", + ], + "description": "The actual Launch Kit or provider prerequisite operation", + }, + "executable": {"type": "string"}, + "argv": {"type": "array", "items": {"type": "string"}}, + "working_directory": { + "type": "string", + "description": "Absolute working directory used for the Launch Kit command", + }, + "exit_code": {"type": "integer"}, + "documents": { + "type": "array", + "items": {"type": "object"}, + "description": "Unmodified JSON documents emitted by l8k", + }, + "sosreport_output_directory": { + "type": "string", + "description": "Absolute directory selected for l8k sosreport output", + }, + "checks": { + "oneOf": [ + {"type": "object"}, + {"type": "array", "items": {"type": "object"}}, + ] + }, + "artifacts": {"type": "object"}, + "error": {"type": "string"}, + "remediation": {"type": "string"}, + }, + "additionalProperties": True, + "description": "Transport envelope around an unmodified Kubernetes Launch Kit CLI operation", + }, # ========================================================================= # Multi-cluster schemas # ========================================================================= diff --git a/isvctl/src/isvctl/config/schema.py b/isvctl/src/isvctl/config/schema.py index 7d4ad1ef1..14e74effd 100644 --- a/isvctl/src/isvctl/config/schema.py +++ b/isvctl/src/isvctl/config/schema.py @@ -83,7 +83,10 @@ class StepConfig(BaseModel): default_factory=list, description="Command arguments (supports Jinja2 templating with {{ steps.prev_step.field }})", ) - timeout: int = Field(default=300, description="Timeout in seconds") + timeout: int | None = Field( + default=300, + description="Timeout in seconds; null disables the orchestration watchdog", + ) env: dict[str, str] = Field(default_factory=dict, description="Additional environment variables") working_dir: str | None = Field(default=None, description="Working directory for command execution") skip: bool = Field(default=False, description="Skip this step") @@ -93,7 +96,23 @@ class StepConfig(BaseModel): "Capability contexts allowed to run this step. Empty delegates capability gating to bound validations." ), ) + requires_selected_validations: list[str] = Field( + default_factory=list, + description=( + "Configured validation names that must be selected after release, capability, label, and suite " + "exclusion filtering for this step to run. A failed step is also reported as an error on these " + "owning validations." + ), + ) continue_on_failure: bool = Field(default=False, description="Continue to next step even if this step fails") + finalizer_for: str | None = Field( + default=None, + min_length=1, + description=( + "Step whose attempted execution activates this finalizer. A finalizer may be in the target phase " + "or in the teardown phase; it runs immediately after the target phase validations." + ), + ) phase: str = Field( default="setup", description="Phase this step belongs to: 'setup', 'test', or 'teardown'", @@ -153,11 +172,75 @@ class PlatformCommands(BaseModel): default_factory=lambda: ["setup", "teardown"], description="Ordered list of phases to execute. Steps are grouped by phase and run in this order.", ) + continue_after_failure: list[str] = Field( + default_factory=list, + description=( + "Phases whose failure must not prevent later phases from running. " + "Use this for independent test cases, never for prerequisite/setup phases." + ), + ) steps: list[StepConfig] = Field( default_factory=list, description="Sequential command steps grouped by phase", ) + @model_validator(mode="after") + def validate_continuation_phases(self) -> "PlatformCommands": + """Reject invalid continuation and linked-finalizer declarations.""" + if len(self.phases) != len(set(self.phases)): + raise ValueError("phases must not contain duplicate names") + unknown = [phase for phase in self.continue_after_failure if phase not in self.phases] + if unknown: + raise ValueError(f"continue_after_failure contains phases not listed in phases: {unknown}") + unsafe = [phase for phase in self.continue_after_failure if phase in {"setup", "teardown"}] + if unsafe: + raise ValueError(f"continue_after_failure cannot contain lifecycle phases: {unsafe}") + if len(self.continue_after_failure) != len(set(self.continue_after_failure)): + raise ValueError("continue_after_failure must not contain duplicate phase names") + + for finalizer in (step for step in self.steps if step.finalizer_for is not None): + targets = [step for step in self.steps if step.name == finalizer.finalizer_for] + if len(targets) != 1: + raise ValueError( + f"step '{finalizer.name}' finalizer_for must name exactly one configured step: " + f"{finalizer.finalizer_for!r}" + ) + target = targets[0] + target_phase = target.phase.lower() + finalizer_phase = finalizer.phase.lower() + if target_phase != finalizer_phase and finalizer_phase != "teardown": + raise ValueError( + f"step '{finalizer.name}' finalizer_for target '{target.name}' must be in the same phase " + "or the finalizer must use phase 'teardown'" + ) + if finalizer_phase == "teardown" and target_phase != finalizer_phase: + normalized_phases = [phase.lower() for phase in self.phases] + if "teardown" not in normalized_phases: + raise ValueError(f"step '{finalizer.name}' uses phase 'teardown', which is not listed in phases") + if target_phase not in normalized_phases: + raise ValueError( + f"step '{finalizer.name}' finalizer_for target '{target.name}' has an unknown phase" + ) + if normalized_phases.index(target_phase) >= normalized_phases.index("teardown"): + raise ValueError(f"step '{finalizer.name}' teardown must be ordered after target '{target.name}'") + if target.finalizer_for is not None: + raise ValueError(f"step '{finalizer.name}' cannot finalize finalizer step '{target.name}'") + gate_fields = ( + "requires", + "requires_selected_validations", + ) + mismatched_gates = [ + field_name + for field_name in gate_fields + if getattr(finalizer, field_name) != getattr(target, field_name) + ] + if mismatched_gates: + raise ValueError( + f"step '{finalizer.name}' must use the same gates as target '{target.name}': " + + ", ".join(mismatched_gates) + ) + return self + class KubernetesNodeOutput(BaseModel): """Schema for a single Kubernetes node in command output.""" diff --git a/isvctl/src/isvctl/config/suite_resolution.py b/isvctl/src/isvctl/config/suite_resolution.py index d0cfbbd75..8e9fdcf9f 100644 --- a/isvctl/src/isvctl/config/suite_resolution.py +++ b/isvctl/src/isvctl/config/suite_resolution.py @@ -40,7 +40,7 @@ def platform_vocabulary(configs_root: Path) -> frozenset[str]: several entry points ask for the vocabulary two or three times per run. """ platforms: set[str] = set() - for path in (configs_root / "suites").glob("*.yaml"): + for path in (configs_root / "suites").rglob("*.yaml"): try: data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} except (OSError, yaml.YAMLError): @@ -55,7 +55,7 @@ def platform_vocabulary(configs_root: Path) -> frozenset[str]: def suite_vocabulary(configs_root: Path) -> frozenset[str]: """Return plain suite names declared by canonical suite YAML.""" declarable = platform_vocabulary(configs_root) - names = {_normalize_name(path.stem) for path in (configs_root / "suites").glob("*.yaml")} + names = {_normalize_name(path.stem) for path in (configs_root / "suites").rglob("*.yaml")} return frozenset(names - declarable) @@ -162,7 +162,8 @@ def resolve_suite(provider: str | None, suite: str, *, configs_root: Path) -> Re # Best-effort per file: one malformed config must not fail `--suite` for the # whole provider. Matches `resolve_suite_name`, which already skips them. classified = [] - for path in sorted(config_dir.glob("*.yaml")): + pattern = "*.yaml" if provider is not None else "**/*.yaml" + for path in sorted(config_dir.glob(pattern)): try: classified.append((path, *_suite_name(path, declarable))) except SuiteResolutionError: diff --git a/isvctl/src/isvctl/doctor/checks/config.py b/isvctl/src/isvctl/doctor/checks/config.py index 386555691..f81f4e85d 100644 --- a/isvctl/src/isvctl/doctor/checks/config.py +++ b/isvctl/src/isvctl/doctor/checks/config.py @@ -46,7 +46,7 @@ def _check_repo_layout(root: Path) -> list[CheckResult]: results: list[CheckResult] = [] suites_dir = root / "isvctl" / "configs" / "suites" - suite_yamls = sorted(suites_dir.glob("*.yaml")) if suites_dir.is_dir() else [] + suite_yamls = sorted(suites_dir.rglob("*.yaml")) if suites_dir.is_dir() else [] if suite_yamls: results.append( CheckResult( diff --git a/isvctl/src/isvctl/orchestrator/commands.py b/isvctl/src/isvctl/orchestrator/commands.py index 373541343..b03177b54 100644 --- a/isvctl/src/isvctl/orchestrator/commands.py +++ b/isvctl/src/isvctl/orchestrator/commands.py @@ -30,6 +30,7 @@ from isvctl.config.schema import CommandConfig, CommandOutput from isvctl.orchestrator.context import _create_jinja_env +from isvctl.orchestrator.process import run_command_process from isvctl.redaction import mask_sensitive_args logger = logging.getLogger(__name__) @@ -132,12 +133,10 @@ def execute( logger.debug(f"Working directory: {cwd}") try: - result = subprocess.run( + result = run_command_process( cmd_parts, cwd=cwd, env=env, - capture_output=True, - text=True, timeout=config.timeout, ) diff --git a/isvctl/src/isvctl/orchestrator/loop.py b/isvctl/src/isvctl/orchestrator/loop.py index 6467b8f8a..242a49384 100644 --- a/isvctl/src/isvctl/orchestrator/loop.py +++ b/isvctl/src/isvctl/orchestrator/loop.py @@ -40,13 +40,14 @@ parse_validations, requirements_satisfied, resolve_entries, + resolve_entry_selection, ) from isvtest.main import run_validations_via_pytest from isvctl.config.schema import RunConfig, StepConfig from isvctl.orchestrator.commands import CommandExecutor from isvctl.orchestrator.context import Context -from isvctl.orchestrator.step_executor import StepExecutor, StepResults +from isvctl.orchestrator.step_executor import StepExecutor, StepResult, StepResults from isvctl.redaction import redact_dict, redact_junit_xml_tree logger = logging.getLogger(__name__) @@ -80,6 +81,7 @@ class PhaseResult: success: bool message: str details: dict[str, Any] | None = None + name: str | None = None @dataclass @@ -279,9 +281,74 @@ def _resolved_entry_to_result_dict(entry: ResolvedEntry) -> dict[str, Any]: "state": entry.state.value if entry.state else None, "skip_reason": entry.skip_reason.value if entry.skip_reason else None, "error_reason": entry.error_reason.value if entry.error_reason else None, + "subtest_summary": { + "total": entry.subtest_summary.total, + "passed": entry.subtest_summary.passed, + "failed": entry.subtest_summary.failed, + "skipped": entry.subtest_summary.skipped, + }, } +def _step_failure_message(result: StepResult) -> str: + """Return an operator-facing diagnostic for one failed workflow step.""" + detail = result.error + if not detail and result.schema_errors: + detail = f"output schema validation failed: {'; '.join(result.schema_errors)}" + if not detail: + detail = f"command exited with code {result.exit_code}" + return f"workflow step '{result.name}' failed: {detail}" + + +def _apply_owned_step_failures( + entries: list[ResolvedEntry], + phase_steps: list[StepConfig], + step_results: StepResults, +) -> list[ResolvedEntry]: + """Turn failed lifecycle steps into errors on their owning validations. + + ``requires_selected_validations`` is both the command-selection gate and + the explicit ownership edge between a workflow step and its selectable + tests. Without this propagation, an early step failure prevents the bound + validation step from producing output and JUnit incorrectly records a + harmless ``step_no_output`` skip. + """ + configs_by_name = {step.name: step for step in phase_steps} + errors_by_validation: dict[str, list[tuple[str, str]]] = {} + + for result in step_results.steps: + if result.success: + continue + step = configs_by_name.get(result.name) + if step is None: + continue + message = _step_failure_message(result) + for validation_name in step.requires_selected_validations: + errors_by_validation.setdefault(validation_name, []).append((step.name, message)) + + propagated: list[ResolvedEntry] = [] + for entry in entries: + step_errors = errors_by_validation.get(entry.entry.name, []) + if entry.is_ready: + # A failed validation-producing step can still return structured + # output that the validation interprets into failures/subtests. + # Only earlier owned lifecycle failures should suppress that run. + step_errors = [(name, message) for name, message in step_errors if name != entry.entry.step] + may_override = entry.is_ready or entry.skip_reason == SkipReason.STEP_NO_OUTPUT + if not step_errors or not may_override: + propagated.append(entry) + continue + propagated.append( + ResolvedEntry( + entry=entry.entry, + state=State.ERROR, + error_reason=ErrorReason.STEP_FAILED, + message="; ".join(message for _, message in step_errors), + ) + ) + return propagated + + def _resolved_entry_success(entry: ResolvedEntry) -> bool: """Return whether a resolved validation outcome should keep the phase successful.""" return entry.state in {State.PASSED, State.SKIPPED} @@ -301,8 +368,7 @@ def _requested_config_phases(config_phases: list[str], requested_phases: list[Ph if Phase.ALL in requested_phases: return config_phases - requested_phase_names = {phase.value for phase in requested_phases} - return [phase for phase in config_phases if phase in requested_phase_names] + return [phase for phase in config_phases if _phase_enum_for_name(phase) in requested_phases] def _has_explicit_pytest_selection(extra_pytest_args: list[str] | None) -> bool: @@ -314,6 +380,48 @@ def _has_explicit_pytest_selection(extra_pytest_args: list[str] | None) -> bool: ) +def _apply_selected_validation_gates( + steps: list[Any], + validation_entries: list[ValidationEntry], + *, + include_labels: set[str], + exclude_labels: set[str], + exclude_tests: set[str], + capability: str | None, +) -> list[Any]: + """Skip lifecycle steps whose required validations are not selected.""" + entries_by_name = {entry.name: entry for entry in validation_entries} + gated_steps: list[Any] = [] + for step in steps: + required_validations = getattr(step, "requires_selected_validations", []) + unselected: list[str] = [] + for validation_name in required_validations: + entry = entries_by_name.get(validation_name) + if entry is None: + unselected.append(f"{validation_name} (not configured)") + continue + result = resolve_entry_selection( + entry, + include_labels=include_labels, + exclude_labels=exclude_labels, + exclude_tests=exclude_tests, + capability=capability, + ) + if result is not None: + unselected.append(f"{validation_name} ({result.message})") + if not unselected: + gated_steps.append(step) + continue + skipped_step = step.model_copy(update={"skip": True}) + logger.info( + "Skipping step '%s' because required validation(s) are not selected: %s", + skipped_step.name, + "; ".join(unselected), + ) + gated_steps.append(skipped_step) + return gated_steps + + def _apply_capability_step_gates( steps: list[Any], validation_entries: list[ValidationEntry], @@ -473,6 +581,7 @@ def _run_steps_mode( phase=_phase_enum_for_name(phase_name), success=True, message=f"SKIPPED: platform '{platform}' is skipped by configuration", + name=phase_name, ) for phase_name in skipped_phases ], @@ -492,6 +601,9 @@ def _run_steps_mode( ], ) + continuation_phases = ( + set(self.config.commands[platform].continue_after_failure) if self.config.commands else set() + ) all_validations = {} if self.config.tests and self.config.tests.validations: all_validations = self.config.tests.validations @@ -510,6 +622,32 @@ def _run_steps_mode( ) ], ) + exclude_labels: list[str] = [] + exclude_tests: list[str] = [] + if self.config.tests and self.config.tests.exclude: + exclude_labels = self.config.tests.exclude.get("labels", []) + exclude_tests = self.config.tests.exclude.get("tests", []) + skip_config_label_exclusions = bool(self._include_labels) or _has_explicit_pytest_selection( + self._extra_pytest_args + ) + resolution_exclude_labels = set(self._exclude_labels) + if not skip_config_label_exclusions: + resolution_exclude_labels.update(exclude_labels) + + steps_before_selection = steps + steps = _apply_selected_validation_gates( + steps, + validation_entries, + include_labels=set(self._include_labels), + exclude_labels=resolution_exclude_labels, + exclude_tests=set(exclude_tests), + capability=self._capability, + ) + selection_skipped_steps = { + selected.name + for original, selected in zip(steps_before_selection, steps, strict=True) + if not original.skip and selected.skip + } steps = _apply_capability_step_gates(steps, validation_entries, self._capability) logger.info(f"Configured phases: {config_phases}") @@ -530,6 +668,8 @@ def _run_steps_mode( steps_by_phase: dict[str, list] = {phase: [] for phase in config_phases} for step in steps: + if step.name in selection_skipped_steps: + continue step_phase = (step.phase or "setup").lower() steps_by_phase[step_phase].append(step) @@ -544,25 +684,30 @@ def _run_steps_mode( step_phase = (step.phase or "setup").lower() self.context.set_step_phase(step.name, step_phase) - resolved_validations_by_index: dict[int, ResolvedEntry] = {} + configured_steps_by_name = {step.name: step for step in steps} + active_steps = [step for phase_steps in steps_by_phase.values() for step in phase_steps] + finalizers_by_target_phase: dict[str, list[StepConfig]] = {} + for finalizer in (step for step in active_steps if step.finalizer_for is not None): + target = configured_steps_by_name[finalizer.finalizer_for] + target_phase = (target.phase or "setup").lower() + finalizers_by_target_phase.setdefault(target_phase, []).append(finalizer) - exclude_labels: list[str] = [] - exclude_tests: list[str] = [] - if self.config.tests and self.config.tests.exclude: - exclude_labels = self.config.tests.exclude.get("labels", []) - exclude_tests = self.config.tests.exclude.get("tests", []) - skip_config_label_exclusions = bool(self._include_labels) or _has_explicit_pytest_selection( - self._extra_pytest_args - ) - resolution_exclude_labels = set(self._exclude_labels) - if not skip_config_label_exclusions: - resolution_exclude_labels.update(exclude_labels) + resolved_validations_by_index: dict[int, ResolvedEntry] = {} phase_results: list[PhaseResult] = [] overall_success = True + block_following_phases = False setup_steps_ran = False requested_phase_names = {p.value for p in requested_phases} + selected_config_phases = _requested_config_phases(config_phases, requested_phases) + selected_config_phase_names = set(selected_config_phases) + selected_finalizer_target_phases = selected_config_phase_names.intersection(finalizers_by_target_phase) + run_finalizers_as_teardown_recovery = ( + "teardown" in selected_config_phase_names and not selected_finalizer_target_phases + ) + attempted_step_names: set[str] = set() + executed_finalizer_names: set[str] = set() # Per-phase JUnit XML files merge at the end so later phases don't # overwrite earlier ones. @@ -574,15 +719,24 @@ def _run_steps_mode( junit_tmpdir = tempfile.mkdtemp(prefix="junit-phases-") for phase_name in config_phases: - if phase_name not in requested_phase_names and Phase.ALL not in requested_phases: + if phase_name not in selected_config_phase_names: continue - phase_steps = steps_by_phase.get(phase_name, []) + configured_phase_steps = steps_by_phase.get(phase_name, []) + phase_steps = [step for step in configured_phase_steps if step.finalizer_for is None] + declared_phase_finalizers = [step for step in configured_phase_steps if step.finalizer_for is not None] + if phase_name == "teardown" and run_finalizers_as_teardown_recovery: + phase_steps.extend(declared_phase_finalizers) + phase_finalizers = [ + step + for step in finalizers_by_target_phase.get(phase_name, []) + if step.name not in executed_finalizer_names + ] phase_enum = _phase_enum_for_name(phase_name) is_teardown = phase_name == "teardown" skip_reason: str | None = None - if not overall_success and not is_teardown: + if block_following_phases and not is_teardown: skip_reason = "previous phase failed" # Teardown gating depends on whether setup was part of this run: @@ -605,6 +759,7 @@ def _run_steps_mode( phase=phase_enum, success=True, message=f"SKIPPED: {skip_reason}", + name=phase_name, ) ) continue @@ -613,6 +768,7 @@ def _run_steps_mode( step_results = self.step_executor.execute_steps(phase_steps, self.context, best_effort=is_teardown) else: step_results = StepResults() + attempted_step_names.update(result.name for result in step_results.steps if result.attempted) # ``step_results.steps`` includes placeholder records for skip:true # steps; require at least one step that wasn't skipped before letting @@ -635,11 +791,16 @@ def _run_steps_mode( phase_entries = [validation_entries[index] for index in phase_entry_indexes] resolved_phase_entries = self._resolve_validation_entries( phase_entries, - requested_phase_names if Phase.ALL not in requested_phases else set(config_phases), + selected_config_phase_names, set(self._include_labels), resolution_exclude_labels, set(exclude_tests), ) + resolved_phase_entries = _apply_owned_step_failures( + resolved_phase_entries, + phase_steps, + step_results, + ) ready_entries = [entry for entry in resolved_phase_entries if entry.is_ready] terminal_before_pytest = [entry for entry in resolved_phase_entries if not entry.is_ready] @@ -686,14 +847,54 @@ def _run_steps_mode( phase_validations = [_resolved_entry_to_result_dict(entry) for entry in terminal_phase_entries] - if phase_steps or phase_validations: + if step_results.steps or phase_validations: phase_results.append( self._create_phase_result(phase_enum, step_results, phase_validations, phase_name) ) + eligible_finalizers = [step for step in phase_finalizers if step.finalizer_for in attempted_step_names] + for finalizer in phase_finalizers: + if finalizer not in eligible_finalizers: + logger.info( + "Skipping finalizer '%s': target step '%s' was not attempted", + finalizer.name, + finalizer.finalizer_for, + ) + finalizer_results = self.step_executor.execute_steps( + eligible_finalizers, + self.context, + best_effort=True, + ) + executed_finalizer_names.update(finalizer.name for finalizer in eligible_finalizers) + if eligible_finalizers: + phase_results.append( + self._create_phase_result( + Phase.TEARDOWN, + finalizer_results, + [], + f"{phase_name}-teardown", + ) + ) + elif phase_finalizers: + target_names = ", ".join(finalizer.finalizer_for or "unknown" for finalizer in phase_finalizers) + phase_results.append( + PhaseResult( + phase=Phase.TEARDOWN, + success=True, + message=f"SKIPPED: target step(s) were not attempted: {target_names}", + details={"steps": [], "validations": []}, + name=f"{phase_name}-teardown", + ) + ) + phase_success = step_results.success and all(v.get("passed", False) for v in phase_validations) if not phase_success: overall_success = False + if phase_name not in continuation_phases: + block_following_phases = True + if not finalizer_results.success: + overall_success = False + block_following_phases = True remaining_entries = [ (index, entry) @@ -703,7 +904,7 @@ def _run_steps_mode( if remaining_entries: terminal_remaining = self._resolve_remaining_validation_entries( remaining_entries, - requested_phase_names if Phase.ALL not in requested_phases else set(config_phases), + selected_config_phase_names, set(self._include_labels), resolution_exclude_labels, set(exclude_tests), @@ -796,6 +997,7 @@ def _create_phase_result( { "name": s.name, "success": s.success, + "attempted": s.attempted, "error": s.error, "output": redact_dict(s.output), "schema_name": s.schema_name, @@ -806,6 +1008,7 @@ def _create_phase_result( ], "validations": validation_results, }, + name=display_name, ) def _resolve_validation_entries( @@ -889,6 +1092,7 @@ def _append_resolution_only_phase_results( "steps": [], "validations": [_resolved_entry_to_result_dict(entry) for entry in resolved_entries], }, + name=phase_name, ) ) diff --git a/isvctl/src/isvctl/orchestrator/process.py b/isvctl/src/isvctl/orchestrator/process.py new file mode 100644 index 000000000..ab39123ac --- /dev/null +++ b/isvctl/src/isvctl/orchestrator/process.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Subprocess execution shared by orchestration command models.""" + +import os +import signal +import subprocess +from collections.abc import Mapping, Sequence +from pathlib import Path + +_TERMINATION_GRACE_SECONDS = 2.0 + + +def run_command_process( + args: Sequence[str], + *, + cwd: str | Path, + env: Mapping[str, str] | None, + timeout: float | None, +) -> subprocess.CompletedProcess[str]: + """Run an orchestration command and terminate its process group on timeout. + + Orchestration steps commonly invoke wrappers which then start a provider + CLI. Killing only the wrapper can leave that CLI running after the step is + reported as timed out. On POSIX, every command therefore starts in a new + session and timeout handling signals the whole process group. Other + platforms fall back to terminating the direct child process. + + Args: + args: Command and arguments to execute without a shell. + cwd: Working directory for the command. + env: Complete process environment, or ``None`` to inherit it. + timeout: Maximum execution time in seconds, or ``None`` for no limit. + + Returns: + The completed process with captured text stdout and stderr. + + Raises: + subprocess.TimeoutExpired: The command exceeded ``timeout``. Captured + stdout and stderr are attached after the process tree is stopped. + OSError: The command could not be started. + """ + command = list(args) + process = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=os.name == "posix", + ) + + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + _terminate_process_tree(process) + try: + stdout, stderr = process.communicate(timeout=_TERMINATION_GRACE_SECONDS) + except subprocess.TimeoutExpired: + _kill_process_tree(process) + stdout, stderr = process.communicate() + else: + # The direct child may exit while a descendant that closed the + # inherited pipes remains alive. Ensure the process group is gone. + _kill_process_tree(process) + + raise subprocess.TimeoutExpired(command, timeout, output=stdout, stderr=stderr) from None + + return subprocess.CompletedProcess(command, process.returncode, stdout, stderr) + + +def _terminate_process_tree(process: subprocess.Popen[str]) -> None: + """Request graceful termination of a process group or direct process.""" + try: + if os.name == "posix": + os.killpg(process.pid, signal.SIGTERM) + else: + process.terminate() + except ProcessLookupError: + pass + + +def _kill_process_tree(process: subprocess.Popen[str]) -> None: + """Force termination of a process group or direct process.""" + try: + if os.name == "posix": + os.killpg(process.pid, signal.SIGKILL) + else: + process.kill() + except ProcessLookupError: + pass diff --git a/isvctl/src/isvctl/orchestrator/step_executor.py b/isvctl/src/isvctl/orchestrator/step_executor.py index 9f8945e44..0a3c7f9e6 100644 --- a/isvctl/src/isvctl/orchestrator/step_executor.py +++ b/isvctl/src/isvctl/orchestrator/step_executor.py @@ -60,6 +60,7 @@ from isvctl.config.output_schemas import get_schema_for_step, validate_output from isvctl.config.schema import StepConfig from isvctl.orchestrator.context import Context, _create_jinja_env +from isvctl.orchestrator.process import run_command_process from isvctl.redaction import mask_sensitive_args, redact_text logger = logging.getLogger(__name__) @@ -199,6 +200,7 @@ class StepResult: schema_errors: Schema validation error messages validation_results: Results from bound validations error: Error message if step failed + attempted: Whether the command process was actually started """ name: str @@ -212,6 +214,7 @@ class StepResult: schema_errors: list[str] = field(default_factory=list) validation_results: list[dict[str, Any]] = field(default_factory=list) error: str | None = None + attempted: bool = True @dataclass @@ -288,6 +291,7 @@ def execute_steps( stdout="", stderr="", error="Step skipped", + attempted=False, ) ) continue @@ -335,6 +339,7 @@ def _execute_step(self, step: StepConfig, context: Context) -> StepResult: stdout="", stderr="", error=f"Skipped: missing step reference steps.{e.missing_path}", + attempted=False, ) # Normalize command - replace python/python3 with current interpreter @@ -378,12 +383,10 @@ def _execute_step(self, step: StepConfig, context: Context) -> StepResult: logger.debug(f"Working directory: {cwd}") try: - result = subprocess.run( + result = run_command_process( cmd_parts, cwd=cwd, env=env, - capture_output=True, - text=True, timeout=step.timeout, ) @@ -430,14 +433,20 @@ def _execute_step(self, step: StepConfig, context: Context) -> StepResult: stderr=e.stderr if isinstance(e.stderr, str) else "", error=f"Command timed out after {step.timeout} seconds", ) - except FileNotFoundError: + except OSError as e: + error = ( + f"Command not found: {step.command}" + if isinstance(e, FileNotFoundError) + else f"Command could not start: {e}" + ) return StepResult( name=step.name, success=False, exit_code=-1, stdout="", stderr="", - error=f"Command not found: {step.command}", + error=error, + attempted=False, ) except Exception as e: return StepResult( diff --git a/isvctl/tests/providers/k8s_launch_kit/__init__.py b/isvctl/tests/providers/k8s_launch_kit/__init__.py new file mode 100644 index 000000000..77ba303a9 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the Kubernetes Launch Kit provider.""" diff --git a/isvctl/tests/providers/k8s_launch_kit/fixtures/launch_kit_scenarios.json b/isvctl/tests/providers/k8s_launch_kit/fixtures/launch_kit_scenarios.json new file mode 100644 index 000000000..c1fe18205 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/fixtures/launch_kit_scenarios.json @@ -0,0 +1,104 @@ +{ + "_copyright": "Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.", + "_license": "Apache-2.0", + "contract": { + "source_commit": "db32e4b98170", + "source_files": [ + "pkg/config/config.go", + "pkg/ui/json_output.go", + "pkg/cmd/clean.go", + "pkg/cmd/deploy.go", + "pkg/cmd/discover.go", + "pkg/cmd/generate.go", + "pkg/cmd/validate.go", + "pkg/networkoperatorplugin/clean.go", + "pkg/networkoperatorplugin/validate.go", + "pkg/networkoperatorplugin/connectivity/connectivity.go", + "pkg/networkoperatorplugin/connectivity/daemonset.go", + "pkg/networkoperatorplugin/connectivity/matrix.go", + "pkg/networkoperatorplugin/connectivity/result.go", + "pkg/networkoperatorplugin/connectivity/rdma.go", + "pkg/networkoperatorplugin/discovery/discover.go", + "profiles/host-device-rdma/40-example-daemonset.yaml", + "profiles/ipoib-rdma-shared/40-example-daemonset.yaml", + "profiles/macvlan-rdma-shared/40-example-daemonset.yaml", + "profiles/sriov-ethernet-rdma/60-example-daemonset.yaml", + "profiles/sriov-ib-rdma/60-example-daemonset.yaml" + ], + "supported_validation_checks": [ + "icmp", + "rping", + "ib_write_bw" + ], + "result_families": [ + "icmp", + "rping", + "ib_write_bw", + "gpudirect_dmabuf" + ], + "notes": [ + "discover and generate emit one ui.JSONResult object", + "successful standalone deploy emits no stdout object", + "validate emits manifest, connectivity, and reportPath JSON documents", + "connectivity DaemonSets declare test-container for RDMA and netshoot for ICMP", + "enabled GPUDirect validation follows ib_write_bw and emits a distinct gpudirect_dmabuf result family", + "clean emits one ui.JSONResult object with a cleanup summary" + ] + }, + "scenarios": { + "roce-sriov": { + "fabric": "ethernet", + "deployment": "sriov", + "profile_name": "SR-IOV Ethernet RDMA", + "network_kind": "SriovNetwork", + "network_api_version": "sriovnetwork.openshift.io/v1", + "requires_sriov": true, + "requires_ib": false + }, + "infiniband-sriov": { + "fabric": "infiniband", + "deployment": "sriov", + "profile_name": "SR-IOV Infiniband RDMA", + "network_kind": "SriovIBNetwork", + "network_api_version": "sriovnetwork.openshift.io/v1", + "requires_sriov": true, + "requires_ib": true + }, + "roce-rdma-shared": { + "fabric": "ethernet", + "deployment": "rdma_shared", + "profile_name": "Macvlan with RDMA shared device", + "network_kind": "MacvlanNetwork", + "network_api_version": "mellanox.com/v1alpha1", + "requires_sriov": false, + "requires_ib": false + }, + "infiniband-rdma-shared": { + "fabric": "infiniband", + "deployment": "rdma_shared", + "profile_name": "IP over Infiniband with RDMA shared device", + "network_kind": "IPoIBNetwork", + "network_api_version": "mellanox.com/v1alpha1", + "requires_sriov": false, + "requires_ib": true + }, + "roce-host-device": { + "fabric": "ethernet", + "deployment": "host_device", + "profile_name": "Host device RDMA", + "network_kind": "HostDeviceNetwork", + "network_api_version": "mellanox.com/v1alpha1", + "requires_sriov": false, + "requires_ib": false + }, + "infiniband-host-device": { + "fabric": "infiniband", + "deployment": "host_device", + "profile_name": "Host device RDMA", + "network_kind": "HostDeviceNetwork", + "network_api_version": "mellanox.com/v1alpha1", + "requires_sriov": false, + "requires_ib": true + } + } +} diff --git a/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_kubectl.py b/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_kubectl.py new file mode 100755 index 000000000..737863593 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_kubectl.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Small kubectl-compatible test double for Launch Kit prerequisite checks.""" + +from __future__ import annotations + +import json +import os +import sys + + +def main(argv: list[str] | None = None) -> int: + """Return Kubernetes version or Ready-node JSON for the requested command.""" + args = list(sys.argv[1:] if argv is None else argv) + if os.environ.get("L8K_MOCK_KUBERNETES_FAIL") == "1": + print("Unable to connect to the server: connection refused", file=sys.stderr) + return 1 + expected_kubeconfig = os.environ.get("L8K_MOCK_EXPECT_KUBECONFIG") + if expected_kubeconfig and os.environ.get("KUBECONFIG") != expected_kubeconfig: + print( + f"expected KUBECONFIG={expected_kubeconfig!r}, got {os.environ.get('KUBECONFIG')!r}", + file=sys.stderr, + ) + return 1 + if "version" in args: + print( + json.dumps( + { + "clientVersion": {"gitVersion": "v1.34.1"}, + "serverVersion": {"gitVersion": "v1.34.1"}, + } + ) + ) + return 0 + if "get" in args and "nodes" in args: + print( + json.dumps( + { + "apiVersion": "v1", + "items": [ + { + "metadata": {"name": "worker-a"}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + }, + { + "metadata": {"name": "worker-b"}, + "status": {"conditions": [{"type": "Ready", "status": "True"}]}, + }, + ], + } + ) + ) + return 0 + print(f"mock kubectl does not support: {' '.join(args)}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py b/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py new file mode 100755 index 000000000..d2f6ae334 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/fixtures/mock_l8k.py @@ -0,0 +1,730 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Executable Launch Kit test double for provider unit tests. + +Values in this file are fixed mock output, not AI Cloud Validation defaults. +The provider passes only real l8k arguments and receives the same distinct +stdout forms used by version, schema, discover, generate, deploy, validate, +clean, and sosreport. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path +from typing import Any + +import yaml + +_TIMESTAMP = "2026-08-05T08:27:43Z" +_SOURCE_COMMIT = "db32e4b98170" +_FIXTURE = Path(__file__).with_name("launch_kit_scenarios.json") +_VALUE_FLAGS: dict[str, set[str]] = { + "version": {"--output"}, + "discover": { + "--kubeconfig", + "--save-cluster-config", + "--user-config", + "--fabric", + "--deployment-type", + "--multirail", + "--node-selector", + "--network-operator-release", + "--output", + }, + "generate": { + "--user-config", + "--save-deployment-files", + "--network-operator-namespace", + "--output", + }, + "deploy": { + "--kubeconfig", + "--user-config", + "--deployment-files", + "--network-operator-namespace", + "--deploy-timeout", + "--output", + }, + "validate": { + "--kubeconfig", + "--user-config", + "--deployment-files", + "--network-operator-namespace", + "--connectivity", + "--connectivity-timeout", + "--validation-mode", + "--validation-checks", + "--rdma-rping-iterations", + "--rdma-ib-write-size", + "--rdma-ib-write-min-bandwidth-gbps", + "--wait", + "--report-path", + "--output", + }, + "clean": { + "--kubeconfig", + "--user-config", + "--network-operator-namespace", + "--keep-helm-chart", + "--output", + }, + "sosreport": { + "--kubeconfig", + "--output-dir", + "--output", + }, +} +_BOOLEAN_FLAGS: dict[str, set[str]] = { + "clean": {"--keep-helm-chart"}, +} + + +def _load_fixture() -> dict[str, Any]: + """Load the pinned mock contract and scenario definitions.""" + value = json.loads(_FIXTURE.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("Launch Kit fixture must contain an object") + return value + + +def _parse_flags(command: str, argv: list[str]) -> dict[str, str]: + """Parse the real flag subset exercised by the provider tests.""" + supported = _VALUE_FLAGS[command] + values: dict[str, str] = {} + index = 0 + while index < len(argv): + token = argv[index] + if not token.startswith("--"): + raise ValueError(f"unexpected positional argument: {token}") + if "=" in token: + flag, value = token.split("=", 1) + else: + flag = token + if flag in _BOOLEAN_FLAGS.get(command, set()): + value = "true" + else: + index += 1 + if index >= len(argv): + raise ValueError(f"missing value for {flag}") + value = argv[index] + if flag not in supported: + raise ValueError(f"unknown flag for l8k {command}: {flag}") + values[flag] = value + index += 1 + return values + + +def _emit(value: dict[str, Any], *, pretty: bool = False) -> None: + """Write one JSON document to stdout.""" + print(json.dumps(value, indent=2 if pretty else None)) + + +def _message(level: str, message: str) -> dict[str, str]: + """Build one ui.LogEntry-compatible record.""" + return {"level": level, "message": message, "timestamp": _TIMESTAMP} + + +def _profile(scenario: dict[str, Any]) -> dict[str, str]: + """Return the profile fields emitted by Launch Kit.""" + return { + "deployment": str(scenario["deployment"]), + "fabric": str(scenario["fabric"]), + "ignoreARP": "false", + "multirail": "true", + "routing": "destination-based", + } + + +def _json_result( + phase: str, + *, + profile: dict[str, str] | None = None, + generated_files: list[str] | None = None, +) -> dict[str, Any]: + """Build a successful ui.JSONResult-compatible object.""" + value: dict[str, Any] = { + "success": True, + "phase": phase, + "deployed": False, + "messages": [ + _message("info", f"Running {phase}"), + _message("success", "Workflow completed successfully"), + ], + } + if profile is not None: + value["profile"] = profile + if generated_files: + value["generatedFiles"] = generated_files + return value + + +def _structured_error(command: str, message: str) -> tuple[dict[str, Any], int]: + """Build the JSON error emitted by a failed Launch Kit command.""" + category = "cluster" if command == "discover" else "deployment" if command == "deploy" else "validation" + exit_code = 3 if category == "cluster" else 4 if category == "deployment" else 2 + return { + "success": False, + "phase": "", + "deployed": False, + "error": { + "code": f"{category.upper()}_ERROR", + "message": message, + "category": category, + "transient": category == "cluster", + "suggestion": "Inspect the preserved Launch Kit logs and correct the reported condition", + }, + "messages": None, + }, exit_code + + +def _scenario_for_profile(fabric: str, deployment: str) -> tuple[str, dict[str, Any]]: + """Find fixture data for one explicit profile.""" + scenarios = _load_fixture().get("scenarios") + if not isinstance(scenarios, dict): + raise ValueError("Launch Kit fixture has no scenarios map") + for name, scenario in scenarios.items(): + if isinstance(scenario, dict) and scenario.get("fabric") == fabric and scenario.get("deployment") == deployment: + return str(name), scenario + raise ValueError(f"unsupported mock profile: fabric={fabric!r}, deployment={deployment!r}") + + +def _load_cluster_config(path: Path) -> dict[str, Any]: + """Load a cluster configuration produced by the mock discovery command.""" + value = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"{path} must contain a YAML object") + return value + + +def _scenario_from_config(flags: dict[str, str]) -> tuple[str, dict[str, Any]]: + """Resolve a scenario from the persisted Launch Kit profile.""" + raw = flags.get("--user-config") + if not raw: + deployment = Path(flags.get("--deployment-files", "deployment")) + raw = str(deployment.parent / "cluster-config.yaml") + config = _load_cluster_config(Path(raw)) + profile = config.get("profile") + if not isinstance(profile, dict): + raise ValueError("cluster config has no profile") + return _scenario_for_profile(str(profile.get("fabric", "")), str(profile.get("deployment", ""))) + + +def _cluster_config_yaml(scenario: dict[str, Any]) -> str: + """Render representative output from l8k discovery.""" + return f"""# Generated by `l8k discover`. +networkOperator: + selectedRelease: "26.4" + version: v26.4.1 + namespace: nvidia-network-operator +validation: + gpuDirect: + enabled: true + gpuResourceType: nvidia.com/gpu + connectivity: true + mode: strict + checks: [icmp, rping, ib_write_bw] + rdma: + rpingIterations: 5 + ibWriteSize: 65536 + ibWriteMinBandwidthGbps: 100 +profile: + fabric: {scenario["fabric"]} + deployment: {scenario["deployment"]} + multirail: true + routing: destination-based +clusterConfig: + - identifier: mock-group + machineType: mock-vm + gpuType: NVIDIA-H100-80GB-HBM3 + workerNodes: [worker-a, worker-b] + nodeSelector: + feature.node.kubernetes.io/pci-15b3.present: "true" + pfs: + - networkInterface: ens5f0np0 + pciAddress: "0000:17:00.0" + rdmaDevice: mlx5_0 + traffic: east-west + rail: 0 + connectedGPU: GPU0 + connectedGPUPCIAddress: "0000:41:00.0" + - networkInterface: ens6f0np0 + pciAddress: "0000:31:00.0" + rdmaDevice: mlx5_1 + traffic: east-west + rail: 1 + connectedGPU: GPU1 + connectedGPUPCIAddress: "0000:71:00.0" +""" + + +def _resource_name(kind: str, rail: int) -> str: + """Return a deterministic resource name for generated mock manifests.""" + prefix = { + "IPPool": "nv-ipam-pool", + "SriovNetwork": "sriov-network", + "SriovIBNetwork": "sriov-ib-network", + "MacvlanNetwork": "macvlan-network", + "IPoIBNetwork": "ipoib-network", + "HostDeviceNetwork": "hostdev-network", + "SriovNetworkNodePolicy": "sriov-policy", + "NicNodePolicy": "nic-node-policy", + }.get(kind, kind.lower()) + return f"{prefix}-rail-{rail}-mock-group" + + +def _manifest_specs(scenario: dict[str, Any]) -> list[tuple[str, str, str, int]]: + """Return generated API/kind/file/count tuples for one profile.""" + specs = [ + ("mellanox.com/v1alpha1", "NicClusterPolicy", "10-nic-cluster-policy", 1), + ("configuration.net.nvidia.com/v1alpha1", "NicNodePolicy", "20-nic-node-policy", 1), + ("nv-ipam.nvidia.com/v1alpha1", "IPPool", "30-ip-pool", 2), + ] + if scenario.get("requires_sriov"): + specs.append(("sriovnetwork.openshift.io/v1", "SriovNetworkNodePolicy", "40-sriov-policy", 2)) + specs.append( + ( + str(scenario["network_api_version"]), + str(scenario["network_kind"]), + "50-secondary-network", + 2, + ) + ) + return specs + + +def _manifest_yaml(api_version: str, kind: str, count: int) -> str: + """Render a valid multi-document mock manifest.""" + documents: list[str] = [] + for rail in range(count): + name = "nic-cluster-policy" if kind == "NicClusterPolicy" else _resource_name(kind, rail) + namespace = "" if kind in {"NicClusterPolicy", "NicNodePolicy"} else " namespace: default\n" + documents.append( + f"apiVersion: {api_version}\nkind: {kind}\nmetadata:\n name: {name}\n{namespace}spec:\n mock: true\n" + ) + return "---\n".join(documents) + + +def _write_generated_files(root: Path, scenario: dict[str, Any]) -> list[str]: + """Materialize profile manifests and return their absolute paths.""" + manifest_dir = root / "network-operator" + manifest_dir.mkdir(parents=True, exist_ok=True) + generated: list[Path] = [] + values = manifest_dir / "values.yaml" + values.write_text("operator:\n namespace: nvidia-network-operator\n", encoding="utf-8") + generated.append(values) + for api_version, kind, stem, count in _manifest_specs(scenario): + path = manifest_dir / f"{stem}.yaml" + path.write_text(_manifest_yaml(api_version, kind, count), encoding="utf-8") + generated.append(path) + example = manifest_dir / "60-example-daemonset-mock-group.yaml" + example.write_text( + "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: l8k-network-test\n namespace: default\n" + "spec:\n" + " selector:\n" + " matchLabels:\n" + " app: l8k-network-test\n" + " template:\n" + " metadata:\n" + " labels:\n" + " app: l8k-network-test\n" + " spec:\n" + " containers:\n" + " - name: test-container\n" + " image: nvcr.io/nvidia/doca/doca:3.3.0-full-rt-host\n" + " command: [/bin/bash, -c, sleep infinity]\n" + " resources:\n" + " requests:\n" + " nvidia.com/gpu: '2'\n" + " limits:\n" + " nvidia.com/gpu: '2'\n" + " - name: netshoot\n" + " image: nicolaka/netshoot:latest\n" + " command: [/bin/bash, -c, sleep infinity]\n", + encoding="utf-8", + ) + generated.append(example) + return [str(path.resolve()) for path in generated] + + +def _manifest_results(scenario: dict[str, Any]) -> list[dict[str, Any]]: + """Build the exported Launch Kit manifest-validation results.""" + results: list[dict[str, Any]] = [] + for api_version, kind, stem, count in _manifest_specs(scenario): + for rail in range(count): + name = "nic-cluster-policy" if kind == "NicClusterPolicy" else _resource_name(kind, rail) + reason = "resource exists and is Ready" + results.append( + { + "Kind": kind, + "APIVersion": api_version, + "Name": name, + "Namespace": "" if kind in {"NicClusterPolicy", "NicNodePolicy"} else "default", + "SourceFile": f"{stem}.yaml", + "State": "success", + "Reason": reason, + "Details": {}, + "Found": True, + "Missing": False, + "Detail": reason, + } + ) + return results + + +def _ping_result( + kind: int, + src_node: str, + dst_node: str, + src_rail: str, + dst_rail: str, + fail_family: str | None, +) -> dict[str, Any]: + """Build one exported connectivity matrix row.""" + family = "icmp" if kind < 2 else "rping" if kind < 4 else "ib_write_bw" if kind < 6 else "gpudirect_dmabuf" + bandwidth_family = family in {"ib_write_bw", "gpudirect_dmabuf"} + cross_rail = src_rail != dst_rail + expectation = "forbidden" if cross_rail else "required" + observed_ok = not cross_rail + ok = True + stderr = "" + stdout = "" + bandwidth = 0.0 + if family == "icmp": + stdout = "1 packets transmitted, 1 received" if observed_ok else "" + stderr = "Network is unreachable" if cross_rail else "" + elif family == "rping": + stdout = "client DISCONNECT EVENT" if observed_ok else "" + stderr = "rping: connection timed out" if cross_rail else "" + elif observed_ok: + bandwidth = 191.25 if family == "gpudirect_dmabuf" else 187.6 + stdout = f"65536 5000 {bandwidth:.2f} {bandwidth:.2f} 0.3578" + else: + stderr = "ib_write_bw: failed to connect" + + if fail_family == family and not cross_rail and src_node == "worker-a" and src_rail == "rail-0": + ok = False + observed_ok = False + if bandwidth_family: + bandwidth = 42.5 + stderr = "observed bandwidth 42.5 Gbps below minimum 100 Gbps" + else: + stderr = f"{family}: connection refused" + + test = { + "Kind": kind, + "SrcPod": f"network-test-{src_node}", + "DstPod": f"network-test-{dst_node}", + "SrcNode": src_node, + "DstNode": dst_node, + "Rail": src_rail if not cross_rail else f"{src_rail}→{dst_rail}", + "SrcIP": "192.168.128.10" if src_node == "worker-a" else "192.168.128.11", + "DstIP": "192.168.128.11" if dst_node == "worker-b" else "192.168.128.10", + "SrcRail": src_rail, + "DstRail": dst_rail, + "SrcIface": "net1" if src_rail == "rail-0" else "net2", + "DstIface": "net1" if dst_rail == "rail-0" else "net2", + "SrcRDMADev": "mlx5_0" if src_rail == "rail-0" else "mlx5_1", + "DstRDMADev": "mlx5_0" if dst_rail == "rail-0" else "mlx5_1", + "Expectation": expectation, + } + if family == "gpudirect_dmabuf": + test.update( + { + "SrcGPUIndex": 0 if src_rail == "rail-0" else 1, + "DstGPUIndex": 0 if dst_rail == "rail-0" else 1, + "SrcGPUPCIAddress": "0000:41:00.0" if src_rail == "rail-0" else "0000:71:00.0", + "DstGPUPCIAddress": "0000:41:00.0" if dst_rail == "rail-0" else "0000:71:00.0", + } + ) + return { + "Test": test, + "Family": family, + "OK": ok, + "ObservedOK": observed_ok, + "Expectation": expectation, + "Route": {"OK": not cross_rail}, + "BandwidthGbps": bandwidth, + "MsgRateMpps": 0.3578 if bandwidth else 0.0, + "MinBandwidthGbps": 100.0 if bandwidth_family else 0.0, + "Stdout": stdout, + "Stderr": stderr, + **({"Error": stderr} if not ok else {}), + } + + +def _connectivity_result(scenario_name: str, fail_family: str | None) -> dict[str, Any]: + """Build a strict two-node, two-rail matrix.""" + rails = ["rail-0", "rail-1"] + rows: list[dict[str, Any]] = [] + for kind in range(8): + for src_node, dst_node in (("worker-a", "worker-b"), ("worker-b", "worker-a")): + pairs = [(rail, rail) for rail in rails] if kind % 2 == 0 else [(rails[0], rails[1]), (rails[1], rails[0])] + for src_rail, dst_rail in pairs: + rows.append(_ping_result(kind, src_node, dst_node, src_rail, dst_rail, fail_family)) + failed = sum(row["OK"] is not True for row in rows) + return { + "DaemonSets": [ + { + "Ref": { + "Namespace": "default", + "Name": f"l8k-network-test-{scenario_name}", + "Container": "test-container", + "RDMAContainer": "test-container", + "ICMPContainer": "netshoot", + "SourceFile": "60-example-daemonset-mock-group.yaml", + }, + "Rollout": {"Desired": 2, "Updated": 2, "Available": 2, "Ready": 2, "NotReady": 0}, + "PodCount": 2, + } + ], + "PingResults": rows, + "Skipped": None, + "Summary": {"TotalTests": len(rows), "Passed": len(rows) - failed, "Failed": failed}, + } + + +def _failure(command: str) -> tuple[bool, str | None]: + """Resolve optional failure injection as ``command[:family]``.""" + parts = os.environ.get("L8K_MOCK_FAIL", "").split(":") + if not parts or parts[0] != command: + return False, None + return len(parts) == 1, parts[1] if len(parts) > 1 else None + + +def _run_version(flags: dict[str, str]) -> int: + """Mock ``l8k version``.""" + if flags.get("--output") == "json": + _emit({"version": "v0.1.0-mock", "gitCommit": _SOURCE_COMMIT, "buildDate": _TIMESTAMP}, pretty=True) + else: + print("l8k v0.1.0-mock") + return 0 + + +def _run_schema() -> int: + """Mock ``l8k schema`` with Launch Kit-owned capabilities and defaults.""" + fixture = _load_fixture() + _emit( + { + "version": "v0.1.0-mock", + "description": "CLI tool for deploying NVIDIA cloud-native networking solutions on Kubernetes", + "commands": { + command: {"description": f"Mock l8k {command}", "example": f"l8k {command}"} + for command in ("discover", "generate", "deploy", "validate", "clean", "sosreport", "schema") + }, + "phases": ["discover", "generate", "deploy"], + "fabrics": sorted({str(value["fabric"]) for value in fixture["scenarios"].values()}), + "deploymentTypes": sorted({str(value["deployment"]) for value in fixture["scenarios"].values()}), + "outputFormats": ["text", "json"], + "supportedNetworkOperatorReleases": ["26.4"], + "exitCodes": {"0": "success", "2": "validation_error", "3": "cluster_error", "4": "deployment_error"}, + "flags": { + "--node-selector": { + "type": "string", + "default": "feature.node.kubernetes.io/pci-15b3.present=true", + "description": "Node selector", + }, + "--validation-mode": { + "type": "string", + "default": "inherit from validation.mode", + "description": "Validation mode", + }, + "--validation-checks": { + "type": "[]string", + "default": "inherit from validation.checks", + "description": ( + "Comma-separated checks: icmp, rping, ib_write_bw. " + "Enabled GPUDirect DMA-BUF validation follows ib_write_bw." + ), + }, + }, + }, + pretty=True, + ) + return 0 + + +def _run_discover(flags: dict[str, str]) -> int: + """Mock ``l8k discover``.""" + fail, _ = _failure("discover") + if fail: + result, exit_code = _structured_error("discover", "cluster discovery failed") + _emit(result, pretty=True) + print("Error: cluster discovery failed", file=sys.stderr) + return exit_code + _, scenario = _scenario_for_profile(flags.get("--fabric", ""), flags.get("--deployment-type", "")) + path = Path(flags.get("--save-cluster-config", "cluster-config.yaml")) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(_cluster_config_yaml(scenario), encoding="utf-8") + _emit(_json_result("discover", profile=_profile(scenario)), pretty=True) + print(f"[success] Configuration saved: {path}", file=sys.stderr) + return 0 + + +def _run_generate(flags: dict[str, str]) -> int: + """Mock ``l8k generate``.""" + fail, _ = _failure("generate") + if fail: + result, exit_code = _structured_error("generate", "deployment file generation failed") + _emit(result, pretty=True) + return exit_code + _, scenario = _scenario_from_config(flags) + root = Path(flags.get("--save-deployment-files", "deployment")) + files = _write_generated_files(root, scenario) + _emit(_json_result("generate", profile=_profile(scenario), generated_files=files), pretty=True) + print(f"[success] Generated {len(files)} files", file=sys.stderr) + return 0 + + +def _run_deploy(flags: dict[str, str]) -> int: + """Mock standalone ``l8k deploy`` including empty success stdout.""" + fail, _ = _failure("deploy") + if fail: + result, exit_code = _structured_error("deploy", "deployment failed") + _emit(result, pretty=True) + print("Error: deployment failed", file=sys.stderr) + return exit_code + _scenario_from_config(flags) + print(f"[success] Deployment completed from {flags.get('--deployment-files', 'deployment')}", file=sys.stderr) + return 0 + + +def _run_validate(flags: dict[str, str]) -> int: + """Mock the current three-document ``l8k validate`` JSON stream.""" + fail, family = _failure("validate") + if fail: + result, exit_code = _structured_error("validate", "failed to create Kubernetes client") + _emit(result) + print("Error: failed to create Kubernetes client", file=sys.stderr) + return exit_code + scenario_name, scenario = _scenario_from_config(flags) + manifests = _manifest_results(scenario) + static = { + "versionCheck": { + "Skipped": False, + "Reason": "", + "SelectedRelease": "26.4", + "ExpectedVersion": "v26.4.1", + "DeployedRelease": { + "Name": "network-operator", + "Namespace": "nvidia-network-operator", + "ChartName": "network-operator", + "ChartVersion": "26.4.1", + "AppVersion": "v26.4.1", + "Revision": 1, + "Status": "deployed", + }, + "Match": True, + }, + "manifests": manifests, + "presetDeviations": [], + "summary": { + "totalManifests": len(manifests), + "successManifests": len(manifests), + "inProgress": 0, + "errorManifests": 0, + "missingManifests": 0, + "versionMatch": True, + "deviationGroups": 0, + "success": True, + }, + } + connectivity = _connectivity_result(scenario_name, family) + deployment = Path(flags.get("--deployment-files", "deployment")) + report = Path(flags.get("--report-path", str(deployment / "k8s-launch-kit-validation-report.html"))).resolve() + report.parent.mkdir(parents=True, exist_ok=True) + verdict = "FAILED" if connectivity["Summary"]["Failed"] else "PASSED" + report.write_text(f"

VALIDATION {verdict}

\n", encoding="utf-8") + _emit(static) + _emit({"connectivity": connectivity}) + _emit({"reportPath": str(report)}) + print(f"HTML report written to {report}", file=sys.stderr) + return 4 if connectivity["Summary"]["Failed"] else 0 + + +def _run_clean(flags: dict[str, str]) -> int: + """Mock the current one-document ``l8k clean`` JSON result.""" + fail, _ = _failure("clean") + if fail: + result, exit_code = _structured_error("clean", "Network Operator cleanup failed") + _emit(result, pretty=True) + print("Error: Network Operator cleanup failed", file=sys.stderr) + return exit_code + keep_helm = flags.get("--keep-helm-chart", "false").lower() == "true" + _emit( + { + "success": True, + "phase": "clean", + "deployed": False, + "cleanup": { + "namespace": flags.get("--network-operator-namespace", "nvidia-network-operator"), + "customResourcesDeleted": 12, + "helmReleaseRemoved": not keep_helm, + "keepHelmChart": keep_helm, + }, + "messages": [], + }, + pretty=True, + ) + return 0 + + +def _run_sosreport(flags: dict[str, str]) -> int: + """Mock the current text-streaming ``l8k sosreport`` command.""" + output_dir = Path(flags.get("--output-dir", "./sosreport")).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + archive = output_dir / "network-operator-sosreport.tar.gz" + archive.write_text("mock Network Operator diagnostic archive\n", encoding="utf-8") + print("Collecting sosreport from cluster...") + print(f" Output: {output_dir}") + fail, _ = _failure("sosreport") + if fail: + print("Error: sosreport collection failed", file=sys.stderr) + return 3 + print(f"\nSosreport collected: {output_dir}") + return 0 + + +def main(argv: list[str] | None = None) -> int: + """Execute one mocked Launch Kit command.""" + args = list(sys.argv[1:] if argv is None else argv) + if not args: + print("mock l8k expects a command", file=sys.stderr) + return 2 + command = args[0] + try: + if command == "schema": + if len(args) != 1: + raise ValueError("l8k schema accepts no arguments") + return _run_schema() + if command not in _VALUE_FLAGS: + raise ValueError(f"unknown l8k command: {command}") + flags = _parse_flags(command, args[1:]) + if command in {"discover", "generate", "deploy", "validate", "clean"} and flags.get("--output") != "json": + raise ValueError("mock workflow commands require --output json") + if command == "version": + return _run_version(flags) + if command == "discover": + return _run_discover(flags) + if command == "generate": + return _run_generate(flags) + if command == "deploy": + return _run_deploy(flags) + if command == "validate": + return _run_validate(flags) + if command == "sosreport": + return _run_sosreport(flags) + return _run_clean(flags) + except (KeyError, OSError, TypeError, ValueError, yaml.YAMLError) as exc: + result, exit_code = _structured_error(command, str(exc)) + _emit(result, pretty=command != "validate") + print(f"Error: {exc}", file=sys.stderr) + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/tests/providers/k8s_launch_kit/test_provider.py b/isvctl/tests/providers/k8s_launch_kit/test_provider.py new file mode 100644 index 000000000..bdba31a57 --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/test_provider.py @@ -0,0 +1,1135 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Contract and framework tests for the generic Kubernetes Launch Kit provider.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest +import yaml +from isvtest.core.resolution import State + +from isvctl.config.merger import merge_yaml_files +from isvctl.config.output_schemas import validate_output +from isvctl.config.schema import RunConfig +from isvctl.orchestrator.loop import Orchestrator, Phase + +_ISVCTL_ROOT = Path(__file__).resolve().parents[3] +_PROVIDERS = _ISVCTL_ROOT / "configs" / "providers" +_LAUNCH_KIT_PROVIDER = _PROVIDERS / "k8s-launch-kit" +_PROVIDER = _LAUNCH_KIT_PROVIDER / "scripts" / "adapter.py" +_FIXTURES = Path(__file__).resolve().parent / "fixtures" +_MOCK_L8K = _FIXTURES / "mock_l8k.py" +_MOCK_KUBECTL = _FIXTURES / "mock_kubectl.py" +_GENERIC_CONFIG = _LAUNCH_KIT_PROVIDER / "config" / "provider.yaml" +_NETWORK_OPERATOR_CONFIG = _LAUNCH_KIT_PROVIDER / "config" / "network-operator.yaml" + + +def _load_provider_module() -> ModuleType: + """Load the provider script for isolated installer tests.""" + spec = importlib.util.spec_from_file_location("k8s_launch_kit_provider", _PROVIDER) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot import {_PROVIDER}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _run_provider( + *arguments: str, + env: dict[str, str] | None = None, +) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: + """Run one provider operation from the same directory used by isvctl.""" + completed = subprocess.run( + [sys.executable, str(_PROVIDER), *arguments], + cwd=_PROVIDERS, + env=env, + check=False, + capture_output=True, + text=True, + ) + try: + output = json.loads(completed.stdout) + except json.JSONDecodeError as error: + raise AssertionError( + f"provider emitted non-JSON stdout (exit {completed.returncode}): " + f"stdout={completed.stdout!r} stderr={completed.stderr!r}" + ) from error + assert isinstance(output, dict) + return completed, output + + +def _run_workflow( + command: str, + arguments: list[str], + *, + working_dir: Path, + artifact_dir: Path, + user_config: Path | None = None, + deployment_files: Path | None = None, + env: dict[str, str] | None = None, +) -> tuple[subprocess.CompletedProcess[str], dict[str, Any]]: + """Run a mocked l8k workflow command through the generic transport.""" + provider_arguments = [ + "run", + "--executable", + str(_MOCK_L8K), + "--command", + command, + "--arguments-json", + json.dumps(arguments), + "--environment-json", + "{}", + "--working-dir", + str(working_dir), + "--artifact-dir", + str(artifact_dir), + ] + if user_config is not None: + provider_arguments.extend(["--user-config", str(user_config)]) + if deployment_files is not None: + provider_arguments.extend(["--deployment-files", str(deployment_files)]) + return _run_provider(*provider_arguments, env=env) + + +def _mocked_network_operator_config(tmp_path: Path) -> RunConfig: + """Load production wiring, then inject test-owned executables and paths.""" + merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) + context = merged["context"]["k8s_launch_kit"] + user_config = tmp_path / "cluster-config.yaml" + user_config.write_text( + """networkOperator: + selectedRelease: "26.4" +profile: + fabric: ethernet + deployment: sriov +clusterConfig: [] +""", + encoding="utf-8", + ) + deployment_files = tmp_path / "deployment" + deployment_files.mkdir() + context["executable"] = str(_MOCK_L8K) + context["user_config"] = str(user_config) + context["deployment_files"] = str(deployment_files) + context["working_dir"] = str(tmp_path / "work") + context["artifact_dir"] = str(tmp_path / "evidence") + return RunConfig.model_validate(merged) + + +def test_generic_provider_has_no_launch_kit_domain_defaults() -> None: + """AI Cloud Validation exposes raw argv while Launch Kit owns domain defaults.""" + merged = merge_yaml_files([_GENERIC_CONFIG]) + config = RunConfig.model_validate(merged) + context = merged["context"]["k8s_launch_kit"] + + assert set(context) == { + "executable", + "installation", + "user_config", + "kubectl_command", + "working_dir", + "artifact_dir", + "environment", + "discover", + "generate", + "deploy", + "validate", + "clean", + } + assert context["user_config"] == "" + assert context["installation"] == { + "mode": "verify", + "version": "", + "installer_ref": "", + "installer_sha256": "", + "prefix": "", + } + assert all( + context[command]["arguments"] == [] for command in ("discover", "generate", "deploy", "validate", "clean") + ) + assert [step.name for step in config.commands["network_operator"].steps] == [ + "launch_kit_prepare", + "launch_kit_verify", + "launch_kit_kubernetes_preflight", + "launch_kit_discover", + "launch_kit_generate", + "launch_kit_deploy", + "launch_kit_validate", + "launch_kit_clean", + ] + assert config.commands["network_operator"].phases == ["setup", "test", "teardown"] + discover_step = next( + step for step in config.commands["network_operator"].steps if step.name == "launch_kit_discover" + ) + prepare_step = next(step for step in config.commands["network_operator"].steps if step.name == "launch_kit_prepare") + assert "--installer-ref={{ context.k8s_launch_kit.installation.installer_ref }}" in prepare_step.args + assert "--installer-sha256={{ context.k8s_launch_kit.installation.installer_sha256 }}" in prepare_step.args + assert "--user-config={{ context.k8s_launch_kit.user_config }}" in discover_step.args + assert config.commands["network_operator"].steps[-1].phase == "teardown" + assert config.commands["network_operator"].steps[-1].finalizer_for == "launch_kit_deploy" + forbidden = { + "namespace", + "node_selector", + "expected_network_operator_version", + "driver_mode", + "rail_names", + "sriov_resource_names", + "ip_pool_names", + "gpu_count", + "validation_mode", + "validation_checks", + "rdma_rping_iterations", + "rdma_ib_write_size", + "rdma_min_bandwidth_gbps", + "timeout_seconds", + } + assert forbidden.isdisjoint(context) + + +def test_network_operator_provider_defaults_to_real_cli_tools() -> None: + """The shipped provider validates once and always collects diagnostics.""" + merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) + config = RunConfig.model_validate(merged) + context = merged["context"]["k8s_launch_kit"] + + assert context == { + "executable": "l8k", + "user_config": "", + "deployment_files": "", + "working_dir": "../../../../../_output/k8s-launch-kit/network-operator/work", + "artifact_dir": "../../../../../_output/k8s-launch-kit/network-operator/evidence", + "environment": {}, + } + assert "mock" not in json.dumps(merged).lower() + assert "poc" not in json.dumps(merged).lower() + command = config.commands["network_operator"] + assert command.phases == ["test"] + assert [step.name for step in command.steps] == ["launch_kit_validate", "launch_kit_sosreport"] + validate_step, sosreport_step = command.steps + assert validate_step.timeout is None + assert "--user-config={{ context.k8s_launch_kit.user_config }}" in validate_step.args + assert "--deployment-files={{ context.k8s_launch_kit.deployment_files }}" in validate_step.args + assert validate_step.requires_selected_validations == ["LaunchKitConnectivityCheck"] + assert sosreport_step.timeout == 1800 + assert sosreport_step.phase == "test" + assert sosreport_step.finalizer_for == "launch_kit_validate" + assert sosreport_step.requires == validate_step.requires + assert sosreport_step.requires_selected_validations == validate_step.requires_selected_validations + + +def test_network_operator_suite_has_one_catalog_check() -> None: + """Fabric and deployment choices are prerequisites, not test entries.""" + merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) + checks = merged["tests"]["validations"]["network_operator"]["checks"] + + assert list(checks) == ["LaunchKitConnectivityCheck"] + assert checks["LaunchKitConnectivityCheck"]["test_id"] == "K8S42-01" + + +def test_kubectl_defaults_to_the_real_binary() -> None: + """An empty provider override resolves to kubectl from PATH.""" + module = _load_provider_module() + + assert module._kubectl_prefix("[]", {}) == ["kubectl"] + + +def test_launch_kit_executable_resolves_from_path(tmp_path: Path, monkeypatch: Any) -> None: + """The production `l8k` setting is resolved as a normal executable.""" + module = _load_provider_module() + executable = tmp_path / "l8k" + executable.write_text("test executable", encoding="utf-8") + monkeypatch.setattr(module.shutil, "which", lambda value: str(executable) if value == "l8k" else None) + + assert module._resolve_executable("l8k") == executable.resolve() + + +def test_prepare_verifies_version_and_schema(tmp_path: Path) -> None: + """Verify mode proves the executable and captures Launch Kit's schema.""" + completed, output = _run_provider( + "prepare", + "--mode", + "verify", + "--executable", + str(_MOCK_L8K), + "--artifact-dir", + str(tmp_path), + ) + + assert completed.returncode == 0 + assert output["success"] is True + assert output["operation"] == "prepare" + assert output["installed"] is False + assert set(output["checks"]) == {"version", "schema"} + assert all(check["passed"] is True for check in output["checks"].values()) + assert validate_output(output, "k8s_launch_kit") == (True, []) + + +def test_verification_rejects_an_unexpected_launch_kit_version(tmp_path: Path) -> None: + """A pinned installation cannot silently verify a different binary on PATH.""" + module = _load_provider_module() + + verification, success, error = module._verify_executable( + _MOCK_L8K, + tmp_path, + "v9.9.9", + ) + + assert success is False + assert verification["checks"]["version"]["passed"] is False + assert error == "l8k version mismatch: expected 'v9.9.9', got 'v0.1.0-mock'" + + +def test_verification_requires_the_launch_kit_clean_command(tmp_path: Path) -> None: + """A pre-clean Launch Kit binary is rejected before deployment begins.""" + module = _load_provider_module() + executable = tmp_path / "l8k" + executable.write_text( + "#!/bin/sh\n" + 'if [ "$1" = version ]; then\n' + ' echo \'{"version": "v0.1.0"}\'\n' + "else\n" + ' echo \'{"commands": {"discover": {}, "generate": {}, "deploy": {}, "validate": {}}}\'\n' + "fi\n", + encoding="utf-8", + ) + executable.chmod(0o755) + + verification, success, error = module._verify_executable(executable, tmp_path) + + assert success is False + assert verification["checks"]["schema"]["passed"] is False + assert error == "l8k schema does not advertise required command(s): clean" + + +def test_installed_executable_is_resolved_from_the_installer_prefix(tmp_path: Path) -> None: + """Install mode verifies the binary written by the installer, not a stale PATH entry.""" + module = _load_provider_module() + executable = tmp_path / "bin" / "l8k" + executable.parent.mkdir(parents=True) + executable.write_text("mock", encoding="utf-8") + + assert module._installed_executable(str(tmp_path)) == executable.resolve() + + +def test_installer_download_verifies_expected_digest(tmp_path: Path, monkeypatch: Any) -> None: + """Install mode verifies an immutable official installer before writing it.""" + module = _load_provider_module() + content = b"#!/bin/sh\nset -eu\n" + installer_ref = "a" * 40 + expected_sha256 = hashlib.sha256(content).hexdigest() + + class Response: + """Minimal context-managed urllib response.""" + + def __enter__(self) -> Response: + return self + + def __exit__(self, *_args: Any) -> None: + return None + + def read(self) -> bytes: + return content + + monkeypatch.setattr(module.urllib.request, "urlopen", lambda *_args, **_kwargs: Response()) + + installer, url = module._download_installer(installer_ref, expected_sha256, tmp_path) + metadata = json.loads((tmp_path / "installer-download.json").read_text(encoding="utf-8")) + + assert installer.read_bytes() == content + assert url.endswith(f"/{installer_ref}/scripts/install.sh") + assert metadata == { + "url": url, + "ref": installer_ref, + "expected_sha256": expected_sha256, + "sha256": expected_sha256, + "verified": True, + } + + +def test_installer_download_rejects_a_digest_mismatch(tmp_path: Path, monkeypatch: Any) -> None: + """A downloaded installer is never persisted when its trusted digest differs.""" + module = _load_provider_module() + content = b"#!/bin/sh\nexit 0\n" + + class Response: + """Minimal context-managed urllib response.""" + + def __enter__(self) -> Response: + return self + + def __exit__(self, *_args: Any) -> None: + return None + + def read(self) -> bytes: + return content + + monkeypatch.setattr(module.urllib.request, "urlopen", lambda *_args, **_kwargs: Response()) + + with pytest.raises(ValueError, match="installer SHA-256 mismatch"): + module._download_installer("b" * 40, "0" * 64, tmp_path) + + metadata = json.loads((tmp_path / "installer-download.json").read_text(encoding="utf-8")) + assert metadata["verified"] is False + assert metadata["sha256"] == hashlib.sha256(content).hexdigest() + assert not (tmp_path / "installer.sh").exists() + + +@pytest.mark.parametrize("installer_ref", ["", "main", "v0.1.0", "a" * 39]) +def test_installer_download_requires_an_immutable_commit_ref(tmp_path: Path, installer_ref: str) -> None: + """Install mode rejects mutable or abbreviated installer references before download.""" + module = _load_provider_module() + + with pytest.raises(ValueError, match="full 40-character Git commit SHA"): + module._download_installer(installer_ref, "0" * 64, tmp_path) + + +def test_install_mode_delegates_to_the_upstream_installer(tmp_path: Path, monkeypatch: Any) -> None: + """The provider does not reimplement Launch Kit archive or checksum logic.""" + module = _load_provider_module() + installer = tmp_path / "installer.sh" + installer.write_text("#!/bin/sh\n", encoding="utf-8") + calls: list[tuple[list[str], dict[str, str]]] = [] + + monkeypatch.setattr( + module, + "_download_installer", + lambda _ref, _sha256, _artifact_dir: (installer, "https://example.invalid/installer.sh"), + ) + monkeypatch.setattr(module, "_installed_executable", lambda _prefix: tmp_path / "bin" / "l8k") + monkeypatch.setattr( + module, + "_verify_executable", + lambda _executable, _artifact_dir, _expected_version, _environment: ( + {"checks": {}, "artifacts": {}}, + True, + None, + ), + ) + + def fake_run(argv: list[str], *, cwd: Path, env: dict[str, str]) -> dict[str, Any]: + del cwd + calls.append((argv, env)) + return {"exit_code": 0, "stdout": "", "stderr": "", "duration_seconds": 0.1} + + monkeypatch.setattr(module, "_run_process", fake_run) + monkeypatch.setattr(module, "_record_process", lambda *_args, **_kwargs: {}) + args = argparse.Namespace( + mode="install", + executable="l8k", + version="v0.1.0", + installer_ref="a" * 40, + installer_sha256="0" * 64, + prefix=str(tmp_path), + environment_json=json.dumps({"HTTPS_PROXY": "http://proxy.example.test"}), + artifact_dir=str(tmp_path / "evidence"), + ) + + output, exit_code = module._prepare(args) + + assert exit_code == 0 + assert output["installed"] is True + assert calls[0][0] == ["/bin/sh", str(installer), "-d", str(tmp_path)] + assert calls[0][1]["L8K_VERSION"] == "v0.1.0" + assert calls[0][1]["HTTPS_PROXY"] == "http://proxy.example.test" + + +def test_provider_runs_the_real_launch_kit_workflow_shape(tmp_path: Path) -> None: + """The transport runs the full Launch Kit lifecycle with raw argv.""" + working_dir = tmp_path / "work" + artifact_dir = tmp_path / "evidence" + kubeconfig = "kubeconfig" + discover_args = [ + "--kubeconfig", + kubeconfig, + "--fabric", + "ethernet", + "--deployment-type", + "sriov", + ] + commands = [ + ("discover", discover_args), + ("generate", []), + ("deploy", ["--kubeconfig", kubeconfig]), + ("validate", ["--kubeconfig", kubeconfig]), + ("clean", ["--kubeconfig", kubeconfig]), + ] + outputs: dict[str, dict[str, Any]] = {} + + for command, arguments in commands: + completed, output = _run_workflow( + command, + arguments, + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + assert completed.returncode == 0 + assert output["success"] is True + assert output["operation"] == command + assert output["working_directory"] == str(working_dir.resolve()) + command_index = output["argv"].index(command) + assert output["argv"][command_index + 1 :] == [*arguments, "--output", "json"] + assert validate_output(output, "k8s_launch_kit") == (True, []) + assert all(Path(path).is_file() for path in output["artifacts"].values()) + outputs[command] = output + + assert len(outputs["discover"]["documents"]) == 1 + assert len(outputs["generate"]["documents"]) == 1 + generated_files = [Path(path) for path in outputs["generate"]["documents"][0]["generatedFiles"]] + daemonset_path = next(path for path in generated_files if "example-daemonset" in path.name) + daemonset = yaml.safe_load(daemonset_path.read_text(encoding="utf-8")) + assert [container["name"] for container in daemonset["spec"]["template"]["spec"]["containers"]] == [ + "test-container", + "netshoot", + ] + test_container = daemonset["spec"]["template"]["spec"]["containers"][0] + assert test_container["resources"]["requests"]["nvidia.com/gpu"] == "2" + assert test_container["resources"]["limits"]["nvidia.com/gpu"] == "2" + assert outputs["deploy"]["documents"] == [] + assert len(outputs["validate"]["documents"]) == 3 + source_report = working_dir / "deployment" / "k8s-launch-kit-validation-report.html" + retained_report = artifact_dir / "k8s-launch-kit-validation-report.html" + assert outputs["validate"]["artifacts"]["validation_report"] == str(retained_report) + assert retained_report.read_bytes() == source_report.read_bytes() + families = {row["Family"] for row in outputs["validate"]["documents"][1]["connectivity"]["PingResults"]} + assert families == {"icmp", "rping", "ib_write_bw", "gpudirect_dmabuf"} + assert outputs["clean"]["documents"][0]["cleanup"] == { + "namespace": "nvidia-network-operator", + "customResourcesDeleted": 12, + "helmReleaseRemoved": True, + "keepHelmChart": False, + } + assert (working_dir / "cluster-config.yaml").is_file() + assert source_report.is_file() + + +def test_sosreport_preserves_text_output_and_registers_its_directory(tmp_path: Path) -> None: + """The adapter retains the text-only sosreport contract as structured evidence.""" + working_dir = tmp_path / "work" + artifact_dir = tmp_path / "evidence" + + completed, output = _run_workflow( + "sosreport", + [], + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + + assert completed.returncode == 0 + assert output["success"] is True + assert output["operation"] == "sosreport" + assert output["documents"] == [] + assert output["argv"][1:] == ["sosreport", "--output-dir", str(artifact_dir / "sosreport")] + assert output["sosreport_output_directory"] == str(artifact_dir / "sosreport") + assert output["artifacts"]["sosreport"] == str(artifact_dir / "sosreport") + assert (artifact_dir / "sosreport" / "network-operator-sosreport.tar.gz").is_file() + assert "Sosreport collected" in Path(output["artifacts"]["stdout"]).read_text(encoding="utf-8") + assert validate_output(output, "k8s_launch_kit") == (True, []) + + +def test_discover_stages_user_config_transiently_without_retaining_secrets(tmp_path: Path) -> None: + """Discovery uses a private staged config but retains only safe input provenance.""" + source = tmp_path / "customer-cluster-config.yaml" + secret_values = ("customer-api-token", "registry-password", "embedded-kubeconfig") + source_contents = """networkOperator: + selectedRelease: "26.4" +profile: + fabric: ethernet + deployment: sriov +clusterConfig: [] +credentials: + token: customer-api-token + registryPassword: registry-password + kubeconfig: embedded-kubeconfig +""" + source.write_text(source_contents, encoding="utf-8") + working_dir = tmp_path / "work" + artifact_dir = tmp_path / "evidence" + + completed, output = _run_workflow( + "discover", + ["--fabric", "ethernet", "--deployment-type", "sriov"], + working_dir=working_dir, + artifact_dir=artifact_dir, + user_config=source, + ) + + staged = working_dir / "user-config.yaml" + discovered = working_dir / "cluster-config.yaml" + assert completed.returncode == 0 + assert output["success"] is True + assert source.read_text(encoding="utf-8") == source_contents + assert not staged.exists() + assert discovered.is_file() + metadata_path = artifact_dir / "inputs" / "user-config.json" + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + assert metadata == { + "source_path": str(source.resolve()), + "staged_path": str(staged.resolve()), + "sha256": hashlib.sha256(source_contents.encode()).hexdigest(), + "size_bytes": len(source_contents.encode()), + "retained": False, + } + assert output["artifacts"]["user_config"] == str(metadata_path.resolve()) + assert output["argv"][-6:] == [ + "--user-config", + str(staged.resolve()), + "--save-cluster-config", + str(discovered.resolve()), + "--output", + "json", + ] + retained_text = "\n".join( + path.read_text(encoding="utf-8", errors="replace") + for root in (working_dir, artifact_dir) + for path in root.rglob("*") + if path.is_file() + ) + assert all(secret not in retained_text for secret in secret_values) + + +def test_user_config_must_not_be_inside_the_retained_working_directory(tmp_path: Path) -> None: + """A source inside the retained output tree is rejected before it can leak as evidence.""" + working_dir = tmp_path / "work" + working_dir.mkdir() + source = working_dir / "customer-cluster-config.yaml" + source.write_text("profile: {}\ncredentials: customer-api-token\n", encoding="utf-8") + + completed, output = _run_workflow( + "discover", + [], + working_dir=working_dir, + artifact_dir=tmp_path / "evidence", + user_config=source, + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert "must be outside the retained provider working directory" in output["error"] + assert not (working_dir / "user-config.yaml").exists() + + +def test_staged_user_config_is_removed_when_discovery_fails(tmp_path: Path) -> None: + """A failed l8k discovery cannot leave the sensitive staged input behind.""" + source = tmp_path / "customer-cluster-config.yaml" + source_contents = "profile: {fabric: ethernet, deployment: sriov}\ncredentials: customer-api-token\n" + source.write_text(source_contents, encoding="utf-8") + working_dir = tmp_path / "work" + + completed, output = _run_workflow( + "discover", + [], + working_dir=working_dir, + artifact_dir=tmp_path / "evidence", + user_config=source, + env={**os.environ, "L8K_MOCK_FAIL": "discover"}, + ) + + assert completed.returncode != 0 + assert output["success"] is False + assert source.read_text(encoding="utf-8") == source_contents + assert not (working_dir / "user-config.yaml").exists() + + +def test_staged_user_config_is_created_with_restricted_permissions(tmp_path: Path) -> None: + """Sensitive input is private from the instant its staged file is created.""" + module = _load_provider_module() + source = tmp_path / "customer-cluster-config.yaml" + source.write_text("credentials: customer-api-token\n", encoding="utf-8") + working_dir = tmp_path / "work" + working_dir.mkdir() + + _, staged, _ = module._stage_user_config(str(source), working_dir, []) + + assert staged is not None + assert staged.stat().st_mode & 0o777 == 0o600 + + +@pytest.mark.parametrize("flag", ["--user-config", "--save-cluster-config"]) +def test_staged_user_config_rejects_conflicting_raw_discovery_paths(tmp_path: Path, flag: str) -> None: + """The first-class input owns both discovery config paths.""" + source = tmp_path / "customer-cluster-config.yaml" + source.write_text("profile: {}\n", encoding="utf-8") + + completed, output = _run_workflow( + "discover", + [flag, str(tmp_path / "raw.yaml")], + working_dir=tmp_path / "work", + artifact_dir=tmp_path / "evidence", + user_config=source, + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert f"cannot be combined with raw discovery flag(s): {flag}" in output["error"] + + +def test_staged_user_config_must_exist(tmp_path: Path) -> None: + """A missing first-class user config fails before l8k starts.""" + working_dir = tmp_path / "work" + + completed, output = _run_workflow( + "discover", + [], + working_dir=working_dir, + artifact_dir=tmp_path / "evidence", + user_config=tmp_path / "missing.yaml", + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert "Launch Kit user config not found" in output["error"] + assert not (working_dir / "user-config.yaml").exists() + + +def test_clean_forwards_launch_kit_boolean_flags_unchanged(tmp_path: Path) -> None: + """The transport accepts Launch Kit's native bare boolean flag syntax.""" + working_dir = tmp_path / "work" + artifact_dir = tmp_path / "evidence" + completed, _ = _run_workflow( + "discover", + ["--fabric", "ethernet", "--deployment-type", "sriov"], + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + assert completed.returncode == 0 + + completed, output = _run_workflow( + "clean", + ["--keep-helm-chart"], + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + + assert completed.returncode == 0 + assert output["argv"][-3:] == ["--keep-helm-chart", "--output", "json"] + assert output["documents"][0]["cleanup"] == { + "namespace": "nvidia-network-operator", + "customResourcesDeleted": 12, + "helmReleaseRemoved": False, + "keepHelmChart": True, + } + + +@pytest.mark.parametrize( + ("fabric", "deployment", "network_kind"), + [ + ("ethernet", "sriov", "SriovNetwork"), + ("infiniband", "sriov", "SriovIBNetwork"), + ("ethernet", "rdma_shared", "MacvlanNetwork"), + ("infiniband", "rdma_shared", "IPoIBNetwork"), + ("ethernet", "host_device", "HostDeviceNetwork"), + ("infiniband", "host_device", "HostDeviceNetwork"), + ], +) +def test_mock_supports_each_launch_kit_profile( + tmp_path: Path, + fabric: str, + deployment: str, + network_kind: str, +) -> None: + """Every pinned profile can traverse the same real command sequence.""" + working_dir = tmp_path / f"{fabric}-{deployment}" + artifact_dir = working_dir / "evidence" + commands = [ + ( + "discover", + [ + "--fabric", + fabric, + "--deployment-type", + deployment, + ], + ), + ("generate", []), + ("deploy", []), + ("validate", []), + ("clean", []), + ] + outputs: dict[str, dict[str, Any]] = {} + + for command, arguments in commands: + completed, output = _run_workflow( + command, + arguments, + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + assert completed.returncode == 0 + outputs[command] = output + + manifest_kinds = {row["Kind"] for row in outputs["validate"]["documents"][0]["manifests"]} + assert network_kind in manifest_kinds + assert outputs["clean"]["documents"][0]["phase"] == "clean" + + +def test_preflight_uses_the_workflow_kubeconfig(tmp_path: Path) -> None: + """kubectl probes target the same explicit kubeconfig supplied to l8k.""" + workflow = { + command: ["--kubeconfig", "partner.kubeconfig"] if command != "generate" else [] + for command in ("discover", "generate", "deploy", "validate", "clean") + } + completed, output = _run_provider( + "preflight", + "--kubectl-command-json", + json.dumps([sys.executable, str(_MOCK_KUBECTL)]), + "--workflow-arguments-json", + json.dumps(workflow), + "--working-dir", + str(tmp_path / "work"), + "--artifact-dir", + str(tmp_path / "evidence"), + ) + + assert completed.returncode == 0 + assert output["success"] is True + assert output["kubeconfig_source"] == "workflow arguments" + assert output["node_count"] == 2 + assert output["ready_node_count"] == 2 + command_file = Path(output["artifacts"]["api_version"]["command"]) + argv = json.loads(command_file.read_text(encoding="utf-8"))["argv"] + kubeconfig_index = argv.index("--kubeconfig") + assert argv[kubeconfig_index : kubeconfig_index + 2] == ["--kubeconfig", "partner.kubeconfig"] + + +def test_preflight_forwards_the_launch_kit_environment(tmp_path: Path) -> None: + """The safety probes use the same environment that the provider gives l8k.""" + workflow = {command: [] for command in ("discover", "generate", "deploy", "validate", "clean")} + completed, output = _run_provider( + "preflight", + "--kubectl-command-json", + json.dumps([sys.executable, str(_MOCK_KUBECTL)]), + "--workflow-arguments-json", + json.dumps(workflow), + "--environment-json", + json.dumps( + { + "KUBECONFIG": "environment.kubeconfig", + "L8K_MOCK_EXPECT_KUBECONFIG": "environment.kubeconfig", + } + ), + "--working-dir", + str(tmp_path / "work"), + "--artifact-dir", + str(tmp_path / "evidence"), + ) + + assert completed.returncode == 0 + assert output["success"] is True + + +def test_preflight_accepts_a_validation_only_workflow(tmp_path: Path) -> None: + """The prerequisite gate follows the caller's actual Launch Kit command subset.""" + workflow = {command: [] for command in ("discover", "generate", "validate")} + completed, output = _run_provider( + "preflight", + "--kubectl-command-json", + json.dumps([sys.executable, str(_MOCK_KUBECTL)]), + "--workflow-arguments-json", + json.dumps(workflow), + "--working-dir", + str(tmp_path / "work"), + "--artifact-dir", + str(tmp_path / "evidence"), + ) + + assert completed.returncode == 0 + assert output["success"] is True + + +def test_preflight_rejects_conflicting_workflow_kubeconfigs(tmp_path: Path) -> None: + """The safety gate fails closed when l8k commands would target different clusters.""" + workflow = { + "discover": ["--kubeconfig", "cluster-a"], + "generate": [], + "deploy": ["--kubeconfig=cluster-b"], + "validate": [], + "clean": [], + } + completed, output = _run_provider( + "preflight", + "--kubectl-command-json", + "[]", + "--workflow-arguments-json", + json.dumps(workflow), + "--working-dir", + str(tmp_path / "work"), + "--artifact-dir", + str(tmp_path / "evidence"), + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert "different kubeconfigs" in output["error"] + + +def test_network_operator_provider_runs_validate_then_sosreport(tmp_path: Path) -> None: + """The production configuration validates once and then collects diagnostics.""" + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + capability="kubernetes", + ) + + assert result.success is True + assert list(result.inventory) == ["launch_kit_validate", "launch_kit_sosreport"] + assert [phase.name for phase in result.phases] == ["test", "test-teardown"] + assert len(result.validations) == 1 + validation = result.validations[0] + assert validation.entry.name == "LaunchKitConnectivityCheck" + assert validation.state is State.PASSED + assert validation.subtest_summary.passed == 32 + assert validation.subtest_summary.failed == 0 + assert validation.subtest_summary.skipped == 0 + + argv = result.inventory["launch_kit_validate"]["argv"] + assert argv[1] == "validate" + assert argv[argv.index("--user-config") + 1] == str((tmp_path / "cluster-config.yaml").resolve()) + assert argv[argv.index("--deployment-files") + 1] == str((tmp_path / "deployment").resolve()) + assert argv[-2:] == ["--output", "json"] + report = tmp_path / "evidence" / "k8s-launch-kit-validation-report.html" + assert result.inventory["launch_kit_validate"]["artifacts"]["validation_report"] == str(report) + assert report.is_file() + sosreport = result.inventory["launch_kit_sosreport"] + assert sosreport["argv"][1:] == ["sosreport", "--output-dir", str(tmp_path / "evidence" / "sosreport")] + assert Path(sosreport["artifacts"]["sosreport"]).is_dir() + + +def test_sosreport_failure_does_not_replace_connectivity_result(tmp_path: Path, monkeypatch: Any) -> None: + """Diagnostic failure is separate while the connectivity assertion stays passed.""" + monkeypatch.setenv("L8K_MOCK_FAIL", "sosreport") + config = _mocked_network_operator_config(tmp_path) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + capability="kubernetes", + ) + + assert result.success is False + assert result.validations[0].state is State.PASSED + assert [(phase.name, phase.success) for phase in result.phases] == [ + ("test", True), + ("test-teardown", False), + ] + assert result.inventory["launch_kit_sosreport"]["success"] is False + assert "sosreport collection failed" in result.inventory["launch_kit_sosreport"]["error"] + + +def test_network_operator_provider_expands_input_paths(tmp_path: Path, monkeypatch: Any) -> None: + """Tilde inputs are resolved before they are supplied to Launch Kit.""" + monkeypatch.setenv("HOME", str(tmp_path)) + user_config = tmp_path / "l8k" / "cluster-config.yaml" + user_config.parent.mkdir() + user_config.write_text( + "profile:\n fabric: ethernet\n deployment: sriov\n", + encoding="utf-8", + ) + deployment_files = tmp_path / "l8k" / "deployment" + deployment_files.mkdir() + + completed, output = _run_workflow( + "validate", + [], + working_dir=tmp_path / "work", + artifact_dir=tmp_path / "evidence", + user_config=Path("~/l8k/cluster-config.yaml"), + deployment_files=Path("~/l8k/deployment"), + ) + + assert completed.returncode == 0 + assert output["argv"][output["argv"].index("--user-config") + 1] == str(user_config) + assert output["argv"][output["argv"].index("--deployment-files") + 1] == str(deployment_files) + + +@pytest.mark.parametrize( + ("user_config", "deployment_files", "expected"), + [ + (None, "deployment", "user_config is required"), + ("cluster-config.yaml", None, "--user-config requires --deployment-files"), + ], +) +def test_validate_requires_both_prerequisite_inputs( + tmp_path: Path, + user_config: str | None, + deployment_files: str | None, + expected: str, +) -> None: + """Partial prerequisite input fails before Launch Kit execution.""" + config_path = tmp_path / "cluster-config.yaml" + config_path.write_text("profile: {}\n", encoding="utf-8") + deployment_path = tmp_path / "deployment" + deployment_path.mkdir() + + completed, output = _run_workflow( + "validate", + [], + working_dir=tmp_path / "work", + artifact_dir=tmp_path / "evidence", + user_config=config_path if user_config is not None else None, + deployment_files=deployment_path if deployment_files is not None else None, + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert expected in output["error"] + + +def test_validate_rejects_duplicate_path_flags(tmp_path: Path) -> None: + """Dedicated inputs cannot silently conflict with raw Launch Kit arguments.""" + user_config = tmp_path / "cluster-config.yaml" + user_config.write_text("profile: {}\n", encoding="utf-8") + deployment_files = tmp_path / "deployment" + deployment_files.mkdir() + + completed, output = _run_workflow( + "validate", + ["--user-config", "other.yaml"], + working_dir=tmp_path / "work", + artifact_dir=tmp_path / "evidence", + user_config=user_config, + deployment_files=deployment_files, + ) + + assert completed.returncode == 1 + assert "cannot be combined with raw flag(s): --user-config" in output["error"] + + +def test_failed_connectivity_is_a_junit_failure(tmp_path: Path, monkeypatch: Any) -> None: + """A failed Launch Kit matrix row is retained as a test failure in JUnit.""" + monkeypatch.setenv("L8K_MOCK_FAIL", "validate:ib_write_bw") + config = _mocked_network_operator_config(tmp_path) + junit_path = tmp_path / "junit.xml" + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + capability="kubernetes", + junitxml=str(junit_path), + ) + + assert result.success is False + assert list(result.inventory) == ["launch_kit_validate", "launch_kit_sosreport"] + report = tmp_path / "evidence" / "k8s-launch-kit-validation-report.html" + assert result.inventory["launch_kit_validate"]["artifacts"]["validation_report"] == str(report) + assert report.is_file() + assert (tmp_path / "evidence" / "sosreport" / "network-operator-sosreport.tar.gz").is_file() + assert result.validations[0].state is State.FAILED + assert result.validations[0].subtest_summary.failed == 1 + case = next( + case + for case in ET.parse(junit_path).getroot().iter("testcase") + if case.get("name") == "LaunchKitConnectivityCheck" + ) + assert case.find("failure") is not None + assert case.find("error") is None + assert case.find("skipped") is None + + +def test_missing_prerequisites_are_a_step_error(tmp_path: Path) -> None: + """An unset prerequisite produces an actionable validation error.""" + merged = merge_yaml_files([_NETWORK_OPERATOR_CONFIG]) + merged["context"]["k8s_launch_kit"]["executable"] = str(_MOCK_L8K) + merged["context"]["k8s_launch_kit"]["working_dir"] = str(tmp_path / "work") + merged["context"]["k8s_launch_kit"]["artifact_dir"] = str(tmp_path / "evidence") + config = RunConfig.model_validate(merged) + + result = Orchestrator(config, working_dir=_NETWORK_OPERATOR_CONFIG.parent).run( + phases=[Phase.TEST], + capability="kubernetes", + ) + + assert result.success is False + assert list(result.inventory) == ["launch_kit_validate", "launch_kit_sosreport"] + assert (tmp_path / "evidence" / "sosreport" / "network-operator-sosreport.tar.gz").is_file() + assert result.validations[0].state is State.FAILED + assert "user_config is required" in result.validations[0].message + + +def test_failed_validate_preserves_documents_and_process_error(tmp_path: Path) -> None: + """A non-zero l8k result retains every JSON document and a clear exit diagnostic.""" + working_dir = tmp_path / "work" + artifact_dir = tmp_path / "evidence" + discover, _ = _run_workflow( + "discover", + [ + "--fabric", + "ethernet", + "--deployment-type", + "sriov", + ], + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + assert discover.returncode == 0 + generate, _ = _run_workflow( + "generate", + [], + working_dir=working_dir, + artifact_dir=artifact_dir, + ) + assert generate.returncode == 0 + env = os.environ.copy() + env["L8K_MOCK_FAIL"] = "validate:ib_write_bw" + + completed, output = _run_workflow( + "validate", + [], + working_dir=working_dir, + artifact_dir=artifact_dir, + env=env, + ) + + assert completed.returncode == 4 + assert output["success"] is False + assert len(output["documents"]) == 3 + assert "l8k validate exited with code 4" in output["error"] + assert Path(output["artifacts"]["validation_report"]).is_file() + assert Path(output["artifacts"]["stdout"]).read_text(encoding="utf-8") + + +def test_missing_advertised_validation_report_is_an_evidence_error(tmp_path: Path) -> None: + """A stale report cannot satisfy a new Launch Kit reportPath document.""" + missing_report = tmp_path / "missing-validation-report.html" + executable = tmp_path / "l8k" + executable.write_text( + f"#!/bin/sh\nprintf '%s\\n' '{{\"reportPath\":\"{missing_report}\"}}'\n", + encoding="utf-8", + ) + executable.chmod(0o755) + artifact_dir = tmp_path / "evidence" + retained_report = artifact_dir / "k8s-launch-kit-validation-report.html" + retained_report.parent.mkdir(parents=True) + retained_report.write_text("stale report\n", encoding="utf-8") + + completed, output = _run_provider( + "run", + "--executable", + str(executable), + "--command", + "validate", + "--arguments-json", + "[]", + "--working-dir", + str(tmp_path / "work"), + "--artifact-dir", + str(artifact_dir), + ) + + assert completed.returncode == 1 + assert output["success"] is False + assert "failed to retain Launch Kit HTML validation report" in output["error"] + assert str(missing_report) in output["error"] + assert "validation_report" not in output["artifacts"] + assert not retained_report.exists() diff --git a/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py b/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py new file mode 100644 index 000000000..1e235304c --- /dev/null +++ b/isvctl/tests/providers/k8s_launch_kit/test_timeout_config.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Regression tests for Launch Kit timeout ownership.""" + +from pathlib import Path +from typing import Any + +import yaml + +PROVIDER_CONFIG_DIR = Path(__file__).parents[3] / "configs" / "providers" / "k8s-launch-kit" / "config" + + +def _steps(config_name: str) -> list[dict[str, Any]]: + """Return Network Operator command steps from a provider configuration.""" + config = yaml.safe_load((PROVIDER_CONFIG_DIR / config_name).read_text()) + return config["commands"]["network_operator"]["steps"] + + +def test_generic_validate_delegates_timeout_to_launch_kit() -> None: + """The generic validate workflow must not preempt l8k's matrix budget.""" + validate_steps = [step for step in _steps("provider.yaml") if step["name"] == "launch_kit_validate"] + + assert len(validate_steps) == 1 + assert validate_steps[0]["timeout"] is None + + +def test_network_operator_validate_delegates_timeout_to_launch_kit() -> None: + """The one Network Operator validation leaves its deadline to l8k.""" + validate_steps = [step for step in _steps("network-operator.yaml") if step["name"] == "launch_kit_validate"] + + assert len(validate_steps) == 1 + assert validate_steps[0]["timeout"] is None + + +def test_network_operator_sosreport_has_a_bounded_watchdog() -> None: + """Diagnostic collection must not hang the orchestration indefinitely.""" + sosreport_steps = [step for step in _steps("network-operator.yaml") if step["name"] == "launch_kit_sosreport"] + + assert len(sosreport_steps) == 1 + assert sosreport_steps[0]["timeout"] == 1800 diff --git a/isvctl/tests/test_orchestrator_loop.py b/isvctl/tests/test_orchestrator_loop.py index 6cd083eff..5d49fa818 100644 --- a/isvctl/tests/test_orchestrator_loop.py +++ b/isvctl/tests/test_orchestrator_loop.py @@ -35,6 +35,7 @@ Orchestrator, Phase, _apply_capability_step_gates, + _apply_selected_validation_gates, _entries_missing_from_junit, _merge_junit_xmls, _write_terminal_junit_xml, @@ -79,6 +80,58 @@ def test_explicit_step_requires_gate_unbound_lifecycle_steps() -> None: assert all(not step.skip for step in kubernetes_steps) +def test_selected_validation_gate_prunes_unselected_lifecycle_steps() -> None: + """Label selection prevents commands owned by another test group from running.""" + steps = [ + StepConfig( + name="run_ethernet", + command="ethernet", + phase="test", + requires_selected_validations=["EthernetCheck"], + ), + StepConfig( + name="run_infiniband", + command="infiniband", + phase="test", + requires_selected_validations=["InfiniBandCheck"], + ), + ] + entries = [ + ValidationEntry( + name="EthernetCheck", + category="network", + params_template={}, + labels=("ethernet",), + ), + ValidationEntry( + name="InfiniBandCheck", + category="network", + params_template={}, + labels=("infiniband",), + ), + ] + + all_steps = _apply_selected_validation_gates( + steps, + entries, + include_labels=set(), + exclude_labels=set(), + exclude_tests=set(), + capability=None, + ) + ethernet_steps = _apply_selected_validation_gates( + steps, + entries, + include_labels={"ethernet"}, + exclude_labels=set(), + exclude_tests=set(), + capability=None, + ) + + assert all(not step.skip for step in all_steps) + assert [step.skip for step in ethernet_steps] == [False, True] + + def test_python_script_path_falls_back_to_current_working_directory( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -371,6 +424,303 @@ def test_run_all_phases_with_failure(self) -> None: assert len(teardown_phases) == 1 assert teardown_phases[0].success + def test_independent_custom_phase_runs_after_an_allowed_failure(self) -> None: + """An opted-in failed use case does not hide results from later independent cases.""" + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig(name="case_one", command="false", phase="case-one"), + StepConfig( + name="case_two", + command="echo", + args=['{"success": true, "platform": "kubernetes"}'], + phase="case-two", + output_schema="generic", + ), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert [(phase.name, phase.success) for phase in result.phases] == [ + ("case-one", False), + ("case-two", True), + ] + + def test_phase_finalizer_runs_after_its_target_fails(self, tmp_path: Path) -> None: + """An attempted mutating step activates cleanup even when the step fails.""" + marker = tmp_path / "cleaned" + cleanup = _write_script( + tmp_path, + "cleanup.sh", + f"#!/bin/sh\ntouch {marker}\necho '{{\"success\": true}}'\n", + ) + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig(name="deploy", command="false", phase="case-one"), + StepConfig( + name="cleanup", + command=cleanup, + phase="case-one", + finalizer_for="deploy", + ), + StepConfig(name="case_two", command="true", phase="case-two"), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert marker.is_file() + assert [step["name"] for step in result.phases[0].details["steps"]] == ["deploy"] + assert [step["name"] for step in result.phases[1].details["steps"]] == ["cleanup"] + assert result.phases[1].phase is Phase.TEARDOWN + assert result.phases[1].name == "case-one-teardown" + assert result.phases[2].name == "case-two" + assert result.phases[2].success is True + + def test_phase_finalizer_skips_when_target_was_not_attempted(self, tmp_path: Path) -> None: + """A prerequisite failure cannot activate destructive cleanup before deployment.""" + marker = tmp_path / "cleaned" + cleanup = _write_script(tmp_path, "cleanup.sh", f"#!/bin/sh\ntouch {marker}\n") + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig(name="preflight", command="false", phase="case-one"), + StepConfig(name="deploy", command="true", phase="case-one"), + StepConfig( + name="cleanup", + command=cleanup, + phase="case-one", + finalizer_for="deploy", + ), + StepConfig(name="case_two", command="true", phase="case-two"), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert not marker.exists() + assert [step["name"] for step in result.phases[0].details["steps"]] == ["preflight"] + assert result.phases[1].name == "case-one-teardown" + assert result.phases[1].message.startswith("SKIPPED: target step(s) were not attempted") + assert result.phases[2].success is True + + def test_phase_finalizer_skips_when_target_process_never_started(self, tmp_path: Path) -> None: + """A command-resolution failure cannot imply that a cluster mutation occurred.""" + marker = tmp_path / "cleaned" + cleanup = _write_script(tmp_path, "cleanup.sh", f"#!/bin/sh\ntouch {marker}\n") + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig( + name="deploy", + command=str(tmp_path / "does-not-exist"), + phase="case-one", + ), + StepConfig( + name="cleanup", + command=cleanup, + phase="case-one", + finalizer_for="deploy", + ), + StepConfig(name="case_two", command="true", phase="case-two"), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert not marker.exists() + assert result.phases[0].details["steps"][0]["attempted"] is False + assert result.phases[1].message.startswith("SKIPPED: target step(s) were not attempted") + assert result.phases[2].success is True + + def test_phase_finalizer_skips_when_target_is_not_executable(self, tmp_path: Path) -> None: + """A permission failure also proves that no cluster mutation occurred.""" + marker = tmp_path / "cleaned" + cleanup = _write_script(tmp_path, "cleanup.sh", f"#!/bin/sh\ntouch {marker}\n") + target = tmp_path / "deploy.sh" + target.write_text("#!/bin/sh\nexit 0\n") + target.chmod(0o644) + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig(name="deploy", command=str(target), phase="case-one"), + StepConfig( + name="cleanup", + command=cleanup, + phase="case-one", + finalizer_for="deploy", + ), + StepConfig(name="case_two", command="true", phase="case-two"), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert not marker.exists() + assert result.phases[0].details["steps"][0]["attempted"] is False + assert result.phases[1].message.startswith("SKIPPED: target step(s) were not attempted") + assert result.phases[2].success is True + + def test_failed_phase_finalizer_blocks_later_independent_phases(self) -> None: + """A failed cleanup leaves unsafe state and overrides continuation policy.""" + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + continue_after_failure=["case-one"], + steps=[ + StepConfig(name="deploy", command="true", phase="case-one"), + StepConfig( + name="cleanup", + command="false", + phase="case-one", + finalizer_for="deploy", + ), + StepConfig(name="case_two", command="true", phase="case-two"), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert [(phase.name, phase.message) for phase in result.phases] == [ + ("case-one", "deploy: passed"), + ("case-one-teardown", "cleanup: failed"), + ("case-two", "SKIPPED: previous phase failed"), + ] + + def test_teardown_finalizer_runs_between_test_phases(self, tmp_path: Path) -> None: + """A step declared in teardown executes directly after its target test phase.""" + marker = tmp_path / "cleaned" + cleanup = _write_script(tmp_path, "cleanup.sh", f"#!/bin/sh\ntouch {marker}\n") + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two", "teardown"], + steps=[ + StepConfig(name="deploy", command="true", phase="case-one"), + StepConfig(name="case_two", command="true", phase="case-two"), + StepConfig( + name="cleanup", + command=cleanup, + phase="teardown", + finalizer_for="deploy", + ), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is True + assert marker.is_file() + assert [(phase.name, phase.phase) for phase in result.phases] == [ + ("case-one", Phase.TEST), + ("case-one-teardown", Phase.TEARDOWN), + ("case-two", Phase.TEST), + ] + + def test_teardown_only_runs_linked_finalizer_as_recovery(self, tmp_path: Path) -> None: + """An explicit teardown-only run does not require an in-memory target attempt.""" + marker = tmp_path / "cleaned" + cleanup = _write_script(tmp_path, "cleanup.sh", f"#!/bin/sh\ntouch {marker}\n") + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["test", "teardown"], + steps=[ + StepConfig(name="deploy", command="true", phase="test"), + StepConfig( + name="cleanup", + command=cleanup, + phase="teardown", + finalizer_for="deploy", + ), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEARDOWN]) + + assert result.success is True + assert marker.is_file() + assert [(phase.name, phase.phase) for phase in result.phases] == [ + ("teardown", Phase.TEARDOWN), + ] + + def test_custom_phase_failure_blocks_later_phases_by_default(self) -> None: + """Without an opt-in, the existing stop-on-failure behavior is unchanged.""" + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["case-one", "case-two"], + steps=[ + StepConfig(name="case_one", command="false", phase="case-one"), + StepConfig( + name="case_two", + command="echo", + args=['{"success": true, "platform": "kubernetes"}'], + phase="case-two", + output_schema="generic", + ), + ], + ) + }, + tests=ValidationConfig(capability="kubernetes"), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST]) + + assert result.success is False + assert [(phase.name, phase.message) for phase in result.phases] == [ + ("case-one", "case_one: failed"), + ("case-two", "SKIPPED: previous phase failed"), + ] + def test_platform_detection_missing(self) -> None: """Test error when platform cannot be detected. @@ -415,6 +765,28 @@ def test_config_without_commands_runs_live_validations_only(self, monkeypatch: p assert [entry.entry.name for entry in result.validations] == ["K8sCsiStorageTypesCheck"] assert result.validations[0].state is State.PASSED + def test_config_without_commands_reports_failed_live_validation(self) -> None: + """A failed commandless validation returns a failed result instead of reading command policy.""" + config = RunConfig( + tests=ValidationConfig( + validations={ + "live_checks": { + "checks": { + "ExistingSystemFieldCheck": { + "compose": [{"FieldExistsCheck": {"field": "missing"}}], + } + }, + } + }, + ), + ) + + result = Orchestrator(config).run(phases=[Phase.TEST], capability="kubernetes") + + assert result.success is False + assert result.validations[0].state is State.FAILED + assert "Missing fields: missing" in result.validations[0].message + def test_config_without_commands_or_validations_is_not_a_pass(self) -> None: """Validations are all a commandless run has, so wiring none asserts nothing.""" orchestrator = Orchestrator(RunConfig(tests=ValidationConfig(validations={}))) @@ -565,9 +937,78 @@ def test_validation_without_step_output_is_reported_as_skipped(self, monkeypatch "state": "skipped", "skip_reason": "step_no_output", "error_reason": None, + "subtest_summary": {"total": 0, "passed": 0, "failed": 0, "skipped": 0}, } ] + def test_failed_owned_step_is_reported_as_validation_error( + self, + tmp_path: Path, + ) -> None: + """An early workflow failure cannot become a harmless missing-output skip.""" + failing_step = _write_script( + tmp_path, + "deploy.sh", + "#!/bin/sh\necho 'driver image not found' >&2\nexit 4\n", + ) + junit_path = tmp_path / "junit.xml" + config = RunConfig( + commands={ + "kubernetes": PlatformCommands( + phases=["use-case"], + steps=[ + StepConfig( + name="deploy_fixture", + command=failing_step, + phase="use-case", + requires_selected_validations=["ProbeSucceededCheck"], + ), + StepConfig( + name="validate_fixture", + command="true", + phase="use-case", + requires_selected_validations=["ProbeSucceededCheck"], + ), + ], + ) + }, + tests=ValidationConfig( + capability="kubernetes", + validations={ + "probe_checks": { + "step": "validate_fixture", + "checks": {"ProbeSucceededCheck": {"compose": ["StepSuccessCheck"]}}, + }, + }, + ), + ) + + result = Orchestrator(config).run( + phases=[Phase.TEST], + capability="kubernetes", + junitxml=str(junit_path), + ) + + assert result.success is False + validation = result.validations[0] + assert validation.state is State.ERROR + assert validation.error_reason is ErrorReason.STEP_FAILED + assert validation.message == ( + "workflow step 'deploy_fixture' failed: Command exited with code 4: driver image not found" + ) + + suite = ET.parse(junit_path).getroot().find("testsuite") + assert suite is not None + assert suite.get("errors") == "1" + assert suite.get("skipped") == "0" + case = suite.find("testcase") + assert case is not None + assert case.get("name") == "ProbeSucceededCheck" + error = case.find("error") + assert error is not None + assert error.get("type") == ErrorReason.STEP_FAILED.value + assert case.find("skipped") is None + def test_validation_template_error_is_reported_as_error( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/isvctl/tests/test_orchestrator_process.py b/isvctl/tests/test_orchestrator_process.py new file mode 100644 index 000000000..803d646e7 --- /dev/null +++ b/isvctl/tests/test_orchestrator_process.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for orchestration subprocess lifecycle handling.""" + +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from isvctl.orchestrator.process import run_command_process + + +def test_run_command_process_captures_output(tmp_path: Path) -> None: + """Successful commands return captured text output.""" + completed = run_command_process( + [sys.executable, "-c", "print('ready')"], + cwd=tmp_path, + env=None, + timeout=5, + ) + + assert completed.returncode == 0 + assert completed.stdout == "ready\n" + assert completed.stderr == "" + + +def test_run_command_process_accepts_no_timeout(tmp_path: Path) -> None: + """A null step timeout waits for a command that owns its deadline.""" + completed = run_command_process( + [sys.executable, "-c", "print('tool-owned-timeout')"], + cwd=tmp_path, + env=None, + timeout=None, + ) + + assert completed.returncode == 0 + assert completed.stdout == "tool-owned-timeout\n" + assert completed.stderr == "" + + +@pytest.mark.skipif(os.name != "posix", reason="process-group behavior is POSIX-specific") +def test_timeout_terminates_descendant_process(tmp_path: Path) -> None: + """A timed-out wrapper must not leave its provider CLI child running.""" + child_pid_path = tmp_path / "child.pid" + wrapper = """ +import subprocess +import sys +import time +from pathlib import Path + +child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) +Path(sys.argv[1]).write_text(str(child.pid)) +print("wrapper-ready", flush=True) +time.sleep(60) +""" + child_pid: int | None = None + + try: + with pytest.raises(subprocess.TimeoutExpired) as exc_info: + run_command_process( + [sys.executable, "-c", wrapper, str(child_pid_path)], + cwd=tmp_path, + env=None, + timeout=0.5, + ) + + assert "wrapper-ready" in (exc_info.value.stdout or "") + child_pid = int(child_pid_path.read_text()) + + deadline = time.monotonic() + 2 + while _process_exists(child_pid) and time.monotonic() < deadline: + time.sleep(0.05) + assert not _process_exists(child_pid) + finally: + if child_pid is not None and _process_exists(child_pid): + os.kill(child_pid, signal.SIGKILL) + + +def _process_exists(pid: int) -> bool: + """Return whether a process currently exists.""" + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + return True diff --git a/isvctl/tests/test_schema.py b/isvctl/tests/test_schema.py index d28e9326b..eb943a1dd 100644 --- a/isvctl/tests/test_schema.py +++ b/isvctl/tests/test_schema.py @@ -76,6 +76,7 @@ def test_minimal_step(self) -> None: assert step.phase == "setup" assert step.skip is False assert step.requires == [] + assert step.requires_selected_validations == [] def test_full_step(self) -> None: """Test creating a fully specified step config.""" @@ -89,6 +90,7 @@ def test_full_step(self) -> None: phase="setup", skip=False, requires=["vm", "bare_metal"], + requires_selected_validations=["SelectedCheck"], continue_on_failure=True, output_schema="vpc", ) @@ -99,9 +101,16 @@ def test_full_step(self) -> None: assert step.env == {"AWS_REGION": "us-west-2"} assert step.phase == "setup" assert step.requires == ["vm", "bare_metal"] + assert step.requires_selected_validations == ["SelectedCheck"] assert step.continue_on_failure is True assert step.output_schema == "vpc" + def test_null_timeout_disables_watchdog(self) -> None: + """A provider may delegate timeout ownership to the invoked tool.""" + step = StepConfig(name="validate", command="l8k", timeout=None) + + assert step.timeout is None + def test_step_rejects_unknown_or_duplicate_requires(self) -> None: """Step requirements use the declarable capability vocabulary.""" with pytest.raises(ValidationError, match="requires must be a list containing only"): @@ -110,6 +119,100 @@ def test_step_rejects_unknown_or_duplicate_requires(self) -> None: StepConfig(name="setup", command="echo", requires=["vm", "vm"]) +class TestPlatformCommands: + """Tests for ordered command phases.""" + + def test_phases_reject_duplicates(self) -> None: + """A duplicate phase would otherwise execute its steps more than once.""" + with pytest.raises(ValidationError, match="phases must not contain duplicate"): + PlatformCommands(phases=["setup", "test", "test"]) + + def test_continue_after_failure_rejects_unknown_phase(self) -> None: + """A typo must not silently restore stop-on-failure behavior.""" + with pytest.raises(ValidationError, match="phases not listed in phases"): + PlatformCommands( + phases=["setup", "use-case"], + continue_after_failure=["use-csae"], + ) + + @pytest.mark.parametrize("phase", ["setup", "teardown"]) + def test_continue_after_failure_rejects_lifecycle_phases(self, phase: str) -> None: + """Setup and teardown are never independent test-case phases.""" + with pytest.raises(ValidationError, match="cannot contain lifecycle phases"): + PlatformCommands( + phases=["setup", "use-case", "teardown"], + continue_after_failure=[phase], + ) + + def test_continue_after_failure_rejects_duplicates(self) -> None: + """Duplicate continuation entries are configuration errors, not useful policy.""" + with pytest.raises(ValidationError, match="must not contain duplicate"): + PlatformCommands( + phases=["setup", "use-case"], + continue_after_failure=["use-case", "use-case"], + ) + + def test_finalizer_requires_one_target_in_its_phase_or_teardown(self) -> None: + """A finalizer target must resolve unambiguously in a supported lifecycle position.""" + with pytest.raises(ValidationError, match="must name exactly one configured step"): + PlatformCommands( + phases=["test"], + steps=[ + StepConfig(name="cleanup", command="clean", phase="test", finalizer_for="deploy"), + ], + ) + + with pytest.raises(ValidationError, match="same phase or the finalizer must use phase 'teardown'"): + PlatformCommands( + phases=["setup", "test", "cleanup"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test"), + StepConfig(name="cleanup", command="clean", phase="cleanup", finalizer_for="deploy"), + ], + ) + + config = PlatformCommands( + phases=["test", "teardown"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test"), + StepConfig(name="cleanup", command="clean", phase="teardown", finalizer_for="deploy"), + ], + ) + assert config.steps[1].finalizer_for == "deploy" + + def test_teardown_finalizer_must_follow_target_and_match_its_gates(self) -> None: + """Linked teardown cannot precede its mutation or be filtered independently.""" + with pytest.raises(ValidationError, match="teardown must be ordered after target"): + PlatformCommands( + phases=["teardown", "test"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test"), + StepConfig(name="cleanup", command="clean", phase="teardown", finalizer_for="deploy"), + ], + ) + + with pytest.raises(ValidationError, match="must use the same gates as target"): + PlatformCommands( + phases=["test", "teardown"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test", requires=["kubernetes"]), + StepConfig(name="cleanup", command="clean", phase="teardown", finalizer_for="deploy"), + ], + ) + + def test_finalizer_cannot_target_another_finalizer(self) -> None: + """Finalizer chains have ambiguous activation and cleanup ordering.""" + with pytest.raises(ValidationError, match="cannot finalize finalizer step"): + PlatformCommands( + phases=["test"], + steps=[ + StepConfig(name="deploy", command="deploy", phase="test"), + StepConfig(name="cleanup", command="clean", phase="test", finalizer_for="deploy"), + StepConfig(name="verify_cleanup", command="verify", phase="test", finalizer_for="cleanup"), + ], + ) + + class TestCommandOutput: """Tests for CommandOutput model (setup command JSON output).""" diff --git a/isvctl/tests/test_stub_contracts.py b/isvctl/tests/test_stub_contracts.py index 686b29088..3c291c5a5 100644 --- a/isvctl/tests/test_stub_contracts.py +++ b/isvctl/tests/test_stub_contracts.py @@ -156,7 +156,7 @@ def _collect_yaml_checks() -> list[StepArgCheck]: checks: list[StepArgCheck] = [] yaml_paths = sorted( [ - *CONFIGS_DIR.glob("suites/*.yaml"), + *CONFIGS_DIR.glob("suites/**/*.yaml"), *CONFIGS_DIR.glob("providers/*.yaml"), # k3s.yaml, microk8s.yaml, minikube.yaml *CONFIGS_DIR.glob("providers/*/config/*.yaml"), # aws/config/*.yaml, my-isv/config/*.yaml ] diff --git a/isvctl/tests/test_suite_resolution.py b/isvctl/tests/test_suite_resolution.py index 86e253c10..0ba72c76e 100644 --- a/isvctl/tests/test_suite_resolution.py +++ b/isvctl/tests/test_suite_resolution.py @@ -52,6 +52,21 @@ def test_one_suite_flag_resolves_canonical_and_provider_suites(tmp_path: Path) - assert canonical_plain.platform is None +def test_canonical_suite_resolution_discovers_nested_suite_yaml(tmp_path: Path) -> None: + """Domain folders under suites remain selectable through the generic resolver.""" + _write_catalog(tmp_path) + domain = tmp_path / "suites" / "launch-kit" + domain.mkdir() + nested = domain / "network-operator.yaml" + nested.write_text("tests:\n validations: {}\n") + + resolved = resolve_suite(None, "network-operator", configs_root=tmp_path) + + assert resolved.config_path == nested + assert resolved.name == "network_operator" + assert resolved.platform is None + + def test_capability_uses_catalog_vocabulary(tmp_path: Path) -> None: """An unknown capability is rejected while omitted context disables filtering.""" _write_catalog(tmp_path) diff --git a/isvctl/tests/test_test_cli_labels.py b/isvctl/tests/test_test_cli_labels.py index f89107f6d..6d65e7ced 100644 --- a/isvctl/tests/test_test_cli_labels.py +++ b/isvctl/tests/test_test_cli_labels.py @@ -52,6 +52,66 @@ def _write_config(tmp_path: Path) -> Path: return config +def test_validation_result_detail_compacts_successful_subtests() -> None: + """Successful validations with probes render an aggregate instead of their long message.""" + detail = test_cli._validation_result_detail( + { + "passed": True, + "skipped": False, + "state": "passed", + "message": "long; member; output", + "subtest_summary": {"total": 6, "passed": 6, "failed": 0, "skipped": 0}, + } + ) + + assert detail == "6 subtests passed" + + +def test_validation_result_detail_preserves_failures() -> None: + """Failed validations keep their actionable message even when they report probes.""" + detail = test_cli._validation_result_detail( + { + "passed": False, + "skipped": False, + "state": "failed", + "message": "RdmaCheck: worker-a -> worker-b timed out", + "subtest_summary": {"total": 6, "passed": 5, "failed": 1, "skipped": 0}, + } + ) + + assert detail == "RdmaCheck: worker-a -> worker-b timed out" + + +def test_validation_result_detail_counts_successful_runs_with_skips() -> None: + """An allowed skipped probe remains visible in the concise success summary.""" + detail = test_cli._validation_result_detail( + { + "passed": True, + "skipped": False, + "state": "passed", + "message": "long output", + "subtest_summary": {"total": 6, "passed": 5, "failed": 0, "skipped": 1}, + } + ) + + assert detail == "6 subtests: 5 passed, 0 failed, 1 skipped" + + +def test_validation_result_detail_derives_total_for_legacy_summaries() -> None: + """Older result payloads remain concise when they omit the explicit total.""" + detail = test_cli._validation_result_detail( + { + "passed": True, + "skipped": False, + "state": "passed", + "message": "long output", + "subtest_summary": {"passed": 5, "failed": 0, "skipped": 1}, + } + ) + + assert detail == "6 subtests: 5 passed, 0 failed, 1 skipped" + + def _write_provider_config(root: Path, provider: str, name: str, suite: str, platform: str) -> Path: """Write a minimal provider config importing one suite.""" config_path = root / "providers" / provider / "config" / name @@ -129,6 +189,49 @@ def test_test_run_forwards_label_filters(monkeypatch: pytest.MonkeyPatch, tmp_pa assert _FakeOrchestrator.captured["include_labels"] == ["gpu", "slow"] +def test_orchestration_summary_hides_filtered_validations_by_default( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The default summary omits phases containing only selection-filtered checks.""" + config = _write_config(tmp_path) + + class FilteredOrchestrator(_FakeOrchestrator): + """Return one resolution-only phase for a filtered validation.""" + + def run(self, **kwargs: Any) -> OrchestratorResult: + """Return the synthetic filtered result.""" + return OrchestratorResult( + success=True, + phases=[ + PhaseResult( + phase=Phase.TEST, + success=True, + message="test phase validations resolved without execution", + details={ + "validations": [ + { + "name": "InfiniBandCheck", + "skipped": True, + "state": "skipped", + "skip_reason": "test_excluded", + "message": "does not match selected label ethernet", + } + ] + }, + ) + ], + ) + + monkeypatch.setattr(test_cli, "Orchestrator", FilteredOrchestrator) + + result = runner.invoke(test_cli.app, ["run", "-f", str(config), "--label", "ethernet", "--no-upload"]) + + assert result.exit_code == 0, result.output + assert "InfiniBandCheck" not in result.output + assert "TEST : test phase validations resolved" not in result.output + + def test_test_run_reports_and_saves_the_complete_catalog_identity( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/isvtest/src/isvtest/catalog.py b/isvtest/src/isvtest/catalog.py index 0ea7994ff..1308cddcb 100644 --- a/isvtest/src/isvtest/catalog.py +++ b/isvtest/src/isvtest/catalog.py @@ -20,7 +20,7 @@ The catalog is version-keyed by the installed isvtest package version. Suite placement and capability requirements come only from canonical -``isvctl/configs/suites/*.yaml`` wiring. +``isvctl/configs/suites/**/*.yaml`` wiring. """ import hashlib @@ -210,7 +210,7 @@ def _iter_suite_docs() -> Iterator[tuple[Path, dict[str, Any]]]: if not configs_dir: logger.warning("Could not locate isvctl/configs/ directory") return - for config_path in sorted((configs_dir / "suites").glob("*.yaml")): + for config_path in sorted((configs_dir / "suites").rglob("*.yaml")): yield config_path, yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} diff --git a/isvtest/src/isvtest/core/composite.py b/isvtest/src/isvtest/core/composite.py index 833fd1774..0b08e4d26 100644 --- a/isvtest/src/isvtest/core/composite.py +++ b/isvtest/src/isvtest/core/composite.py @@ -42,6 +42,8 @@ from typing import Any, ClassVar +import pytest + from isvtest.core.validation import BaseValidation, get_validation_class COMPOSE_KEY = "compose" @@ -86,7 +88,7 @@ class CompositeCheck(BaseValidation): _exclude_from_discovery: ClassVar[bool] = True def run(self) -> None: - """Run every configured member and fail the composite on invalid or failed members.""" + """Run every member, retaining skips and failing on invalid or failed members.""" raw = self.config.get(COMPOSE_KEY) members = composed_members(raw) if not members: @@ -115,9 +117,23 @@ def run(self) -> None: member = member_class(runner=self.runner, config={**shared, **member_params}) member.name = member_name - result = member.execute() + try: + result = member.execute() + except pytest.skip.Exception as exc: + reason = str(exc) + self.report_subtest(member_name, False, reason, skipped=True) + outputs.append(f"{member_name}: skipped - {reason}") + continue message = result["output"] if result["passed"] else result["error"] self.report_subtest(member_name, result["passed"], message, duration=result["duration"]) + for nested in result.get("subtests", []): + self.report_subtest( + f"{member_name}/{nested['name']}", + bool(nested.get("passed")), + str(nested.get("message", "")), + skipped=bool(nested.get("skipped")), + duration=nested.get("duration"), + ) if result["passed"]: outputs.append(f"{member_name}: {message}" if message else member_name) else: diff --git a/isvtest/src/isvtest/core/resolution.py b/isvtest/src/isvtest/core/resolution.py index c3401294f..492d0619e 100644 --- a/isvtest/src/isvtest/core/resolution.py +++ b/isvtest/src/isvtest/core/resolution.py @@ -20,7 +20,7 @@ import logging from collections.abc import Iterable, Mapping from collections.abc import Set as AbstractSet -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import StrEnum from functools import cache from typing import Any @@ -92,6 +92,7 @@ class ErrorReason(StrEnum): INVALID_CONFIG = "invalid_config" RUNTIME_EXCEPTION = "runtime_exception" + STEP_FAILED = "step_failed" TEMPLATE_RENDER_FAILED = "template_render_failed" @@ -108,6 +109,20 @@ class ValidationEntry: requires: tuple[str, ...] = () +@dataclass(frozen=True) +class SubtestSummary: + """Aggregate counts for the subtests reported by one validation.""" + + passed: int = 0 + failed: int = 0 + skipped: int = 0 + + @property + def total(self) -> int: + """Return the total number of reported subtests.""" + return self.passed + self.failed + self.skipped + + @dataclass class ResolvedEntry: """Lifecycle record for a single validation entry.""" @@ -119,6 +134,7 @@ class ResolvedEntry: error_reason: ErrorReason | None = None message: str = "" duration_seconds: float = 0.0 + subtest_summary: SubtestSummary = field(default_factory=SubtestSummary) @property def is_ready(self) -> bool: @@ -239,6 +255,56 @@ def parse_validations(raw_config: Mapping[str, Any]) -> list[ValidationEntry]: return entries +def resolve_entry_selection( + entry: ValidationEntry, + *, + include_labels: AbstractSet[str], + exclude_labels: AbstractSet[str], + exclude_tests: AbstractSet[str], + capability: str | None = None, +) -> ResolvedEntry | None: + """Return a terminal result when selection excludes an entry, otherwise ``None``. + + This is the provider-neutral selection boundary shared by validation + execution and lifecycle steps gated with ``requires_selected_validations``. + It deliberately stops before phase, step-output, and template resolution. + """ + config_error = _validate_entry_shape(entry) + if config_error: + return _error(entry, ErrorReason.INVALID_CONFIG, config_error) + + if entry.name in exclude_tests: + return _skip(entry, SkipReason.EXCLUDED, f"validation '{entry.name}' is excluded by name") + + if capability is not None and not requirements_satisfied(entry.requires, capability): + requirement_list = ", ".join(entry.requires) or "(none)" + return _skip( + entry, + SkipReason.CAPABILITY_REQUIREMENT, + f"requires {requirement_list} (context: {capability})", + ) + + missing_include_labels = sorted(set(include_labels).difference(entry.labels)) + if missing_include_labels: + label_list = ", ".join(sorted(include_labels)) + return _skip( + entry, + SkipReason.EXCLUDED, + f"validation '{entry.name}' does not match all selected labels: {label_list}", + ) + + label_matches = sorted(set(entry.labels).intersection(exclude_labels)) + if label_matches: + label_list = ", ".join(label_matches) + return _skip( + entry, + SkipReason.EXCLUDED, + f"validation '{entry.name}' is excluded by label: {label_list}", + ) + + return None + + def resolve_entries( entries: list[ValidationEntry], *, @@ -275,49 +341,15 @@ def resolve_entries( env = _create_jinja_env() for entry in entries: - config_error = _validate_entry_shape(entry) - if config_error: - resolved.append(_error(entry, ErrorReason.INVALID_CONFIG, config_error)) - continue - - if entry.name in exclude_tests: - resolved.append(_skip(entry, SkipReason.EXCLUDED, f"validation '{entry.name}' is excluded by name")) - continue - - if capability is not None and not requirements_satisfied(entry.requires, capability): - requirement_list = ", ".join(entry.requires) or "(none)" - context_list = capability - resolved.append( - _skip( - entry, - SkipReason.CAPABILITY_REQUIREMENT, - f"requires {requirement_list} (context: {context_list})", - ) - ) - continue - - missing_include_labels = sorted(set(include_labels).difference(entry.labels)) - if missing_include_labels: - label_list = ", ".join(sorted(include_labels)) - resolved.append( - _skip( - entry, - SkipReason.EXCLUDED, - f"validation '{entry.name}' does not match all selected labels: {label_list}", - ) - ) - continue - - label_matches = sorted(set(entry.labels).intersection(exclude_labels)) - if label_matches: - label_list = ", ".join(label_matches) - resolved.append( - _skip( - entry, - SkipReason.EXCLUDED, - f"validation '{entry.name}' is excluded by label: {label_list}", - ) - ) + selection_result = resolve_entry_selection( + entry, + include_labels=include_labels, + exclude_labels=exclude_labels, + exclude_tests=exclude_tests, + capability=capability, + ) + if selection_result is not None: + resolved.append(selection_result) continue if entry.step and entry.step in skipped_steps: diff --git a/isvtest/src/isvtest/main.py b/isvtest/src/isvtest/main.py index 44806cadd..f2b723946 100644 --- a/isvtest/src/isvtest/main.py +++ b/isvtest/src/isvtest/main.py @@ -36,7 +36,7 @@ from isvtest.config.loader import ConfigLoader from isvtest.core import runners as reframe_runner from isvtest.core.logger import setup_logger -from isvtest.core.resolution import ErrorReason, ResolvedEntry, SkipReason, State +from isvtest.core.resolution import ErrorReason, ResolvedEntry, SkipReason, State, SubtestSummary from isvtest.tests.test_validations import ( clear_validation_results, get_validation_results, @@ -223,6 +223,12 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R """Convert a captured pytest validation result to a terminal resolved entry.""" message = str(result.get("message", "")) duration = float(result.get("duration", 0.0) or 0.0) + raw_subtests = result.get("subtest_summary", {}) + subtest_summary = SubtestSummary( + passed=int(raw_subtests.get("passed", 0)) if isinstance(raw_subtests, dict) else 0, + failed=int(raw_subtests.get("failed", 0)) if isinstance(raw_subtests, dict) else 0, + skipped=int(raw_subtests.get("skipped", 0)) if isinstance(raw_subtests, dict) else 0, + ) if result.get("skipped"): return ResolvedEntry( entry=entry.entry, @@ -231,6 +237,7 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R skip_reason=SkipReason.RUNTIME_SKIP, message=message, duration_seconds=duration, + subtest_summary=subtest_summary, ) if result.get("passed", False): return ResolvedEntry( @@ -239,6 +246,7 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R state=State.PASSED, message=message, duration_seconds=duration, + subtest_summary=subtest_summary, ) if result.get("error_reason") == ErrorReason.RUNTIME_EXCEPTION.value: return ResolvedEntry( @@ -248,6 +256,7 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R error_reason=ErrorReason.RUNTIME_EXCEPTION, message=message, duration_seconds=duration, + subtest_summary=subtest_summary, ) return ResolvedEntry( entry=entry.entry, @@ -255,6 +264,7 @@ def _result_to_resolved_entry(entry: ResolvedEntry, result: dict[str, Any]) -> R state=State.FAILED, message=message, duration_seconds=duration, + subtest_summary=subtest_summary, ) diff --git a/isvtest/src/isvtest/testing/subtests.py b/isvtest/src/isvtest/testing/subtests.py index 09f889fbb..e7cd5fb83 100644 --- a/isvtest/src/isvtest/testing/subtests.py +++ b/isvtest/src/isvtest/testing/subtests.py @@ -509,11 +509,6 @@ def _inject_subtests_into_junit(junit_path: Path, reports: list[SubTestReport]) name = tc.get("name", "") testcase_indices[name] = idx - # Track counts for updating testsuite attributes - added_tests = 0 - added_failures = 0 - added_skipped = 0 - # Insert subtests after their parent, in reverse order of parent index # to avoid index shifting issues insertions: list[tuple[int, list[ET.Element]]] = [] @@ -540,16 +535,12 @@ def _inject_subtests_into_junit(junit_path: Path, reports: list[SubTestReport]) testcase.set("classname", "") # Match pytest's default testcase.set("time", f"{report.duration:.3f}") - added_tests += 1 - if report.failed: - added_failures += 1 failure = ET.SubElement(testcase, "failure") failure.set("message", f"Subtest {subtest_desc} failed") if report.longrepr: failure.text = str(report.longrepr)[:2000] # Limit length elif report.skipped: - added_skipped += 1 skipped_elem = ET.SubElement(testcase, "skipped") # longrepr from both our skipped-flag path and pytest.skip() # is a (file, line, msg) tuple; surface the msg so the @@ -578,14 +569,15 @@ def _inject_subtests_into_junit(junit_path: Path, reports: list[SubTestReport]) for i, elem in enumerate(subtest_elements): testsuite.insert(insert_pos + i, elem) - # Update testsuite counts - current_tests = int(testsuite.get("tests", "0")) - current_failures = int(testsuite.get("failures", "0")) - current_skipped = int(testsuite.get("skipped", "0")) - - testsuite.set("tests", str(current_tests + added_tests)) - testsuite.set("failures", str(current_failures + added_failures)) - testsuite.set("skipped", str(current_skipped + added_skipped)) + # pytest counts subtest reports in the testsuite attributes even though + # its JUnit plugin does not serialize them as testcase elements. Adding + # our testcase nodes and incrementing those attributes would therefore + # double-count every subtest. Reconcile counters with the serialized XML. + serialized_cases = testsuite.findall("testcase") + testsuite.set("tests", str(len(serialized_cases))) + testsuite.set("failures", str(sum(case.find("failure") is not None for case in serialized_cases))) + testsuite.set("errors", str(sum(case.find("error") is not None for case in serialized_cases))) + testsuite.set("skipped", str(sum(case.find("skipped") is not None for case in serialized_cases))) # Write back tree.write(junit_path, encoding="utf-8", xml_declaration=True) diff --git a/isvtest/src/isvtest/tests/test_validations.py b/isvtest/src/isvtest/tests/test_validations.py index 425171a93..d4497c10b 100644 --- a/isvtest/src/isvtest/tests/test_validations.py +++ b/isvtest/src/isvtest/tests/test_validations.py @@ -273,6 +273,18 @@ def test_validation( "category": category, "duration": result.get("duration", 0.0), "error_reason": result.get("error_reason"), + "subtest_summary": { + "total": len(result.get("subtests", [])), + "passed": sum( + 1 for subtest in result.get("subtests", []) if subtest.get("passed") and not subtest.get("skipped") + ), + "failed": sum( + 1 + for subtest in result.get("subtests", []) + if not subtest.get("passed") and not subtest.get("skipped") + ), + "skipped": sum(1 for subtest in result.get("subtests", []) if subtest.get("skipped")), + }, } ) diff --git a/isvtest/src/isvtest/validations/k8s_launch_kit/__init__.py b/isvtest/src/isvtest/validations/k8s_launch_kit/__init__.py new file mode 100644 index 000000000..953b56014 --- /dev/null +++ b/isvtest/src/isvtest/validations/k8s_launch_kit/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Kubernetes Launch Kit validation checks.""" diff --git a/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py b/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py new file mode 100644 index 000000000..c4d3e6678 --- /dev/null +++ b/isvtest/src/isvtest/validations/k8s_launch_kit/checks.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Connectivity assertions over unmodified Kubernetes Launch Kit output.""" + +from __future__ import annotations + +from typing import Any, ClassVar + +from isvtest.core.validation import BaseValidation + +_FAMILY_BY_KIND = { + 0: "icmp", + 1: "icmp", + 2: "rping", + 3: "rping", + 4: "ib_write_bw", + 5: "ib_write_bw", + 6: "gpudirect_dmabuf", + 7: "gpudirect_dmabuf", +} + + +def _mapping(value: Any) -> dict[str, Any]: + """Return ``value`` as a mapping or an empty mapping.""" + return value if isinstance(value, dict) else {} + + +def _sequence(value: Any) -> list[Any]: + """Return ``value`` as a sequence or an empty sequence.""" + return value if isinstance(value, list) else [] + + +class LaunchKitConnectivityCheck(BaseValidation): + """Report every connectivity result emitted by ``l8k validate``.""" + + description: ClassVar[str] = "Check the Kubernetes Launch Kit connectivity matrix" + + def _connectivity(self) -> dict[str, Any] | None: + """Find the connectivity document in the bound provider output.""" + output = self.config.get("step_output") + if not isinstance(output, dict): + self.set_failed("Missing Launch Kit step_output") + return None + if output.get("operation") != "validate": + self.set_failed(f"Expected Launch Kit operation 'validate', got {output.get('operation')!r}") + return None + for document in _sequence(output.get("documents")): + connectivity = _mapping(document).get("connectivity") + if isinstance(connectivity, dict): + return connectivity + provider_error = output.get("error") + suffix = f": {provider_error}" if isinstance(provider_error, str) and provider_error else "" + self.set_failed(f"Launch Kit validate output has no connectivity matrix{suffix}") + return None + + @staticmethod + def _probe(row: dict[str, Any], index: int) -> dict[str, Any]: + """Convert one Launch Kit matrix row into one informative subtest.""" + test = _mapping(row.get("Test")) + explicit_family = row.get("Family") + family = explicit_family if isinstance(explicit_family, str) and explicit_family else None + if family is None: + family = _FAMILY_BY_KIND.get(test.get("Kind"), f"kind-{test.get('Kind', 'unknown')}") + + source = str(test.get("SrcNode") or test.get("SrcPod") or "unknown-source") + destination = str(test.get("DstNode") or test.get("DstPod") or "unknown-destination") + source_rail = str(test.get("SrcRail") or test.get("Rail") or "unknown-rail") + destination_rail = str(test.get("DstRail") or test.get("Rail") or "unknown-rail") + expectation = str(row.get("Expectation") or test.get("Expectation") or "required") + details = [f"expectation={expectation}", f"observedOK={row.get('ObservedOK')}"] + + bandwidth = row.get("BandwidthGbps") + minimum = row.get("MinBandwidthGbps") + if bandwidth is not None or minimum is not None: + details.extend([f"bandwidthGbps={bandwidth}", f"minimumGbps={minimum}"]) + + source_gpu = test.get("SrcGPUIndex") + destination_gpu = test.get("DstGPUIndex") + if source_gpu is not None or destination_gpu is not None: + details.append(f"gpuIndices={source_gpu}->{destination_gpu}") + source_gpu_pci = test.get("SrcGPUPCIAddress") + destination_gpu_pci = test.get("DstGPUPCIAddress") + if source_gpu_pci: + details.append(f"sourceGpuPci={source_gpu_pci}") + if destination_gpu_pci: + details.append(f"destinationGpuPci={destination_gpu_pci}") + + stderr = str(row.get("Stderr") or "").strip() + error = str(row.get("Error") or "").strip() + if stderr: + details.append(f"stderr={stderr}") + if error and error != stderr: + details.append(f"error={error}") + + return { + "name": f"{family}/{source}->{destination}/{source_rail}->{destination_rail}", + "passed": row.get("OK") is True, + "message": ", ".join(details) or f"matrix row {index}", + } + + def run(self) -> None: + """Expose the complete Launch Kit connectivity matrix as subtests.""" + connectivity = self._connectivity() + if connectivity is None: + return + probes = [ + self._probe(row, index) + for index, row in enumerate(_sequence(connectivity.get("PingResults")), start=1) + if isinstance(row, dict) + ] + if not probes: + self.set_failed("Launch Kit connectivity matrix produced no results") + return + + failures: list[str] = [] + for probe in probes: + self.report_subtest( + probe["name"], + passed=probe["passed"], + message=probe["message"], + ) + if not probe["passed"]: + failures.append(f"{probe['name']}: {probe['message']}") + if failures: + self.set_failed("Launch Kit connectivity failed: " + "; ".join(failures)) + return + self.set_passed(f"Launch Kit connectivity passed ({len(probes)} results)") diff --git a/isvtest/tests/k8s_launch_kit/test_checks.py b/isvtest/tests/k8s_launch_kit/test_checks.py new file mode 100644 index 000000000..51324e490 --- /dev/null +++ b/isvtest/tests/k8s_launch_kit/test_checks.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for Kubernetes Launch Kit connectivity interpretation.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from isvtest.validations.k8s_launch_kit.checks import LaunchKitConnectivityCheck + +pytestmark = pytest.mark.unit + + +def _matrix_row( + family: str | None, + kind: int, + *, + passed: bool = True, + destination_rail: str = "rail-0", +) -> dict[str, Any]: + """Build one Launch Kit connectivity result.""" + bandwidth = family in {"ib_write_bw", "gpudirect_dmabuf"} + return { + "Test": { + "Kind": kind, + "SrcNode": "worker-a", + "DstNode": "worker-b", + "SrcRail": "rail-0", + "DstRail": destination_rail, + "Expectation": "required", + **( + { + "SrcGPUIndex": 2, + "DstGPUIndex": 5, + "SrcGPUPCIAddress": "0000:41:00.0", + "DstGPUPCIAddress": "0000:71:00.0", + } + if family == "gpudirect_dmabuf" + else {} + ), + }, + **({"Family": family} if family is not None else {}), + "OK": passed, + "ObservedOK": passed, + "Expectation": "required", + **( + { + "BandwidthGbps": 187.6 if passed else 42.5, + "MinBandwidthGbps": 100.0, + } + if bandwidth + else {} + ), + **( + { + "Stderr": f"{family}: connection refused on rail-0", + "Error": f"{family} validation failed", + } + if not passed + else {} + ), + } + + +def _validate(rows: list[dict[str, Any]]) -> dict[str, Any]: + """Wrap matrix rows in the provider contract consumed by the check.""" + failed = sum(row.get("OK") is not True for row in rows) + return { + "success": failed == 0, + "platform": "kubernetes", + "operation": "validate", + "exit_code": 0 if failed == 0 else 4, + "documents": [ + {"versionCheck": {}, "manifests": [], "summary": {}}, + { + "connectivity": { + "PingResults": rows, + "Summary": {"TotalTests": len(rows), "Failed": failed}, + } + }, + ], + **({"error": "one or more connectivity rows failed"} if failed else {}), + } + + +def _execute(output: dict[str, Any]) -> dict[str, Any]: + """Execute the catalog check against one provider envelope.""" + return LaunchKitConnectivityCheck(config={"step_output": output}).execute() + + +def test_reports_every_emitted_connectivity_family() -> None: + """The single check exposes every Launch Kit matrix row in stream order.""" + rows = [ + _matrix_row("icmp", 0), + _matrix_row("rping", 2), + _matrix_row("ib_write_bw", 4), + _matrix_row("gpudirect_dmabuf", 6), + ] + + result = _execute(_validate(rows)) + + assert result["passed"] is True + assert [subtest["name"] for subtest in result["subtests"]] == [ + f"{family}/worker-a->worker-b/rail-0->rail-0" for family in ("icmp", "rping", "ib_write_bw", "gpudirect_dmabuf") + ] + + +def test_failure_preserves_endpoints_rails_bandwidth_and_stderr() -> None: + """A failed row retains the diagnostics emitted by Launch Kit.""" + result = _execute(_validate([_matrix_row("ib_write_bw", 4, passed=False)])) + + assert result["passed"] is False + assert result["subtests"][0]["passed"] is False + assert "ib_write_bw/worker-a->worker-b/rail-0->rail-0" in result["error"] + assert "bandwidthGbps=42.5" in result["error"] + assert "minimumGbps=100.0" in result["error"] + assert "connection refused on rail-0" in result["error"] + + +def test_gpudirect_details_are_reported_without_a_separate_expected_check() -> None: + """GPUDirect rows are ordinary connectivity rows when Launch Kit emits them.""" + result = _execute(_validate([_matrix_row("gpudirect_dmabuf", 6)])) + + assert result["passed"] is True + message = result["subtests"][0]["message"] + assert "gpuIndices=2->5" in message + assert "sourceGpuPci=0000:41:00.0" in message + assert "destinationGpuPci=0000:71:00.0" in message + + +def test_disabled_gpudirect_requires_no_skip_or_placeholder() -> None: + """Families disabled by user config are absent instead of becoming skipped tests.""" + result = _execute(_validate([_matrix_row("icmp", 0), _matrix_row("rping", 2)])) + + assert result["passed"] is True + assert all("gpudirect" not in subtest["name"] for subtest in result["subtests"]) + assert all(subtest["skipped"] is False for subtest in result["subtests"]) + + +def test_explicit_future_family_is_not_filtered_out() -> None: + """The wrapper forwards new Launch Kit families without a catalog update.""" + result = _execute(_validate([_matrix_row("future_connectivity", 999)])) + + assert result["passed"] is True + assert result["subtests"][0]["name"].startswith("future_connectivity/") + + +def test_legacy_numeric_kind_resolves_a_family() -> None: + """Older Launch Kit output without Family remains readable.""" + result = _execute(_validate([_matrix_row(None, 3)])) + + assert result["passed"] is True + assert result["subtests"][0]["name"].startswith("rping/") + + +@pytest.mark.parametrize( + "output", + [ + {"operation": "validate", "documents": [], "error": "Kubernetes client failed"}, + {"operation": "discover", "documents": [{"connectivity": {"PingResults": []}}]}, + ], +) +def test_rejects_missing_or_wrong_validate_output(output: dict[str, Any]) -> None: + """Transport failures remain actionable instead of passing vacuously.""" + result = _execute(output) + + assert result["passed"] is False + + +def test_empty_connectivity_matrix_fails() -> None: + """A validate response with no connectivity rows is not evidence of success.""" + result = _execute(_validate([])) + + assert result["passed"] is False + assert result["error"] == "Launch Kit connectivity matrix produced no results" diff --git a/isvtest/tests/test_catalog.py b/isvtest/tests/test_catalog.py index 988340bb6..5d530761c 100644 --- a/isvtest/tests/test_catalog.py +++ b/isvtest/tests/test_catalog.py @@ -55,6 +55,8 @@ def test_derives_suite_vocabulary_from_plain_suites(self) -> None: """Plain suite YAML files are listed separately from platform suites.""" suites = build_suite_vocabulary() assert "iam" in suites + assert "network_operator" in suites + assert "network_operator_use_cases" not in suites assert "storage" in suites assert "kubernetes" not in suites assert "vm" not in suites @@ -112,6 +114,10 @@ def test_entries_have_suite_contract(self) -> None: assert isinstance(entry["requires"], list) if entry["capability"]: assert entry["requires"] == [] + assert "LaunchKitConnectivityCheck" in names + launch_kit = next(entry for entry in catalog if entry["name"] == "LaunchKitConnectivityCheck") + assert launch_kit["suite"] == "network_operator" + assert launch_kit["test_ids"] == ["K8S42-01"] def test_extract_checks_supports_direct_dict_category_form(self, tmp_path) -> None: """Direct dict category wiring is included in catalog config scans.""" diff --git a/isvtest/tests/test_composite.py b/isvtest/tests/test_composite.py index 0d8c16f75..283fd1684 100644 --- a/isvtest/tests/test_composite.py +++ b/isvtest/tests/test_composite.py @@ -19,9 +19,11 @@ import pytest +import isvtest.core.composite as composite_module from isvtest.core.composite import CompositeCheck, composed_members, is_composite from isvtest.core.discovery import discover_all_tests from isvtest.core.resolution import parse_validations +from isvtest.core.validation import BaseValidation from isvtest.validations.generic import ( CrudOperationsCheck, FieldExistsCheck, @@ -144,6 +146,52 @@ def test_reports_each_member_as_a_subtest(self) -> None: ("FieldExistsCheck", True), ] + def test_forwards_member_subtests_with_member_qualified_names(self, monkeypatch: pytest.MonkeyPatch) -> None: + """A composite preserves member probes for terminal and JUnit diagnostics.""" + + class NestedCheck(BaseValidation): + def run(self) -> None: + self.report_subtest("probe-a", True, "probe passed") + self.report_subtest("probe-b", False, "not applicable", skipped=True) + self.set_passed("nested check passed") + + monkeypatch.setattr(composite_module, "get_validation_class", lambda name: NestedCheck) + composite = CompositeCheck(config=_config(["NestedCheck"])) + + result = composite.execute() + + assert result["passed"] is True + assert [(sub["name"], sub["skipped"]) for sub in result["subtests"]] == [ + ("NestedCheck", False), + ("NestedCheck/probe-a", False), + ("NestedCheck/probe-b", True), + ] + + def test_skipped_member_does_not_skip_or_fail_the_composite(self, monkeypatch: pytest.MonkeyPatch) -> None: + """An inapplicable member is skipped while later members still run.""" + + class SkippedCheck(BaseValidation): + def run(self) -> None: + pytest.skip("only one rail discovered") + + class PassingCheck(BaseValidation): + def run(self) -> None: + self.set_passed("later member passed") + + classes = {"SkippedCheck": SkippedCheck, "PassingCheck": PassingCheck} + monkeypatch.setattr(composite_module, "get_validation_class", classes.get) + composite = CompositeCheck(config=_config(["SkippedCheck", "PassingCheck"])) + + result = composite.execute() + + assert result["passed"] is True + assert [(sub["name"], sub["passed"], sub["skipped"]) for sub in result["subtests"]] == [ + ("SkippedCheck", False, True), + ("PassingCheck", True, False), + ] + assert "SkippedCheck: skipped - only one rail discovered" in result["output"] + assert "PassingCheck: later member passed" in result["output"] + def test_fails_naming_the_failing_member(self) -> None: """A failing member fails the composite and is named in the error.""" result = CompositeCheck( diff --git a/isvtest/tests/test_main.py b/isvtest/tests/test_main.py index efbe688c7..4b833835d 100644 --- a/isvtest/tests/test_main.py +++ b/isvtest/tests/test_main.py @@ -22,6 +22,7 @@ from isvtest.main import ( _entries_with_pytest_names, _resolved_entries_to_pytest_validations, + _result_to_resolved_entry, run_validations_via_pytest, ) @@ -102,6 +103,27 @@ def test_run_validations_via_pytest_updates_ready_entries() -> None: assert results[1].message == "NIM was not deployed" +def test_result_to_resolved_entry_preserves_subtest_summary() -> None: + """Subtest counts remain structured until the presentation layer renders them.""" + entry = _ready("StepSuccessCheck", "setup_checks", {"step_output": {"success": True}}) + + result = _result_to_resolved_entry( + entry, + { + "passed": True, + "skipped": False, + "message": "detailed validation output", + "subtest_summary": {"passed": 4, "failed": 0, "skipped": 2}, + }, + ) + + assert result.message == "detailed validation output" + assert result.subtest_summary.total == 6 + assert result.subtest_summary.passed == 4 + assert result.subtest_summary.failed == 0 + assert result.subtest_summary.skipped == 2 + + def test_run_validations_via_pytest_skips_structured_step_skips() -> None: """A step-level structured skip should skip all dependent validations.""" step_output = {"success": True, "skipped": True, "skip_reason": "No VPCs found at site"} diff --git a/isvtest/tests/test_subtests_junit.py b/isvtest/tests/test_subtests_junit.py index 3d83d20e6..767e333f6 100644 --- a/isvtest/tests/test_subtests_junit.py +++ b/isvtest/tests/test_subtests_junit.py @@ -23,9 +23,12 @@ from isvtest.testing.subtests import SubTestReport, _inject_subtests_into_junit -def _parent_junit(tmp_path: Path, parent_name: str) -> Path: +def _parent_junit(tmp_path: Path, parent_name: str, *, reported_tests: int = 1) -> Path: """Write a minimal pytest-style JUnit with a single parent testcase.""" - suite = ET.Element("testsuite", attrib={"name": "phase", "tests": "1", "skipped": "0", "failures": "0"}) + suite = ET.Element( + "testsuite", + attrib={"name": "phase", "tests": str(reported_tests), "skipped": "0", "failures": "0"}, + ) ET.SubElement(suite, "testcase", attrib={"name": parent_name, "classname": "", "time": "0.000"}) path = tmp_path / "junit.xml" ET.ElementTree(suite).write(path, encoding="utf-8", xml_declaration=True) @@ -92,3 +95,18 @@ def test_skipped_subtest_falls_back_when_longrepr_is_missing(tmp_path: Path) -> skipped = subtest_case.find("skipped") assert skipped is not None assert skipped.get("message") == "Subtest noisy-subtest skipped" + + +def test_injected_subtests_reconcile_precounted_junit_totals(tmp_path: Path) -> None: + """pytest's pre-counted subtest report must not be counted again after injection.""" + junit = _parent_junit(tmp_path, "ParentCheck", reported_tests=2) + report = _stub_report("::ParentCheck", "probe", failed=True, longrepr="probe failed") + + _inject_subtests_into_junit(junit, cast(list[SubTestReport], [report])) + + suite = ET.parse(junit).getroot() + assert len(suite.findall("testcase")) == 2 + assert suite.get("tests") == "2" + assert suite.get("failures") == "1" + assert suite.get("errors") == "0" + assert suite.get("skipped") == "0" diff --git a/isvtest/tests/test_validation.py b/isvtest/tests/test_validation.py index cf549e059..6662556f6 100644 --- a/isvtest/tests/test_validation.py +++ b/isvtest/tests/test_validation.py @@ -77,6 +77,16 @@ def run(self) -> None: self.set_failed("Test failed", "Error output") +class SubtestValidation(BaseValidation): + """Validation that reports passing and skipped probes.""" + + def run(self) -> None: + """Report two probe outcomes and pass the parent validation.""" + self.report_subtest("ready", True, "ready") + self.report_subtest("optional", False, "not applicable", skipped=True) + self.set_passed("All required probes passed") + + class ExceptionValidation(BaseValidation): """Validation that raises an exception.""" @@ -1999,6 +2009,16 @@ def test_passed_validation_captured(self) -> None: assert r["skipped"] is False assert r["passed"] is True + def test_subtest_summary_captured(self) -> None: + """Probe counts cross the pytest bridge without replacing the parent message.""" + subtests = MagicMock() + + run_validation_entry_point(SubtestValidation, {"_category": "test_cat"}, "SubtestValidation", subtests) + + result = _validation_results[0] + assert result["message"] == "All required probes passed" + assert result["subtest_summary"] == {"total": 2, "passed": 1, "failed": 0, "skipped": 1} + def test_failed_validation_captured(self) -> None: """Failed validations must appear with passed=False.""" config = {"_category": "test_cat"} diff --git a/scripts/requirements_source_to_md.py b/scripts/requirements_source_to_md.py index 3c5ca3923..3653cb5f6 100644 --- a/scripts/requirements_source_to_md.py +++ b/scripts/requirements_source_to_md.py @@ -17,7 +17,8 @@ """Render a source-requirements YAML to a publishable Markdown listing. YAML is the source of record (queryable, key-clean); Markdown is the published, -Google-Docs-friendly view. Handles both the `offtake` and `reference` sources. +Google-Docs-friendly view. Handles the registered offtake, reference, storage, +and project-PRD sources. Usage: python3 scripts/requirements_source_to_md.py docs/requirements/offtake-requirements.yaml @@ -38,6 +39,7 @@ REQ_DIR / "offtake-requirements.yaml", REQ_DIR / "software-reference-requirements.yaml", REQ_DIR / "storage-acceptance-requirements.yaml", + REQ_DIR / "network-operator-readiness-requirements.yaml", ] GENERATED_BANNER = "" @@ -149,7 +151,40 @@ def render_storage(doc: dict[str, Any], src_name: str) -> str: return "\n".join(out) + "\n" -RENDERERS = {"offtake": render_offtake, "reference": render_reference, "storage": render_storage} +def render_project_prd(doc: dict[str, Any], src_name: str) -> str: + """Render a project PRD listing, grouped by section.""" + out = [ + GENERATED_BANNER.format(src=src_name), + "", + f"# {doc.get('title', 'Project Requirements')}", + "", + f"> Structured source of record: `{src_name}` (version {doc.get('version', 'n/a')}).", + f"> Owner: {doc.get('owner', 'not specified')}.", + "> Edit the YAML, not this file.", + "", + ] + section = None + for requirement in doc.get("requirements", []): + if requirement.get("section") != section: + section = requirement.get("section") + heading(out, f"## {section}") + out += [ + "| Req ID | Requirement Area | Description | Status |", + "| :----- | :--------------- | :---------- | :----- |", + ] + out.append( + f"| {cell(requirement.get('req_id'))} | {cell(requirement.get('area'))} " + f"| {cell(requirement.get('description'))} | {cell(requirement.get('status', 'active'))} |" + ) + return "\n".join(out) + "\n" + + +RENDERERS = { + "offtake": render_offtake, + "project-prd": render_project_prd, + "reference": render_reference, + "storage": render_storage, +} def render(path: Path) -> None: @@ -158,9 +193,14 @@ def render(path: Path) -> None: if not isinstance(doc, dict): raise ValueError(f"{path} must contain a single mapping document") source = doc.get("source") - renderer = RENDERERS.get(source) + if not isinstance(source, str): + raise ValueError(f"{path} must declare a string source") + document_format = doc.get("format", source) + if not isinstance(document_format, str): + raise ValueError(f"{path} format must be a string") + renderer = RENDERERS.get(document_format) if renderer is None: - raise ValueError(f"{path} has unsupported source {source!r} (expected one of {sorted(RENDERERS)})") + raise ValueError(f"{path} has unsupported format {document_format!r} (expected one of {sorted(RENDERERS)})") out_path = path.with_suffix(".md") out_path.write_text(renderer(doc, path.name), encoding="utf-8") print(f"Wrote {out_path}") diff --git a/scripts/test_plan_coverage.py b/scripts/test_plan_coverage.py index ce5fc14d1..8fab388cf 100644 --- a/scripts/test_plan_coverage.py +++ b/scripts/test_plan_coverage.py @@ -131,7 +131,7 @@ def config_test_id_map(suites_dir: Path = SUITES_DIR) -> dict[str, list[str]]: both bare_metal and vm), so values still aggregate to a set across configs. """ out: dict[str, set[str]] = defaultdict(set) - for path in sorted(suites_dir.glob("*.yaml")): + for path in sorted(suites_dir.rglob("*.yaml")): for name, params in iter_config_checks(path): tid = params.get("test_id") if isinstance(tid, str) and tid: @@ -148,7 +148,7 @@ def config_label_map(suites_dir: Path = SUITES_DIR) -> dict[str, list[str]]: union of its labels. """ out: dict[str, set[str]] = defaultdict(set) - for path in sorted(suites_dir.glob("*.yaml")): + for path in sorted(suites_dir.rglob("*.yaml")): for name, params in iter_config_checks(path): out[name].update(_normalize_labels(params.get("labels"))) return {name: sorted(labels) for name, labels in out.items()} @@ -166,7 +166,7 @@ def _normalize_labels(value: Any) -> list[str]: def config_test_label_instances(suites_dir: Path = SUITES_DIR) -> list[tuple[str, str, str, list[str]]]: """Return ``(source, check_name, test_id, labels)`` for each mapped suite check.""" instances: list[tuple[str, str, str, list[str]]] = [] - for path in sorted(suites_dir.glob("*.yaml")): + for path in sorted(suites_dir.rglob("*.yaml")): try: source = path.relative_to(REPO_ROOT).as_posix() except ValueError: diff --git a/scripts/tests/test_requirements_source_to_md.py b/scripts/tests/test_requirements_source_to_md.py new file mode 100644 index 000000000..d5d8f7093 --- /dev/null +++ b/scripts/tests/test_requirements_source_to_md.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the structured-requirements Markdown renderer.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest +import yaml + +_SCRIPT = Path(__file__).resolve().parent.parent / "requirements_source_to_md.py" +_spec = importlib.util.spec_from_file_location("requirements_source_to_md", _SCRIPT) +assert _spec and _spec.loader +requirements_source_to_md = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(requirements_source_to_md) + + +def test_project_prd_format_renders_a_uniquely_named_source() -> None: + """A project PRD keeps a unique trace source while sharing a generic renderer.""" + source_path = requirements_source_to_md.REQ_DIR / "network-operator-readiness-requirements.yaml" + document = yaml.safe_load(source_path.read_text(encoding="utf-8")) + + rendered = requirements_source_to_md.RENDERERS[document["format"]](document, source_path.name) + + assert source_path in requirements_source_to_md.DEFAULT_SOURCES + assert document["source"] == "network-operator-prd" + assert document["format"] == "project-prd" + assert "# Enterprise RA Network Operator Self-Validation Integration PRD" in rendered + assert "> Owner: NVIDIA Network Operator team." in rendered + assert "## Network Validation" in rendered + assert "| ENT-REQ-008 | GPUDirect RDMA |" in rendered + + +def test_render_rejects_a_non_string_source(tmp_path: Path) -> None: + """A malformed source fails clearly before renderer dispatch.""" + source_path = tmp_path / "malformed-requirements.yaml" + source_path.write_text("source: null\nrequirements: []\n", encoding="utf-8") + + with pytest.raises(ValueError, match="must declare a string source"): + requirements_source_to_md.render(source_path) + + +def test_render_rejects_an_unknown_document_format(tmp_path: Path) -> None: + """A unique source may select only a documented reusable renderer format.""" + source_path = tmp_path / "malformed-requirements.yaml" + source_path.write_text( + "source: team-prd\nformat: not-a-renderer\nrequirements: []\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="unsupported format 'not-a-renderer'"): + requirements_source_to_md.render(source_path) diff --git a/scripts/tests/test_validate_suite_wiring.py b/scripts/tests/test_validate_suite_wiring.py index ac6ef32b5..ee4536070 100644 --- a/scripts/tests/test_validate_suite_wiring.py +++ b/scripts/tests/test_validate_suite_wiring.py @@ -95,6 +95,19 @@ def test_wiring_errors_reports_yaml_parse_failures(tmp_path: Path) -> None: assert "failed to read/parse" in errors[0] +def test_wiring_errors_scans_nested_suite_directories(tmp_path: Path) -> None: + """Domain-organized suites receive the same metadata guardrails as root suites.""" + nested = tmp_path / "launch-kit" + nested.mkdir() + (nested / "network-operator.yaml").write_text( + "tests:\n validations:\n sample:\n checks:\n MissingMetadata: {}\n" + ) + + errors = validate_suite_wiring.wiring_errors(tmp_path) + + assert any("launch-kit/network-operator.yaml" in error and "MissingMetadata" in error for error in errors) + + def test_find_check_line_numbers_supports_list_form() -> None: """List-form wiring reports each repeated check at its own line.""" lines = """ diff --git a/scripts/validate_suite_wiring.py b/scripts/validate_suite_wiring.py index a07cc007e..2fdfa5a4b 100644 --- a/scripts/validate_suite_wiring.py +++ b/scripts/validate_suite_wiring.py @@ -16,7 +16,7 @@ """Validate suite identity and check resolution in canonical and provider YAML. -Suite configs under ``isvctl/configs/suites/`` are the source of truth for +Suite configs recursively under ``isvctl/configs/suites/`` are the source of truth for validation metadata on this branch. Each wired check must declare: * ``test_id`` - a plan id from ``docs/test-plan.yaml``, or ``"N/A"`` when the @@ -190,7 +190,7 @@ def wiring_errors(suites_dir: Path = SUITES_DIR) -> list[str]: # Read and parse each suite once; both the dead-requirement pre-pass and the # per-check loop below work off these parsed documents. parsed: list[tuple[Path, list[str], dict[str, Any]]] = [] - for path in sorted(suites_dir.glob("*.yaml")): + for path in sorted(suites_dir.rglob("*.yaml")): try: text = path.read_text() parsed.append((path, text.splitlines(), yaml.safe_load(text) or {}))