diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c5247be180f..9b3bbf0af223 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1660,6 +1660,17 @@ repos: entry: ./scripts/check_test_list.py --validate language: script pass_filenames: false + - id: check-binding-stubs + name: Verify collection binding stubs cover compiled extensions + entry: python scripts/check_binding_stubs.py + language: python + pass_filenames: false + files: > + (?x)^( + tests/integration/defs/stubify_bindings.py| + scripts/check_binding_stubs.py| + setup.py + )$ - id: DCO check name: Checks the commit message for a developer certificate of origin signature entry: ./scripts/dco_check.py diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 1ef2aac37c97..f18fe673b877 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -666,6 +666,23 @@ def launchReleaseCheck(pipeline, globalVars) }) } +def launchTestListCheck(pipeline, globalVars) +{ + def key = "Check Test List" + def image = globalVars["LLM_DOCKER_IMAGE"] + trtllm_utils.launchKubernetesPod(pipeline, createKubernetesPodConfig(image, "package"), "trt-llm", { + stage("[${key}] Run") { + echoNodeAndGpuInfo(pipeline, key) + sh "git config --global --add safe.directory \"*\"" + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true) + + def llmPath = sh(script: "realpath ${LLM_ROOT}", returnStdout: true).trim() + sh "NVIDIA_TRITON_SERVER_VERSION=26.05 LLM_ROOT=${llmPath} LLM_BACKEND_ROOT=${llmPath}/triton_backend " + + "python3 ${llmPath}/scripts/check_test_list.py --l0 --qa --waive --validate --parity --check-duplicate-waives" + } + }) +} + def getGitlabMRChangedFile(pipeline, function, filePath="") { def result = null def pageId = 0 @@ -1915,6 +1932,27 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) launchReleaseCheck(this, globalVars) } }, + "Check Test List": { + script { + if (testFilter[INFRA_DRY_RUN]) { + echo "Skipping Check Test List for the infrastructure dry run." + return + } else if (GEN_POST_MERGE_BUILDS_ONLY) { + echo "Skipping Check Test List (GenPostMergeBuilds mode: builds only)" + return + } else if (runMode == "nightly_release") { + echo "Skipping Check Test List for nightly_release." + return + } else if (testFilter[(ONLY_ONE_GROUP_CHANGED)] == "Docs") { + echo "Skipping Check Test List for Docs-only changes." + return + } else if (env.JOB_NAME ==~ /.*BuildDockerImageSanityTest.*/) { + echo "Skipping Check Test List for BuildDockerImageSanityTest." + return + } + launchTestListCheck(this, globalVars) + } + }, "OSS-Compliance-Check": { script { stage("[OSS-Compliance-Check] Run") { @@ -2549,10 +2587,18 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "Will run job to build ngc containers and running in-pipeline scanning for them" } + def alwaysFailFastStages = ["Release-Check", "Check Test List"] as Set parallelJobs = stages.collectEntries{key, value -> [key, { script { stage(key) { - value() + if (enableFailFast || key in alwaysFailFastStages) { + value() + } else { + // Avoid interrupting other stages on failure. + catchError(catchInterruptions: false) { + value() + } + } } } }]} @@ -2636,15 +2682,34 @@ pipeline { steps { script { if (isReleaseCheckMode) { - stage("Release-Check") { - script { - if (testFilter[INFRA_DRY_RUN]) { - echo "Skipping Release-Check for the infrastructure dry run." - } else { - launchReleaseCheck(this, globalVars) + def releaseCheckStages = [ + "Release-Check": { + stage("Release-Check") { + if (testFilter[INFRA_DRY_RUN]) { + echo "Skipping Release-Check for the infrastructure dry run." + } else { + launchReleaseCheck(this, globalVars) + } } - } - } + }, + "Check Test List": { + stage("Check Test List") { + if (testFilter[INFRA_DRY_RUN]) { + echo "Skipping Check Test List for the infrastructure dry run." + } else if (runMode == "nightly_release") { + echo "Skipping Check Test List for nightly_release." + } else if (testFilter[(ONLY_ONE_GROUP_CHANGED)] == "Docs") { + echo "Skipping Check Test List for Docs-only changes." + } else if (env.JOB_NAME ==~ /.*BuildDockerImageSanityTest.*/) { + echo "Skipping Check Test List for BuildDockerImageSanityTest." + } else { + launchTestListCheck(this, globalVars) + } + } + }, + ] + releaseCheckStages.failFast = true + parallel releaseCheckStages } else { // globalVars[CACHED_CHANGED_FILE_LIST] is only used in setupPipelineEnvironment // Remove it to workaround the "Argument list too long" error diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index d9b7211226f1..b7d64046855d 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -4015,32 +4015,6 @@ def runLLMAgentFlowTest(pipeline, stageName) sh "cd ${WORKSPACE}/${stageName} && sed -i 's/testsuite name=\"pytest\"/testsuite name=\"${stageName}\"/g' results.xml || true" } -def launchTestListCheck(pipeline) -{ - stageName = "Test List Check" - trtllm_utils.launchKubernetesPod(pipeline, createKubernetesPodConfig(LLM_DOCKER_IMAGE, "a10"), "trt-llm", { - try { - echoNodeAndGpuInfo(pipeline, stageName) - sh "nvidia-smi && nvidia-smi -q && nvidia-smi topo -m" - // download TRT-LLM tarfile - def tarName = BUILD_CONFIGS[VANILLA_CONFIG][TARNAME] - def llmTarfile = "https://urm.nvidia.com/artifactory/${ARTIFACT_PATH}/${tarName}" - trtllm_utils.llmExecStepWithRetry(pipeline, script: "pwd && wget -nv -O '${tarName}' '${llmTarfile}' && ls -alh") - sh "tar -zxf ${tarName}" - def llmPath = sh (script: "realpath .", returnStdout: true).trim() - def llmSrc = "${llmPath}/TensorRT-LLM/src" - trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 install -r ${llmSrc}/requirements-dev.txt") - // --validate --parity: after --l0/--qa generate the collectable lists, assert every - // statically-verified parametrize ID is actually collectable (validate<->collection parity). - sh "NVIDIA_TRITON_SERVER_VERSION=26.05 LLM_ROOT=${llmSrc} LLM_BACKEND_ROOT=${llmSrc}/triton_backend python3 ${llmSrc}/scripts/check_test_list.py --l0 --qa --waive --validate --parity" - } catch (InterruptedException e) { - throw e - } catch (Exception e) { - throw e - } - }) -} - def generateTimeoutTestResultXml(pipeline, stageName) { def scriptPath = sh( script: "find . -name generate_timeout_xml.py | head -n 1 | xargs realpath", @@ -4263,7 +4237,8 @@ def renderTestDB(pipeline, testContext, llmSrc, stageName, preDefinedMakoOpts=nu "Test-db blocks conditioned on those properties (e.g. linux_distribution_name: ubuntu*) " + "will NOT be selected." } - sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==1.8.5+bc6df7" + def ciVersions = readProperties file: "${llmSrc}/jenkins/ci_versions.properties" + sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==${ciVersions.TRT_TEST_DB_VERSION}" // CBTS Layer 3: download the pre-built cbts_test_db/ tarball that the // orchestrator uploaded to Artifactory (see getCbtsResult in // L0_MergeRequest.groovy). This avoids re-running main.py locally and @@ -7229,25 +7204,6 @@ pipeline { } } } - stage("Check Test List") - { - when { - expression { - // Only run the test list validation when necessary - globalVars[RUN_MODE] != "nightly_release" && - env.targetArch == X86_64_TRIPLE && - testFilter[ONLY_ONE_GROUP_CHANGED] != "Docs" && - !(env.JOB_NAME ==~ /.*Multi-GPU.*/) && - !(env.JOB_NAME ==~ /.*BuildDockerImageSanityTest.*/) - } - } - steps - { - script { - launchTestListCheck(this) - } - } - } stage("Test") { steps { script { diff --git a/jenkins/ci_versions.properties b/jenkins/ci_versions.properties new file mode 100644 index 000000000000..b1e0cca84bd8 --- /dev/null +++ b/jenkins/ci_versions.properties @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# CI tool versions shared across Jenkins pipelines and Python scripts. +# Format: KEY=VALUE (no quotes, no spaces around '='). +# +# Consumed by: +# - jenkins/L0_Test.groovy (readProperties) +# - scripts/check_test_list.py (key=value parse) +TRT_TEST_DB_VERSION=1.8.5+bc6df7 diff --git a/scripts/check_binding_stubs.py b/scripts/check_binding_stubs.py new file mode 100644 index 000000000000..284d6c0aad36 --- /dev/null +++ b/scripts/check_binding_stubs.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Fail when collection binding stubs miss a compiled extension. + +``tests/integration/defs/stubify_bindings.py`` is a factory: most symbols are +invented on demand. Collection only needs ``_STUB_ROOTS`` to cover every +compiled Python extension the package ships so ``pytest --co`` can import +without a wheel. + +This script is stdlib-only so pre-commit and GitHub Release Checks can run it +without a wheel, GPU, or torch. +""" + +from __future__ import annotations + +import ast +import importlib.util +from pathlib import Path, PurePosixPath + +REPO_ROOT = Path(__file__).resolve().parent.parent +STUB_PATH = REPO_ROOT / "tests" / "integration" / "defs" / "stubify_bindings.py" +SETUP_PATH = REPO_ROOT / "setup.py" +_SKIP_PACKAGE_DATA_PREFIXES = ("libs/", "include/", "runtime/") +_EXTENSION_SUFFIXES = frozenset({".so", ".pyd", ".dll"}) + + +def _stub_roots() -> set[str]: + """Load ``_STUB_ROOTS`` by importing the collection stub plugin. + + Import runs ``install_bindings_stub()`` (pytest ``-p`` needs that on + import). This checker then exits, so the meta-path finder is harmless. + """ + spec = importlib.util.spec_from_file_location("stubify_bindings", STUB_PATH) + if spec is None or spec.loader is None: + raise SystemExit(f"cannot load {STUB_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return set(module._STUB_ROOTS) + + +def _const_strings(node: ast.expr) -> list[str]: + """Extract string literals from a list or a single constant.""" + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return [node.value] + if not isinstance(node, ast.List): + return [] + return [ + elt.value + for elt in node.elts + if isinstance(elt, ast.Constant) and isinstance(elt.value, str) + ] + + +def _setup_package_data_patterns(tree: ast.AST) -> list[str]: + """Collect ``package_data`` glob strings assigned in setup.py via AST.""" + patterns: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "package_data" for target in node.targets + ): + patterns.extend(_const_strings(node.value)) + elif ( + isinstance(node, ast.AugAssign) + and isinstance(node.target, ast.Name) + and node.target.id == "package_data" + ): + patterns.extend(_const_strings(node.value)) + elif isinstance(node, ast.Call): + func = node.func + if ( + isinstance(func, ast.Attribute) + and func.attr == "append" + and isinstance(func.value, ast.Name) + and func.value.id == "package_data" + ): + for arg in node.args: + patterns.extend(_const_strings(arg)) + return patterns + + +def _root_extension_module(pattern: str) -> str | None: + """Map a setuptools package_data glob to ``tensorrt_llm.`` or None. + + Top-level compiled extensions: ``bindings.*.so`` → ``tensorrt_llm.bindings``. + One-level Python packages: ``flash_mla/*.py`` → ``tensorrt_llm.flash_mla``. + Nested native libs (``libs/*.so``) and mypyc trees (``runtime/...``) are + skipped. + """ + posix = pattern.replace("\\", "/") + if posix.startswith(_SKIP_PACKAGE_DATA_PREFIXES): + return None + path = PurePosixPath(posix) + if len(path.parts) == 1: + if path.suffix.lower() not in _EXTENSION_SUFFIXES: + return None + stem = path.stem.removesuffix(".*").rstrip("*") + if not stem.isidentifier(): + return None + return f"tensorrt_llm.{stem}" + if len(path.parts) == 2 and path.parts[1] == "*.py" and path.parts[0].isidentifier(): + return f"tensorrt_llm.{path.parts[0]}" + return None + + +def _compiled_extension_roots(setup_source: str) -> set[str]: + tree = ast.parse(setup_source, filename=str(SETUP_PATH)) + roots: set[str] = set() + for pattern in _setup_package_data_patterns(tree): + module = _root_extension_module(pattern) + if module is not None: + roots.add(module) + return roots + + +def main() -> int: + stub_roots = _stub_roots() + compiled_roots = _compiled_extension_roots(SETUP_PATH.read_text(encoding="utf-8")) + + missing_roots = sorted(compiled_roots - stub_roots) + if not missing_roots: + print("OK: collection binding stubs cover compiled extensions.") + return 0 + + stub_rel = STUB_PATH.relative_to(REPO_ROOT) + print("Collection binding stubs are out of date:\n") + print( + f" - {stub_rel}: _STUB_ROOTS is missing compiled modules " + f"{missing_roots}. Add them when you introduce a new bindings/.so " + "package so Check Test List can collect without a wheel." + ) + print( + "\nUpdate tests/integration/defs/stubify_bindings.py in this change " + "so Jenkins Check Test List keeps working." + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_test_list.py b/scripts/check_test_list.py index 2a5a44367052..80be4710e85e 100755 --- a/scripts/check_test_list.py +++ b/scripts/check_test_list.py @@ -26,6 +26,15 @@ pytest -- i.e. assert validate-accepts is a subset of collectable. Catches resolver soundness bugs and stale entries. +Collection stub (``--l0`` / ``--qa`` / ``--waive``): + These modes run ``pytest --co`` with ``tests/integration/defs/stubify_bindings.py`` + (loaded only via ``-p stubify_bindings``, not by default) so TensorRT-LLM need + not be compiled and no ``tensorrt_llm`` wheel is downloaded. The stub fabricates + the compiled modules on demand; its ``_EXPLICIT`` table is only for symbols + whose real *value* is consumed at import time. Full local builds will not catch + stub gaps — watch Jenkins Check Test List. Pre-commit still runs only + ``--validate`` / waive duplicate checks (no stubbed ``--co``). + Note: All the perf tests will be excluded since they are generated dynamically. """ @@ -36,6 +45,7 @@ import re import subprocess import sys +import tempfile from collections import defaultdict from itertools import product from pathlib import Path @@ -897,23 +907,82 @@ def compute_parity( # ============================================================================= -# L0 / QA / Waive verification (runtime, requires pytest + model weights) +# L0 / QA / Waive verification (runtime pytest --co with bindings collection stub) # ============================================================================= -def install_python_dependencies(llm_src): - subprocess.run(f"cd {llm_src} && pip3 install -r requirements-dev.txt", - shell=True, - check=True) +def _get_trt_test_db_version() -> str: + """Read TRT_TEST_DB_VERSION from jenkins/ci_versions.properties.""" + props_file = Path( + __file__).resolve().parent.parent / "jenkins" / "ci_versions.properties" + with open(props_file) as f: + for line in f: + line = line.strip() + if line.startswith("TRT_TEST_DB_VERSION="): + return line.split("=", 1)[1] + raise RuntimeError(f"TRT_TEST_DB_VERSION not found in {props_file}") + + +def install_python_dependencies(llm_src: str) -> None: + """Install collection dependencies without TRT-LLM binaries.""" subprocess.run( - f"pip3 install --force-reinstall --no-deps {llm_src}/../tensorrt_llm-*.whl", - shell=True, - check=True) + [sys.executable, "-m", "pip", "install", "-r", "requirements-dev.txt"], + cwd=llm_src, + check=True, + ) + trt_test_db_ver = _get_trt_test_db_version() subprocess.run( - "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple " - "--ignore-installed trt-test-db==1.8.5+bc6df7", - shell=True, - check=True) + [ + sys.executable, + "-m", + "pip", + "install", + "--extra-index-url", + "https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple", + "--ignore-installed", + f"trt-test-db=={trt_test_db_ver}", + ], + check=True, + ) + + +def _collection_pytest_env(llm_src: str, models_root: str) -> dict[str, str]: + """Env for stubbed ``pytest --co``: PYTHONPATH + bindings stub flags.""" + existing = os.environ.get("PYTHONPATH", "") + pythonpath = os.pathsep.join(p for p in (llm_src, existing) if p) + return { + **os.environ, + "PYTHONPATH": pythonpath, + "TRT_LLM_NO_LIB_INIT": "1", + # Override any caller LLM_MODELS_ROOT. Collection only interpolates the + # path into class-body constants; an empty directory keeps the models + # NFS share off the Check Test List pod. + "LLM_MODELS_ROOT": models_root, + } + + +def _run_collection_pytest(llm_src: str, test_list: str) -> None: + """Run pytest --co with the collection bindings stub plugin.""" + defs_dir = os.path.join(llm_src, "tests", "integration", "defs") + with tempfile.TemporaryDirectory( + prefix="trtllm-collection-models-root-") as models_root: + subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-p", + "stubify_bindings", + f"--test-list={test_list}", + f"--output-dir={llm_src}", + "-s", + "--co", + "-q", + ], + check=True, + cwd=defs_dir, + env=_collection_pytest_env(llm_src, models_root), + ) def verify_l0_test_lists(llm_src): @@ -965,11 +1034,7 @@ def verify_l0_test_lists(llm_src): with open(test_list, "w") as f: f.writelines(f"{line}\n" for line in sorted(cleaned_lines)) - subprocess.run( - f"cd {llm_src}/tests/integration/defs && " - f"pytest --test-list={test_list} --output-dir={llm_src} -s --co -q", - shell=True, - check=True) + _run_collection_pytest(llm_src, test_list) def verify_qa_test_lists(llm_src): @@ -986,11 +1051,7 @@ def verify_qa_test_lists(llm_src): test_def_files = subprocess.check_output( f"ls -d {test_qa_path}/*.txt", shell=True).decode().strip().split('\n') for test_def_file in test_def_files: - subprocess.run( - f"cd {llm_src}/tests/integration/defs && " - f"pytest --test-list={test_def_file} --output-dir={llm_src} -s --co -q", - shell=True, - check=True) + _run_collection_pytest(llm_src, test_def_file) # append all the test_def_file to qa_test.txt with open(f"{llm_src}/qa_test.txt", "a") as f: with open(test_def_file, "r") as test_file: @@ -1101,11 +1162,7 @@ def verify_waive_list(llm_src, args): with open(tmp_waives_file, "w") as f: f.writelines(f"{line}\n" for line in sorted(processed_lines)) - subprocess.run( - f"cd {llm_src}/tests/integration/defs && " - f"pytest --test-list={tmp_waives_file} --output-dir={llm_src} -s --co -q", - shell=True, - check=True) + _run_collection_pytest(llm_src, tmp_waives_file) def main(): @@ -1176,7 +1233,6 @@ def main(): script_dir = os.path.dirname(os.path.realpath(__file__)) llm_src = os.path.abspath(os.path.join(script_dir, "../")) - # Only skip installing dependencies if ONLY --check-duplicates or --validate is used if args.l0 or args.qa or args.waive: install_python_dependencies(llm_src) diff --git a/tests/integration/defs/accuracy/test_llm_api_autodeploy.py b/tests/integration/defs/accuracy/test_llm_api_autodeploy.py index 6d166da2c38e..492636947278 100644 --- a/tests/integration/defs/accuracy/test_llm_api_autodeploy.py +++ b/tests/integration/defs/accuracy/test_llm_api_autodeploy.py @@ -195,7 +195,6 @@ def reduced_model_kwargs(num_hidden_layers: int, class TestLlama3_1_8B(LlmapiAccuracyTestHarness): MODEL_NAME = "meta-llama/Llama-3.1-8B" - MODEL_PATH = hf_id_to_local_model_dir(MODEL_NAME) # Configuration presets for different attention backends ATTN_BACKEND_CONFIGS = { @@ -298,10 +297,11 @@ def get_default_sampling_params(self): ], ) def test_auto_dtype(self, world_size, enable_chunked_prefill, attn_backend): + model_path = hf_id_to_local_model_dir(self.MODEL_NAME) kwargs = self.get_default_kwargs(enable_chunked_prefill, attn_backend) sampling_params = self.get_default_sampling_params() - with AutoDeployLLM(model=self.MODEL_PATH, - tokenizer=self.MODEL_PATH, + with AutoDeployLLM(model=model_path, + tokenizer=model_path, world_size=world_size, **kwargs) as llm: task = CnnDailymail(self.MODEL_NAME) @@ -317,12 +317,13 @@ def test_auto_dtype(self, world_size, enable_chunked_prefill, attn_backend): ]) def test_attention_dp(self, world_size): """Test attention data parallelism mode where TP sharding is disabled.""" + model_path = hf_id_to_local_model_dir(self.MODEL_NAME) kwargs = self.get_default_kwargs(enable_chunked_prefill=True) # Enable attention DP - this disables TP sharding kwargs["transforms"]["detect_sharding"] = {"enable_attention_dp": True} sampling_params = self.get_default_sampling_params() - with AutoDeployLLM(model=self.MODEL_PATH, - tokenizer=self.MODEL_PATH, + with AutoDeployLLM(model=model_path, + tokenizer=model_path, world_size=world_size, **kwargs) as llm: task = CnnDailymail(self.MODEL_NAME) @@ -335,15 +336,13 @@ class TestLlama3_1_8B_Instruct_Eagle3(LlmapiAccuracyTestHarness): """Accuracy test for Eagle3 one-model speculative decoding with AutoDeploy.""" MODEL_NAME = "meta-llama/Llama-3.1-8B-Instruct" - MODEL_PATH = hf_id_to_local_model_dir(MODEL_NAME) - EAGLE_MODEL_PATH = hf_id_to_local_model_dir( - "yuhuili/EAGLE3-LLaMA3.1-Instruct-8B") + EAGLE_MODEL_NAME = "yuhuili/EAGLE3-LLaMA3.1-Instruct-8B" def get_default_kwargs(self, attn_backend="flashinfer"): yaml_paths, _ = _get_registry_yaml_extra(self.MODEL_NAME) speculative_config = Eagle3DecodingConfig( max_draft_len=3, - speculative_model=self.EAGLE_MODEL_PATH, + speculative_model=hf_id_to_local_model_dir(self.EAGLE_MODEL_NAME), eagle3_one_model=True, eagle3_layers_to_capture={1, 15, 28}, ) @@ -391,11 +390,12 @@ def check_acceptance_rate(self, llm, min_acceptance_rate: float): @pytest.mark.parametrize("attn_backend", ["flashinfer", "trtllm"]) def test_eagle3_one_model(self, attn_backend): """Test Eagle3 one-model speculative decoding accuracy on GSM8K.""" + model_path = hf_id_to_local_model_dir(self.MODEL_NAME) kwargs = self.get_default_kwargs(attn_backend=attn_backend) with AutoDeployLLM( - model=self.MODEL_PATH, - tokenizer=self.MODEL_PATH, + model=model_path, + tokenizer=model_path, **kwargs, ) as llm: task = GSM8K(self.MODEL_NAME) @@ -480,13 +480,10 @@ class TestNemotronNanoV3(LlmapiAccuracyTestHarness): CONFIG_YAML = str( Path(get_llm_root()) / "examples" / "auto_deploy" / "nano_v3.yaml") - MODEL_PATHS = { - "bf16": - hf_id_to_local_model_dir("nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"), - "fp8": - hf_id_to_local_model_dir("nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"), - "nvfp4": - hf_id_to_local_model_dir("nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4"), + MODEL_NAMES = { + "bf16": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", + "fp8": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", + "nvfp4": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4", } def get_default_sampling_params(self): @@ -519,7 +516,7 @@ def test_accuracy(self, model_id, world_size, enable_attention_dp, # max_dp_num_tokens path; on world_size=1 it's a no-op. if enable_attention_dp and world_size < 2: pytest.skip("attention_dp requires world_size >= 2") - model_path = self.MODEL_PATHS[model_id] + model_path = hf_id_to_local_model_dir(self.MODEL_NAMES[model_id]) kwargs = {} device_memory_mib = get_device_memory() # bf16 always needs low-memory overrides; below H100-class total @@ -554,16 +551,10 @@ class TestNemotronSuperV3(LlmapiAccuracyTestHarness): MODEL_NAME = "nvidia/Nemotron-Super-V3" CONFIG_YAML = str( Path(get_llm_root()) / "examples" / "auto_deploy" / "super_v3.yaml") - MODEL_PATHS = { - "bf16": - hf_id_to_local_model_dir( - "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"), - "fp8": - hf_id_to_local_model_dir( - "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"), - "nvfp4": - hf_id_to_local_model_dir( - "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"), + MODEL_NAMES = { + "bf16": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", + "fp8": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", + "nvfp4": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", } def get_default_sampling_params(self): @@ -599,7 +590,7 @@ def test_accuracy(self, model_id, world_size, enable_attention_dp, if model_id == "bf16" and world_size < 4: pytest.skip("bf16 Super V3 requires at least 4 GPUs") - model_path = self.MODEL_PATHS[model_id] + model_path = hf_id_to_local_model_dir(self.MODEL_NAMES[model_id]) kwargs = {} if model_id == "bf16": low_memory_overrides(kwargs) @@ -644,7 +635,7 @@ def test_functional_small(self, dtype): No accuracy threshold is checked — the truncated model is not expected to produce meaningful text. """ - model_path = self.MODEL_PATHS[dtype] + model_path = hf_id_to_local_model_dir(self.MODEL_NAMES[dtype]) kwargs = {} kwargs.update( reduced_model_kwargs(num_hidden_layers=16, model_path=model_path)) @@ -715,7 +706,7 @@ def test_functional_small(self, dtype): ) def test_mtp(self, world_size, attn_backend, model_id): - model_path = self.MODEL_PATHS[model_id] + model_path = hf_id_to_local_model_dir(self.MODEL_NAMES[model_id]) kwargs = {} # TODO: gate for bf16 only after replay lands low_memory_overrides( @@ -769,8 +760,8 @@ class TestNemotronUltraV3(LlmapiAccuracyTestHarness): CONFIG_YAML = str( Path(get_llm_root()) / "examples" / "auto_deploy" / "model_registry" / "configs" / "ultra_v3.yaml") - MODEL_PATHS = { - "nvfp4": hf_id_to_local_model_dir("nvidia/Nemotron-Ultra-V3-NVFP4"), + MODEL_NAMES = { + "nvfp4": "nvidia/Nemotron-Ultra-V3-NVFP4", } def get_default_sampling_params(self): @@ -788,7 +779,7 @@ def test_accuracy(self, model_id, world_size): if get_device_count() < world_size: pytest.skip(f"Not enough devices for world_size={world_size}") - model_path = self.MODEL_PATHS[model_id] + model_path = hf_id_to_local_model_dir(self.MODEL_NAMES[model_id]) print_memory_usage("test start") with AutoDeployLLM( model=model_path, @@ -950,7 +941,6 @@ class TestQwen3_5_397B_MoE(LlmapiAccuracyTestHarness): MODEL_NAME = "Qwen/Qwen3.5-397B-A17B" MODEL_NAME_NVFP4 = "nvidia/Qwen3.5-397B-A17B-NVFP4" MODEL_NAME_SMALL = "Qwen/Qwen3.5-35B-A3B" - MODEL_PATH_SMALL = hf_id_to_local_model_dir(MODEL_NAME_SMALL) GSM8K_MAX_OUTPUT_LEN = 512 EXTRA_EVALUATOR_KWARGS = dict( apply_chat_template=True, @@ -1036,8 +1026,9 @@ def test_bf16_small(self, world_size): if get_device_count() < world_size: pytest.skip("Not enough devices for world size, skipping test") sampling_params = self.get_default_sampling_params() - with AutoDeployLLM(model=self.MODEL_PATH_SMALL, - tokenizer=self.MODEL_PATH_SMALL, + model_path = hf_id_to_local_model_dir(self.MODEL_NAME_SMALL) + with AutoDeployLLM(model=model_path, + tokenizer=model_path, dtype="bfloat16", world_size=world_size, **config) as llm: @@ -1061,7 +1052,6 @@ class TestMiniMaxM2(LlmapiAccuracyTestHarness): """ MODEL_NAME = "MiniMaxAI/MiniMax-M2" - MODEL_PATH = hf_id_to_local_model_dir(MODEL_NAME) # Set minimum possible seq len + small buffer, for test speed & memory usage MAX_SEQ_LEN = max(MMLU.MAX_INPUT_LEN + MMLU.MAX_OUTPUT_LEN, GSM8K.MAX_INPUT_LEN + GSM8K.MAX_OUTPUT_LEN) @@ -1090,9 +1080,10 @@ def get_default_kwargs(self): @skip_pre_hopper @pytest.mark.skip_less_device(4) def test_finegrained_fp8(self): + model_path = hf_id_to_local_model_dir(self.MODEL_NAME) kwargs = self.get_default_kwargs() - with AutoDeployLLM(model=self.MODEL_PATH, - tokenizer=self.MODEL_PATH, + with AutoDeployLLM(model=model_path, + tokenizer=model_path, world_size=4, **kwargs) as llm: task = MMLU(self.MODEL_NAME) @@ -1265,7 +1256,6 @@ class TestGemma4MoE(LlmapiAccuracyTestHarness): """Bench-run coverage for Gemma4 MoE via AutoDeploy.""" MODEL_NAME = "google/gemma-4-26B-A4B-it" - MODEL_PATH = hf_id_to_local_model_dir(MODEL_NAME) EXTRA_EVALUATOR_KWARGS = { "apply_chat_template": True, } @@ -1289,8 +1279,9 @@ def test_bf16(self): pytest.skip("Not enough devices for world size, skipping test") sampling_params = self.get_default_sampling_params() - with AutoDeployLLM(model=self.MODEL_PATH, - tokenizer=self.MODEL_PATH, + model_path = hf_id_to_local_model_dir(self.MODEL_NAME) + with AutoDeployLLM(model=model_path, + tokenizer=model_path, world_size=registry_world_size, yaml_extra=yaml_paths) as llm: task = MMMU(self.MODEL_NAME) # noqa: F821 diff --git a/tests/integration/defs/conftest.py b/tests/integration/defs/conftest.py index bcea8e086953..2205bbe02e7c 100644 --- a/tests/integration/defs/conftest.py +++ b/tests/integration/defs/conftest.py @@ -48,9 +48,6 @@ # is harmless. from test_common import session_prefetcher_hooks as _prefetch_hooks -from tensorrt_llm.bindings import ipc_nvls_supported -from tensorrt_llm.llmapi.mpi_session import get_mpi_world_size - from .perf.gpu_clock_lock import GPUClockLock from .perf.session_data_writer import SessionDataWriter from .test_list_parser import (TestCorrectionMode, apply_waives, @@ -1210,6 +1207,13 @@ def skip_by_device_count(request): f"Device count {device_count} is less than {expected_count}") +def get_mpi_world_size() -> int: + """Lazy wrapper so conftest import does not load MPI bindings.""" + from tensorrt_llm.llmapi.mpi_session import \ + get_mpi_world_size as _get_mpi_world_size + return _get_mpi_world_size() + + @pytest.fixture(autouse=True) def skip_by_mpi_world_size(request): "fixture for skip less mpi world size" @@ -1284,8 +1288,9 @@ def is_ipc_nvls_supported(): if not torch.cuda.is_available(): return False try: + from tensorrt_llm.bindings import ipc_nvls_supported return ipc_nvls_supported() - except RuntimeError: + except (RuntimeError, ImportError): return False diff --git a/tests/integration/defs/stubify_bindings.py b/tests/integration/defs/stubify_bindings.py new file mode 100644 index 000000000000..b11f07f88d0c --- /dev/null +++ b/tests/integration/defs/stubify_bindings.py @@ -0,0 +1,355 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Pytest plugin: pure-Python stubs of TensorRT-LLM's compiled modules. + +Used only by ``scripts/check_test_list.py`` so ``pytest --co`` can import product +code without a compiled ``bindings*.so`` or a prebuilt wheel. Covers the +build-generated Python modules listed in ``_STUB_ROOTS`` plus the +``libth_common.so`` surfaces torch exposes (see ``stub_torch_extensions``). + +Factory-first. The factory fabricates modules, classes and values on demand, +and deliberately exposes *empty* introspection surfaces (``dir()`` has no +public non-callable names, ``__members__`` is empty) so that PybindMirror's +field/enum mirroring in ``llm_args`` passes without per-symbol entries. + +Add ``_EXPLICIT`` entries only when Check Test List fails because a +callable or concrete type is required at import time. Dummy return +values (``-1``) are enough for collection. Prefer removing eager +imports in tests/conftest over growing this table. + +Collection-only — never a substitute for running tests. +""" + +from __future__ import annotations + +import sys +import types +from abc import ABCMeta +from collections.abc import Callable, Iterator, Sequence +from importlib import abc +from importlib.machinery import ModuleSpec +from types import ModuleType +from typing import TYPE_CHECKING, ParamSpec, TypeVar, cast + +if TYPE_CHECKING: + import pytest + +_P = ParamSpec("_P") +_R = TypeVar("_R") + +# --------------------------------------------------------------------------- +# Escape hatch +# --------------------------------------------------------------------------- +# Keyed by fully qualified name. The factory can fake any *shape*, but not a +# concrete value that product code consumes at import time. ``llm_args`` +# evaluates these lookahead getters in a class body to seed pydantic field +# defaults, so they must return a value. +# (``kDefaultLookaheadDecoding*`` in cpp/include/tensorrt_llm/executor/executor.h). + + +class LookaheadDecodingConfig: + """Explicit stub: dummy lookahead getters for collection-time imports.""" + + @staticmethod + def get_default_lookahead_decoding_window() -> int: + return -1 + + @staticmethod + def get_default_lookahead_decoding_ngram() -> int: + return -1 + + @staticmethod + def get_default_lookahead_decoding_verification_set() -> int: + return -1 + + +_EXPLICIT: dict[str, type[LookaheadDecodingConfig]] = { + "tensorrt_llm.bindings.executor.LookaheadDecodingConfig": LookaheadDecodingConfig, +} + +# --------------------------------------------------------------------------- +# Factory +# --------------------------------------------------------------------------- + +_BINDINGS = "tensorrt_llm.bindings" + +# Everything the C++ build generates under tensorrt_llm/ and that product code +# imports at module scope (see .gitignore / setup.py package_data). +_STUB_ROOTS = ( + _BINDINGS, + "tensorrt_llm.deep_ep", + "tensorrt_llm.deep_ep_cpp_tllm", + "tensorrt_llm.deep_gemm", + "tensorrt_llm.deep_gemm_cpp_tllm", + "tensorrt_llm.flash_mla", + "tensorrt_llm.flash_mla_cpp_tllm", + "tensorrt_llm.pg_utils_bindings", + "tensorrt_llm.tensorrt_llm_transfer_agent_binding", +) + +# Submodules that must resolve to modules even though they are not lower_case, +# so attribute access does not hit the CapWords "this is a class" rule. +_FORCED_SUBMODULES = (f"{_BINDINGS}.BuildInfo",) + +# torch::class_ namespaces registered by libs/libth_common.so, which +# TRT_LLM_NO_LIB_INIT=1 skips loading. Probing an unregistered class raises +# RuntimeError (not AttributeError), so hasattr() checks in product code blow +# up; an empty namespace makes them report "unavailable", which is the truth +# for a no-compile checkout. +_TORCH_CLASS_NAMESPACES = ("trtllm",) + + +class _StubMeta(ABCMeta): + """Metaclass for fabricated binding classes. + + Derives from ``ABCMeta`` so product classes can inherit from both a stubbed + binding type and an ABC without a metaclass conflict. + """ + + def __getattr__(cls, name: str) -> type[_StubBase]: + # Never fabricate dunders: Python and pydantic probe them to decide + # which protocols a type supports. + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + + children = cast( + dict[str, type[_StubBase]] | None, + cls.__dict__.get("_stub_children"), + ) + if children is None: + children = {} + type.__setattr__(cls, "_stub_children", children) + if name not in children: + # Cached and distinct per name so enum-style members stay usable as + # dict keys (e.g. the DataType maps built in tensorrt_llm/_utils.py). + children[name] = _make_stub_class(f"{cls.__name__}.{name}", cls.__module__) + return children[name] + + @property + def __members__(cls) -> dict[str, type[_StubBase]]: + # PybindMirror.mirror_pybind_enum iterates the C++ members and requires + # each to exist on the Python enum; an empty mapping trivially passes. + return {} + + def __iter__(cls) -> Iterator[type[_StubBase]]: + # Product code materializes some binding sequences at import time + # (e.g. tuple(KVCacheIterationStatsDelta._field_names)). + return iter(()) + + def __int__(cls) -> int: + # Stubbed enum members are coerced at import time + # (e.g. int(BufferKind.DEFAULT) in cute_dsl_custom_ops.py). + return 0 + + def __index__(cls) -> int: + return 0 + + +class _StubBase(metaclass=_StubMeta): + """Base for fabricated binding classes; only dunders, so ``dir()`` is clean.""" + + def __init__(self, *args: object, **kwargs: object) -> None: + pass + + def __getattr__(self, name: str) -> type[_StubBase]: + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + return _make_stub_class(f"{type(self).__name__}.{name}", type(self).__module__) + + def __call__(self, *args: object, **kwargs: object) -> bool: + return False + + def __bool__(self) -> bool: + return False + + def __iter__(self) -> Iterator[type[_StubBase]]: + return iter(()) + + def __int__(self) -> int: + return 0 + + def __index__(self) -> int: + return 0 + + +def _make_stub_class(name: str, module: str) -> type[_StubBase]: + return cast( + type[_StubBase], + _StubMeta(name, (_StubBase,), {"__module__": module}), + ) + + +class StubModule(types.ModuleType): + """Fake (sub)module of a stubbed root, resolving attributes on demand.""" + + def __init__(self, fullname: str) -> None: + super().__init__(fullname) + self.__file__ = f"" + self.__package__ = fullname + self.__path__ = [] # marks this as a package for nested imports + object.__setattr__(self, "_stub_cache", {}) + + def __getattr__( + self, name: str + ) -> int | type[LookaheadDecodingConfig] | type[_StubBase] | StubModule: + cache = cast( + dict[ + str, + int | type[LookaheadDecodingConfig] | type[_StubBase] | StubModule, + ], + object.__getattribute__(self, "_stub_cache"), + ) + if name in cache: + return cache[name] + + fq = f"{self.__name__}.{name}" + value: int | type[LookaheadDecodingConfig] | type[_StubBase] | StubModule + if fq in _EXPLICIT: + value = _EXPLICIT[fq] + elif name.isupper(): + # Module-level constants such as BuildInfo.ENABLE_MULTI_DEVICE: + # falsy keeps mpi_session and communicator off their mpi4py paths. + value = 0 + elif name[:1].isupper(): + value = _make_stub_class(name, self.__name__) + else: + # Lower case: either a submodule or a free function. A module is + # both importable and callable, so it covers each case. + value = StubModule(fq) + sys.modules[fq] = value + + cache[name] = value + return value + + def __call__(self, *args: object, **kwargs: object) -> bool: + return False + + def __bool__(self) -> bool: + return False + + def __repr__(self) -> str: + return f"" + + +class _StubFinder(abc.MetaPathFinder): + """Meta path finder fabricating any module under a stubbed root. + + Attribute access alone is not enough: ``import a.b.c`` and + ``from a.b import c`` go through the import system, which never consults a + parent module's ``__getattr__``. + """ + + @staticmethod + def find_spec( + fullname: str, + path: Sequence[str] | None = None, + target: ModuleType | None = None, + ) -> ModuleSpec | None: + del path, target + if not any(fullname == root or fullname.startswith(f"{root}.") for root in _STUB_ROOTS): + return None + return ModuleSpec(fullname, _StubLoader(), is_package=True) + + +class _StubLoader(abc.Loader): + @staticmethod + def create_module(spec: ModuleSpec) -> StubModule: + return StubModule(spec.name) + + @staticmethod + def exec_module(module: ModuleType) -> None: + del module + + +def install_bindings_stub() -> StubModule: + """Install stub modules for every ``_STUB_ROOTS`` entry into ``sys.modules``. + + Always wins over a compiled wheel already on ``sys.path``: Check Test List + must collect against this checkout's Python, not against installed binaries + that may expose a different API. + """ + installed = sys.modules.get(_BINDINGS) + if isinstance(installed, StubModule): + # Idempotent: reinstalling would orphan the stub classes already + # captured by imported product modules. + return installed + + finder = _StubFinder() + sys.meta_path[:] = [f for f in sys.meta_path if not isinstance(f, _StubFinder)] + sys.meta_path.insert(0, finder) + + for root in _STUB_ROOTS: + sys.modules[root] = StubModule(root) + + for sub in _FORCED_SUBMODULES: + child = StubModule(sub) + sys.modules[sub] = child + parent_name, _, attr = sub.rpartition(".") + parent = sys.modules.get(parent_name) + if isinstance(parent, StubModule): + object.__getattribute__(parent, "_stub_cache")[attr] = child + + bindings = sys.modules[_BINDINGS] + assert isinstance(bindings, StubModule) + return bindings + + +def stub_torch_extensions() -> None: + """Neutralize the parts of torch that expect ``libth_common.so`` to be loaded.""" + try: + import torch + except ImportError: + return + + for namespace in _TORCH_CLASS_NAMESPACES: + if namespace not in torch.classes.__dict__: + setattr(torch.classes, namespace, types.ModuleType(f"torch.classes.{namespace}")) + + # Product modules register fake kernels for C++ ops at import time; without + # the library the schemas are missing, so make those registrations no-ops. + register_fake = torch.library.register_fake + if getattr(register_fake, "_trtllm_collection_stub", False): + return + + def tolerant_register_fake( + op: str, + func: Callable[_P, _R] | None = None, + /, + **kwargs: object, + ) -> Callable[_P, _R] | Callable[[Callable[_P, _R]], Callable[_P, _R]]: + def apply(fn: Callable[_P, _R]) -> Callable[_P, _R]: + try: + return cast(Callable[_P, _R], register_fake(op, fn, **kwargs)) + except RuntimeError as exc: + if "does not exist" not in str(exc): + raise + return fn + + return apply(func) if func is not None else apply + + setattr(tolerant_register_fake, "_trtllm_collection_stub", True) + torch.library.register_fake = tolerant_register_fake + + +# Install on import so `pytest -p stubify_bindings` wins the race with +# conftest collection. +install_bindings_stub() + + +def pytest_configure(config: pytest.Config) -> None: + """Re-assert the stubs early in the pytest session.""" + del config + install_bindings_stub() + stub_torch_extensions() diff --git a/tests/integration/test_lists/test-db/README.md b/tests/integration/test_lists/test-db/README.md index 74e8a137cceb..fc7bdf709d45 100644 --- a/tests/integration/test_lists/test-db/README.md +++ b/tests/integration/test_lists/test-db/README.md @@ -4,10 +4,10 @@ This folder contains test definition which is consumed by `trt-test-db` tool bas ## Installation -Install `trt-test-db` using the following command: +Install `trt-test-db` using the following command (substitute `TRT_TEST_DB_VERSION` with the version from `jenkins/ci_versions.properties`): ```bash -pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==1.8.5+bc6df7 +pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db== ``` ## Test Definition @@ -58,6 +58,20 @@ pytest -v --test-list=/TensorRT-LLM/src/l0_e2e.txt --output-dir=/tmp/logs This command runs the tests specified in the test list and outputs the results to the specified directory. +## Check Test List (CI) + +Jenkins **Check Test List** runs +`scripts/check_test_list.py --l0 --qa --waive --validate --parity`. +L0/QA/waive collection uses pure-Python stubs of the compiled modules +(`tests/integration/defs/stubify_bindings.py`, loaded only via +`-p stubify_bindings`), so no TensorRT-LLM wheel or C++ +compile or binary download is required. `--parity` asserts that statically +verified parametrize IDs are a subset of what `pytest --co` collects. +Pre-commit runs check_test_list.py with `--validate` and +`--check-duplicate-waives`, plus `scripts/check_binding_stubs.py` to fail when +`stubify_bindings.py` `_STUB_ROOTS` misses a compiled extension listed in +`setup.py` `package_data`. That last check does not require stubs or a wheel. + ## Additional Information - The `--context` parameter in the `trt-test-db` command specifies which context to search in the YAML files. - The `--match-exact` parameter provides system information used to filter tests based on the conditions defined in the YAML files. diff --git a/tests/unittest/scripts/test_slurm_install.py b/tests/unittest/scripts/test_slurm_install.py index 77a01c72a580..92643ea77181 100644 --- a/tests/unittest/scripts/test_slurm_install.py +++ b/tests/unittest/scripts/test_slurm_install.py @@ -97,14 +97,3 @@ def test_slurm_install_requires_artifact_inputs(missing: str) -> None: assert result.returncode != 0 assert f"{missing} is required" in result.stderr - - -def test_l0_artifact_download_retries_overwrite_archive() -> None: - download_commands = [ - line.strip() - for line in L0_TEST.read_text().splitlines() - if "wget -nv" in line and "llmTarfile" in line - ] - - assert len(download_commands) == 4 - assert all("wget -nv -O" in command for command in download_commands) diff --git a/tests/unittest/utils/util.py b/tests/unittest/utils/util.py index 08ce24000241..10b5833486b1 100644 --- a/tests/unittest/utils/util.py +++ b/tests/unittest/utils/util.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -56,8 +56,16 @@ def ASSERT_DRV(err): # ref: https://github.com/NVIDIA/cuda-python/blob/main/examples/extra/jit_program_test.py def getSMVersion(): # Init - err, = cuda.cuInit(0) + try: + err, = cuda.cuInit(0) + except RuntimeError as exc: + # Missing/broken driver. Returning inf makes skip_pre_* true and + # skip_post_* false so collection can proceed on CPU hosts. + print(f"WARNING: CUDA driver init failed in getSMVersion(): {exc}") + return math.inf if err == cuda.CUresult.CUDA_ERROR_NO_DEVICE: + print("WARNING: CUDA reports no device in getSMVersion(). " + "Tests that require a GPU will be skipped.") return -1 ASSERT_DRV(err)