Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1660,6 +1660,17 @@ repos:
entry: ./scripts/check_test_list.py --validate
language: script
pass_filenames: false
- id: check-binding-stubs
name: Verify collection binding stubs cover compiled extensions
entry: python scripts/check_binding_stubs.py
language: python
pass_filenames: false
files: >
(?x)^(
tests/integration/defs/stubify_bindings.py|
scripts/check_binding_stubs.py|
setup.py
)$
- id: DCO check
name: Checks the commit message for a developer certificate of origin signature
entry: ./scripts/dco_check.py
Expand Down
83 changes: 74 additions & 9 deletions jenkins/L0_MergeRequest.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -666,6 +666,23 @@ def launchReleaseCheck(pipeline, globalVars)
})
}

def launchTestListCheck(pipeline, globalVars)
{
def key = "Check Test List"
def image = globalVars["LLM_DOCKER_IMAGE"]
trtllm_utils.launchKubernetesPod(pipeline, createKubernetesPodConfig(image, "package"), "trt-llm", {
stage("[${key}] Run") {
echoNodeAndGpuInfo(pipeline, key)
sh "git config --global --add safe.directory \"*\""
trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, false, true)

def llmPath = sh(script: "realpath ${LLM_ROOT}", returnStdout: true).trim()
sh "NVIDIA_TRITON_SERVER_VERSION=26.05 LLM_ROOT=${llmPath} LLM_BACKEND_ROOT=${llmPath}/triton_backend " +
"python3 ${llmPath}/scripts/check_test_list.py --l0 --qa --waive --validate --parity --check-duplicate-waives"
}
})
}

def getGitlabMRChangedFile(pipeline, function, filePath="") {
def result = null
def pageId = 0
Expand Down Expand Up @@ -1915,6 +1932,27 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
launchReleaseCheck(this, globalVars)
}
},
"Check Test List": {
Comment thread
brnguyen2 marked this conversation as resolved.
Comment thread
tburt-nv marked this conversation as resolved.
script {
if (testFilter[INFRA_DRY_RUN]) {
echo "Skipping Check Test List for the infrastructure dry run."
return
} else if (GEN_POST_MERGE_BUILDS_ONLY) {
echo "Skipping Check Test List (GenPostMergeBuilds mode: builds only)"
return
} else if (runMode == "nightly_release") {
echo "Skipping Check Test List for nightly_release."
return
} else if (testFilter[(ONLY_ONE_GROUP_CHANGED)] == "Docs") {
echo "Skipping Check Test List for Docs-only changes."
return
} else if (env.JOB_NAME ==~ /.*BuildDockerImageSanityTest.*/) {
echo "Skipping Check Test List for BuildDockerImageSanityTest."
return
}
launchTestListCheck(this, globalVars)
}
},
"OSS-Compliance-Check": {
script {
stage("[OSS-Compliance-Check] Run") {
Expand Down Expand Up @@ -2549,10 +2587,18 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
echo "Will run job to build ngc containers and running in-pipeline scanning for them"
}

def alwaysFailFastStages = ["Release-Check", "Check Test List"] as Set
parallelJobs = stages.collectEntries{key, value -> [key, {
script {
stage(key) {
value()
if (enableFailFast || key in alwaysFailFastStages) {
value()
} else {
// Avoid interrupting other stages on failure.
catchError(catchInterruptions: false) {
Comment thread
brnguyen2 marked this conversation as resolved.
value()
}
}
}
}
}]}
Expand Down Expand Up @@ -2636,15 +2682,34 @@ pipeline {
steps {
script {
if (isReleaseCheckMode) {
stage("Release-Check") {
script {
if (testFilter[INFRA_DRY_RUN]) {
echo "Skipping Release-Check for the infrastructure dry run."
} else {
launchReleaseCheck(this, globalVars)
def releaseCheckStages = [
"Release-Check": {
stage("Release-Check") {
if (testFilter[INFRA_DRY_RUN]) {
echo "Skipping Release-Check for the infrastructure dry run."
} else {
launchReleaseCheck(this, globalVars)
}
Comment thread
tburt-nv marked this conversation as resolved.
}
}
}
},
"Check Test List": {
stage("Check Test List") {
if (testFilter[INFRA_DRY_RUN]) {
echo "Skipping Check Test List for the infrastructure dry run."
} else if (runMode == "nightly_release") {
echo "Skipping Check Test List for nightly_release."
} else if (testFilter[(ONLY_ONE_GROUP_CHANGED)] == "Docs") {
echo "Skipping Check Test List for Docs-only changes."
} else if (env.JOB_NAME ==~ /.*BuildDockerImageSanityTest.*/) {
echo "Skipping Check Test List for BuildDockerImageSanityTest."
} else {
launchTestListCheck(this, globalVars)
}
}
},
]
releaseCheckStages.failFast = true
parallel releaseCheckStages
} else {
// globalVars[CACHED_CHANGED_FILE_LIST] is only used in setupPipelineEnvironment
// Remove it to workaround the "Argument list too long" error
Expand Down
48 changes: 2 additions & 46 deletions jenkins/L0_Test.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -4015,32 +4015,6 @@ def runLLMAgentFlowTest(pipeline, stageName)
sh "cd ${WORKSPACE}/${stageName} && sed -i 's/testsuite name=\"pytest\"/testsuite name=\"${stageName}\"/g' results.xml || true"
}

def launchTestListCheck(pipeline)
{
stageName = "Test List Check"
trtllm_utils.launchKubernetesPod(pipeline, createKubernetesPodConfig(LLM_DOCKER_IMAGE, "a10"), "trt-llm", {
try {
echoNodeAndGpuInfo(pipeline, stageName)
sh "nvidia-smi && nvidia-smi -q && nvidia-smi topo -m"
// download TRT-LLM tarfile
def tarName = BUILD_CONFIGS[VANILLA_CONFIG][TARNAME]
def llmTarfile = "https://urm.nvidia.com/artifactory/${ARTIFACT_PATH}/${tarName}"
trtllm_utils.llmExecStepWithRetry(pipeline, script: "pwd && wget -nv -O '${tarName}' '${llmTarfile}' && ls -alh")
sh "tar -zxf ${tarName}"
def llmPath = sh (script: "realpath .", returnStdout: true).trim()
def llmSrc = "${llmPath}/TensorRT-LLM/src"
trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 install -r ${llmSrc}/requirements-dev.txt")
// --validate --parity: after --l0/--qa generate the collectable lists, assert every
// statically-verified parametrize ID is actually collectable (validate<->collection parity).
sh "NVIDIA_TRITON_SERVER_VERSION=26.05 LLM_ROOT=${llmSrc} LLM_BACKEND_ROOT=${llmSrc}/triton_backend python3 ${llmSrc}/scripts/check_test_list.py --l0 --qa --waive --validate --parity"
} catch (InterruptedException e) {
throw e
} catch (Exception e) {
throw e
}
})
}

def generateTimeoutTestResultXml(pipeline, stageName) {
def scriptPath = sh(
script: "find . -name generate_timeout_xml.py | head -n 1 | xargs realpath",
Expand Down Expand Up @@ -4263,7 +4237,8 @@ def renderTestDB(pipeline, testContext, llmSrc, stageName, preDefinedMakoOpts=nu
"Test-db blocks conditioned on those properties (e.g. linux_distribution_name: ubuntu*) " +
"will NOT be selected."
}
sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==1.8.5+bc6df7"
def ciVersions = readProperties file: "${llmSrc}/jenkins/ci_versions.properties"
sh "pip3 install --extra-index-url https://urm.nvidia.com/artifactory/api/pypi/sw-tensorrt-pypi/simple --ignore-installed trt-test-db==${ciVersions.TRT_TEST_DB_VERSION}"
// CBTS Layer 3: download the pre-built cbts_test_db/ tarball that the
// orchestrator uploaded to Artifactory (see getCbtsResult in
// L0_MergeRequest.groovy). This avoids re-running main.py locally and
Expand Down Expand Up @@ -7229,25 +7204,6 @@ pipeline {
}
}
}
stage("Check Test List")
{
when {
expression {
// Only run the test list validation when necessary
globalVars[RUN_MODE] != "nightly_release" &&
env.targetArch == X86_64_TRIPLE &&
testFilter[ONLY_ONE_GROUP_CHANGED] != "Docs" &&
!(env.JOB_NAME ==~ /.*Multi-GPU.*/) &&
!(env.JOB_NAME ==~ /.*BuildDockerImageSanityTest.*/)
}
}
steps
{
script {
launchTestListCheck(this)
}
}
}
stage("Test") {
steps {
script {
Expand Down
10 changes: 10 additions & 0 deletions jenkins/ci_versions.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# CI tool versions shared across Jenkins pipelines and Python scripts.
# Format: KEY=VALUE (no quotes, no spaces around '=').
#
# Consumed by:
# - jenkins/L0_Test.groovy (readProperties)
# - scripts/check_test_list.py (key=value parse)
TRT_TEST_DB_VERSION=1.8.5+bc6df7
Comment thread
tburt-nv marked this conversation as resolved.
140 changes: 140 additions & 0 deletions scripts/check_binding_stubs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
"""Fail when collection binding stubs miss a compiled extension.

``tests/integration/defs/stubify_bindings.py`` is a factory: most symbols are
invented on demand. Collection only needs ``_STUB_ROOTS`` to cover every
compiled Python extension the package ships so ``pytest --co`` can import
without a wheel.

This script is stdlib-only so pre-commit and GitHub Release Checks can run it
without a wheel, GPU, or torch.
"""

from __future__ import annotations

import ast
import importlib.util
from pathlib import Path, PurePosixPath

REPO_ROOT = Path(__file__).resolve().parent.parent
STUB_PATH = REPO_ROOT / "tests" / "integration" / "defs" / "stubify_bindings.py"
SETUP_PATH = REPO_ROOT / "setup.py"
_SKIP_PACKAGE_DATA_PREFIXES = ("libs/", "include/", "runtime/")
_EXTENSION_SUFFIXES = frozenset({".so", ".pyd", ".dll"})


def _stub_roots() -> set[str]:
"""Load ``_STUB_ROOTS`` by importing the collection stub plugin.

Import runs ``install_bindings_stub()`` (pytest ``-p`` needs that on
import). This checker then exits, so the meta-path finder is harmless.
"""
spec = importlib.util.spec_from_file_location("stubify_bindings", STUB_PATH)
if spec is None or spec.loader is None:
raise SystemExit(f"cannot load {STUB_PATH}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return set(module._STUB_ROOTS)


def _const_strings(node: ast.expr) -> list[str]:
"""Extract string literals from a list or a single constant."""
if isinstance(node, ast.Constant) and isinstance(node.value, str):
return [node.value]
if not isinstance(node, ast.List):
return []
return [
elt.value
for elt in node.elts
if isinstance(elt, ast.Constant) and isinstance(elt.value, str)
]


def _setup_package_data_patterns(tree: ast.AST) -> list[str]:
"""Collect ``package_data`` glob strings assigned in setup.py via AST."""
patterns: list[str] = []
for node in ast.walk(tree):
if isinstance(node, ast.Assign) and any(
isinstance(target, ast.Name) and target.id == "package_data" for target in node.targets
):
patterns.extend(_const_strings(node.value))
elif (
isinstance(node, ast.AugAssign)
and isinstance(node.target, ast.Name)
and node.target.id == "package_data"
):
patterns.extend(_const_strings(node.value))
elif isinstance(node, ast.Call):
func = node.func
if (
isinstance(func, ast.Attribute)
and func.attr == "append"
and isinstance(func.value, ast.Name)
and func.value.id == "package_data"
):
for arg in node.args:
patterns.extend(_const_strings(arg))
return patterns


def _root_extension_module(pattern: str) -> str | None:
"""Map a setuptools package_data glob to ``tensorrt_llm.<ext>`` or None.

Top-level compiled extensions: ``bindings.*.so`` → ``tensorrt_llm.bindings``.
One-level Python packages: ``flash_mla/*.py`` → ``tensorrt_llm.flash_mla``.
Nested native libs (``libs/*.so``) and mypyc trees (``runtime/...``) are
skipped.
"""
posix = pattern.replace("\\", "/")
if posix.startswith(_SKIP_PACKAGE_DATA_PREFIXES):
return None
path = PurePosixPath(posix)
if len(path.parts) == 1:
if path.suffix.lower() not in _EXTENSION_SUFFIXES:
return None
stem = path.stem.removesuffix(".*").rstrip("*")
if not stem.isidentifier():
return None
return f"tensorrt_llm.{stem}"
if len(path.parts) == 2 and path.parts[1] == "*.py" and path.parts[0].isidentifier():
return f"tensorrt_llm.{path.parts[0]}"
return None


def _compiled_extension_roots(setup_source: str) -> set[str]:
tree = ast.parse(setup_source, filename=str(SETUP_PATH))
roots: set[str] = set()
for pattern in _setup_package_data_patterns(tree):
module = _root_extension_module(pattern)
if module is not None:
roots.add(module)
return roots


def main() -> int:
stub_roots = _stub_roots()
compiled_roots = _compiled_extension_roots(SETUP_PATH.read_text(encoding="utf-8"))

missing_roots = sorted(compiled_roots - stub_roots)
if not missing_roots:
print("OK: collection binding stubs cover compiled extensions.")
return 0

stub_rel = STUB_PATH.relative_to(REPO_ROOT)
print("Collection binding stubs are out of date:\n")
print(
f" - {stub_rel}: _STUB_ROOTS is missing compiled modules "
f"{missing_roots}. Add them when you introduce a new bindings/.so "
"package so Check Test List can collect without a wheel."
)
print(
"\nUpdate tests/integration/defs/stubify_bindings.py in this change "
"so Jenkins Check Test List keeps working."
)
return 1


if __name__ == "__main__":
raise SystemExit(main())
Loading
Loading