diff --git a/.github/workflows/auto-full-premerge-approval.yml b/.github/workflows/auto-full-premerge-approval.yml
new file mode 100644
index 000000000000..9a1d1ef04bc4
--- /dev/null
+++ b/.github/workflows/auto-full-premerge-approval.yml
@@ -0,0 +1,256 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Auto Full Pre-Merge Approval Label
+
+on:
+ status:
+ pull_request_review:
+ types: [submitted]
+
+permissions:
+ contents: read
+
+jobs:
+ auto-full-premerge-approval:
+ if: >-
+ github.repository == 'NVIDIA/TensorRT-LLM' &&
+ ((github.event_name == 'status' &&
+ github.event.context == 'full-single-gpu-tests' &&
+ github.event.state == 'success') ||
+ (github.event_name == 'pull_request_review' &&
+ github.event.review.state == 'approved'))
+ concurrency:
+ group: auto-full-premerge-${{ github.event.pull_request.head.sha || github.event.sha }}
+ cancel-in-progress: false
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ steps:
+ - name: Require agent token
+ env:
+ AGENT_TOKEN: ${{ secrets.TRTLLM_AGENT_SHARED_TOKEN }}
+ run: |
+ if [[ -z "${AGENT_TOKEN}" ]]; then
+ echo "::error::TRTLLM_AGENT_SHARED_TOKEN is unavailable for this event."
+ exit 1
+ fi
+
+ - name: Add full pre-merge approval label when eligible
+ uses: actions/github-script@v8
+ with:
+ github-token: ${{ secrets.TRTLLM_AGENT_SHARED_TOKEN }}
+ script: |
+ const approvalLabel = 'ci: full pre-merge approved';
+ const fullSingleStatus = 'full-single-gpu-tests';
+ const owner = context.repo.owner;
+ const repo = context.repo.repo;
+
+ const sleep = (milliseconds) =>
+ new Promise((resolve) => setTimeout(resolve, milliseconds));
+
+ async function withRetry(description, operation) {
+ const attempts = 3;
+ for (let attempt = 1; attempt <= attempts; attempt += 1) {
+ try {
+ return await operation();
+ } catch (error) {
+ if (attempt === attempts) {
+ throw error;
+ }
+ core.warning(
+ `${description} failed (attempt ${attempt}/${attempts}): ${error.message}`
+ );
+ await sleep(1000 * attempt);
+ }
+ }
+ }
+
+ async function getCandidatePullRequests() {
+ if (context.eventName === 'pull_request_review') {
+ return [context.payload.pull_request.number];
+ }
+
+ const pullRequests = await withRetry(
+ `Find pull requests associated with ${context.payload.sha}`,
+ () =>
+ github.paginate(
+ 'GET /repos/{owner}/{repo}/commits/{commit_sha}/pulls',
+ {
+ owner,
+ repo,
+ commit_sha: context.payload.sha,
+ per_page: 100,
+ }
+ )
+ );
+ const repository = `${owner}/${repo}`.toLowerCase();
+ return [
+ ...new Set(
+ pullRequests
+ .filter(
+ (pullRequest) =>
+ pullRequest.state === 'open' &&
+ pullRequest.base.repo.full_name.toLowerCase() === repository
+ )
+ .map((pullRequest) => pullRequest.number)
+ ),
+ ];
+ }
+
+ const pullRequestQuery = `
+ query($owner: String!, $repo: String!, $number: Int!) {
+ repository(owner: $owner, name: $repo) {
+ pullRequest(number: $number) {
+ headRefOid
+ labels(first: 100) {
+ nodes {
+ name
+ }
+ }
+ reviewDecision
+ state
+ }
+ }
+ }
+ `;
+
+ async function getPullRequestState(pullNumber) {
+ const result = await withRetry(
+ `Read pull request #${pullNumber}`,
+ () =>
+ github.graphql(pullRequestQuery, {
+ owner,
+ repo,
+ number: pullNumber,
+ })
+ );
+ return result.repository.pullRequest;
+ }
+
+ function hasApprovalLabel(pullRequest) {
+ return pullRequest.labels.nodes.some(
+ (label) =>
+ label.name.toLowerCase() === approvalLabel.toLowerCase()
+ );
+ }
+
+ async function getLatestFullSingleStatus(sha) {
+ const statuses = await withRetry(
+ `Read commit statuses for ${sha}`,
+ () =>
+ github.paginate(
+ github.rest.repos.listCommitStatusesForRef,
+ {
+ owner,
+ repo,
+ ref: sha,
+ per_page: 100,
+ }
+ )
+ );
+ return statuses.find(
+ (status) => status.context === fullSingleStatus
+ );
+ }
+
+ async function findSuccessfulFullSingleStatus(pullNumber) {
+ const commits = await withRetry(
+ `Read commits for pull request #${pullNumber}`,
+ () =>
+ github.paginate(github.rest.pulls.listCommits, {
+ owner,
+ repo,
+ pull_number: pullNumber,
+ per_page: 100,
+ })
+ );
+ const commitShas = commits.map((commit) => commit.sha).reverse();
+ const triggeringSha =
+ context.eventName === 'status' ? context.payload.sha : null;
+
+ if (triggeringSha && commitShas.includes(triggeringSha)) {
+ commitShas.splice(commitShas.indexOf(triggeringSha), 1);
+ commitShas.unshift(triggeringSha);
+ }
+
+ for (const sha of commitShas) {
+ const status = await getLatestFullSingleStatus(sha);
+ if (status?.state === 'success') {
+ return sha;
+ }
+ }
+ return null;
+ }
+
+ const pullNumbers = await getCandidatePullRequests();
+ if (pullNumbers.length === 0) {
+ console.log('No open pull request is associated with this event.');
+ return;
+ }
+
+ for (const pullNumber of pullNumbers) {
+ const initialState = await getPullRequestState(pullNumber);
+ if (!initialState || initialState.state !== 'OPEN') {
+ console.log(`PR #${pullNumber} is not open; skipping.`);
+ continue;
+ }
+ if (hasApprovalLabel(initialState)) {
+ console.log(`PR #${pullNumber} already has the approval label.`);
+ continue;
+ }
+ if (initialState.reviewDecision !== 'APPROVED') {
+ console.log(`PR #${pullNumber} is not fully approved; skipping.`);
+ continue;
+ }
+
+ const successfulCommit =
+ await findSuccessfulFullSingleStatus(pullNumber);
+ if (!successfulCommit) {
+ console.log(
+ `PR #${pullNumber} has no successful ${fullSingleStatus} status in its current commit history.`
+ );
+ continue;
+ }
+
+ const finalState = await getPullRequestState(pullNumber);
+ if (!finalState || finalState.state !== 'OPEN') {
+ console.log(`PR #${pullNumber} is no longer open; skipping.`);
+ continue;
+ }
+ if (hasApprovalLabel(finalState)) {
+ console.log(`PR #${pullNumber} already has the approval label.`);
+ continue;
+ }
+ if (finalState.reviewDecision !== 'APPROVED') {
+ console.log(`PR #${pullNumber} is no longer fully approved; skipping.`);
+ continue;
+ }
+ if (finalState.headRefOid !== initialState.headRefOid) {
+ console.log(`PR #${pullNumber} changed during validation; skipping.`);
+ continue;
+ }
+
+ await withRetry(`Add approval label to PR #${pullNumber}`, () =>
+ github.rest.issues.addLabels({
+ owner,
+ repo,
+ issue_number: pullNumber,
+ labels: [approvalLabel],
+ })
+ );
+ console.log(
+ `Added ${approvalLabel} to PR #${pullNumber}; ${fullSingleStatus} succeeded on ${successfulCommit}.`
+ );
+ }
diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy
index 1ef2aac37c97..48792ff99ed5 100644
--- a/jenkins/L0_MergeRequest.groovy
+++ b/jenkins/L0_MergeRequest.groovy
@@ -135,6 +135,8 @@ def ONLY_MULTI_GPU_TEST = "only_multi_gpu_test"
@Field
def DISABLE_MULTI_GPU_TEST = "disable_multi_gpu_test"
@Field
+def FULL_SINGLE_GPU_RUN = "full_single_gpu_run"
+@Field
def EXTRA_STAGE_LIST = "extra_stage"
@Field
def MULTI_GPU_FILE_CHANGED = "multi_gpu_file_changed"
@@ -201,6 +203,7 @@ def testFilter = [
(ADD_MULTI_GPU_TEST): gitlabParamsFromBot.get((ADD_MULTI_GPU_TEST), false),
(ONLY_MULTI_GPU_TEST): gitlabParamsFromBot.get((ONLY_MULTI_GPU_TEST), false) || gitlabParamsFromBot.get((ENABLE_MULTI_GPU_TEST), false),
(DISABLE_MULTI_GPU_TEST): gitlabParamsFromBot.get((DISABLE_MULTI_GPU_TEST), false),
+ (FULL_SINGLE_GPU_RUN): gitlabParamsFromBot.get((FULL_SINGLE_GPU_RUN), false),
(EXTRA_STAGE_LIST): trimForStageList(gitlabParamsFromBot.get((EXTRA_STAGE_LIST), null)?.tokenize(',')),
(MULTI_GPU_FILE_CHANGED): false,
(ONLY_ONE_GROUP_CHANGED): "",
@@ -778,51 +781,234 @@ def getGithubMRChangedFile(pipeline, githubPrApiUrl, function, filePath="") {
return result
}
-// Gate multi-GPU stages behind 'ci: full pre-merge approved' label.
-// Uses trtllm_utils.validatePRLabelApproval() from the shared lib to verify
-// both label existence and that the labeler is an active team member.
-// Exempt: PostMerge pipelines and GitLab MR builds (no GITHUB_PR_API_URL).
-def requireMultiGpuApprovalLabel(pipeline, globalVars, String arch) {
+def getGithubPrNumber(globalVars) {
+ def prMatch = (globalVars[GITHUB_PR_API_URL] =~ /\/pulls?\/(\d+)/)
+ return prMatch ? prMatch[0][1] : null
+}
+
+// Check the existing approval label without deciding whether the new automatic
+// full-approval path may be used. API failures retain the existing fail-open
+// behavior so this feature does not make the current label gate less available.
+def checkMultiGpuApprovalLabel(pipeline, globalVars, String arch) {
if (!globalVars[GITHUB_PR_API_URL]) {
- echo "[requireMultiGpuApprovalLabel] Skipping label check: not a GitHub PR (no GITHUB_PR_API_URL)"
- return false
+ echo "[checkMultiGpuApprovalLabel] Skipping label check: not a GitHub PR (no GITHUB_PR_API_URL)"
+ return [allowed: true]
}
if (env.JOB_NAME ==~ /.*PostMerge.*/) {
- echo "[requireMultiGpuApprovalLabel] Skipping label check: PostMerge pipeline is exempt"
- return false
+ echo "[checkMultiGpuApprovalLabel] Skipping label check: PostMerge pipeline is exempt"
+ return [allowed: true]
}
- def prMatch = (globalVars[GITHUB_PR_API_URL] =~ /\/pulls?\/(\d+)/)
- if (!prMatch) {
- echo "[requireMultiGpuApprovalLabel] Could not extract PR number from ${globalVars[GITHUB_PR_API_URL]}. Failing open."
- return false
+ def prNumber = getGithubPrNumber(globalVars)
+ if (!prNumber) {
+ echo "[checkMultiGpuApprovalLabel] Could not extract PR number from ${globalVars[GITHUB_PR_API_URL]}. Failing open."
+ return [allowed: true]
}
- def prNumber = prMatch[0][1]
def result = trtllm_utils.validatePRLabelApproval(pipeline, prNumber, "ci: full pre-merge approved")
if (!result.checkCompleted) {
- // API error — fail-open: do not block CI if the label check itself fails
- echo "[requireMultiGpuApprovalLabel] Label validation incomplete (${result.error}). Failing open."
- return false
+ echo "[checkMultiGpuApprovalLabel] Label validation incomplete (${result.error}). Failing open."
+ return [allowed: true]
}
if (result.labelExists && result.authorized) {
- return false
+ return [allowed: true]
}
- // Label missing or unauthorized — write description marker for wrapper
- // to surface in PR comment, and return the block reason string.
- def existingDesc = currentBuild.description ?: ""
- currentBuild.description = existingDesc + (existingDesc ? "
" : "") +
- "" +
- "Multi-GPU tests require label 'ci: full pre-merge approved'" +
- ""
def reason = !result.labelExists
? "label 'ci: full pre-merge approved' is not present on this PR"
: "label 'ci: full pre-merge approved' was applied by '${result.actor}' who is not an active member of NVIDIA/trt-llm-ci-approvers"
- def blockMsg = "${arch} Multi-GPU tests blocked: ${reason}. " +
- "Ask a member of NVIDIA/trt-llm-ci-approvers to add the label, then re-trigger CI."
- echo "[requireMultiGpuApprovalLabel] ${blockMsg}"
- return blockMsg
+ echo "[checkMultiGpuApprovalLabel] ${arch} Multi-GPU tests are not label-approved: ${reason}."
+ return [allowed: false, prNumber: prNumber, blockReason: reason]
+}
+
+def getMultiGpuLabelBlockMessage(String arch, String reason) {
+ return "${arch} Multi-GPU tests blocked: ${reason}. " +
+ "Ask a member of NVIDIA/trt-llm-ci-approvers to add the label, then re-trigger CI."
+}
+
+// Query GitHub's aggregate required-review decision. This is fail-closed for
+// the new automatic path: only an exact APPROVED result can bypass the label.
+def checkPullRequestFullApproval(pipeline, prNumber) {
+ def result = [checkCompleted: false, approved: false, error: null]
+ if (!(prNumber?.toString() ==~ /\d+/)) {
+ result.error = "Invalid PR number: ${prNumber}"
+ return result
+ }
+
+ def query = '''
+ query($owner: String!, $repo: String!, $number: Int!) {
+ repository(owner: $owner, name: $repo) {
+ pullRequest(number: $number) {
+ reviewDecision
+ }
+ }
+ }
+ '''
+ def requestBody = JsonOutput.toJson([
+ query: query,
+ variables: [owner: "NVIDIA", repo: "TensorRT-LLM", number: prNumber.toInteger()],
+ ])
+
+ try {
+ withCredentials([
+ usernamePassword(
+ credentialsId: 'github-cred-trtllm-ci',
+ usernameVariable: 'NOT_USED_YET',
+ passwordVariable: 'GITHUB_API_TOKEN'
+ ),
+ ]) {
+ def responseJson = pipeline.sh(
+ script: """curl --silent --fail --show-error --connect-timeout 10 --max-time 30 \\
+ --request POST \\
+ --header "Authorization: Bearer \${GITHUB_API_TOKEN}" \\
+ --header "Accept: application/vnd.github+json" \\
+ --header "Content-Type: application/json" \\
+ --data '${requestBody}' \\
+ --url "https://api.github.com/graphql" """,
+ returnStdout: true
+ )
+ def response = readJSON text: responseJson, returnPojo: true
+ if (response.get("errors")) {
+ result.error = "GraphQL errors: ${JsonOutput.toJson(response.get('errors'))}"
+ echo "[checkPullRequestFullApproval] ${result.error}"
+ return result
+ }
+ def reviewDecision = response.get("data")?.get("repository")?.get("pullRequest")?.get("reviewDecision")
+ result.checkCompleted = true
+ result.approved = (reviewDecision == "APPROVED")
+ echo "[checkPullRequestFullApproval] PR #${prNumber} reviewDecision: ${reviewDecision}"
+ }
+ } catch (FlowInterruptedException e) {
+ throw e
+ } catch (Exception e) {
+ result.error = e.toString()
+ echo "[checkPullRequestFullApproval] Check failed: ${result.error}"
+ }
+ return result
+}
+
+def getWrapperBuildInfo(globalVars) {
+ def parents = globalVars[ACTION_INFO]?.get("parents", []) ?: []
+ // setupPipelineDescription appends the current L0 build, so the wrapper is
+ // the immediately preceding parent.
+ return parents.size() >= 2 ? parents[-2] : null
+}
+
+def appendMarkerToWrapper(pipeline, globalVars, marker, markerName) {
+ def wrapperBuild = getWrapperBuildInfo(globalVars)
+ if (!wrapperBuild) {
+ echo "[${markerName}] No wrapper parent found; marker was not published."
+ return
+ }
+ trtllm_utils.appendBuildDescription(pipeline, wrapperBuild, marker)
+}
+
+def recordSingleGpuResult(pipeline, singleGpuState, stateLockName, globalVars, String arch, String result) {
+ def publishFullSingleMarker = false
+ pipeline.lock(resource: stateLockName) {
+ if (singleGpuState.singleResults[arch] == "PENDING") {
+ singleGpuState.singleResults[arch] = result
+ echo "[fullSingleGpuGate] ${arch} single-GPU result: ${result}"
+ }
+ if (!singleGpuState.fullSingleMarkerPublished &&
+ singleGpuState.singleResults["x86_64"] == "SUCCESS" &&
+ singleGpuState.singleResults["SBSA"] == "SUCCESS") {
+ singleGpuState.fullSingleMarkerPublished = true
+ publishFullSingleMarker = true
+ }
+ }
+
+ if (publishFullSingleMarker) {
+ appendMarkerToWrapper(
+ pipeline,
+ globalVars,
+ 'Both x86_64 and SBSA single-GPU jobs succeeded',
+ "fullSingleGpuGate"
+ )
+ }
+}
+
+def publishMultiGpuLabelRequiredMarker(pipeline, singleGpuState, stateLockName, globalVars) {
+ def publishMarker = false
+ pipeline.lock(resource: stateLockName) {
+ if (!singleGpuState.labelRequiredMarkerPublished) {
+ singleGpuState.labelRequiredMarkerPublished = true
+ publishMarker = true
+ }
+ }
+ if (!publishMarker) {
+ return
+ }
+
+ def marker = "" +
+ "Multi-GPU tests require label 'ci: full pre-merge approved'" +
+ ""
+ def existingDesc = currentBuild.description ?: ""
+ currentBuild.description = existingDesc + (existingDesc ? "
" : "") + marker
+ appendMarkerToWrapper(pipeline, globalVars, marker, "multiGpuLabelGate")
+}
+
+def resolveMultiGpuGate(pipeline, singleGpuState, stateLockName, globalVars,
+ boolean isFullSingleGpuRun, String arch) {
+ def gate = null
+ // Serialize the one-time API decision separately from the short-lived state
+ // lock, so a slow GitHub request cannot prevent the sibling from recording
+ // its completed single-GPU result.
+ pipeline.lock(resource: "${stateLockName}-gate") {
+ def shouldEvaluate = false
+ pipeline.lock(resource: stateLockName) {
+ shouldEvaluate = (singleGpuState.gateDecision == "UNCHECKED")
+ }
+ if (shouldEvaluate) {
+ def labelCheck = checkMultiGpuApprovalLabel(pipeline, globalVars, arch)
+ def decision = null
+ def reason = ""
+ if (labelCheck.allowed) {
+ decision = "LABEL_ALLOWED"
+ } else if (!isFullSingleGpuRun) {
+ decision = "DENIED"
+ reason = labelCheck.blockReason
+ } else {
+ def approval = checkPullRequestFullApproval(pipeline, labelCheck.prNumber)
+ if (approval.checkCompleted && approval.approved) {
+ decision = "AUTO_ALLOWED"
+ } else {
+ decision = "DENIED"
+ reason = labelCheck.blockReason
+ if (!approval.checkCompleted) {
+ echo "[fullSingleGpuGate] Full approval could not be verified (${approval.error}); automatic multi-GPU dispatch is disabled."
+ }
+ }
+ }
+ pipeline.lock(resource: stateLockName) {
+ singleGpuState.gateDecision = decision
+ singleGpuState.gateReason = reason
+ }
+ echo "[fullSingleGpuGate] Gate decision: ${decision}"
+ }
+ pipeline.lock(resource: stateLockName) {
+ gate = [decision: singleGpuState.gateDecision, reason: singleGpuState.gateReason]
+ }
+ }
+ return gate
+}
+
+def waitForBothSingleGpuResults(pipeline, singleGpuState, stateLockName, String arch) {
+ echo "[fullSingleGpuGate] ${arch} is waiting for both single-GPU jobs to finish."
+ while (true) {
+ def results = null
+ pipeline.lock(resource: stateLockName) {
+ results = new LinkedHashMap(singleGpuState.singleResults)
+ }
+ if (results.values().every { it == "SUCCESS" }) {
+ return "SUCCESS"
+ }
+ if (results.values().any { it == "NON_SUCCESS" }) {
+ echo "[fullSingleGpuGate] Single-GPU results are not both successful: ${results}"
+ return "NON_SUCCESS"
+ }
+ pipeline.sleep(time: 30, unit: "SECONDS")
+ }
}
def getMergeRequestChangedFileList(pipeline, globalVars) {
@@ -1050,9 +1236,9 @@ def getCbtsResult(pipeline, testFilter, globalVars)
def _cbtsMultiGpuLabelGateOpen(pipeline, globalVars)
{
try {
- def blockReason = requireMultiGpuApprovalLabel(
+ def labelCheck = checkMultiGpuApprovalLabel(
pipeline, globalVars, "CBTS telemetry")
- return !blockReason
+ return labelCheck.allowed
} catch (InterruptedException e) {
throw e
} catch (Exception e) {
@@ -1902,6 +2088,17 @@ def launchInfraDryRunTestJob(pipeline, arch, testFilter, globalVars, platform, i
def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
{
+ def singleGpuState = [
+ singleResults: ["x86_64": "PENDING", "SBSA": "PENDING"],
+ gateDecision: "UNCHECKED",
+ gateReason: "",
+ fullSingleMarkerPublished: false,
+ labelRequiredMarkerPublished: false,
+ ]
+ def stateLockName = "trtllm-full-single-gpu-${env.JOB_NAME}-${env.BUILD_NUMBER}"
+ .replaceAll(/[^A-Za-z0-9_.-]/, "-")
+ boolean isFullSingleGpuRun = testFilter[FULL_SINGLE_GPU_RUN]?.toString()?.toBoolean() ?: false
+
stages = [
"Release-Check": {
script {
@@ -1971,6 +2168,8 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
},
"x86_64-Linux": {
script {
+ def singleGpuResultRecorded = false
+ try {
// CBTS deliberately does NOT short-circuit at the arch / Build
// layer. Build always runs so a wheel exists for sanity checks
// and post-merge consumers; case-level narrowing happens later
@@ -2033,6 +2232,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
testStageName = "[Test-x86_64-Single-GPU] Remote Run"
def singleGpuTestFailed = false
def singleGpuInfraIncomplete = false
+ def singleGpuResult = "NON_SUCCESS"
stage(testStageName) {
if (X86_TEST_CHOICE == STAGE_CHOICE_SKIP) {
echo "x86_64 test job is skipped due to Jenkins configuration"
@@ -2051,6 +2251,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
// sub-job was infra-incomplete: only infra aborts, no real failure.
def singleGpuStatus = launchJob(pipeline, "L0_Test-x86_64-Single-GPU", false, enableFailFast, globalVars, "x86_64", additionalParameters)
singleGpuInfraIncomplete = (singleGpuStatus == "UNSTABLE")
+ singleGpuResult = (singleGpuStatus == "SUCCESS") ? "SUCCESS" : "NON_SUCCESS"
} catch (InterruptedException e) {
throw e
} catch (Exception e) {
@@ -2070,6 +2271,8 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
}
}
}
+ recordSingleGpuResult(pipeline, singleGpuState, stateLockName, globalVars, "x86_64", singleGpuResult)
+ singleGpuResultRecorded = true
uploadArchCoverage("x86_64", pipeline, testFilter)
def requireMultiGpuTesting = currentBuild.description?.contains("Require x86_64 Multi-GPU Testing") ?: false
@@ -2110,14 +2313,31 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
}
}
- // Label gate: check before entering the Remote Run stage so a
- // missing/unauthorized label shows as "Blocked" (not a Remote Run
- // failure) and does not trigger fail-fast.
- def x86LabelBlock = requireMultiGpuApprovalLabel(pipeline, globalVars, "x86_64")
- if (x86LabelBlock) {
+ // An existing valid label preserves the current per-architecture
+ // behavior. Without one, a full run may proceed only after both
+ // single-GPU jobs pass and GitHub reports full approval.
+ def x86Gate = resolveMultiGpuGate(
+ pipeline,
+ singleGpuState,
+ stateLockName,
+ globalVars,
+ isFullSingleGpuRun,
+ "x86_64"
+ )
+ if (x86Gate.decision == "DENIED") {
+ publishMultiGpuLabelRequiredMarker(pipeline, singleGpuState, stateLockName, globalVars)
+ stage("[Test-x86_64-Multi-GPU] Blocked") {
+ catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
+ error getMultiGpuLabelBlockMessage("x86_64", x86Gate.reason)
+ }
+ }
+ return
+ }
+ if (x86Gate.decision == "AUTO_ALLOWED" &&
+ waitForBothSingleGpuResults(pipeline, singleGpuState, stateLockName, "x86_64") != "SUCCESS") {
stage("[Test-x86_64-Multi-GPU] Blocked") {
catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
- error x86LabelBlock
+ error "x86_64 Multi-GPU tests require both x86_64 and SBSA single-GPU jobs to succeed."
}
}
return
@@ -2154,10 +2374,18 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
}
}
}
+ } finally {
+ if (!singleGpuResultRecorded) {
+ recordSingleGpuResult(
+ pipeline, singleGpuState, stateLockName, globalVars, "x86_64", "NON_SUCCESS")
+ }
+ }
}
},
"SBSA-Linux": {
script {
+ def singleGpuResultRecorded = false
+ try {
if (testFilter[(ONLY_ONE_GROUP_CHANGED)] == "Docs") {
echo "SBSA build job is skipped due to Jenkins configuration or conditional pipeline run"
return
@@ -2240,6 +2468,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
testStageName = "[Test-SBSA-Single-GPU] Remote Run"
def singleGpuTestFailed = false
def singleGpuInfraIncomplete = false
+ def singleGpuResult = "NON_SUCCESS"
stage(testStageName) {
if (SBSA_TEST_CHOICE == STAGE_CHOICE_SKIP) {
echo "SBSA test job is skipped due to Jenkins configuration"
@@ -2257,6 +2486,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
// sub-job was infra-incomplete: only infra aborts, no real failure.
def singleGpuStatus = launchJob(pipeline, "L0_Test-SBSA-Single-GPU", false, enableFailFast, globalVars, "SBSA", additionalParameters)
singleGpuInfraIncomplete = (singleGpuStatus == "UNSTABLE")
+ singleGpuResult = (singleGpuStatus == "SUCCESS") ? "SUCCESS" : "NON_SUCCESS"
} catch (InterruptedException e) {
throw e
} catch (Exception e) {
@@ -2276,6 +2506,8 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
}
}
}
+ recordSingleGpuResult(pipeline, singleGpuState, stateLockName, globalVars, "SBSA", singleGpuResult)
+ singleGpuResultRecorded = true
uploadArchCoverage("SBSA", pipeline, testFilter)
@@ -2317,11 +2549,28 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
}
}
- def sbsaLabelBlock = requireMultiGpuApprovalLabel(pipeline, globalVars, "SBSA")
- if (sbsaLabelBlock) {
+ def sbsaGate = resolveMultiGpuGate(
+ pipeline,
+ singleGpuState,
+ stateLockName,
+ globalVars,
+ isFullSingleGpuRun,
+ "SBSA"
+ )
+ if (sbsaGate.decision == "DENIED") {
+ publishMultiGpuLabelRequiredMarker(pipeline, singleGpuState, stateLockName, globalVars)
+ stage("[Test-SBSA-Multi-GPU] Blocked") {
+ catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
+ error getMultiGpuLabelBlockMessage("SBSA", sbsaGate.reason)
+ }
+ }
+ return
+ }
+ if (sbsaGate.decision == "AUTO_ALLOWED" &&
+ waitForBothSingleGpuResults(pipeline, singleGpuState, stateLockName, "SBSA") != "SUCCESS") {
stage("[Test-SBSA-Multi-GPU] Blocked") {
catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
- error sbsaLabelBlock
+ error "SBSA Multi-GPU tests require both x86_64 and SBSA single-GPU jobs to succeed."
}
}
return
@@ -2357,6 +2606,12 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
}
}
}
+ } finally {
+ if (!singleGpuResultRecorded) {
+ recordSingleGpuResult(
+ pipeline, singleGpuState, stateLockName, globalVars, "SBSA", "NON_SUCCESS")
+ }
+ }
}
},
]
diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index 317a2e4874b6..3061f7a5aae8 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -3012,6 +3012,10 @@ def testFilter = [
(INFRA_DRY_RUN): false,
]
+def isPipelineMonitorSingleGpuTestMode() {
+ return env.JOB_NAME ==~ /LLM\/PipelineMonitor\/L0_Test-(x86_64|SBSA)-Single-GPU/
+}
+
@Field
def GITHUB_PR_API_URL = "github_pr_api_url"
@Field
@@ -7224,6 +7228,10 @@ pipeline {
stage("Test") {
steps {
script {
+ if (isPipelineMonitorSingleGpuTestMode()) {
+ echo "[TEST MODE] Skipping GPU test execution in ${env.JOB_NAME} and returning SUCCESS."
+ return
+ }
// Default scope map so the image-sanity path (which does not
// build one) still has a value for runBranchesWithInfraDefer;
// launchTestJobs overwrites this with per-stage scopes.