[https://nvbugs/6720250][fix] Use adjusted clock for VisualGen timing

tensorrt_llm/serve/openai_server.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 import array
 import asyncio
 import base64
@@ -3161,7 +3164,7 @@ async def openai_image_generation(self, request: ImageGenerationRequest,
             logger.info(f"Image {image_id} generated and encoded: "
                         f"latency={latency:.3f}s generation={generation:.3f}s "
                         f"denoise={denoise:.3f}s")
-            total = get_steady_clock_now_in_seconds() - request_received
+            total = self._adjusted_steady_clock.now() - request_received
             headers = build_visual_gen_timing_headers(
                 build_visual_gen_server_timings(metrics, total=total))
 
@@ -3381,7 +3384,7 @@ async def openai_image_edit(self, raw_request: Request) -> Response:
             logger.info(f"Image {image_id} edited and encoded: "
                         f"latency={latency:.3f}s generation={generation:.3f}s "
                         f"denoise={denoise:.3f}s")
-            total = get_steady_clock_now_in_seconds() - request_received
+            total = self._adjusted_steady_clock.now() - request_received
             headers = build_visual_gen_timing_headers(
                 build_visual_gen_server_timings(metrics, total=total))
 

tensorrt_llm/serve/openai_video_routes.py

@@ -26,7 +26,6 @@
 from fastapi.responses import FileResponse, JSONResponse, Response
 from pydantic import ValidationError
 
-from tensorrt_llm._utils import get_steady_clock_now_in_seconds
 from tensorrt_llm.logger import logger
 from tensorrt_llm.media.encoding import resolve_video_format
 from tensorrt_llm.media.tensor_payload import is_tensor_format
@@ -134,7 +133,8 @@ class _VideoRoutesMixin:
 
     Concrete subclasses (``OpenAIServer``) supply ``self.generator``,
     ``self.media_storage_path``, ``self.video_gen_tasks``, ``self.model``,
-    and ``self.create_error_response`` from their own initializer.
+    ``self._adjusted_steady_clock``, and ``self.create_error_response`` from
+    their own initializer.
     """
 
     async def openai_video_generation_sync(self, raw_request: Request) -> Response:
@@ -214,7 +214,7 @@ async def openai_video_generation_sync(self, raw_request: Request) -> Response:
                     f"Video {video_id} serialized as tensor: latency={latency:.3f}s "
                     f"generation={getattr(output.metrics, 'generation', 0.0):.3f}s"
                 )
-                total = get_steady_clock_now_in_seconds() - request_received
+                total = self._adjusted_steady_clock.now() - request_received
                 headers = build_visual_gen_timing_headers(
                     build_visual_gen_server_timings(output.metrics, total=total)
                 )
@@ -253,7 +253,7 @@ async def openai_video_generation_sync(self, raw_request: Request) -> Response:
                 f"latency={latency:.3f}s generation={generation:.3f}s "
                 f"denoise={denoise:.3f}s"
             )
-            total = get_steady_clock_now_in_seconds() - request_received
+            total = self._adjusted_steady_clock.now() - request_received
             headers = build_visual_gen_timing_headers(
                 build_visual_gen_server_timings(metrics, total=total)
             )
@@ -558,7 +558,7 @@ async def _generate_video_background(
                 # from the status wire). ``total`` spans POST arrival ->
                 # completion.
                 total = (
-                    get_steady_clock_now_in_seconds() - job.request_started
+                    self._adjusted_steady_clock.now() - job.request_started
                     if job.request_started is not None
                     else None
                 )

tests/integration/test_lists/waives.txt

@@ -331,7 +331,6 @@ unittest/_torch/visual_gen/multi_gpu/test_ring_attention.py::TestRingAttention::
 unittest/_torch/visual_gen/multi_gpu/test_ulysses_async.py::test_capture_smoke SKIP (https://nvbugs/6385134)
 unittest/_torch/visual_gen/multi_gpu/test_ulysses_attention.py SKIP (https://nvbugs/6311866)
 unittest/_torch/visual_gen/multi_gpu/test_wan_pipeline_parallel.py::TestWanPipelineParallel::test_cfg2_attn2d2x1_ulysses2_pvae8 SKIP (https://nvbugs/6644465)
-unittest/_torch/visual_gen/test_trtllm_serve_endpoints.py SKIP (https://nvbugs/6720250)
 unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_wait_in_progress_on_zero_timeout SKIP (https://nvbugs/6517836)
 unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_write_transfer_gpu_tensor SKIP (https://nvbugs/6517836)
 unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_write_transfer_multiple_chunks SKIP (https://nvbugs/6517836)

tests/unittest/_torch/visual_gen/test_trtllm_serve_endpoints.py

@@ -1,3 +1,6 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 """trtllm-serve visual_gen endpoints tests.
 
 Tests all endpoints registered for the VISUAL_GEN server role
@@ -31,6 +34,7 @@
 from fastapi.testclient import TestClient
 from PIL import Image
 
+from tensorrt_llm._utils import AdjustedSteadyClock
 from tensorrt_llm.serve.openai_protocol import VideoJob
 from tensorrt_llm.serve.openai_server import _normalize_image_output
 from tensorrt_llm.serve.visual_gen_metrics import SERVER_TIMING_HEADER
@@ -131,6 +135,20 @@ def _server_timing_ms(headers, name: str) -> float:
     raise AssertionError(f"{name!r} not in Server-Timing: {server_timing!r}")
 
 
+def _set_deterministic_server_clock(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Make the adjusted clock read 10 at arrival and 15 thereafter."""
+    is_arrival = True
+
+    def now(_self: AdjustedSteadyClock) -> float:
+        nonlocal is_arrival
+        if is_arrival:
+            is_arrival = False
+            return 10.0
+        return 15.0
+
+    monkeypatch.setattr(AdjustedSteadyClock, "now", now)
+
+
 def _drive_job_to_completion(client, video_id, timeout: float = 5.0):
     """Poll ``GET /v1/videos/{id}`` until the job reaches a terminal state.
 
@@ -666,25 +684,15 @@ def test_image_generation_server_timing_has_total(self, image_client):
         assert _server_timing_ms(resp.headers, "total") > 0
 
     def test_image_generation_total_anchored_to_server_arrival(self, image_client, monkeypatch):
-        """``total`` measures from the middleware's arrival stamp, not from a
-        handler-local clock — this route is handed an already-parsed
-        ``ImageGenerationRequest``, so stamping in the handler would silently
-        exclude body parsing.
-
-        Backdating only the middleware's clock leaves the handler's end
-        reading on the real one, so ``total`` must absorb the full offset.
-        """
-        import tensorrt_llm.serve.responses_utils as _ru
-
-        real = _ru.get_steady_clock_now_in_seconds
-        monkeypatch.setattr(_ru, "get_steady_clock_now_in_seconds", lambda: real() - 5.0)
+        """``total`` uses the shared adjusted clock from arrival to completion."""
+        _set_deterministic_server_clock(monkeypatch)
 
         resp = image_client.post(
             "/v1/images/generations",
             json={"prompt": "timing", "response_format": "b64_json", "size": "64x64"},
         )
         assert resp.status_code == 200
-        assert _server_timing_ms(resp.headers, "total") >= 5000.0
+        assert _server_timing_ms(resp.headers, "total") == 5000.0
 
     def test_image_generation_with_optional_params(self, image_client):
         resp = image_client.post(
@@ -1128,8 +1136,9 @@ def test_image_edit_default_url_returns_fetchable_output(self, tmp_path, monkeyp
         assert content.headers["content-type"] == "image/png"
 
     def test_image_edit_server_timing_has_total(self, tmp_path, monkeypatch):
-        """The edit route reports ``total`` too (real wall-clock, so > 0)."""
+        """The edit route reports ``total`` using the shared adjusted clock."""
         client, _ = self._client(tmp_path, monkeypatch)
+        _set_deterministic_server_clock(monkeypatch)
 
         resp = client.post(
             "/v1/images/edits",
@@ -1143,7 +1152,7 @@ def test_image_edit_server_timing_has_total(self, tmp_path, monkeypatch):
         assert resp.status_code == 200
         assert _server_timing_ms(resp.headers, "generation") == 1250.0
         assert _server_timing_ms(resp.headers, "denoise") == 750.0
-        assert _server_timing_ms(resp.headers, "total") > 0
+        assert _server_timing_ms(resp.headers, "total") == 5000.0
 
     def test_image_edit_response_format_path_returns_on_disk_path(self, tmp_path, monkeypatch):
         """``response_format='path'`` returns the server-side output path
@@ -1496,9 +1505,10 @@ def test_basic_sync_video_generation(self, video_client):
         assert resp.headers["content-type"] == "video/mp4"
         _assert_visual_gen_server_timing(resp.headers)
 
-    def test_sync_video_server_timing_has_total(self, video_client):
+    def test_sync_video_server_timing_has_total(self, video_client, monkeypatch):
         """The sync Server-Timing header carries generation, denoise, and the
-        new ``total`` (full server time; real wall-clock, so only checked > 0)."""
+        full server time from the shared adjusted clock."""
+        _set_deterministic_server_clock(monkeypatch)
         resp = video_client.post(
             "/v1/videos/sync",
             json={
@@ -1513,7 +1523,7 @@ def test_sync_video_server_timing_has_total(self, video_client):
         assert resp.status_code == 200
         assert _server_timing_ms(resp.headers, "generation") == 1250.0
         assert _server_timing_ms(resp.headers, "denoise") == 750.0
-        assert _server_timing_ms(resp.headers, "total") > 0
+        assert _server_timing_ms(resp.headers, "total") == 5000.0
         assert len(resp.content) > 0
 
     def test_deprecated_generations_alias_routes_to_sync(self, video_client):
@@ -2553,11 +2563,15 @@ def test_sync_tensor_file_returns_file_with_correct_suffix(self, video_audio_cli
         assert "video" in loaded
 
     @pytest.mark.parametrize("fmt", ["safetensors", "pt"])
-    def test_sync_tensor_path_returns_readable_output_path(self, video_audio_client, fmt):
+    def test_sync_tensor_path_returns_readable_output_path(
+        self, video_audio_client, fmt, monkeypatch
+    ):
+        _set_deterministic_server_clock(monkeypatch)
         resp = self._post_sync(video_audio_client, fmt, "path")
         assert resp.status_code == 200
         # path responses carry the Server-Timing metrics too.
         _assert_visual_gen_server_timing(resp.headers)
+        assert _server_timing_ms(resp.headers, "total") == 5000.0
         data = resp.json()
         assert set(data) >= {"id", "output_path"}
         # Co-located client reads the returned server-side path directly.
@@ -3067,10 +3081,7 @@ async def test_async_total_anchored_to_server_arrival(self, async_video_client,
         ``VideoJob.request_started`` and closes ``total`` out in a background
         task, so the stamp and the end reading must stay on one clock.
         """
-        import tensorrt_llm.serve.responses_utils as _ru
-
-        real = _ru.get_steady_clock_now_in_seconds
-        monkeypatch.setattr(_ru, "get_steady_clock_now_in_seconds", lambda: real() - 5.0)
+        _set_deterministic_server_clock(monkeypatch)
 
         resp = await async_video_client.post(
             "/v1/videos",
@@ -3089,7 +3100,7 @@ async def test_async_total_anchored_to_server_arrival(self, async_video_client,
 
         content = await async_video_client.get(f"/v1/videos/{video_id}/content")
         assert content.status_code == 200
-        assert _server_timing_ms(content.headers, "total") >= 5000.0
+        assert _server_timing_ms(content.headers, "total") == 5000.0
 
     @pytest.mark.asyncio
     async def test_async_file_still_returns_file_response(self, async_video_client):