diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index df8bd45ad3ec..c0f0cfd418d6 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -1,4 +1,7 @@ #!/usr/bin/env python +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import array import asyncio import base64 @@ -3161,7 +3164,7 @@ async def openai_image_generation(self, request: ImageGenerationRequest, logger.info(f"Image {image_id} generated and encoded: " f"latency={latency:.3f}s generation={generation:.3f}s " f"denoise={denoise:.3f}s") - total = get_steady_clock_now_in_seconds() - request_received + total = self._adjusted_steady_clock.now() - request_received headers = build_visual_gen_timing_headers( build_visual_gen_server_timings(metrics, total=total)) @@ -3381,7 +3384,7 @@ async def openai_image_edit(self, raw_request: Request) -> Response: logger.info(f"Image {image_id} edited and encoded: " f"latency={latency:.3f}s generation={generation:.3f}s " f"denoise={denoise:.3f}s") - total = get_steady_clock_now_in_seconds() - request_received + total = self._adjusted_steady_clock.now() - request_received headers = build_visual_gen_timing_headers( build_visual_gen_server_timings(metrics, total=total)) diff --git a/tensorrt_llm/serve/openai_video_routes.py b/tensorrt_llm/serve/openai_video_routes.py index ba61e83a2d57..8ca3b69c9ef2 100644 --- a/tensorrt_llm/serve/openai_video_routes.py +++ b/tensorrt_llm/serve/openai_video_routes.py @@ -26,7 +26,6 @@ from fastapi.responses import FileResponse, JSONResponse, Response from pydantic import ValidationError -from tensorrt_llm._utils import get_steady_clock_now_in_seconds from tensorrt_llm.logger import logger from tensorrt_llm.media.encoding import resolve_video_format from tensorrt_llm.media.tensor_payload import is_tensor_format @@ -134,7 +133,8 @@ class _VideoRoutesMixin: Concrete subclasses (``OpenAIServer``) supply ``self.generator``, ``self.media_storage_path``, ``self.video_gen_tasks``, ``self.model``, - and ``self.create_error_response`` from their own initializer. + ``self._adjusted_steady_clock``, and ``self.create_error_response`` from + their own initializer. """ async def openai_video_generation_sync(self, raw_request: Request) -> Response: @@ -214,7 +214,7 @@ async def openai_video_generation_sync(self, raw_request: Request) -> Response: f"Video {video_id} serialized as tensor: latency={latency:.3f}s " f"generation={getattr(output.metrics, 'generation', 0.0):.3f}s" ) - total = get_steady_clock_now_in_seconds() - request_received + total = self._adjusted_steady_clock.now() - request_received headers = build_visual_gen_timing_headers( build_visual_gen_server_timings(output.metrics, total=total) ) @@ -253,7 +253,7 @@ async def openai_video_generation_sync(self, raw_request: Request) -> Response: f"latency={latency:.3f}s generation={generation:.3f}s " f"denoise={denoise:.3f}s" ) - total = get_steady_clock_now_in_seconds() - request_received + total = self._adjusted_steady_clock.now() - request_received headers = build_visual_gen_timing_headers( build_visual_gen_server_timings(metrics, total=total) ) @@ -558,7 +558,7 @@ async def _generate_video_background( # from the status wire). ``total`` spans POST arrival -> # completion. total = ( - get_steady_clock_now_in_seconds() - job.request_started + self._adjusted_steady_clock.now() - job.request_started if job.request_started is not None else None ) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 301fc947e3b1..5350341ef215 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -331,11 +331,6 @@ unittest/_torch/visual_gen/multi_gpu/test_ring_attention.py::TestRingAttention:: unittest/_torch/visual_gen/multi_gpu/test_ulysses_async.py::test_capture_smoke SKIP (https://nvbugs/6385134) unittest/_torch/visual_gen/multi_gpu/test_ulysses_attention.py SKIP (https://nvbugs/6311866) unittest/_torch/visual_gen/multi_gpu/test_wan_pipeline_parallel.py::TestWanPipelineParallel::test_cfg2_attn2d2x1_ulysses2_pvae8 SKIP (https://nvbugs/6644465) -unittest/_torch/visual_gen/test_trtllm_serve_e2e.py::TestFlux1TextToImage::test_t2i_sync_b64 SKIP (https://nvbugs/6720250) -unittest/_torch/visual_gen/test_trtllm_serve_e2e.py::TestFlux1TextToImage::test_t2i_sync_with_optional_params SKIP (https://nvbugs/6720250) -unittest/_torch/visual_gen/test_trtllm_serve_e2e.py::TestFlux2TextToImage::test_t2i_sync_b64 SKIP (https://nvbugs/6720250) -unittest/_torch/visual_gen/test_trtllm_serve_e2e.py::TestFlux2TextToImage::test_t2i_sync_with_optional_params SKIP (https://nvbugs/6720250) -unittest/_torch/visual_gen/test_trtllm_serve_endpoints.py SKIP (https://nvbugs/6720250) unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_wait_in_progress_on_zero_timeout SKIP (https://nvbugs/6517836) unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_write_transfer_gpu_tensor SKIP (https://nvbugs/6517836) unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_write_transfer_multiple_chunks SKIP (https://nvbugs/6517836) diff --git a/tests/unittest/_torch/visual_gen/test_trtllm_serve_endpoints.py b/tests/unittest/_torch/visual_gen/test_trtllm_serve_endpoints.py index 48e48cf744d5..90095af22418 100644 --- a/tests/unittest/_torch/visual_gen/test_trtllm_serve_endpoints.py +++ b/tests/unittest/_torch/visual_gen/test_trtllm_serve_endpoints.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """trtllm-serve visual_gen endpoints tests. Tests all endpoints registered for the VISUAL_GEN server role @@ -31,6 +34,7 @@ from fastapi.testclient import TestClient from PIL import Image +from tensorrt_llm._utils import AdjustedSteadyClock from tensorrt_llm.serve.openai_protocol import VideoJob from tensorrt_llm.serve.openai_server import _normalize_image_output from tensorrt_llm.serve.visual_gen_metrics import SERVER_TIMING_HEADER @@ -131,6 +135,23 @@ def _server_timing_ms(headers, name: str) -> float: raise AssertionError(f"{name!r} not in Server-Timing: {server_timing!r}") +def _set_deterministic_server_clock(monkeypatch: pytest.MonkeyPatch) -> None: + """Hold the clock at 10 until mock generation starts, then advance to 15.""" + current_time = 10.0 + original_generate_async = MockVisualGen.generate_async + + def now(_self: AdjustedSteadyClock) -> float: + return current_time + + def generate_async(self, inputs=None, params=None) -> "MockVisualGenResult": + nonlocal current_time + current_time = 15.0 + return original_generate_async(self, inputs=inputs, params=params) + + monkeypatch.setattr(AdjustedSteadyClock, "now", now) + monkeypatch.setattr(MockVisualGen, "generate_async", generate_async) + + def _drive_job_to_completion(client, video_id, timeout: float = 5.0): """Poll ``GET /v1/videos/{id}`` until the job reaches a terminal state. @@ -666,25 +687,17 @@ def test_image_generation_server_timing_has_total(self, image_client): assert _server_timing_ms(resp.headers, "total") > 0 def test_image_generation_total_anchored_to_server_arrival(self, image_client, monkeypatch): - """``total`` measures from the middleware's arrival stamp, not from a - handler-local clock — this route is handed an already-parsed - ``ImageGenerationRequest``, so stamping in the handler would silently - exclude body parsing. - - Backdating only the middleware's clock leaves the handler's end - reading on the real one, so ``total`` must absorb the full offset. - """ - import tensorrt_llm.serve.responses_utils as _ru - - real = _ru.get_steady_clock_now_in_seconds - monkeypatch.setattr(_ru, "get_steady_clock_now_in_seconds", lambda: real() - 5.0) + """``total`` uses the shared adjusted clock from arrival to completion.""" + _set_deterministic_server_clock(monkeypatch) + # An unrelated read must not advance the clock before request arrival. + assert AdjustedSteadyClock().now() == 10.0 resp = image_client.post( "/v1/images/generations", json={"prompt": "timing", "response_format": "b64_json", "size": "64x64"}, ) assert resp.status_code == 200 - assert _server_timing_ms(resp.headers, "total") >= 5000.0 + assert _server_timing_ms(resp.headers, "total") == 5000.0 def test_image_generation_with_optional_params(self, image_client): resp = image_client.post( @@ -1128,8 +1141,9 @@ def test_image_edit_default_url_returns_fetchable_output(self, tmp_path, monkeyp assert content.headers["content-type"] == "image/png" def test_image_edit_server_timing_has_total(self, tmp_path, monkeypatch): - """The edit route reports ``total`` too (real wall-clock, so > 0).""" + """The edit route reports ``total`` using the shared adjusted clock.""" client, _ = self._client(tmp_path, monkeypatch) + _set_deterministic_server_clock(monkeypatch) resp = client.post( "/v1/images/edits", @@ -1143,7 +1157,7 @@ def test_image_edit_server_timing_has_total(self, tmp_path, monkeypatch): assert resp.status_code == 200 assert _server_timing_ms(resp.headers, "generation") == 1250.0 assert _server_timing_ms(resp.headers, "denoise") == 750.0 - assert _server_timing_ms(resp.headers, "total") > 0 + assert _server_timing_ms(resp.headers, "total") == 5000.0 def test_image_edit_response_format_path_returns_on_disk_path(self, tmp_path, monkeypatch): """``response_format='path'`` returns the server-side output path @@ -1496,9 +1510,10 @@ def test_basic_sync_video_generation(self, video_client): assert resp.headers["content-type"] == "video/mp4" _assert_visual_gen_server_timing(resp.headers) - def test_sync_video_server_timing_has_total(self, video_client): + def test_sync_video_server_timing_has_total(self, video_client, monkeypatch): """The sync Server-Timing header carries generation, denoise, and the - new ``total`` (full server time; real wall-clock, so only checked > 0).""" + full server time from the shared adjusted clock.""" + _set_deterministic_server_clock(monkeypatch) resp = video_client.post( "/v1/videos/sync", json={ @@ -1513,7 +1528,7 @@ def test_sync_video_server_timing_has_total(self, video_client): assert resp.status_code == 200 assert _server_timing_ms(resp.headers, "generation") == 1250.0 assert _server_timing_ms(resp.headers, "denoise") == 750.0 - assert _server_timing_ms(resp.headers, "total") > 0 + assert _server_timing_ms(resp.headers, "total") == 5000.0 assert len(resp.content) > 0 def test_deprecated_generations_alias_routes_to_sync(self, video_client): @@ -2553,11 +2568,15 @@ def test_sync_tensor_file_returns_file_with_correct_suffix(self, video_audio_cli assert "video" in loaded @pytest.mark.parametrize("fmt", ["safetensors", "pt"]) - def test_sync_tensor_path_returns_readable_output_path(self, video_audio_client, fmt): + def test_sync_tensor_path_returns_readable_output_path( + self, video_audio_client, fmt, monkeypatch + ): + _set_deterministic_server_clock(monkeypatch) resp = self._post_sync(video_audio_client, fmt, "path") assert resp.status_code == 200 # path responses carry the Server-Timing metrics too. _assert_visual_gen_server_timing(resp.headers) + assert _server_timing_ms(resp.headers, "total") == 5000.0 data = resp.json() assert set(data) >= {"id", "output_path"} # Co-located client reads the returned server-side path directly. @@ -3067,10 +3086,7 @@ async def test_async_total_anchored_to_server_arrival(self, async_video_client, ``VideoJob.request_started`` and closes ``total`` out in a background task, so the stamp and the end reading must stay on one clock. """ - import tensorrt_llm.serve.responses_utils as _ru - - real = _ru.get_steady_clock_now_in_seconds - monkeypatch.setattr(_ru, "get_steady_clock_now_in_seconds", lambda: real() - 5.0) + _set_deterministic_server_clock(monkeypatch) resp = await async_video_client.post( "/v1/videos", @@ -3089,7 +3105,7 @@ async def test_async_total_anchored_to_server_arrival(self, async_video_client, content = await async_video_client.get(f"/v1/videos/{video_id}/content") assert content.status_code == 200 - assert _server_timing_ms(content.headers, "total") >= 5000.0 + assert _server_timing_ms(content.headers, "total") == 5000.0 @pytest.mark.asyncio async def test_async_file_still_returns_file_response(self, async_video_client): diff --git a/tests/unittest/llmapi/test_llm.py b/tests/unittest/llmapi/test_llm.py index 65a936d9e890..fcea6626a701 100644 --- a/tests/unittest/llmapi/test_llm.py +++ b/tests/unittest/llmapi/test_llm.py @@ -28,6 +28,7 @@ import transformers from tensorrt_llm import LLM +from tensorrt_llm._utils import AdjustedSteadyClock from tensorrt_llm.bindings import executor as tllm from tensorrt_llm.executor import GenerationResultBase, RequestError from tensorrt_llm.llmapi import (KvCacheConfig, KvCacheRetentionConfig, @@ -1412,6 +1413,7 @@ async def run_request(): server.metrics_collector = None server._collect_perf_metrics = False server._input_proc_executor = None + server._adjusted_steady_clock = AdjustedSteadyClock() request = CompletionRequest(model="test-model", prompt=["A", "B"],