From 99ff98a81d14f8216dbabab3e147d84c8ca7517b Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Thu, 3 Sep 2026 12:30:05 -0700 Subject: [PATCH 1/4] [None][fix] Use the single custom-tokenizer alias table in llm_args `tensorrt_llm/llmapi/llm_args.py` kept its own copy of `TOKENIZER_ALIASES` next to the canonical table in `tensorrt_llm/tokenizer`, where built-in custom tokenizers register their aliases. The copy had drifted: it lacked `mistral_common`, so `LlmArgs(custom_tokenizer="mistral_common")` failed with "not enough values to unpack" (the unresolved alias was split as if it were a dotted import path) while the same alias loaded fine through `load_custom_tokenizer`. Import the canonical table instead of duplicating it, so every alias that `tensorrt_llm.tokenizer` knows also resolves through `LlmArgs`. The `llm_args.TOKENIZER_ALIASES` name is kept as a re-export for existing importers. Add a unit test that pins the two names to one table, checks every registered alias names an importable `TokenizerBase` subclass, constructs `TorchLlmArgs` with each alias (with the class's `from_pretrained` stubbed) and asserts the alias reaches that loader, and checks that an unknown identifier still fails with the existing error. The test is registered in `tests/integration/test_lists/test-db/l0_cpu.yml` so pre-merge CI runs it. Signed-off-by: Michal Guzek --- tensorrt_llm/llmapi/llm_args.py | 8 +- .../integration/test_lists/test-db/l0_cpu.yml | 1 + .../llmapi/test_custom_tokenizer_aliases.py | 80 +++++++++++++++++++ 3 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 tests/unittest/llmapi/test_custom_tokenizer_aliases.py diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 343cef0fbd63..dd50e3d78ba3 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -73,6 +73,7 @@ from ..mapping import CpType, Mapping from ..models.modeling_utils import QuantAlgo, QuantConfig from ..sampling_params import BatchedLogitsProcessor +from ..tokenizer import TOKENIZER_ALIASES # also exported from here from ..usage.config import UsageContext # noqa: F401 from ..usage.config import TelemetryConfig, TelemetryField from .tokenizer import TokenizerBase, tokenizer_factory @@ -1507,13 +1508,6 @@ class MoeConfig(StrictBaseModel): Nvfp4Backend = Literal['cutlass', 'cublaslt', 'cutedsl', 'cuda_core', 'marlin'] -# Short aliases for built-in custom tokenizers. -# Maps alias → full import path (module.ClassName). -TOKENIZER_ALIASES = { - 'deepseek_v32': 'tensorrt_llm.tokenizer.deepseek_v32.DeepseekV32Tokenizer', - 'deepseek_v4': 'tensorrt_llm.tokenizer.deepseek_v4.DeepseekV4Tokenizer', -} - class Nvfp4GemmConfig(StrictBaseModel): """Configuration for NVFP4 GEMM backend selection.""" diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 1fa059baca8d..63103c93622b 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -101,6 +101,7 @@ l0_cpu: - unittest/llmapi/test_whisper_suppress_tokens_processor.py - unittest/llmapi/test_kv_cache_dtype_override.py - unittest/llmapi/test_llm_args.py + - unittest/llmapi/test_custom_tokenizer_aliases.py - unittest/llmapi/test_llm_quant.py - unittest/llmapi/test_llm_telemetry.py - unittest/llmapi/test_llm_utils.py diff --git a/tests/unittest/llmapi/test_custom_tokenizer_aliases.py b/tests/unittest/llmapi/test_custom_tokenizer_aliases.py new file mode 100644 index 000000000000..bcee6a5e11e0 --- /dev/null +++ b/tests/unittest/llmapi/test_custom_tokenizer_aliases.py @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Custom-tokenizer alias resolution through ``LlmArgs``. + +``tensorrt_llm.tokenizer.TOKENIZER_ALIASES`` is the one place where built-in +custom tokenizers register a short alias. ``llm_args`` used to carry its own +copy of that table, and the copy drifted: an alias present only in the +canonical table loaded fine through ``load_custom_tokenizer`` but made +``LlmArgs(custom_tokenizer=)`` fail with "not enough values to +unpack", because the unresolved alias was split as if it were a dotted import +path. These tests pin the two tables to one object and drive every registered +alias through the ``LlmArgs`` validator. +""" + +import importlib +from unittest import mock + +import pytest + +import tensorrt_llm.llmapi.llm_args as llm_args_mod +from tensorrt_llm.llmapi.llm_args import TorchLlmArgs +from tensorrt_llm.llmapi.tokenizer import TokenizerBase +from tensorrt_llm.tokenizer import TOKENIZER_ALIASES + +pytestmark = pytest.mark.cpu_only + +DUMMY_MODEL = "/tmp/dummy_model" + + +def _resolve(alias: str): + module_path, class_name = TOKENIZER_ALIASES[alias].rsplit(".", 1) + return getattr(importlib.import_module(module_path), class_name) + + +def test_llm_args_uses_the_canonical_alias_table(): + """One table, not a copy that can drift.""" + assert llm_args_mod.TOKENIZER_ALIASES is TOKENIZER_ALIASES + + +@pytest.mark.parametrize("alias", sorted(TOKENIZER_ALIASES)) +def test_every_alias_names_an_importable_tokenizer_class(alias): + tokenizer_class = _resolve(alias) + assert issubclass(tokenizer_class, TokenizerBase) + assert callable(getattr(tokenizer_class, "from_pretrained", None)) + + +@pytest.mark.parametrize("alias", sorted(TOKENIZER_ALIASES)) +def test_llm_args_resolves_every_registered_alias(alias): + """``custom_tokenizer=`` reaches the aliased class's loader. + + ``from_pretrained`` is stubbed so no checkpoint is read; the point is that + the alias is resolved to the class rather than split as an import path. + """ + tokenizer_class = _resolve(alias) + loaded = mock.Mock(spec=TokenizerBase) + with mock.patch.object( + tokenizer_class, "from_pretrained", return_value=loaded + ) as from_pretrained: + args = TorchLlmArgs(model=DUMMY_MODEL, custom_tokenizer=alias) + + from_pretrained.assert_called_once() + assert from_pretrained.call_args.args[0] == DUMMY_MODEL + assert args.tokenizer is loaded + + +def test_unknown_custom_tokenizer_is_still_rejected(): + with pytest.raises(ValueError, match="Failed to load custom tokenizer"): + TorchLlmArgs(model=DUMMY_MODEL, custom_tokenizer="not_a_registered_alias") From fff728dc21c83376c7c0671e5ffbf424069816ac Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Thu, 3 Sep 2026 13:42:27 -0700 Subject: [PATCH 2/4] Annotate the return types of the alias tests Requested in review: every function carries a return annotation, so the helper returns `type[TokenizerBase]` and the test functions `None`. Signed-off-by: Michal Guzek --- tests/unittest/llmapi/test_custom_tokenizer_aliases.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unittest/llmapi/test_custom_tokenizer_aliases.py b/tests/unittest/llmapi/test_custom_tokenizer_aliases.py index bcee6a5e11e0..d5603b13ac89 100644 --- a/tests/unittest/llmapi/test_custom_tokenizer_aliases.py +++ b/tests/unittest/llmapi/test_custom_tokenizer_aliases.py @@ -39,25 +39,25 @@ DUMMY_MODEL = "/tmp/dummy_model" -def _resolve(alias: str): +def _resolve(alias: str) -> type[TokenizerBase]: module_path, class_name = TOKENIZER_ALIASES[alias].rsplit(".", 1) return getattr(importlib.import_module(module_path), class_name) -def test_llm_args_uses_the_canonical_alias_table(): +def test_llm_args_uses_the_canonical_alias_table() -> None: """One table, not a copy that can drift.""" assert llm_args_mod.TOKENIZER_ALIASES is TOKENIZER_ALIASES @pytest.mark.parametrize("alias", sorted(TOKENIZER_ALIASES)) -def test_every_alias_names_an_importable_tokenizer_class(alias): +def test_every_alias_names_an_importable_tokenizer_class(alias: str) -> None: tokenizer_class = _resolve(alias) assert issubclass(tokenizer_class, TokenizerBase) assert callable(getattr(tokenizer_class, "from_pretrained", None)) @pytest.mark.parametrize("alias", sorted(TOKENIZER_ALIASES)) -def test_llm_args_resolves_every_registered_alias(alias): +def test_llm_args_resolves_every_registered_alias(alias: str) -> None: """``custom_tokenizer=`` reaches the aliased class's loader. ``from_pretrained`` is stubbed so no checkpoint is read; the point is that @@ -75,6 +75,6 @@ def test_llm_args_resolves_every_registered_alias(alias): assert args.tokenizer is loaded -def test_unknown_custom_tokenizer_is_still_rejected(): +def test_unknown_custom_tokenizer_is_still_rejected() -> None: with pytest.raises(ValueError, match="Failed to load custom tokenizer"): TorchLlmArgs(model=DUMMY_MODEL, custom_tokenizer="not_a_registered_alias") From e68d8ee6a42f061a19079015a6b69e5e59248bd9 Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Thu, 3 Sep 2026 13:46:48 -0700 Subject: [PATCH 3/4] Document the alias test helpers Short docstrings for the helper and the two test functions that had none. Signed-off-by: Michal Guzek --- tests/unittest/llmapi/test_custom_tokenizer_aliases.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unittest/llmapi/test_custom_tokenizer_aliases.py b/tests/unittest/llmapi/test_custom_tokenizer_aliases.py index d5603b13ac89..aa3d4024ef4e 100644 --- a/tests/unittest/llmapi/test_custom_tokenizer_aliases.py +++ b/tests/unittest/llmapi/test_custom_tokenizer_aliases.py @@ -40,6 +40,7 @@ def _resolve(alias: str) -> type[TokenizerBase]: + """Import the tokenizer class an alias maps to.""" module_path, class_name = TOKENIZER_ALIASES[alias].rsplit(".", 1) return getattr(importlib.import_module(module_path), class_name) @@ -51,6 +52,7 @@ def test_llm_args_uses_the_canonical_alias_table() -> None: @pytest.mark.parametrize("alias", sorted(TOKENIZER_ALIASES)) def test_every_alias_names_an_importable_tokenizer_class(alias: str) -> None: + """Each alias target is an importable ``TokenizerBase`` with a loader.""" tokenizer_class = _resolve(alias) assert issubclass(tokenizer_class, TokenizerBase) assert callable(getattr(tokenizer_class, "from_pretrained", None)) @@ -76,5 +78,6 @@ def test_llm_args_resolves_every_registered_alias(alias: str) -> None: def test_unknown_custom_tokenizer_is_still_rejected() -> None: + """An identifier that is neither an alias nor an import path errors out.""" with pytest.raises(ValueError, match="Failed to load custom tokenizer"): TorchLlmArgs(model=DUMMY_MODEL, custom_tokenizer="not_a_registered_alias") From 1e4dfbdf2653e39596a6ade807e88e2236bafe42 Mon Sep 17 00:00:00 2001 From: Michal Guzek Date: Thu, 3 Sep 2026 15:31:04 -0700 Subject: [PATCH 4/4] Use single backticks for code names in the alias test docstrings Signed-off-by: Michal Guzek --- .../llmapi/test_custom_tokenizer_aliases.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/unittest/llmapi/test_custom_tokenizer_aliases.py b/tests/unittest/llmapi/test_custom_tokenizer_aliases.py index aa3d4024ef4e..7a8ad4efafde 100644 --- a/tests/unittest/llmapi/test_custom_tokenizer_aliases.py +++ b/tests/unittest/llmapi/test_custom_tokenizer_aliases.py @@ -12,16 +12,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Custom-tokenizer alias resolution through ``LlmArgs``. +"""Custom-tokenizer alias resolution through `LlmArgs`. -``tensorrt_llm.tokenizer.TOKENIZER_ALIASES`` is the one place where built-in -custom tokenizers register a short alias. ``llm_args`` used to carry its own +`tensorrt_llm.tokenizer.TOKENIZER_ALIASES` is the one place where built-in +custom tokenizers register a short alias. `llm_args` used to carry its own copy of that table, and the copy drifted: an alias present only in the -canonical table loaded fine through ``load_custom_tokenizer`` but made -``LlmArgs(custom_tokenizer=)`` fail with "not enough values to +canonical table loaded fine through `load_custom_tokenizer` but made +`LlmArgs(custom_tokenizer=)` fail with "not enough values to unpack", because the unresolved alias was split as if it were a dotted import path. These tests pin the two tables to one object and drive every registered -alias through the ``LlmArgs`` validator. +alias through the `LlmArgs` validator. """ import importlib @@ -52,7 +52,7 @@ def test_llm_args_uses_the_canonical_alias_table() -> None: @pytest.mark.parametrize("alias", sorted(TOKENIZER_ALIASES)) def test_every_alias_names_an_importable_tokenizer_class(alias: str) -> None: - """Each alias target is an importable ``TokenizerBase`` with a loader.""" + """Each alias target is an importable `TokenizerBase` with a loader.""" tokenizer_class = _resolve(alias) assert issubclass(tokenizer_class, TokenizerBase) assert callable(getattr(tokenizer_class, "from_pretrained", None)) @@ -60,9 +60,9 @@ def test_every_alias_names_an_importable_tokenizer_class(alias: str) -> None: @pytest.mark.parametrize("alias", sorted(TOKENIZER_ALIASES)) def test_llm_args_resolves_every_registered_alias(alias: str) -> None: - """``custom_tokenizer=`` reaches the aliased class's loader. + """`custom_tokenizer=` reaches the aliased class's loader. - ``from_pretrained`` is stubbed so no checkpoint is read; the point is that + `from_pretrained` is stubbed so no checkpoint is read; the point is that the alias is resolved to the class rather than split as an import path. """ tokenizer_class = _resolve(alias)