From f3fca65544d37bceeed1b2e9f4f52810a95ffd79 Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Wed, 16 Sep 2026 09:29:31 -0700 Subject: [PATCH 01/12] Add IQ post-training quantization recipes Signed-off-by: Hung-Yueh Chiang --- modelopt_recipes/configs/numerics/iq1_s.yaml | 14 +++++++++++++ modelopt_recipes/configs/numerics/iq2_xs.yaml | 14 +++++++++++++ .../configs/ptq/presets/model/iq1_s.yaml | 20 +++++++++++++++++++ .../configs/ptq/presets/model/iq2_xs.yaml | 20 +++++++++++++++++++ modelopt_recipes/general/ptq/iq1_s.yaml | 15 ++++++++++++++ modelopt_recipes/general/ptq/iq2_xs.yaml | 15 ++++++++++++++ modelopt_recipes/ptq.md | 8 +++++++- 7 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 modelopt_recipes/configs/numerics/iq1_s.yaml create mode 100644 modelopt_recipes/configs/numerics/iq2_xs.yaml create mode 100644 modelopt_recipes/configs/ptq/presets/model/iq1_s.yaml create mode 100644 modelopt_recipes/configs/ptq/presets/model/iq2_xs.yaml create mode 100644 modelopt_recipes/general/ptq/iq1_s.yaml create mode 100644 modelopt_recipes/general/ptq/iq2_xs.yaml diff --git a/modelopt_recipes/configs/numerics/iq1_s.yaml b/modelopt_recipes/configs/numerics/iq1_s.yaml new file mode 100644 index 00000000000..4f126417c22 --- /dev/null +++ b/modelopt_recipes/configs/numerics/iq1_s.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# IQ1_S weight quantizer using the built-in GGML IQ auto search. + +# modelopt-schema: modelopt.torch.quantization.config.QuantizerAttributeConfig +num_bits: iq1_s +# Storage cost: 50 packed bytes * 8 / 256 weights. Keep in sync with IQ1_S_BLOCK_BYTES. +effective_bits: 1.5625 +block_sizes: + -1: 256 +backend: ggml +backend_extra_args: + search_impl: auto diff --git a/modelopt_recipes/configs/numerics/iq2_xs.yaml b/modelopt_recipes/configs/numerics/iq2_xs.yaml new file mode 100644 index 00000000000..a05622d2775 --- /dev/null +++ b/modelopt_recipes/configs/numerics/iq2_xs.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# IQ2_XS weight quantizer using the built-in GGML IQ auto search. + +# modelopt-schema: modelopt.torch.quantization.config.QuantizerAttributeConfig +num_bits: iq2_xs +# Storage cost: 74 packed bytes * 8 / 256 weights. Keep in sync with IQ2_XS_BLOCK_BYTES. +effective_bits: 2.3125 +block_sizes: + -1: 256 +backend: ggml +backend_extra_args: + search_impl: auto diff --git a/modelopt_recipes/configs/ptq/presets/model/iq1_s.yaml b/modelopt_recipes/configs/ptq/presets/model/iq1_s.yaml new file mode 100644 index 00000000000..31fab0f7a44 --- /dev/null +++ b/modelopt_recipes/configs/ptq/presets/model/iq1_s.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# QuantizeConfig preset for IQ1_S weight-only quantization. + +# modelopt-schema: modelopt.torch.quantization.config.QuantizeConfig +imports: + base_disable_all: configs/ptq/units/base_disable_all + default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers + iq1_s: configs/numerics/iq1_s + +algorithm: +quant_cfg: + - $import: base_disable_all + - quantizer_name: '*weight_quantizer' + cfg: + $import: iq1_s + - quantizer_name: '*input_quantizer' + enable: false + - $import: default_disabled_quantizers diff --git a/modelopt_recipes/configs/ptq/presets/model/iq2_xs.yaml b/modelopt_recipes/configs/ptq/presets/model/iq2_xs.yaml new file mode 100644 index 00000000000..96938c79f09 --- /dev/null +++ b/modelopt_recipes/configs/ptq/presets/model/iq2_xs.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# QuantizeConfig preset for IQ2_XS weight-only quantization. + +# modelopt-schema: modelopt.torch.quantization.config.QuantizeConfig +imports: + base_disable_all: configs/ptq/units/base_disable_all + default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers + iq2_xs: configs/numerics/iq2_xs + +algorithm: +quant_cfg: + - $import: base_disable_all + - quantizer_name: '*weight_quantizer' + cfg: + $import: iq2_xs + - quantizer_name: '*input_quantizer' + enable: false + - $import: default_disabled_quantizers diff --git a/modelopt_recipes/general/ptq/iq1_s.yaml b/modelopt_recipes/general/ptq/iq1_s.yaml new file mode 100644 index 00000000000..0eb80fcf29d --- /dev/null +++ b/modelopt_recipes/general/ptq/iq1_s.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# IQ1_S weight-only PTQ. + +imports: + preset: configs/ptq/presets/model/iq1_s + +metadata: + recipe_type: ptq + description: >- + Applies uniform GGML-compatible IQ1_S weight-only quantization to eligible linear layers. + This is not a mixed per-tensor precision preset. No calibration data is required. +quantize: + $import: preset diff --git a/modelopt_recipes/general/ptq/iq2_xs.yaml b/modelopt_recipes/general/ptq/iq2_xs.yaml new file mode 100644 index 00000000000..c0edc2d5351 --- /dev/null +++ b/modelopt_recipes/general/ptq/iq2_xs.yaml @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# IQ2_XS weight-only PTQ. + +imports: + preset: configs/ptq/presets/model/iq2_xs + +metadata: + recipe_type: ptq + description: >- + Applies uniform GGML-compatible IQ2_XS weight-only quantization to eligible linear layers. + This is not a mixed per-tensor precision preset. No calibration data is required. +quantize: + $import: preset diff --git a/modelopt_recipes/ptq.md b/modelopt_recipes/ptq.md index 2ad373ffcab..d2738aef8a0 100644 --- a/modelopt_recipes/ptq.md +++ b/modelopt_recipes/ptq.md @@ -28,7 +28,7 @@ supported combinations. ### The shipped recipes
-All 26 general/ptq/ recipes (click to expand) +All 28 general/ptq/ recipes (click to expand) | Recipe | Model body | KV cache | Calibration | |--------|-----------|----------|-------------| @@ -58,6 +58,8 @@ supported combinations. | `int4_blockwise_weight_only` | INT4 W4A16, block 128, weights only | none | max | | `nvfp4_mlp_weight_only` | NVFP4 W4A16 (block 32), MLP + MoE weights only | none | max | | `mxfp4_mlp_weight_only` | MXFP4 W4A16, MLP + MoE weights only | none | none (no calibration) | +| `iq1_s` | IQ1_S W1A16, all linears | none | GGML IQ auto search (no calibration) | +| `iq2_xs` | IQ2_XS W2A16, all linears | none | GGML IQ auto search (no calibration) |
@@ -136,6 +138,10 @@ activations and tensor-core math are what deliver the throughput. - **`mxfp4_mlp_weight_only`** — MXFP4 weights on MLP/MoE layers only, BF16 activations. Needs no calibration forward pass; the QAT starting point for the GPT-OSS family (see `examples/gpt-oss`). +- **`iq1_s` / `iq2_xs`** — GGML-compatible IQ1_S or IQ2_XS weights on all linear + layers, with BF16 activations. No calibration data is required. Quantized weights must have a + final dimension divisible by 256. These recipes configure simulated weight quantization only; + packed checkpoint export is added separately. --- From b9dfae349472fdd68752f1e8f28be57f2d592ce5 Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Wed, 16 Sep 2026 14:55:37 -0700 Subject: [PATCH 02/12] Document IQ quantization recipes Signed-off-by: Hung-Yueh Chiang --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 46168b7fe0f..5d54114f457 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,7 @@ Changelog *Quantization* +- Add IQ1_S and IQ2_XS weight-only fake quantization with GGML-compatible 256-value block encoders and built-in ``iq1_s`` / ``iq2_xs`` PTQ recipes. - Add ``layerwise.export_dir``: layerwise calibration writes each decoder layer to its own quantized checkpoint shard as it finishes, so no separate ``export_hf_checkpoint()`` pass is needed and, with ``layerwise.checkpoint_dir``, an interrupted run resumes without redoing finished layers. Calibration writes the layer shards; ``finalize()`` on the exporter left on the model adds the tail shard, the index and the config artifacts, and the checkpoint does not load until it runs. ``examples/hf_ptq`` does this for you. Supports FP8 and NVFP4 on single-process models, resident or offloaded, including multimodal models and models with MTP layers; other formats and placements raise ``NotImplementedError`` before calibration starts. - Add support for quantizing and calibrating enabled operators outside the transformer layers, such as ``lm_head``, when using layerwise calibration. - Add an end-to-end BEVFormer ONNX PTQ example with temporal calibration data generation, INT8 and FP8 quantization, TensorRT engine building, and nuScenes accuracy evaluation. See `examples/onnx_ptq/bevformer/README.md `_ for details. From d658c5f2550b478385aa066f089670f7a66badff Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Wed, 16 Sep 2026 16:32:01 -0700 Subject: [PATCH 03/12] Remove unused IQ search recipe option Signed-off-by: Hung-Yueh Chiang --- modelopt_recipes/configs/numerics/iq1_s.yaml | 4 +--- modelopt_recipes/configs/numerics/iq2_xs.yaml | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/modelopt_recipes/configs/numerics/iq1_s.yaml b/modelopt_recipes/configs/numerics/iq1_s.yaml index 4f126417c22..b1aaf4a8af5 100644 --- a/modelopt_recipes/configs/numerics/iq1_s.yaml +++ b/modelopt_recipes/configs/numerics/iq1_s.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# IQ1_S weight quantizer using the built-in GGML IQ auto search. +# IQ1_S weight quantizer using the built-in fixed-scale codebook search. # modelopt-schema: modelopt.torch.quantization.config.QuantizerAttributeConfig num_bits: iq1_s @@ -10,5 +10,3 @@ effective_bits: 1.5625 block_sizes: -1: 256 backend: ggml -backend_extra_args: - search_impl: auto diff --git a/modelopt_recipes/configs/numerics/iq2_xs.yaml b/modelopt_recipes/configs/numerics/iq2_xs.yaml index a05622d2775..353bca0471d 100644 --- a/modelopt_recipes/configs/numerics/iq2_xs.yaml +++ b/modelopt_recipes/configs/numerics/iq2_xs.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# IQ2_XS weight quantizer using the built-in GGML IQ auto search. +# IQ2_XS weight quantizer using the built-in fixed-scale codebook search. # modelopt-schema: modelopt.torch.quantization.config.QuantizerAttributeConfig num_bits: iq2_xs @@ -10,5 +10,3 @@ effective_bits: 2.3125 block_sizes: -1: 256 backend: ggml -backend_extra_args: - search_impl: auto From a2849c6f31f9088e4e13031b0ab601271b673672 Mon Sep 17 00:00:00 2001 From: Hung-Yueh Chiang Date: Fri, 18 Sep 2026 13:36:58 -0700 Subject: [PATCH 04/12] [OMNIML-5899] Pin IQ recipe packing contracts Signed-off-by: Hung-Yueh Chiang --- modelopt_recipes/configs/numerics/iq1_s.yaml | 12 +++++++++ modelopt_recipes/configs/numerics/iq2_xs.yaml | 12 +++++++++ .../configs/ptq/presets/model/iq1_s.yaml | 12 +++++++++ .../configs/ptq/presets/model/iq2_xs.yaml | 12 +++++++++ modelopt_recipes/general/ptq/iq1_s.yaml | 12 +++++++++ modelopt_recipes/general/ptq/iq2_xs.yaml | 12 +++++++++ tests/unit/recipe/test_presets.py | 27 +++++++++++++++++++ 7 files changed, 99 insertions(+) diff --git a/modelopt_recipes/configs/numerics/iq1_s.yaml b/modelopt_recipes/configs/numerics/iq1_s.yaml index b1aaf4a8af5..ec96dea7e47 100644 --- a/modelopt_recipes/configs/numerics/iq1_s.yaml +++ b/modelopt_recipes/configs/numerics/iq1_s.yaml @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # IQ1_S weight quantizer using the built-in fixed-scale codebook search. diff --git a/modelopt_recipes/configs/numerics/iq2_xs.yaml b/modelopt_recipes/configs/numerics/iq2_xs.yaml index 353bca0471d..837943f1be3 100644 --- a/modelopt_recipes/configs/numerics/iq2_xs.yaml +++ b/modelopt_recipes/configs/numerics/iq2_xs.yaml @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # IQ2_XS weight quantizer using the built-in fixed-scale codebook search. diff --git a/modelopt_recipes/configs/ptq/presets/model/iq1_s.yaml b/modelopt_recipes/configs/ptq/presets/model/iq1_s.yaml index 31fab0f7a44..57bddbe7180 100644 --- a/modelopt_recipes/configs/ptq/presets/model/iq1_s.yaml +++ b/modelopt_recipes/configs/ptq/presets/model/iq1_s.yaml @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # QuantizeConfig preset for IQ1_S weight-only quantization. diff --git a/modelopt_recipes/configs/ptq/presets/model/iq2_xs.yaml b/modelopt_recipes/configs/ptq/presets/model/iq2_xs.yaml index 96938c79f09..4fcf1c29213 100644 --- a/modelopt_recipes/configs/ptq/presets/model/iq2_xs.yaml +++ b/modelopt_recipes/configs/ptq/presets/model/iq2_xs.yaml @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # QuantizeConfig preset for IQ2_XS weight-only quantization. diff --git a/modelopt_recipes/general/ptq/iq1_s.yaml b/modelopt_recipes/general/ptq/iq1_s.yaml index 0eb80fcf29d..a3c98b92df3 100644 --- a/modelopt_recipes/general/ptq/iq1_s.yaml +++ b/modelopt_recipes/general/ptq/iq1_s.yaml @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # IQ1_S weight-only PTQ. diff --git a/modelopt_recipes/general/ptq/iq2_xs.yaml b/modelopt_recipes/general/ptq/iq2_xs.yaml index c0edc2d5351..6a9a08d17a9 100644 --- a/modelopt_recipes/general/ptq/iq2_xs.yaml +++ b/modelopt_recipes/general/ptq/iq2_xs.yaml @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # IQ2_XS weight-only PTQ. diff --git a/tests/unit/recipe/test_presets.py b/tests/unit/recipe/test_presets.py index df90fcfd7d3..2bf899cf650 100644 --- a/tests/unit/recipe/test_presets.py +++ b/tests/unit/recipe/test_presets.py @@ -33,6 +33,12 @@ from modelopt.recipe.presets import RecipeSupersededAction from modelopt.torch.opt.config_loader import BUILTIN_CONFIG_ROOT from modelopt.torch.quantization.config import LocalHessianCalibConfig, QuantizeConfig +from modelopt.torch.quantization.ggml import ( + IQ1_S_BLOCK_SIZE, + IQ1_S_EFFECTIVE_BITS, + IQ2_XS_BLOCK_SIZE, + IQ2_XS_EFFECTIVE_BITS, +) def _yaml_basenames(subdir: str) -> set[str]: @@ -125,6 +131,27 @@ def test_mlp_weight_only_recipe_matches_its_mtq_cfg(recipe_name, cfg_name): assert recipe_cfg == mtq_cfg +@pytest.mark.parametrize( + ("qformat", "block_size", "effective_bits"), + [ + ("iq1_s", IQ1_S_BLOCK_SIZE, IQ1_S_EFFECTIVE_BITS), + ("iq2_xs", IQ2_XS_BLOCK_SIZE, IQ2_XS_EFFECTIVE_BITS), + ], +) +def test_iq_recipe_matches_packing_contract(qformat, block_size, effective_bits): + recipe = load_recipe(f"general/ptq/{qformat}") + quant_cfg = recipe.quantize.model_dump(exclude_unset=True)["quant_cfg"] + weight_cfg = next( + entry["cfg"] for entry in quant_cfg if entry.get("quantizer_name") == "*weight_quantizer" + ) + + assert qformat in presets.QUANT_CFG_CHOICES + assert weight_cfg["backend"] == "ggml" + assert weight_cfg["num_bits"] == qformat + assert weight_cfg["block_sizes"][-1] == block_size + assert weight_cfg["effective_bits"] == effective_bits + + # --- RecipeSupersededAction: the flags --recipe replaces ---------------------------------------- From 6b5d60362f318a3c4aab107cff5e2a07a7a64106 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 21 Sep 2026 16:40:04 +0000 Subject: [PATCH 05/12] Update the IQ recipe docs now that export has landed Both statements were written while #2447 was still open and are stale as of its merge in ed5c5ed36. ptq.md claimed these recipes "configure simulated weight quantization only; packed checkpoint export is added separately". Unified HF export now writes the packed GGML blocks, so say that instead, along with the two limits a recipe user will otherwise discover by hitting an exception: Megatron export requires tensor and pipeline parallel sizes of 1, and does not support fused-MoE experts. The CHANGELOG entry said "fake quantization". This PR carries the only entry for the whole feature -- #2446, #2447 and #2448 each added none -- so it has to describe the shipped surface, which now includes export. Verified against main rather than the PR descriptions: the TP and PP guards and the fused-MoE rejection are present in unified_export_megatron.py, and unified_export_hf.py calls quantize_iq1_s/quantize_iq2_xs. No rebase was needed -- the branch already carries main at b311c054d. The recipe configs check out: the validator loads both recipes cleanly and the 12 preset tests pass, including the one asserting the recipe matches IQ1_S_BLOCK_SIZE / IQ1_S_EFFECTIVE_BITS. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 2 +- modelopt_recipes/ptq.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 16e1f9fe355..70d6bbc0359 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,7 +13,7 @@ Changelog *Quantization* -- Add IQ1_S and IQ2_XS weight-only fake quantization with GGML-compatible 256-value block encoders and built-in ``iq1_s`` / ``iq2_xs`` PTQ recipes. +- Add IQ1_S and IQ2_XS weight-only quantization with GGML-compatible 256-value block encoders, built-in ``iq1_s`` / ``iq2_xs`` PTQ recipes, and unified HF and Megatron export of the packed blocks. Quantized weights must have a final dimension divisible by 256, and Megatron export requires tensor and pipeline parallel sizes of 1. - Add ``layerwise.export_dir``: layerwise calibration writes each decoder layer to its own quantized checkpoint shard as it finishes, so no separate ``export_hf_checkpoint()`` pass is needed and, with ``layerwise.checkpoint_dir``, an interrupted run resumes without redoing finished layers. Calibration writes the layer shards; ``finalize()`` on the exporter left on the model adds the tail shard, the index and the config artifacts, and the checkpoint does not load until it runs. ``examples/hf_ptq`` does this for you. Supports FP8 and NVFP4 on single-process models, resident or offloaded, including multimodal models and models with MTP layers; other formats and placements raise ``NotImplementedError`` before calibration starts. - Add support for quantizing and calibrating enabled operators outside the transformer layers, such as ``lm_head``, when using layerwise calibration. - Add an end-to-end BEVFormer ONNX PTQ example with temporal calibration data generation, INT8 and FP8 quantization, TensorRT engine building, and nuScenes accuracy evaluation. See `examples/onnx_ptq/bevformer/README.md `_ for details. diff --git a/modelopt_recipes/ptq.md b/modelopt_recipes/ptq.md index d2738aef8a0..683690b0a15 100644 --- a/modelopt_recipes/ptq.md +++ b/modelopt_recipes/ptq.md @@ -140,8 +140,9 @@ activations and tensor-core math are what deliver the throughput. GPT-OSS family (see `examples/gpt-oss`). - **`iq1_s` / `iq2_xs`** — GGML-compatible IQ1_S or IQ2_XS weights on all linear layers, with BF16 activations. No calibration data is required. Quantized weights must have a - final dimension divisible by 256. These recipes configure simulated weight quantization only; - packed checkpoint export is added separately. + final dimension divisible by 256. Unified HF export writes the packed GGML blocks; Megatron + export additionally requires tensor and pipeline parallel sizes of 1, and does not support + fused-MoE experts. --- From eaee22741ac66d0c6aed6a711e2da1d44245ca6d Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 21 Sep 2026 18:53:01 +0000 Subject: [PATCH 06/12] Add end-to-end hf_ptq coverage for IQ1_S and IQ2_XS The formats had unit, GPU-kernel and export coverage but nothing driving the whole pipeline the way a user does. These two cases run scripts/huggingface_example.sh through the shipped recipes, so they cover recipe resolution, quantization and checkpoint export together. Driven by recipe= rather than quant=, since that is how these formats are exposed -- there is no --quant name for them. That path is already exercised by the auto_quantize entry above. TinyLlama-1.1B is a valid target: these encoders require every weight's input dimension to be a multiple of 256, and its hidden_size 2048 and intermediate_size 5632 both are, as is the 32000 vocab. Checked against the published config rather than assumed, since a non-conforming layer would make validate_weight raise rather than skip. kv_cache_quant="none" matches the neighbouring weight-only entries; the IQ recipes disable input quantizers. Verified locally that both PTQCommands construct, produce sane test ids, and that their recipes resolve to num_bits/backend/block_sizes/effective_bits matching the packing contract. The runs themselves need CI: this environment cannot execute them, its transformers install is broken and pytest collection of the examples tree fails on a libcufile/GLIBCXX link error. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- tests/examples/hf_ptq/test_llm_ptq.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/examples/hf_ptq/test_llm_ptq.py b/tests/examples/hf_ptq/test_llm_ptq.py index d307868c372..9d5e07872f7 100644 --- a/tests/examples/hf_ptq/test_llm_ptq.py +++ b/tests/examples/hf_ptq/test_llm_ptq.py @@ -76,6 +76,10 @@ def test_ptq_whisper(command): PTQCommand(quant="int8_weight_only", kv_cache_quant="none"), PTQCommand(quant="int4_awq", kv_cache_quant="none"), PTQCommand(quant="w4a8_awq_beta", kv_cache_quant="none"), + # GGML IQ weight-only, recipe-driven. These encoders require every weight's input + # dimension to be a multiple of 256; TinyLlama's 2048 and 5632 both are. + PTQCommand(recipe="general/ptq/iq1_s", kv_cache_quant="none"), + PTQCommand(recipe="general/ptq/iq2_xs", kv_cache_quant="none"), PTQCommand(quant="nvfp4"), PTQCommand(quant="nvfp4_awq_lite"), # autoquant (recipe-driven) From 80209353622b42c9117b833fcb6718217f557a4b Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 21 Sep 2026 20:26:26 +0000 Subject: [PATCH 07/12] Give the IQ end-to-end cases an explicit timeout Verified on 2xH100 via the debug server: both cases pass, 2 passed in 1097.46s (18m17s), covering quantization and checkpoint export through scripts/huggingface_example.sh. They need more than the 300s default that tests/conftest.py applies to tests/examples/, which they were failing on. 1800s each leaves headroom; an explicit marker wins over the directory default by design. The cost is the codebook search running over all 556 quantizers, once on the first calibration forward and again at export. It is not a CPU fallback -- instrumenting quantize_iq1_s inside a real run shows device=cuda:0 and ext_loaded=True, so the packing does use the CUDA extension. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- tests/examples/hf_ptq/test_llm_ptq.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/examples/hf_ptq/test_llm_ptq.py b/tests/examples/hf_ptq/test_llm_ptq.py index 9d5e07872f7..4b488fde023 100644 --- a/tests/examples/hf_ptq/test_llm_ptq.py +++ b/tests/examples/hf_ptq/test_llm_ptq.py @@ -78,8 +78,17 @@ def test_ptq_whisper(command): PTQCommand(quant="w4a8_awq_beta", kv_cache_quant="none"), # GGML IQ weight-only, recipe-driven. These encoders require every weight's input # dimension to be a multiple of 256; TinyLlama's 2048 and 5632 both are. - PTQCommand(recipe="general/ptq/iq1_s", kv_cache_quant="none"), - PTQCommand(recipe="general/ptq/iq2_xs", kv_cache_quant="none"), + # Measured 18m17s for the pair on 2xH100, over the 300s tests/examples default. The + # packing does run on GPU; the cost is the codebook search over every linear, once on + # the first calibration forward and again at export. + pytest.param( + PTQCommand(recipe="general/ptq/iq1_s", kv_cache_quant="none"), + marks=pytest.mark.timeout(1800), + ), + pytest.param( + PTQCommand(recipe="general/ptq/iq2_xs", kv_cache_quant="none"), + marks=pytest.mark.timeout(1800), + ), PTQCommand(quant="nvfp4"), PTQCommand(quant="nvfp4_awq_lite"), # autoquant (recipe-driven) From 3390b98895b701ee90cea4cd0ca98ccac650b078 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 21 Sep 2026 22:24:01 +0000 Subject: [PATCH 08/12] Reuse the packed IQ weight across forwards instead of re-encoding it Note this touches merged #2446 code rather than the recipes this PR is about; carried here deliberately to keep it in one review. The packed-weight cache never hit. TensorQuantizer hands the backend a fresh view of the weight on every forward, so weakref.ref(inputs) pointed at a temporary that died as soon as that forward returned, and the ``cache.input_ref() is inputs`` guard could never match again. Measured with an instrumented run: ref_alive=False on every call while key_equal=True on every call -- the part that actually identifies the weight matched, and the identity check threw the result away. So the codebook search reran on every forward. On TinyLlama-1.1B: 154 weights packed exactly 100 times each, 15400 calls, 96.89B elements encoded for a 1.03B-parameter model, 499s of a 537s hf_ptq run. Per weight, IQ1_S costs 30.96 ms per forward against NVFP4's 0.27 ms, so the shared per-forward fake-quant design that is free for NVFP4 is ruinous here. Hold a strong reference and drop the identity check. The weakref was guarding against data_ptr recycling; a strong reference does that properly by pinning the storage, and costs nothing since the storage is the module's weight. Measured on an H100, 21 forwards of one 2048x2048 weight: before: packs=21/21, 30.00 ms/forward after: packs= 1/21, 5.37 ms/forward The residue is the dequantize, which genuinely does run every forward. Tests: one per format asserting a single pack across five forwards under inference_mode, which is what generate() runs under and where this bites. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- modelopt/torch/quantization/ggml/common.py | 18 +++++++--- .../torch/quantization/test_ggml_backend.py | 33 +++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/modelopt/torch/quantization/ggml/common.py b/modelopt/torch/quantization/ggml/common.py index 80e8cf2e2e2..e84b0c2f69a 100644 --- a/modelopt/torch/quantization/ggml/common.py +++ b/modelopt/torch/quantization/ggml/common.py @@ -16,7 +16,6 @@ """Shared validation for GGML-compatible block quantizers.""" import math -import weakref from collections.abc import Callable from dataclasses import dataclass @@ -27,7 +26,19 @@ @dataclass class _PackedWeightCache: - input_ref: weakref.ReferenceType + """One weight's packed payload, reused across forwards. + + ``input_tensor`` is a strong reference on purpose. It pins the storage, so the ``data_ptr`` + inside ``input_key`` cannot be recycled by a later allocation while this entry is live -- + which is the hazard a weakref was meant to cover. It costs nothing in practice: the storage + belongs to the module's weight, which outlives the quantizer anyway. + + A weakref cannot do that job here. TensorQuantizer hands the backend a fresh view of the + weight on every forward, so a weakref to it dies as soon as that forward returns, and an + identity check against it never matches again. + """ + + input_tensor: torch.Tensor input_key: tuple[object, ...] format_name: str block_chunk_size: int @@ -66,7 +77,6 @@ def fake_quantize_with_cache( if ( isinstance(cache, _PackedWeightCache) and input_key is not None - and cache.input_ref() is inputs and cache.input_key == input_key and cache.format_name == format_name and cache.block_chunk_size == block_chunk_size @@ -76,7 +86,7 @@ def fake_quantize_with_cache( packed_weights, weight_shape = quantize(inputs, block_chunk_size=block_chunk_size) if input_key is not None: quantizer._quantizer_cache = _PackedWeightCache( - input_ref=weakref.ref(inputs), + input_tensor=inputs, input_key=input_key, format_name=format_name, block_chunk_size=block_chunk_size, diff --git a/tests/unit/torch/quantization/test_ggml_backend.py b/tests/unit/torch/quantization/test_ggml_backend.py index ee78bab5713..8820a968c9a 100644 --- a/tests/unit/torch/quantization/test_ggml_backend.py +++ b/tests/unit/torch/quantization/test_ggml_backend.py @@ -22,8 +22,10 @@ import modelopt.torch.quantization.ggml.backend as backend_module import modelopt.torch.quantization.ggml.iq1_s as iq1_s_module import modelopt.torch.quantization.ggml.iq2_xs as iq2_xs_module +from modelopt.torch.quantization.config import QuantizerAttributeConfig from modelopt.torch.quantization.ggml.backend import ggml_fake_quant from modelopt.torch.quantization.ggml.common import narrow_to_float32 +from modelopt.torch.quantization.nn import TensorQuantizer @pytest.mark.parametrize("num_bits", ["iq1_s", "iq2_xs"]) @@ -135,3 +137,34 @@ def test_narrow_to_float32_matches_the_cuda_load_float_policy(): assert torch.equal( narrowed, torch.tensor([0.0, 0.0, 0.0, largest, -largest, 1.5], dtype=torch.float32) ) + + +@pytest.mark.parametrize( + ("num_bits", "module"), [("iq1_s", iq1_s_module), ("iq2_xs", iq2_xs_module)] +) +def test_ggml_weight_is_packed_once_across_forwards(monkeypatch, num_bits, module): + """The packed weight is reused across forwards rather than re-encoded each time. + + TensorQuantizer hands the backend a fresh view of the weight on every forward, so a cache + that checked tensor identity never hit: the codebook search reran on every forward, roughly + 100x during a generate loop and over 90% of a PTQ run's wall clock. + """ + packer = f"quantize_{num_bits}" + original = getattr(module, packer) + calls = [] + + def counting(weight, **kwargs): + calls.append(tuple(weight.shape)) + return original(weight, **kwargs) + + monkeypatch.setattr(module, packer, counting) + quantizer = TensorQuantizer( + QuantizerAttributeConfig(num_bits=num_bits, block_sizes={-1: 256}, backend="ggml") + ) + weight = torch.randn(4, 256) + + with torch.inference_mode(): # what generate() runs under + for _ in range(5): + quantizer(weight) + + assert calls == [(4, 256)], f"expected one pack, got {len(calls)}" From f11eb630a9741af232a1bae841e1bb1889d72cba Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 21 Sep 2026 23:15:26 +0000 Subject: [PATCH 09/12] Tie the packed IQ weight cache to the parameter, not the forward's view The previous entry held a strong reference to the tensor the backend was handed. That pins full-precision storage for the life of the quantizer and defeats offloaded and meta-device flows, where releasing the weight after packing is the point. Hold a weakref to inputs._base instead -- the parameter the per-forward view is taken from. It is stable across forwards, so the identity check matches and the cache hits; and it is weak, so the payload stops being reused exactly when the weight it came from is released. Measured on TinyLlama IQ1_S hf_ptq (2xH100): packer calls 15400 -> 154 (one per weight), packing 499.0s -> 6.2s, wall clock 8m57s -> 3m21s. Also drop the stale W4A16 label from the weight-only heading in ptq.md, which now covers W1A16 and W2A16 too, and stop describing the IQ recipes' calibration column as an "auto search" -- those recipes set algorithm: null and run no calibration pass at all. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- modelopt/torch/quantization/ggml/common.py | 31 +++++++++++++++------- modelopt_recipes/ptq.md | 6 ++--- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/modelopt/torch/quantization/ggml/common.py b/modelopt/torch/quantization/ggml/common.py index e84b0c2f69a..9f5bdf14662 100644 --- a/modelopt/torch/quantization/ggml/common.py +++ b/modelopt/torch/quantization/ggml/common.py @@ -16,6 +16,7 @@ """Shared validation for GGML-compatible block quantizers.""" import math +import weakref from collections.abc import Callable from dataclasses import dataclass @@ -28,17 +29,17 @@ class _PackedWeightCache: """One weight's packed payload, reused across forwards. - ``input_tensor`` is a strong reference on purpose. It pins the storage, so the ``data_ptr`` - inside ``input_key`` cannot be recycled by a later allocation while this entry is live -- - which is the hazard a weakref was meant to cover. It costs nothing in practice: the storage - belongs to the module's weight, which outlives the quantizer anyway. + ``base_ref`` points at the parameter, not at the tensor the backend was handed. + TensorQuantizer passes a fresh view of the weight on every forward, so a weakref to that + view dies as soon as the forward returns and an identity check against it never matches + again -- which is what kept this cache from ever hitting. - A weakref cannot do that job here. TensorQuantizer hands the backend a fresh view of the - weight on every forward, so a weakref to it dies as soon as that forward returns, and an - identity check against it never matches again. + Keeping it a weakref matters: a strong reference would pin full-precision storage alive and + defeat offloaded or meta-device flows. Tying the entry to the parameter's lifetime means the + payload stops being reused exactly when the weight it came from is released. """ - input_tensor: torch.Tensor + base_ref: weakref.ReferenceType input_key: tuple[object, ...] format_name: str block_chunk_size: int @@ -46,6 +47,16 @@ class _PackedWeightCache: weight_shape: torch.Tensor +def _cache_base(inputs: torch.Tensor) -> torch.Tensor: + """The tensor whose lifetime the cached payload should follow. + + ``inputs`` is a per-forward view; ``inputs._base`` is the parameter behind it, which lives + as long as the module does. + """ + base = inputs._base + return inputs if base is None else base + + def _input_cache_key(inputs: torch.Tensor) -> tuple[object, ...] | None: try: version = inputs._version @@ -73,10 +84,12 @@ def fake_quantize_with_cache( ) -> torch.Tensor: """Fake-quantize a weight while caching its compact packed representation.""" input_key = _input_cache_key(inputs) + cache_base = _cache_base(inputs) cache = getattr(quantizer, "_quantizer_cache", None) if ( isinstance(cache, _PackedWeightCache) and input_key is not None + and cache.base_ref() is cache_base and cache.input_key == input_key and cache.format_name == format_name and cache.block_chunk_size == block_chunk_size @@ -86,7 +99,7 @@ def fake_quantize_with_cache( packed_weights, weight_shape = quantize(inputs, block_chunk_size=block_chunk_size) if input_key is not None: quantizer._quantizer_cache = _PackedWeightCache( - input_tensor=inputs, + base_ref=weakref.ref(cache_base), input_key=input_key, format_name=format_name, block_chunk_size=block_chunk_size, diff --git a/modelopt_recipes/ptq.md b/modelopt_recipes/ptq.md index 683690b0a15..27a217d6f8a 100644 --- a/modelopt_recipes/ptq.md +++ b/modelopt_recipes/ptq.md @@ -58,8 +58,8 @@ supported combinations. | `int4_blockwise_weight_only` | INT4 W4A16, block 128, weights only | none | max | | `nvfp4_mlp_weight_only` | NVFP4 W4A16 (block 32), MLP + MoE weights only | none | max | | `mxfp4_mlp_weight_only` | MXFP4 W4A16, MLP + MoE weights only | none | none (no calibration) | -| `iq1_s` | IQ1_S W1A16, all linears | none | GGML IQ auto search (no calibration) | -| `iq2_xs` | IQ2_XS W2A16, all linears | none | GGML IQ auto search (no calibration) | +| `iq1_s` | IQ1_S W1A16, all linears | none | none (no calibration) | +| `iq2_xs` | IQ2_XS W2A16, all linears | none | none (no calibration) | @@ -115,7 +115,7 @@ activations are quantized too** (W4A4/W8A8 vs weight-only W4A16). > keeps that sensitive path at a safer precision than NVFP4 while still halving those > weights vs. BF16. -#### Weight-only schemes (W4A16 — activations stay BF16) +#### Weight-only schemes (activations stay BF16) Quantize weights only; activations run in BF16. This shrinks the model (memory-bound decode win) with much lower accuracy risk than W4A4, and **needs no From 871d4ee9fcb30f558713916afe20fb39cc18bb10 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Mon, 21 Sep 2026 23:24:33 +0000 Subject: [PATCH 10/12] Say what the IQ recipes actually quantize and who reads effective_bits The IQ rows and bullet in ptq.md claimed "all linears", but the presets import default_disabled_quantizers like every other preset, so lm_head, MoE routers, conv1d and the vision branch stay in BF16. Say "eligible linears" and name what is held back, matching the recipes' own description. The effective_bits comment explained the arithmetic but not the consumer, which was the question it kept drawing. It is cost metadata for AutoQuantize's compression estimate and drives no packing or numerics; the generic estimator cannot derive it because num_bits is a format string. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- modelopt_recipes/configs/numerics/iq1_s.yaml | 4 +++- modelopt_recipes/configs/numerics/iq2_xs.yaml | 4 +++- modelopt_recipes/ptq.md | 14 ++++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/modelopt_recipes/configs/numerics/iq1_s.yaml b/modelopt_recipes/configs/numerics/iq1_s.yaml index ec96dea7e47..78e309fd433 100644 --- a/modelopt_recipes/configs/numerics/iq1_s.yaml +++ b/modelopt_recipes/configs/numerics/iq1_s.yaml @@ -17,7 +17,9 @@ # modelopt-schema: modelopt.torch.quantization.config.QuantizerAttributeConfig num_bits: iq1_s -# Storage cost: 50 packed bytes * 8 / 256 weights. Keep in sync with IQ1_S_BLOCK_BYTES. +# Cost metadata for AutoQuantize's compression estimate only; it drives no packing or +# numerics. num_bits is the string "iq1_s", so the generic estimator cannot derive the +# storage cost: 50 packed bytes * 8 / 256 weights. Keep in sync with IQ1_S_BLOCK_BYTES. effective_bits: 1.5625 block_sizes: -1: 256 diff --git a/modelopt_recipes/configs/numerics/iq2_xs.yaml b/modelopt_recipes/configs/numerics/iq2_xs.yaml index 837943f1be3..e30102ada82 100644 --- a/modelopt_recipes/configs/numerics/iq2_xs.yaml +++ b/modelopt_recipes/configs/numerics/iq2_xs.yaml @@ -17,7 +17,9 @@ # modelopt-schema: modelopt.torch.quantization.config.QuantizerAttributeConfig num_bits: iq2_xs -# Storage cost: 74 packed bytes * 8 / 256 weights. Keep in sync with IQ2_XS_BLOCK_BYTES. +# Cost metadata for AutoQuantize's compression estimate only; it drives no packing or +# numerics. num_bits is the string "iq2_xs", so the generic estimator cannot derive the +# storage cost: 74 packed bytes * 8 / 256 weights. Keep in sync with IQ2_XS_BLOCK_BYTES. effective_bits: 2.3125 block_sizes: -1: 256 diff --git a/modelopt_recipes/ptq.md b/modelopt_recipes/ptq.md index 27a217d6f8a..f96a889cf29 100644 --- a/modelopt_recipes/ptq.md +++ b/modelopt_recipes/ptq.md @@ -58,8 +58,8 @@ supported combinations. | `int4_blockwise_weight_only` | INT4 W4A16, block 128, weights only | none | max | | `nvfp4_mlp_weight_only` | NVFP4 W4A16 (block 32), MLP + MoE weights only | none | max | | `mxfp4_mlp_weight_only` | MXFP4 W4A16, MLP + MoE weights only | none | none (no calibration) | -| `iq1_s` | IQ1_S W1A16, all linears | none | none (no calibration) | -| `iq2_xs` | IQ2_XS W2A16, all linears | none | none (no calibration) | +| `iq1_s` | IQ1_S W1A16, eligible linears | none | none (no calibration) | +| `iq2_xs` | IQ2_XS W2A16, eligible linears | none | none (no calibration) | @@ -138,10 +138,12 @@ activations and tensor-core math are what deliver the throughput. - **`mxfp4_mlp_weight_only`** — MXFP4 weights on MLP/MoE layers only, BF16 activations. Needs no calibration forward pass; the QAT starting point for the GPT-OSS family (see `examples/gpt-oss`). -- **`iq1_s` / `iq2_xs`** — GGML-compatible IQ1_S or IQ2_XS weights on all linear - layers, with BF16 activations. No calibration data is required. Quantized weights must have a - final dimension divisible by 256. Unified HF export writes the packed GGML blocks; Megatron - export additionally requires tensor and pipeline parallel sizes of 1, and does not support +- **`iq1_s` / `iq2_xs`** — GGML-compatible IQ1_S or IQ2_XS weights on the eligible + linear layers, with BF16 activations; `lm_head`, MoE routers, `conv1d` and the + vision branch stay in BF16 like every other preset. No calibration data is + required. Quantized weights must have a final dimension divisible by 256. + Unified HF export writes the packed GGML blocks; Megatron export additionally + requires tensor and pipeline parallel sizes of 1, and does not support fused-MoE experts. --- From e707df76587b40640e870b6cf99be2b060e540d5 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Tue, 22 Sep 2026 00:26:23 +0000 Subject: [PATCH 11/12] Size the IQ decode chunk separately from the encode chunk One constant served two loops with opposite characteristics. The encode fallback holds the large codebook-search temporaries and runs once per weight, so it wants a small chunk -- and IQ2_XS needs a smaller one than IQ1_S because its search sweeps sixteen local scales per grid tile. The decode's temporaries are tiny, but it runs on every forward and is never cached, so a small chunk just multiplies kernel launches. IQ2_XS therefore inherited a chunk four times smaller than IQ1_S on the path that runs most, which is where its end-to-end cost came from. Decoding a 2048x5632 weight: 91.4 ms at 256 blocks, 22.9 ms at 1024, 5.8 ms at 4096. Give the decode its own _DEFAULT_DECODE_CHUNK_SIZE of 4096 and thread it through fake_quantize_with_cache. The encode bounds are unchanged, so the memory ceiling stays where it was aimed; the decode's transient peak goes from +24 MiB to +56 MiB on that weight. Also flatten the seven-iteration sign-parity loop in dequantize_iq2_xs into a three-step XOR fold, which drops fifteen elementwise passes per chunk from the same per-forward path. tests/examples/hf_ptq on 2xH100, per case: IQ1_S 259.65s -> 94.64s, IQ2_XS 1081.05s -> 102.77s. Both now fit the 300s tests/examples default, so drop the explicit 1800s timeouts, and correct the comment that credited the cost to a calibration forward these recipes never run. Tests pin the parts that were easy to get wrong: the parity fold over five sign indices, that the decode receives the decode chunk rather than the encode one, and that decode output is invariant to chunk size. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- modelopt/torch/quantization/ggml/common.py | 5 ++- modelopt/torch/quantization/ggml/iq1_s.py | 12 ++++- modelopt/torch/quantization/ggml/iq2_xs.py | 26 ++++++++--- tests/examples/hf_ptq/test_llm_ptq.py | 18 +++----- .../torch/quantization/test_ggml_backend.py | 45 +++++++++++++++++++ tests/unit/torch/quantization/test_iq2_xs.py | 26 +++++++++++ 6 files changed, 111 insertions(+), 21 deletions(-) diff --git a/modelopt/torch/quantization/ggml/common.py b/modelopt/torch/quantization/ggml/common.py index 9f5bdf14662..574db9bcac8 100644 --- a/modelopt/torch/quantization/ggml/common.py +++ b/modelopt/torch/quantization/ggml/common.py @@ -79,6 +79,7 @@ def fake_quantize_with_cache( *, format_name: str, block_chunk_size: int, + decode_chunk_size: int, quantize: Callable[..., tuple[torch.Tensor, torch.Tensor]], dequantize: Callable[..., torch.Tensor], ) -> torch.Tensor: @@ -109,11 +110,13 @@ def fake_quantize_with_cache( else: quantizer._quantizer_cache = None + # Sized separately from the encode chunk: packing happens once per weight and is bounded + # by its search temporaries, while this runs on every forward and is bounded by launches. reconstructed = dequantize( packed_weights, weight_shape, dtype=inputs.dtype, - block_chunk_size=block_chunk_size, + block_chunk_size=decode_chunk_size, ) return inputs + (reconstructed - inputs).detach() diff --git a/modelopt/torch/quantization/ggml/iq1_s.py b/modelopt/torch/quantization/ggml/iq1_s.py index 804d27af058..a43e1699db1 100644 --- a/modelopt/torch/quantization/ggml/iq1_s.py +++ b/modelopt/torch/quantization/ggml/iq1_s.py @@ -61,8 +61,14 @@ _IQ1_S_DELTA = 0.125 _IQ1_S_NATIVE_MAX = 16.875 _IQ1_S_SCALE_ANCHOR = 0.61 -# At 1024 blocks, each largest IQ1_S search temporary is about 16 MiB in FP32. +# Bounds the torch encode fallback, whose codebook search holds the large temporaries: at +# 1024 blocks each is about 16 MiB in FP32. The CUDA encoder ignores this entirely. _DEFAULT_BLOCK_CHUNK_SIZE = 1024 +# The decode's temporaries are far smaller, so it is launch-bound rather than memory-bound +# and wants a bigger chunk -- and unlike packing it is not cached, so it runs on every +# forward. Measured decoding a 2048x5632 weight: 66.5 ms at 256 blocks, 4.2 ms at 4096, +# where the transient peak is +42 MiB. +_DEFAULT_DECODE_CHUNK_SIZE = 4096 _GRID_CACHE: dict[torch.device, torch.Tensor] = {} @@ -198,7 +204,7 @@ def dequantize_iq1_s( weight_shape: torch.Tensor, *, dtype: torch.dtype = torch.bfloat16, - block_chunk_size: int = _DEFAULT_BLOCK_CHUNK_SIZE, + block_chunk_size: int = _DEFAULT_DECODE_CHUNK_SIZE, ) -> torch.Tensor: """Decode GGML-compatible IQ1_S payload bytes.""" shape = validate_packed_weights( @@ -234,6 +240,7 @@ def iq1_s_fake_quant( quantizer, *, block_chunk_size: int = _DEFAULT_BLOCK_CHUNK_SIZE, + decode_chunk_size: int = _DEFAULT_DECODE_CHUNK_SIZE, ) -> torch.Tensor: """IQ1_S weight backend for TensorQuantizer, with pass-through backward.""" if getattr(quantizer, "num_bits", None) != "iq1_s": @@ -243,6 +250,7 @@ def iq1_s_fake_quant( quantizer, format_name="iq1_s", block_chunk_size=block_chunk_size, + decode_chunk_size=decode_chunk_size, quantize=quantize_iq1_s, dequantize=dequantize_iq1_s, ) diff --git a/modelopt/torch/quantization/ggml/iq2_xs.py b/modelopt/torch/quantization/ggml/iq2_xs.py index 2f43e71d479..e2adc2fa44f 100644 --- a/modelopt/torch/quantization/ggml/iq2_xs.py +++ b/modelopt/torch/quantization/ggml/iq2_xs.py @@ -60,8 +60,17 @@ _IQ2_XS_SCALE_ANCHOR_MIN = 0.65 _IQ2_XS_SCALE_ANCHOR_MAX = 0.92 _IQ2_XS_PEAK_TO_RMS_TAPER = 0.035 -# At 256 blocks, the largest IQ2_XS search temporary is about 16 MiB in FP32. +# Bounds the torch encode fallback, whose codebook search holds the large temporaries: at +# 256 blocks the largest is about 16 MiB in FP32. The CUDA encoder ignores this entirely. +# This is four times smaller than the IQ1_S bound because the IQ2_XS search sweeps sixteen +# local scales per grid tile. _DEFAULT_BLOCK_CHUNK_SIZE = 256 +# The decode's temporaries are far smaller, so it is launch-bound rather than memory-bound +# and wants a bigger chunk -- and unlike packing it is not cached, so it runs on every +# forward. Sharing the encode bound above is what made IQ2_XS four times slower end to end +# than IQ1_S. Measured decoding a 2048x5632 weight: 91.4 ms at 256 blocks, 22.9 ms at 1024, +# 5.8 ms at 4096, where the transient peak is +56 MiB. +_DEFAULT_DECODE_CHUNK_SIZE = 4096 _SCALE_BLOCK_CHUNK_SIZE = 4096 @@ -204,7 +213,7 @@ def dequantize_iq2_xs( weight_shape: torch.Tensor, *, dtype: torch.dtype = torch.bfloat16, - block_chunk_size: int = _DEFAULT_BLOCK_CHUNK_SIZE, + block_chunk_size: int = _DEFAULT_DECODE_CHUNK_SIZE, ) -> torch.Tensor: """Decode GGML-compatible IQ2_XS payload bytes.""" shape = validate_packed_weights( @@ -225,10 +234,13 @@ def dequantize_iq2_xs( ) entries = codes & 0x1FF sign_index = codes >> 9 - parity = torch.zeros_like(sign_index) - for bit in range(7): - parity ^= (sign_index >> bit) & 1 - sign_mask = sign_index | (parity << 7) + # XOR-fold the seven payload bits down to bit 0 to recover the eighth sign bit. + # The loop this replaces cost seven elementwise passes per chunk, and unlike packing + # the decode is not cached -- it runs again on every forward. + folded = sign_index ^ (sign_index >> 4) + folded ^= folded >> 2 + folded ^= folded >> 1 + sign_mask = sign_index | ((folded & 1) << 7) signs = 1.0 - 2.0 * ((sign_mask.unsqueeze(-1) >> bit_positions) & 1).float() scale_bytes = block_chunk[:, 66:].to(torch.int64) @@ -248,6 +260,7 @@ def iq2_xs_fake_quant( quantizer, *, block_chunk_size: int = _DEFAULT_BLOCK_CHUNK_SIZE, + decode_chunk_size: int = _DEFAULT_DECODE_CHUNK_SIZE, ) -> torch.Tensor: """IQ2_XS weight backend for TensorQuantizer, with pass-through backward.""" if getattr(quantizer, "num_bits", None) != "iq2_xs": @@ -257,6 +270,7 @@ def iq2_xs_fake_quant( quantizer, format_name="iq2_xs", block_chunk_size=block_chunk_size, + decode_chunk_size=decode_chunk_size, quantize=quantize_iq2_xs, dequantize=dequantize_iq2_xs, ) diff --git a/tests/examples/hf_ptq/test_llm_ptq.py b/tests/examples/hf_ptq/test_llm_ptq.py index 4b488fde023..6ae697b3c09 100644 --- a/tests/examples/hf_ptq/test_llm_ptq.py +++ b/tests/examples/hf_ptq/test_llm_ptq.py @@ -77,18 +77,12 @@ def test_ptq_whisper(command): PTQCommand(quant="int4_awq", kv_cache_quant="none"), PTQCommand(quant="w4a8_awq_beta", kv_cache_quant="none"), # GGML IQ weight-only, recipe-driven. These encoders require every weight's input - # dimension to be a multiple of 256; TinyLlama's 2048 and 5632 both are. - # Measured 18m17s for the pair on 2xH100, over the 300s tests/examples default. The - # packing does run on GPU; the cost is the codebook search over every linear, once on - # the first calibration forward and again at export. - pytest.param( - PTQCommand(recipe="general/ptq/iq1_s", kv_cache_quant="none"), - marks=pytest.mark.timeout(1800), - ), - pytest.param( - PTQCommand(recipe="general/ptq/iq2_xs", kv_cache_quant="none"), - marks=pytest.mark.timeout(1800), - ), + # dimension to be a multiple of 256; TinyLlama's 2048 and 5632 both are. Neither + # recipe calibrates -- both set algorithm: null -- so the only IQ-specific cost is + # packing each weight once and decoding it on each forward. 95s and 103s on 2xH100, + # inside the 300s tests/examples default. + PTQCommand(recipe="general/ptq/iq1_s", kv_cache_quant="none"), + PTQCommand(recipe="general/ptq/iq2_xs", kv_cache_quant="none"), PTQCommand(quant="nvfp4"), PTQCommand(quant="nvfp4_awq_lite"), # autoquant (recipe-driven) diff --git a/tests/unit/torch/quantization/test_ggml_backend.py b/tests/unit/torch/quantization/test_ggml_backend.py index 8820a968c9a..1370fdd312b 100644 --- a/tests/unit/torch/quantization/test_ggml_backend.py +++ b/tests/unit/torch/quantization/test_ggml_backend.py @@ -168,3 +168,48 @@ def counting(weight, **kwargs): quantizer(weight) assert calls == [(4, 256)], f"expected one pack, got {len(calls)}" + + +@pytest.mark.parametrize( + ("num_bits", "module"), [("iq1_s", iq1_s_module), ("iq2_xs", iq2_xs_module)] +) +def test_ggml_decode_chunk_is_sized_independently_of_the_encode_chunk( + monkeypatch, num_bits, module +): + """The decode runs every forward; the encode runs once and holds the big temporaries. + + Sharing one constant between them is what made IQ2_XS four times slower end to end than + IQ1_S, so pin that the decode gets its own, larger chunk. + """ + seen = {} + original = getattr(module, f"dequantize_{num_bits}") + + def recording(packed_weights, weight_shape, **kwargs): + seen["block_chunk_size"] = kwargs["block_chunk_size"] + return original(packed_weights, weight_shape, **kwargs) + + monkeypatch.setattr(module, f"dequantize_{num_bits}", recording) + quantizer = TensorQuantizer( + QuantizerAttributeConfig(num_bits=num_bits, block_sizes={-1: 256}, backend="ggml") + ) + quantizer(torch.randn(4, 256)) + + assert seen["block_chunk_size"] == module._DEFAULT_DECODE_CHUNK_SIZE + assert module._DEFAULT_DECODE_CHUNK_SIZE > module._DEFAULT_BLOCK_CHUNK_SIZE + + +@pytest.mark.parametrize( + ("num_bits", "module"), [("iq1_s", iq1_s_module), ("iq2_xs", iq2_xs_module)] +) +def test_ggml_decode_is_invariant_to_chunk_size(num_bits, module): + """Chunking the decode is a memory bound, not a numerical choice.""" + torch.manual_seed(0) + weight = torch.randn(3, 1024, dtype=torch.bfloat16) + packed, shape = getattr(module, f"quantize_{num_bits}")(weight) + dequantize = getattr(module, f"dequantize_{num_bits}") + + reference = dequantize(packed, shape, dtype=weight.dtype, block_chunk_size=1) + for chunk in (2, 7, 4096): + assert torch.equal( + dequantize(packed, shape, dtype=weight.dtype, block_chunk_size=chunk), reference + ) diff --git a/tests/unit/torch/quantization/test_iq2_xs.py b/tests/unit/torch/quantization/test_iq2_xs.py index 73a1b352c8a..3dde5a8a902 100644 --- a/tests/unit/torch/quantization/test_iq2_xs.py +++ b/tests/unit/torch/quantization/test_iq2_xs.py @@ -102,6 +102,32 @@ def test_iq2_xs_dequantizes_pinned_scale_factor(): assert torch.equal(decoded, torch.full((1, 256), 43 * 31 / 8, dtype=torch.float32)) +@pytest.mark.parametrize("sign_index", [0b0000001, 0b0000011, 0b0000111, 0b1010101, 0b1111111]) +def test_iq2_xs_dequantizes_the_implied_eighth_sign_bit(sign_index): + """The eighth sign is not stored; it is the parity of the seven that are. + + The pinned-scale test above only covers sign_index 0, where every value is positive + whether or not the implied bit is derived correctly. These indices vary which payload + bits are set so the parity actually has to be computed. + """ + codes = 511 | (sign_index << 9) # entry 511 holds eight 43s + packed = torch.zeros((1, 1, 74), dtype=torch.uint8) + packed[0, 0, :2] = torch.tensor([1.0], dtype=torch.float16).view(torch.uint8) + packed[0, 0, 2:66:2] = codes & 0xFF + packed[0, 0, 3:66:2] = codes >> 8 + packed[0, 0, 66:] = 0xFF # local code 15 in both nibbles + + decoded = dequantize_iq2_xs(packed, torch.tensor([1, 256]), dtype=torch.float32) + + payload_bits = [(sign_index >> bit) & 1 for bit in range(7)] + magnitude = 43 * 31 / 8 # entry value 43, local code 15 -> (2 * 15 + 1) / 8 + expected_group = torch.tensor( + [-magnitude if bit else magnitude for bit in (*payload_bits, sum(payload_bits) % 2)], + dtype=torch.float32, + ) + assert torch.equal(decoded.reshape(32, 8), expected_group.expand(32, 8)) + + def test_iq2_xs_requires_complete_last_dimension_blocks(): with pytest.raises(ValueError, match="last weight dimension"): quantize_iq2_xs(torch.ones(2, 257)) From bcabea8eb7b9b0f3960856b04ac12815fabcf571 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Tue, 22 Sep 2026 02:09:10 +0000 Subject: [PATCH 12/12] Migrate the IQ recipes off the deprecated recipe_type The main merge deprecated metadata.recipe_type -- a file now declares its kind with a '# modelopt-schema:' comment instead -- and migrated the other 26 general/ptq recipes. These two were on a branch at the time, so they were the only files left declaring the old field. Dropping the field alone is not enough: the loader rejects a recipe that states its kind by none of the three accepted means, so these take the schema comment the siblings use. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- modelopt_recipes/general/ptq/iq1_s.yaml | 2 +- modelopt_recipes/general/ptq/iq2_xs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modelopt_recipes/general/ptq/iq1_s.yaml b/modelopt_recipes/general/ptq/iq1_s.yaml index a3c98b92df3..4bc31221b79 100644 --- a/modelopt_recipes/general/ptq/iq1_s.yaml +++ b/modelopt_recipes/general/ptq/iq1_s.yaml @@ -15,11 +15,11 @@ # IQ1_S weight-only PTQ. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: preset: configs/ptq/presets/model/iq1_s metadata: - recipe_type: ptq description: >- Applies uniform GGML-compatible IQ1_S weight-only quantization to eligible linear layers. This is not a mixed per-tensor precision preset. No calibration data is required. diff --git a/modelopt_recipes/general/ptq/iq2_xs.yaml b/modelopt_recipes/general/ptq/iq2_xs.yaml index 6a9a08d17a9..f690c55cf09 100644 --- a/modelopt_recipes/general/ptq/iq2_xs.yaml +++ b/modelopt_recipes/general/ptq/iq2_xs.yaml @@ -15,11 +15,11 @@ # IQ2_XS weight-only PTQ. +# modelopt-schema: modelopt.recipe.config.ModelOptPTQRecipe imports: preset: configs/ptq/presets/model/iq2_xs metadata: - recipe_type: ptq description: >- Applies uniform GGML-compatible IQ2_XS weight-only quantization to eligible linear layers. This is not a mixed per-tensor precision preset. No calibration data is required.