diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py index 8ffcd35d9303..57a22f1f5e54 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @@ -33,7 +33,6 @@ from ..conftest import ( get_sm_version, - is_sm_100f, llm_models_root, skip_post_blackwell_ultra, skip_pre_blackwell, @@ -228,10 +227,7 @@ def test_nvfp4_prequantized(self): task.evaluate(llm, sampling_params=self.sampling_params) -@pytest.mark.skipif( - not is_sm_100f(), - reason="Gemma4 shared-KV MTP requires FlashInfer trtllm-gen on the SM100 family", -) +@pytest.mark.skip_device_not_contain(["B200", "GB10"]) class TestGemma4_26B_A4B(LlmapiAccuracyTestHarness): MODEL_NAME = "google/gemma-4-26B-A4B-it" MODEL_PATH = f"{llm_models_root()}/gemma/nvidia-Gemma-4-26B-A4B-NVFP4" @@ -274,6 +270,22 @@ def test_nvfp4(self): extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS, ) + def test_nvfp4_no_mtp(self): + # Same checkpoint without MTP drafting. + with LLM( + self.MODEL_PATH, + max_batch_size=16, + kv_cache_config=self.kv_cache_config, + enable_chunked_prefill=True, + ) as llm: + assert llm.args.quant_config.quant_algo == QuantAlgo.NVFP4 + task = MMMU(self.MODEL_NAME) + task.evaluate( + llm, + sampling_params=self.sampling_params, + extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS, + ) + class TestQwen3VL_MOE(LlmapiAccuracyTestHarness): MODEL_NAME = "Qwen/Qwen3-VL-30B-A3B-Instruct" diff --git a/tests/integration/defs/perf/_model_paths.py b/tests/integration/defs/perf/_model_paths.py index c143aba4cfff..a47d96d45468 100644 --- a/tests/integration/defs/perf/_model_paths.py +++ b/tests/integration/defs/perf/_model_paths.py @@ -68,8 +68,10 @@ "qwen3.5_122b_a10b": "Qwen3.5-122B-A10B", "qwen3.5_397b_a17b_fp8": "Qwen3.5-397B-A17B-FP8", "qwen3.5_397b_a17b_fp4": "Qwen3.5-397B-A17B-NVFP4", - # Qwen3.6 (GDN linear-attn MoE, NVFP4) + # Qwen3.6 (GDN linear-attn MoE, NVFP4). The _mtp key is the same checkpoint + # benchmarked with one-model MTP drafting, so the two can be compared. "qwen3.6_35b_a3b_fp4": "Qwen3.6-35B-A3B-NVFP4", + "qwen3.6_35b_a3b_fp4_mtp": "Qwen3.6-35B-A3B-NVFP4", # DeepSeek V4 "deepseek_v4_pro_fp4": "DeepSeek-V4-Pro", "deepseek_v4_flash": "DeepSeek-V4-Flash", diff --git a/tests/integration/defs/perf/pytorch_model_config.py b/tests/integration/defs/perf/pytorch_model_config.py index 5b8cb5cd89d0..e59738699e8c 100644 --- a/tests/integration/defs/perf/pytorch_model_config.py +++ b/tests/integration/defs/perf/pytorch_model_config.py @@ -314,6 +314,28 @@ def get_model_yaml_config(model_label: str, }, } }, + # Qwen3.6-35B-A3B NVFP4 one-model MTP drafting + { + 'patterns': ['qwen3.6_35b_a3b_fp4_mtp-bench-pytorch-streaming'], + 'config': { + 'trust_remote_code': True, + 'moe_config': { + 'backend': 'CUTLASS', + }, + 'enable_chunked_prefill': True, + 'cuda_graph_config': { + 'enable_padding': True, + }, + 'kv_cache_config': { + 'enable_block_reuse': False, + 'dtype': 'fp8', + }, + 'speculative_config': { + 'decoding_type': 'MTP', + 'max_draft_len': 1, + }, + } + }, # MiniMax-M3 MXFP8 block-sparse MoE: sparse backend, no KV reuse, trust_remote_code, capped max_seq_len to avoid the 1M-default CUDA-graph OOM. { 'patterns': ['minimax_m3_mxfp8'], diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index cf39d3e5fe72..1da7601291d2 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -63,6 +63,7 @@ "glm_5_fp8", "minimax_m3_mxfp8", "qwen3.6_35b_a3b_fp4", + "qwen3.6_35b_a3b_fp4_mtp", "nemotron_3_nano_omni_nvfp4", "nemotron_3_nano_omni_nvfp4_image", "nemotron_nano_12b_v2", @@ -96,6 +97,7 @@ "qwen3_235b_a22b_fp4_eagle3", "gpt_oss_120b_eagle3", "gpt_oss_120b_eagle3_throughput", + "qwen3.6_35b_a3b_fp4_mtp", *SPEC_DEC_REAL_DATASET_MODELS, } diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index 6a7836b390c6..1c6894850a3b 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -916,6 +916,9 @@ def test_ptp_quickstart(llm_root, llm_venv): pytest.param('Nemotron-Nano-9B-v2-nvfp4', 'NVIDIA-Nemotron-Nano-9B-v2-NVFP4', marks=skip_pre_blackwell), + pytest.param('Qwen3.6-35B-A3B-nvfp4', + 'Qwen3.6-35B-A3B-NVFP4', + marks=skip_pre_blackwell), ]) def test_ptp_quickstart_advanced(llm_root, llm_venv, model_name, model_path): print(f"Testing {model_name}.") @@ -924,7 +927,9 @@ def test_ptp_quickstart_advanced(llm_root, llm_venv, model_name, model_path): llm_venv.run_cmd([ str(example_root / "quickstart_advanced.py"), "--disable_kv_cache_reuse", + "--trust_remote_code", "--max_batch_size=8", + "--use_kv_cache_manager_v2=false", "--model_dir", f"{llm_models_root()}/{model_path}", ]) @@ -943,6 +948,10 @@ def test_ptp_quickstart_advanced(llm_root, llm_venv, model_name, model_path): ] if "Qwen3" in model_name: cmds.append("--kv_cache_fraction=0.6") + if "Qwen3.6-35B-A3B" in model_name: + # Hybrid linear-attention model: the Mamba cache preallocates a + # recurrent state per sequence slot, so cap the batch size. + cmds.append("--max_batch_size=1") llm_venv.run_cmd(cmds) diff --git a/tests/integration/test_lists/qa/llm_spark_core.txt b/tests/integration/test_lists/qa/llm_spark_core.txt deleted file mode 100644 index e9250acc0a84..000000000000 --- a/tests/integration/test_lists/qa/llm_spark_core.txt +++ /dev/null @@ -1,30 +0,0 @@ -test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8] -test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP4-modelopt-hf-model-hub/Llama-3.1-8B-Instruct-fp4] -test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-bf16-instruct-llama-3.1-model/Llama-3.1-8B-Instruct] -test_e2e.py::test_ptp_quickstart_advanced[GPT-OSS-20B-gpt_oss/gpt-oss-20b] -test_e2e.py::test_ptp_quickstart_advanced[GPT-OSS-120B-gpt_oss/gpt-oss-120b] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-8b-fp8-Qwen3/nvidia-Qwen3-8B-FP8] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-8b-nvfp4-Qwen3/nvidia-Qwen3-8B-NVFP4] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-8B-bf16-Qwen3/Qwen3-8B] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-14b-fp8-Qwen3/nvidia-Qwen3-14B-FP8] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-14b-nvfp4-Qwen3/nvidia-Qwen3-14B-NVFP4] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-14B-bf16-Qwen3/Qwen3-14B] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-32B-bf16-Qwen3/Qwen3-32B] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-32b-nvfp4-Qwen3/nvidia-Qwen3-32B-NVFP4] -test_e2e.py::test_ptp_quickstart_advanced[Nemotron-Nano-9B-v2-nvfp4-NVIDIA-Nemotron-Nano-9B-v2-NVFP4] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_nvfp4_hf-Qwen3/saved_models_Qwen3-30B-A3B_nvfp4_hf] - -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_auto_dtype -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 -accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False] -accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=True] - -test_e2e.py::test_trtllm_benchmark_serving[gpt_oss/gpt-oss-20b] -examples/serve/test_serve.py::test_nemotron3_super_120b_nvfp4 -examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[text_reasoning_on] -examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[text_reasoning_off] -examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[text_streaming] -examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[tool_calling] -examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[image] -examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[video] diff --git a/tests/integration/test_lists/qa/llm_spark_func.yml b/tests/integration/test_lists/qa/llm_spark_func.yml index f38f2fed1482..a59385112d68 100644 --- a/tests/integration/test_lists/qa/llm_spark_func.yml +++ b/tests/integration/test_lists/qa/llm_spark_func.yml @@ -28,6 +28,7 @@ llm_spark_func: - test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_nvfp4_hf-Qwen3/saved_models_Qwen3-30B-A3B_nvfp4_hf] - test_e2e.py::test_ptp_quickstart_advanced[Nemotron-Nano-9B-v2-nvfp4-NVIDIA-Nemotron-Nano-9B-v2-NVFP4] - test_e2e.py::test_ptp_quickstart_advanced_eagle3[GPT-OSS-120B-Eagle3-gpt_oss/gpt-oss-120b-gpt_oss/gpt-oss-120b-Eagle3] + - test_e2e.py::test_ptp_quickstart_advanced[Qwen3.6-35B-A3B-nvfp4-Qwen3.6-35B-A3B-NVFP4] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_auto_dtype - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized @@ -35,6 +36,8 @@ llm_spark_func: - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_12BInstruct::test_auto_dtype - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_12BInstruct::test_fp8_prequantized - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_12BInstruct::test_nvfp4_prequantized + - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma4_26B_A4B::test_nvfp4_no_mtp + - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma4_26B_A4B::test_nvfp4 - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=True] - test_e2e.py::test_trtllm_benchmark_serving[gpt_oss/gpt-oss-20b] diff --git a/tests/integration/test_lists/qa/llm_spark_perf.yml b/tests/integration/test_lists/qa/llm_spark_perf.yml index 2214aa6122e2..966db8a62ef9 100644 --- a/tests/integration/test_lists/qa/llm_spark_perf.yml +++ b/tests/integration/test_lists/qa/llm_spark_perf.yml @@ -44,6 +44,10 @@ llm_spark_perf: - perf/test_perf.py::test_perf[gemma_3_12b_it-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[gemma_3_12b_it_fp8-bench-pytorch-streaming-float8-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[gemma_3_12b_it_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,128-reqs:1-con:1] + - perf/test_perf.py::test_perf[gemma_4_26b_a4b_nvfp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,128-reqs:1-con:1] + # Qwen3.6-35B-A3B NVFP4: baseline and the same checkpoint with MTP drafting + - perf/test_perf.py::test_perf[qwen3.6_35b_a3b_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,256-reqs:1-con:1] + - perf/test_perf.py::test_perf[qwen3.6_35b_a3b_fp4_mtp-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,256-reqs:1-con:1] # =============================================================================== # 2: Multi-GPU (2 GPUs) Spark perf cases with multinode support # ===============================================================================