From b352bce91ccdf99a714887d774cb73ea35f41c04 Mon Sep 17 00:00:00 2001 From: Jenny Liu Date: Thu, 3 Sep 2026 09:42:11 +0000 Subject: [PATCH 1/2] add func and perf for Qwen3.6-35B-A3B and gemma4 on spark Signed-off-by: Jenny Liu --- .../test_llm_api_pytorch_multimodal.py | 18 ++++++++++- tests/integration/defs/perf/_model_paths.py | 4 ++- .../defs/perf/pytorch_model_config.py | 22 +++++++++++++ tests/integration/defs/perf/test_perf.py | 2 ++ tests/integration/defs/test_e2e.py | 7 ++++ .../test_lists/qa/llm_spark_core.txt | 32 ------------------- .../test_lists/qa/llm_spark_func.yml | 3 ++ .../test_lists/qa/llm_spark_perf.yml | 4 +++ 8 files changed, 58 insertions(+), 34 deletions(-) delete mode 100644 tests/integration/test_lists/qa/llm_spark_core.txt diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py index ef018713ffa5..57a22f1f5e54 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch_multimodal.py @@ -227,7 +227,7 @@ def test_nvfp4_prequantized(self): task.evaluate(llm, sampling_params=self.sampling_params) -@pytest.mark.skip_device_not_contain(["B200"]) +@pytest.mark.skip_device_not_contain(["B200", "GB10"]) class TestGemma4_26B_A4B(LlmapiAccuracyTestHarness): MODEL_NAME = "google/gemma-4-26B-A4B-it" MODEL_PATH = f"{llm_models_root()}/gemma/nvidia-Gemma-4-26B-A4B-NVFP4" @@ -270,6 +270,22 @@ def test_nvfp4(self): extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS, ) + def test_nvfp4_no_mtp(self): + # Same checkpoint without MTP drafting. + with LLM( + self.MODEL_PATH, + max_batch_size=16, + kv_cache_config=self.kv_cache_config, + enable_chunked_prefill=True, + ) as llm: + assert llm.args.quant_config.quant_algo == QuantAlgo.NVFP4 + task = MMMU(self.MODEL_NAME) + task.evaluate( + llm, + sampling_params=self.sampling_params, + extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS, + ) + class TestQwen3VL_MOE(LlmapiAccuracyTestHarness): MODEL_NAME = "Qwen/Qwen3-VL-30B-A3B-Instruct" diff --git a/tests/integration/defs/perf/_model_paths.py b/tests/integration/defs/perf/_model_paths.py index dd52496fea99..6f283b7d4527 100644 --- a/tests/integration/defs/perf/_model_paths.py +++ b/tests/integration/defs/perf/_model_paths.py @@ -75,8 +75,10 @@ "qwen3.5_122b_a10b": "Qwen3.5-122B-A10B", "qwen3.5_397b_a17b_fp8": "Qwen3.5-397B-A17B-FP8", "qwen3.5_397b_a17b_fp4": "Qwen3.5-397B-A17B-NVFP4", - # Qwen3.6 (GDN linear-attn MoE, NVFP4) + # Qwen3.6 (GDN linear-attn MoE, NVFP4). The _mtp key is the same checkpoint + # benchmarked with one-model MTP drafting, so the two can be compared. "qwen3.6_35b_a3b_fp4": "Qwen3.6-35B-A3B-NVFP4", + "qwen3.6_35b_a3b_fp4_mtp": "Qwen3.6-35B-A3B-NVFP4", # DeepSeek V4 "deepseek_v4_pro_fp4": "DeepSeek-V4-Pro", "deepseek_v4_flash": "DeepSeek-V4-Flash", diff --git a/tests/integration/defs/perf/pytorch_model_config.py b/tests/integration/defs/perf/pytorch_model_config.py index 373af821b377..de4d783bffaf 100644 --- a/tests/integration/defs/perf/pytorch_model_config.py +++ b/tests/integration/defs/perf/pytorch_model_config.py @@ -314,6 +314,28 @@ def get_model_yaml_config(model_label: str, }, } }, + # Qwen3.6-35B-A3B NVFP4 one-model MTP drafting + { + 'patterns': ['qwen3.6_35b_a3b_fp4_mtp-bench-pytorch-streaming'], + 'config': { + 'trust_remote_code': True, + 'moe_config': { + 'backend': 'CUTLASS', + }, + 'enable_chunked_prefill': True, + 'cuda_graph_config': { + 'enable_padding': True, + }, + 'kv_cache_config': { + 'enable_block_reuse': False, + 'dtype': 'fp8', + }, + 'speculative_config': { + 'decoding_type': 'MTP', + 'max_draft_len': 1, + }, + } + }, # MiniMax-M3 MXFP8 block-sparse MoE: sparse backend, no KV reuse, trust_remote_code, capped max_seq_len to avoid the 1M-default CUDA-graph OOM. { 'patterns': ['minimax_m3_mxfp8'], diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index a62d445021f5..b72c685b5bef 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -63,6 +63,7 @@ "glm_5_fp8", "minimax_m3_mxfp8", "qwen3.6_35b_a3b_fp4", + "qwen3.6_35b_a3b_fp4_mtp", "nemotron_3_nano_omni_nvfp4", "nemotron_3_nano_omni_nvfp4_image", "nemotron_nano_12b_v2", @@ -96,6 +97,7 @@ "qwen3_235b_a22b_fp4_eagle3", "gpt_oss_120b_eagle3", "gpt_oss_120b_eagle3_throughput", + "qwen3.6_35b_a3b_fp4_mtp", *SPEC_DEC_REAL_DATASET_MODELS, } diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index 9775d56e9a1a..d1913bbdc1af 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -958,6 +958,9 @@ def test_ptp_quickstart(llm_root, llm_venv): pytest.param('Nemotron-Nano-9B-v2-nvfp4', 'NVIDIA-Nemotron-Nano-9B-v2-NVFP4', marks=skip_pre_blackwell), + pytest.param('Qwen3.6-35B-A3B-nvfp4', + 'Qwen3.6-35B-A3B-NVFP4', + marks=skip_pre_blackwell), ]) def test_ptp_quickstart_advanced(llm_root, llm_venv, model_name, model_path): print(f"Testing {model_name}.") @@ -985,6 +988,10 @@ def test_ptp_quickstart_advanced(llm_root, llm_venv, model_name, model_path): ] if "Qwen3" in model_name: cmds.append("--kv_cache_fraction=0.6") + if "Qwen3.6-35B-A3B" in model_name: + # Hybrid linear-attention model: the Mamba cache preallocates a + # recurrent state per sequence slot, so cap the batch size. + cmds.append("--max_batch_size=1") if "Llama3.3-70B" in model_name: cmds.append("--max_num_tokens=1024") if "Llama-4" in model_name: diff --git a/tests/integration/test_lists/qa/llm_spark_core.txt b/tests/integration/test_lists/qa/llm_spark_core.txt deleted file mode 100644 index 33851b1eb2be..000000000000 --- a/tests/integration/test_lists/qa/llm_spark_core.txt +++ /dev/null @@ -1,32 +0,0 @@ -test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP8-llama-3.1-model/Llama-3.1-8B-Instruct-FP8] -test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-FP4-modelopt-hf-model-hub/Llama-3.1-8B-Instruct-fp4] -test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-bf16-instruct-llama-3.1-model/Llama-3.1-8B-Instruct] -test_e2e.py::test_ptp_quickstart_advanced[GPT-OSS-20B-gpt_oss/gpt-oss-20b] -test_e2e.py::test_ptp_quickstart_advanced[GPT-OSS-120B-gpt_oss/gpt-oss-120b] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-8b-fp8-Qwen3/nvidia-Qwen3-8B-FP8] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-8b-nvfp4-Qwen3/nvidia-Qwen3-8B-NVFP4] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-8B-bf16-Qwen3/Qwen3-8B] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-14b-fp8-Qwen3/nvidia-Qwen3-14B-FP8] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-14b-nvfp4-Qwen3/nvidia-Qwen3-14B-NVFP4] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-14B-bf16-Qwen3/Qwen3-14B] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-32B-bf16-Qwen3/Qwen3-32B] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-32b-nvfp4-Qwen3/nvidia-Qwen3-32B-NVFP4] -test_e2e.py::test_ptp_quickstart_advanced[Nemotron-Nano-9B-v2-nvfp4-NVIDIA-Nemotron-Nano-9B-v2-NVFP4] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B-Qwen3/Qwen3-30B-A3B] -test_e2e.py::test_ptp_quickstart_advanced[Qwen3-30B-A3B_nvfp4_hf-Qwen3/saved_models_Qwen3-30B-A3B_nvfp4_hf] -test_e2e.py::test_ptp_quickstart_advanced[Llama3.3-70B-FP8-modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp8] -test_e2e.py::test_ptp_quickstart_advanced[Llama3.3-70B-FP4-modelopt-hf-model-hub/Llama-3.3-70B-Instruct-fp4] - -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_auto_dtype -accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 -accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False] -accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=True] - -test_e2e.py::test_trtllm_benchmark_serving[gpt_oss/gpt-oss-20b] -examples/serve/test_serve.py::test_nemotron3_super_120b_nvfp4 -examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[text_reasoning_on] -examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[text_reasoning_off] -examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[text_streaming] -examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[tool_calling] -examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[image] -examples/serve/test_serve.py::test_nemotron3_nano_omni_nvfp4[video] diff --git a/tests/integration/test_lists/qa/llm_spark_func.yml b/tests/integration/test_lists/qa/llm_spark_func.yml index b87ae1af0f99..1b88082341f8 100644 --- a/tests/integration/test_lists/qa/llm_spark_func.yml +++ b/tests/integration/test_lists/qa/llm_spark_func.yml @@ -31,6 +31,7 @@ llm_spark_func: - test_e2e.py::test_ptp_quickstart_advanced[Llama-4-Scout-17B-16E-FP4-llama4-models/Llama-4-Scout-17B-16E-Instruct-FP4] - test_e2e.py::test_ptp_quickstart_advanced[Nemotron-Nano-9B-v2-nvfp4-NVIDIA-Nemotron-Nano-9B-v2-NVFP4] - test_e2e.py::test_ptp_quickstart_advanced_eagle3[GPT-OSS-120B-Eagle3-gpt_oss/gpt-oss-120b-gpt_oss/gpt-oss-120b-Eagle3] + - test_e2e.py::test_ptp_quickstart_advanced[Qwen3.6-35B-A3B-nvfp4-Qwen3.6-35B-A3B-NVFP4] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_auto_dtype - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_27BInstruct::test_fp8_prequantized @@ -38,6 +39,8 @@ llm_spark_func: - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_12BInstruct::test_auto_dtype - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_12BInstruct::test_fp8_prequantized - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma3_12BInstruct::test_nvfp4_prequantized + - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma4_26B_A4B::test_nvfp4_no_mtp + - accuracy/test_llm_api_pytorch_multimodal.py::TestGemma4_26B_A4B::test_nvfp4 - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[latency_moe_cutlass-torch_compile=True] - test_e2e.py::test_trtllm_benchmark_serving[gpt_oss/gpt-oss-20b] diff --git a/tests/integration/test_lists/qa/llm_spark_perf.yml b/tests/integration/test_lists/qa/llm_spark_perf.yml index 24975f87ff66..95afc090f0dd 100644 --- a/tests/integration/test_lists/qa/llm_spark_perf.yml +++ b/tests/integration/test_lists/qa/llm_spark_perf.yml @@ -48,6 +48,10 @@ llm_spark_perf: - perf/test_perf.py::test_perf[gemma_3_12b_it-bench-pytorch-streaming-bfloat16-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[gemma_3_12b_it_fp8-bench-pytorch-streaming-float8-maxbs:1-input_output_len:2048,128-reqs:1-con:1] - perf/test_perf.py::test_perf[gemma_3_12b_it_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,128-reqs:1-con:1] + - perf/test_perf.py::test_perf[gemma_4_26b_a4b_nvfp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,128-reqs:1-con:1] + # Qwen3.6-35B-A3B NVFP4: baseline and the same checkpoint with MTP drafting + - perf/test_perf.py::test_perf[qwen3.6_35b_a3b_fp4-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,256-reqs:1-con:1] + - perf/test_perf.py::test_perf[qwen3.6_35b_a3b_fp4_mtp-bench-pytorch-streaming-float4-maxbs:1-input_output_len:2048,256-reqs:1-con:1] # =============================================================================== # 2: Multi-GPU (2 GPUs) Spark perf cases with multinode support # =============================================================================== From 9976066e1f044b2acc0b3946c90ccb9bdef9ee0c Mon Sep 17 00:00:00 2001 From: Jenny Liu Date: Fri, 4 Sep 2026 03:22:32 +0000 Subject: [PATCH 2/2] for bug 6556429 add trust_remote_code and use_kv_cache_manager_v2=false Signed-off-by: Jenny Liu --- tests/integration/defs/test_e2e.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index a9915a61a568..1c6894850a3b 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -927,7 +927,9 @@ def test_ptp_quickstart_advanced(llm_root, llm_venv, model_name, model_path): llm_venv.run_cmd([ str(example_root / "quickstart_advanced.py"), "--disable_kv_cache_reuse", + "--trust_remote_code", "--max_batch_size=8", + "--use_kv_cache_manager_v2=false", "--model_dir", f"{llm_models_root()}/{model_path}", ])