diff --git a/README.md b/README.md index e391dfd..40df38b 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ The repository currently includes: | `recipes/qwen36-27b/` | Universal causal LLM quantization with llmcompressor | `uv sync --extra qwen36-27b` | | `recipes/qwen36-moe-35b-nvfp4/` | INT8, FP8, and NVFP4 quantization for hybrid MoE models | `uv sync --extra qwen36-moe-35b-nvfp4` | | `recipes/cosmos-reason2/` | NVFP4 quantization for Cosmos Reason2 (2B, 8B) with llmcompressor and Hugging Face export | `uv sync --extra cosmos-reason2` | +| `recipes/internvla-n1-dualvln/` | FP8 quantization and TensorRT-Edge-LLM deployment for InternVLA-N1-DualVLN, a dual-system vision-language navigation model, on NVIDIA Jetson Thor | `uv sync --extra internvla-n1-dualvln` | ## Architecture @@ -51,6 +52,7 @@ uv sync --extra qwen3-asr uv sync --extra qwen36-27b uv sync --extra qwen36-moe-35b-nvfp4 uv sync --extra cosmos-reason2 +uv sync --extra internvla-n1-dualvln ``` ## Quick Start @@ -124,6 +126,27 @@ MODEL_PATH=/path/to/Cosmos-Reason2-2B OUTPUT_PATH=/path/to/output ./quantize.sh MODEL_PATH=/path/to/Cosmos-Reason2-8B OUTPUT_PATH=/path/to/output ./quantize.sh ``` +### InternVLA-N1 DualVLN + +```bash +cd recipes/internvla-n1-dualvln + +export INTERNVLA_CKPT=/path/to/InternVLA-N1-DualVLN +make repackage # strip System 1 -> stock Qwen2.5-VL System 2 checkpoint +make quantize-fp8 # FP8 W8A8, LLM backbone only +make export-build # ONNX export + TensorRT-Edge-LLM engines +make verify-latents # acceptance gate: System2 -> System1 bridge fidelity +``` + +The recipe is split by dependency boundary: + +- `quantize/` turns the checkpoint into a quantized stock Qwen2.5-VL — needs no InternNav +- `trt-edgellm/` builds and verifies the engines; only System 1 export and the agent-level + checks require `INTERNNAV_PATH` + +Acceptance is measured on `z_latents` cosine (the System 2 → System 1 bridge), not on text +fluency — a checkpoint can caption correctly and still navigate wrongly. + ## Benchmark and Results Representative results from `recipes/qwen3-asr/`: @@ -155,6 +178,7 @@ For the edge results, Jetson measurements use unified memory while RTX measureme │ ├── _template/ │ ├── cosmos-reason2/ │ ├── gemma4/ +│ ├── internvla-n1-dualvln/ │ ├── qwen3-asr/ │ ├── qwen36-27b/ │ └── qwen36-moe-35b-nvfp4/ diff --git a/pyproject.toml b/pyproject.toml index 7add770..ae713e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,14 @@ cosmos-reason2 = [ "tqdm==4.67.3", ] +# `build_calib_jsonl.py` is stdlib-only (argparse, gzip, json, random). The one real +# dependency is the `huggingface-cli` entrypoint used to fetch the calibration source data. +# Quantize/export/build run against TensorRT-Edge-LLM's own environment, not this one -- +# see recipes/internvla-n1-dualvln/README.md. +internvla-n1-dualvln = [ + "huggingface-hub>=0.36.0", +] + [tool.ruff] target-version = "py312" diff --git a/recipes/internvla-n1-dualvln/.gitignore b/recipes/internvla-n1-dualvln/.gitignore new file mode 100644 index 0000000..292337d --- /dev/null +++ b/recipes/internvla-n1-dualvln/.gitignore @@ -0,0 +1,8 @@ +# Recipe artifacts. These are multi-gigabyte and reproducible; nothing in the root +# .gitignore covers them, so a stray WORK_DIR=. would otherwise stage ~40 GB. +work/ +*.engine +*.onnx +*.onnx.data +bench_imgs/ +reports/ diff --git a/recipes/internvla-n1-dualvln/Makefile b/recipes/internvla-n1-dualvln/Makefile new file mode 100644 index 0000000..4c1b340 --- /dev/null +++ b/recipes/internvla-n1-dualvln/Makefile @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 VinRobotics +# SPDX-License-Identifier: BSD-3-Clause +# Makefile — internvla-n1-dualvln +# +# Builds the navigation-domain calibration set this recipe is responsible for. Quantize, +# export, and build are the standard TensorRT-Edge-LLM CLI, run in that repo's own +# environment -- see README.md for why they are not wrapped here. +# +# Override paths via environment variables, e.g.: +# make build-calib CALIB_DATA_ROOT=/mnt/scratch/calib + +INTERNVLA_CKPT ?= $(HOME)/InternNav/checkpoints/InternVLA-N1-DualVLN +CALIB_DATA_ROOT ?= $(HOME)/vln-opt-work/calib +TRAIN_JSON := $(CALIB_DATA_ROOT)/vln_ce/raw_data/r2r/train/train.json.gz +NAV_CALIB_JSONL := $(CALIB_DATA_ROOT)/nav_calib.jsonl +NUM_SAMPLES ?= 512 + +.PHONY: fetch-calib-source build-calib all clean help + +fetch-calib-source: + huggingface-cli download InternRobotics/InternData-N1 \ + vln_ce/raw_data/r2r/train/train.json.gz --repo-type dataset \ + --local-dir $(CALIB_DATA_ROOT) + +build-calib: $(TRAIN_JSON) + python quantize/build_calib_jsonl.py \ + --train_json $(TRAIN_JSON) \ + --output $(NAV_CALIB_JSONL) \ + --num_samples $(NUM_SAMPLES) + +all: fetch-calib-source build-calib + @echo "" + @echo "Calibration set ready: $(NAV_CALIB_JSONL)" + @echo "Quantize/export/build with the TensorRT-Edge-LLM CLI -- see README.md:" + @echo " export EDGELLM_QUANT_DATASET_CNN_DAILYMAIL=$(NAV_CALIB_JSONL)" + @echo " tensorrt-edgellm-quantize llm --model_dir $(INTERNVLA_CKPT) \\" + @echo " --output_dir --quantization {fp8,nvfp4}" + +clean: + rm -rf $(CALIB_DATA_ROOT) + +help: + @echo "internvla-n1-dualvln — targets" + @echo "" + @echo " fetch-calib-source download InternData-N1's R2R train split (2.4 MB, gated)" + @echo " build-calib build the navigation-prompt calibration JSONL" + @echo " all both of the above, then print the CLI commands to run next" + @echo " clean remove CALIB_DATA_ROOT" + @echo "" + @echo " Variables: INTERNVLA_CKPT CALIB_DATA_ROOT NUM_SAMPLES" + @echo "" + @echo " Quantize / export / build / run are the standard TensorRT-Edge-LLM CLI," + @echo " not wrapped here -- see README.md." diff --git a/recipes/internvla-n1-dualvln/README.md b/recipes/internvla-n1-dualvln/README.md new file mode 100644 index 0000000..3d5d872 --- /dev/null +++ b/recipes/internvla-n1-dualvln/README.md @@ -0,0 +1,98 @@ +# internvla-n1-dualvln + +FP8 / NVFP4 quantization and TensorRT-Edge-LLM deployment for InternVLA-N1-DualVLN, a +dual-system vision-language navigation model, on NVIDIA Jetson Thor. + +## Status + +Native support landed in TensorRT-Edge-LLM: +[NVIDIA/TensorRT-Edge-LLM#193](https://github.com/NVIDIA/TensorRT-Edge-LLM/pull/193) +(pending review, tracked by +[NVIDIA/TensorRT-Edge-LLM#190](https://github.com/NVIDIA/TensorRT-Edge-LLM/issues/190)). This +recipe used to repackage the checkpoint and compute the `z_latents` bridge on the host in +Python; both steps are gone now that TensorRT-Edge-LLM exports the checkpoint directly and +folds the bridge (`final_norm` + `cond_projector`) into the graph. What is left here is the one +thing that stays outside TensorRT-Edge-LLM: building a navigation-domain calibration set. + +**Measured on Jetson Thor, 199 R2R val_unseen episodes, closed-loop:** + +| | prefill | decode | control rate | engine | SR vs PyTorch (69.8%) | +|---|---|---|---|---|---| +| TensorRT FP8 | 90.6 ms | 32.8 ms | 61.3 ms (16.3 Hz) | 7.10 GB | 68.3% (p = 0.728) | +| TensorRT NVFP4 | 75.4 ms | 20.3 ms | 55.4 ms (18.0 Hz) | 4.45 GB | 67.8% (p = 0.572) | + +Neither differs from PyTorch significantly. **This replaces the recipe's earlier +recommendation.** The old version gated acceptance on `z_latents` cosine and, on that basis, +recommended FP8 and ruled NVFP4 out (cosine 0.647 against a 0.99 gate). Closed-loop SR shows +that gate does not predict the outcome it stands in for — full validation and the "no offline +metric predicts SR" finding are in the PR. Pick NVFP4 for speed and size, FP8 for a wider +margin; both are viable. + +## What this recipe does + +Build a calibration set of realistic navigation prompts. Calibrating the quantized backbone on +its own prompt domain, rather than generic news text, measurably changes activation scales — +the same FP8 recipe moved from trajectory cosine 0.909 (`cnn_dailymail`, the CLI's default) to +0.978 (navigation prompts) in earlier testing on this model. + +```bash +huggingface-cli download InternRobotics/InternData-N1 \ + vln_ce/raw_data/r2r/train/train.json.gz --repo-type dataset \ + --local-dir $CALIB_DATA_ROOT +# gated on Hugging Face -- accept the dataset terms and `huggingface-cli login` first + +python quantize/build_calib_jsonl.py \ + --train_json $CALIB_DATA_ROOT/vln_ce/raw_data/r2r/train/train.json.gz \ + --output $CALIB_DATA_ROOT/nav_calib.jsonl +``` + +Everything after that is the standard TensorRT-Edge-LLM flow, in its own environment: + +```bash +# Point the CLI's default text_dataset at the local file instead of the Hub. +export EDGELLM_QUANT_DATASET_CNN_DAILYMAIL=$CALIB_DATA_ROOT/nav_calib.jsonl + +tensorrt-edgellm-quantize llm --model_dir $INTERNVLA_CKPT \ + --output_dir $QUANT_CKPT --quantization {fp8,nvfp4} +tensorrt-edgellm-export $QUANT_CKPT $ONNX_DIR + +export EDGELLM_PLUGIN_PATH=.../libNvInfer_edgellm_plugin.so +export __LUNOWUD="-cask_fusion:max_num_epilogues=1" # NVFP4 at maxBatchSize 1 only +llm_build --onnxDir $ONNX_DIR/llm --engineDir $ENGINE_DIR/llm \ + --maxBatchSize 1 --maxInputLen 3072 --maxKVCacheCapacity 4096 +``` + +System 1 (the trajectory expert) is not part of this checkpoint's quantization — it stays +BF16, built separately with `trtexec` — and the async runtime +(`internvla_n1_dual_system_inference` / `internvla_n1_dual_system_server`) that drives both +systems is not part of this repo either; it ships with TensorRT-Edge-LLM. See +[NVIDIA/TensorRT-Edge-LLM#193](https://github.com/NVIDIA/TensorRT-Edge-LLM/pull/193) for the +full export → build → run flow and the resident-server protocol for driving it from a Python +agent. + +## Why calibration text needs no special tokens here + +An earlier version of this pipeline appended four trajectory-query placeholder tokens to every +calibration prompt, because its own driver ran the System-2 → System-1 bridge forward pass +during calibration. `tensorrt-edgellm-quantize` does not: it loads System 2 as a stock +Qwen2.5-VL (see `internvla_n1_loader.py` in TensorRT-Edge-LLM) and calibrates with ordinary +text forward passes, never touching the bridge. Appending tokens the tokenizer does not even +have registered yet — they are added later, at export time — would only add noise. + +## What is and is not quantized + +Quantized: the System 2 LLM backbone. Never quantized: System 1 (`traj_dit`, memory block), +and the bridge (`cond_projector`, `latent_queries`) — four rows through a +Linear/GELU/Linear, kept at source precision because quantizing them saves nothing measurable +and puts error directly on the tensor System 1 steers by. + +**NVFP4 with the vision tower is blocked, permanently.** The Qwen2.5-VL ViT MLP has +`intermediate_size = 3420`, and 3420 / 16 = 213.75 — not divisible by the NVFP4 block size. +Only the LLM backbone is quantized here, so this does not apply, but it is worth knowing if you +extend this to a strategy that includes the vision tower. + +## Tested environment + +Jetson Thor, JetPack 7.1 (TensorRT 10.13.3.9, CUDA 13). `pip install -e ".[tools]"` in the +TensorRT-Edge-LLM checkout pulls `nvidia-modelopt` and `datasets`; without it +`tensorrt-edgellm-quantize` fails at import. diff --git a/recipes/internvla-n1-dualvln/quantize/build_calib_jsonl.py b/recipes/internvla-n1-dualvln/quantize/build_calib_jsonl.py new file mode 100644 index 0000000..23401bc --- /dev/null +++ b/recipes/internvla-n1-dualvln/quantize/build_calib_jsonl.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 VinRobotics +# SPDX-License-Identifier: BSD-3-Clause +"""Build a navigation-prompt calibration JSONL for `tensorrt-edgellm-quantize`. + +The native `tensorrt-edgellm-quantize llm --text_dataset NAME` flag only accepts a +*registered* dataset name (the default is `cnn_dailymail`), but any built-in can be pointed +at a local file with `EDGELLM_QUANT_DATASET_=/path/to/file.jsonl` -- see +`tensorrt_edgellm/quantization/datasets/__init__.py::local_override_path`. This script writes +that file in the schema the override loader expects: one `{"article": "..."}` object per line +(the field name `cnn_dailymail()` reads, since that is the dataset being overridden). + +Why bother, instead of the CLI's cnn_dailymail default: calibrating on the deployment prompt's +own domain measurably changes activation scales. The same FP8 recipe went from trajectory +cosine 0.909 (cnn_dailymail) to 0.978 (this) in earlier testing -- calibration text that never +resembles a navigation instruction leaves the backbone's activation ranges fitted to news +articles instead of the prompts it actually deploys on. + +One thing this does *not* need, unlike an earlier version of this recipe: the four trailing +trajectory-query tokens (`<|latent_q0..3|>`). Those only matter for calibrating the +System-2 -> System-1 bridge forward pass, and `tensorrt-edgellm-quantize` never runs that -- +it loads System 2 as a stock Qwen2.5-VL (see `internvla_n1_loader.py` in TensorRT-Edge-LLM) +and calibrates with ordinary text forward passes. Appending tokens the tokenizer does not yet +know about (they are registered later, at export time) would only add noise. + +Usage: + huggingface-cli download InternRobotics/InternData-N1 \\ + vln_ce/raw_data/r2r/train/train.json.gz --repo-type dataset \\ + --local-dir $CALIB_DATA_ROOT + python build_calib_jsonl.py \\ + --train_json $CALIB_DATA_ROOT/vln_ce/raw_data/r2r/train/train.json.gz \\ + --output $CALIB_DATA_ROOT/nav_calib.jsonl +""" +import argparse +import gzip +import json +import random + +PROMPT_TEMPLATE = ( + "You are an autonomous navigation assistant. Your task is to {instruction} " + "Where should you go next to stay on track? Please output the next waypoint's " + "coordinates in the image. Please output STOP when you have successfully completed " + "the task.") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--train_json", required=True, + help="InternData-N1 vln_ce/raw_data/r2r/train/train.json.gz") + ap.add_argument("--output", required=True, help="Destination JSONL") + ap.add_argument("--num_samples", type=int, default=512, + help="Matches the native CLI's calibration sample count (default: 512)") + ap.add_argument("--seed", type=int, default=0) + args = ap.parse_args() + + with gzip.open(args.train_json) as f: + episodes = json.load(f)["episodes"] + + random.Random(args.seed).shuffle(episodes) + + written = 0 + with open(args.output, "w") as out: + for ep in episodes: + if written >= args.num_samples: + break + text = ep["instruction"]["instruction_text"].strip() + if not text: + continue + instruction = text.rstrip(". ") + prompt = PROMPT_TEMPLATE.format(instruction=instruction + ".") + out.write(json.dumps({"article": prompt}) + "\n") + written += 1 + + print(f"wrote {written} navigation calibration prompts to {args.output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/recipes/internvla-n1-dualvln/requirements.txt b/recipes/internvla-n1-dualvln/requirements.txt new file mode 100644 index 0000000..a77cb51 --- /dev/null +++ b/recipes/internvla-n1-dualvln/requirements.txt @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 VinRobotics +# SPDX-License-Identifier: BSD-3-Clause +# +# InternVLA-N1-DualVLN — Python dependencies (Jetson Thor / aarch64, Python 3.12). +# PyTorch is intentionally absent; see requirements-torch.txt. +# +# pip install -r requirements.txt + +# Hugging Face Ecosystem +transformers==4.51.3 +accelerate==1.13.0 +safetensors==0.8.0 +huggingface-hub>=0.23.0 + +# Diffusion (System 1 trajectory head) +diffusers==0.33.1 + +# Numerics — pinned below numpy 2 so OpenCV and diffusers keep working on Jetson +numpy==1.26.4 +scipy==1.13.1 +pandas>=2.0.0 +pillow>=10.0.0 + +# ONNX export +onnx==1.22.0 +onnxscript==0.7.1 +onnx-graphsurgeon==0.6.1 + +# Quantization +nvidia-modelopt==0.44.0 + +# Utilities +numpy-quaternion>=2023.0.0 +pyyaml>=6.0.0 +tqdm>=4.66.0 + +# Calibration dataset loader — installing this may pull numpy>=2; if it does, +# re-pin numpy==1.26.4 and scipy==1.13.1 afterwards. +# datasets==2.19.0 + +# ============================================================================= +# Not available on PyPI for aarch64 — install these separately: +# +# TensorRT 10.13 provided by JetPack, imported from /usr/lib/python3.12/dist-packages +# tensorrt-edgellm build from source, then: pip install --no-deps -e $TRT_EDGE_LLM +# OpenCV (cv2) system package on Jetson +# InternNav git clone; export INTERNNAV_PATH. Needed only for System 1 +# export and the agent-level verifications. +# flash-attn optional, only for the PyTorch reference baseline +# =============================================================================