From e723644b6ccd267180f36412316a73737cc3d75d Mon Sep 17 00:00:00 2001 From: Thanos Stratikopoulos Date: Wed, 22 Jul 2026 13:12:21 +0300 Subject: [PATCH 1/8] [feat] Auto-detect TornadoVM backend in llama-tornado/llamaTornado --- llama-tornado | 99 ++++++++++++++++++++++++++++++++++----------------- llamaTornado | 85 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 141 insertions(+), 43 deletions(-) diff --git a/llama-tornado b/llama-tornado index 78388295..21ef80e3 100755 --- a/llama-tornado +++ b/llama-tornado @@ -2,6 +2,9 @@ """ llama-tornado: GPU-accelerated Java LLM runner with TornadoVM Run LLM models using OpenCL, PTX, CUDA, or Metal backends. + +The backend is auto-detected from the installed TornadoVM SDK +(TORNADOVM_HOME/etc/tornado.backend) - no need to select it manually. """ import argparse @@ -23,6 +26,18 @@ class Backend(Enum): METAL = "metal" +# Maps the names TornadoVM writes to etc/tornado.backend to our Backend enum. +BACKEND_NAME_MAP: Dict[str, Backend] = { + "opencl-backend": Backend.OPENCL, + "ptx-backend": Backend.PTX, + "cuda-backend": Backend.CUDA, + "metal-backend": Backend.METAL, +} + +# When an SDK is built with more than one backend, prefer in this order. +BACKEND_PRIORITY: List[Backend] = [Backend.CUDA, Backend.PTX, Backend.OPENCL, Backend.METAL] + + class LlamaRunner: """Main class for managing LLM execution with GPU acceleration.""" @@ -50,6 +65,35 @@ class LlamaRunner: print(f"Error: {name} path does not exist: {path}") sys.exit(1) + def detect_backend(self) -> Backend: + """Detect the TornadoVM backend from the installed SDK's tornado.backend file.""" + backend_file = Path(self.tornado_sdk) / "etc" / "tornado.backend" + if not backend_file.exists(): + print(f"Error: Could not detect TornadoVM backend - {backend_file} not found") + print("Note: this file is written by the TornadoVM installer; make sure TORNADOVM_HOME points at a valid SDK") + sys.exit(1) + + installed_names: List[str] = [] + for line in backend_file.read_text().splitlines(): + line = line.strip() + if line.startswith("tornado.backends="): + installed_names = [b.strip() for b in line.split("=", 1)[1].split(",") if b.strip()] + break + + if not installed_names: + print(f"Error: No backends declared in {backend_file}") + sys.exit(1) + + detected = [BACKEND_NAME_MAP[n] for n in installed_names if n in BACKEND_NAME_MAP] + if not detected: + print(f"Error: Unsupported backend(s) in {backend_file}: {', '.join(installed_names)}") + sys.exit(1) + + for backend in BACKEND_PRIORITY: + if backend in detected: + return backend + return detected[0] + @staticmethod def module_path_colon_sep(paths: List[str]) -> str: """Return OS-specific separator for Java module paths.""" @@ -354,12 +398,28 @@ def load_env_from_script(): sys.exit(1) +class _RemovedBackendFlag(argparse.Action): + """Gives a clear error for the removed --opencl/--ptx/--cuda/--metal flags + instead of silently abbreviating to an unrelated option (e.g. --cuda -> --cuda-graphs).""" + + def __init__(self, option_strings, dest, **kwargs): + kwargs["nargs"] = 0 + super().__init__(option_strings, dest, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + parser.error( + f"{option_string} has been removed - the TornadoVM backend is now " + f"auto-detected from TORNADOVM_HOME/etc/tornado.backend" + ) + + def create_parser() -> argparse.ArgumentParser: """Create and configure the argument parser.""" parser = argparse.ArgumentParser( prog="llama-tornado", description="GPU-accelerated LLM runner using TornadoVM", formatter_class=argparse.ArgumentDefaultsHelpFormatter, + allow_abbrev=False, ) # Required arguments @@ -423,34 +483,8 @@ def create_parser() -> argparse.ArgumentParser: hw_group.add_argument( "--gpu", dest="use_gpu", action="store_true", help="Enable GPU acceleration" ) - hw_group.add_argument( - "--opencl", - dest="backend", - action="store_const", - const=Backend.OPENCL, - help="Use OpenCL backend (default)", - ) - hw_group.add_argument( - "--ptx", - dest="backend", - action="store_const", - const=Backend.PTX, - help="Use PTX backend", - ) - hw_group.add_argument( - "--cuda", - dest="backend", - action="store_const", - const=Backend.CUDA, - help="Use CUDA backend (requires TornadoVM built with the CUDA backend)", - ) - hw_group.add_argument( - "--metal", - dest="backend", - action="store_const", - const=Backend.METAL, - help="Use Apple Metal backend (macOS only, requires TornadoVM 4.0+)", - ) + for flag in ("--opencl", "--ptx", "--cuda", "--metal"): + hw_group.add_argument(flag, action=_RemovedBackendFlag, help=argparse.SUPPRESS) hw_group.add_argument("--gpu-memory", default="14GB", help="GPU memory allocation") hw_group.add_argument("--heap-min", default="20g", help="Minimum JVM heap size") hw_group.add_argument("--heap-max", default="20g", help="Maximum JVM heap size") @@ -576,16 +610,15 @@ def main(): if args.seed is None: args.seed = int(time.time()) - # Set default backend to OpenCL if not specified - if not hasattr(args, "backend") or args.backend is None: - args.backend = Backend.OPENCL - # Handle mode selection logic if args.interactive: args.instruct = False - # Create and run the LLaMA runner + # Create the LLaMA runner and auto-detect the TornadoVM backend runner = LlamaRunner() + args.backend = runner.detect_backend() + print(f"Detected TornadoVM backend: {args.backend.value} (from {runner.tornado_sdk}/etc/tornado.backend)") + return runner.run(args) diff --git a/llamaTornado b/llamaTornado index 068c7946..1073614f 100755 --- a/llamaTornado +++ b/llamaTornado @@ -5,7 +5,10 @@ import module java.logging; String name = MethodHandles.lookup().lookupClass().getName(); String version = "2026-04-11.1"; -enum Backend { OPENCL, PTX, METAL } +enum Backend { OPENCL, PTX, CUDA, METAL } + +// When an SDK is built with more than one backend, prefer in this order. +List BACKEND_PRIORITY = List.of(Backend.CUDA, Backend.PTX, Backend.OPENCL, Backend.METAL); record Config( String modelPath, String prompt, String systemPrompt, @@ -20,7 +23,7 @@ record Config( String openclFlags, int maxWaitEvents, boolean verbose ) {} -Config parseArgs(String[] args) { +Config parseArgs(String[] args, Backend detectedBackend) { String modelPath = null; String prompt = null; String systemPrompt = null; @@ -33,7 +36,7 @@ Config parseArgs(String[] args) { boolean interactive = false; boolean instruct = true; boolean useGpu = false; - Backend backend = Backend.OPENCL; + Backend backend = detectedBackend; String gpuMemory = "14GB"; String heapMin = "20g"; String heapMax = "20g"; @@ -66,9 +69,11 @@ Config parseArgs(String[] args) { case "-i", "--interactive" -> { interactive = true; instruct = false; } case "--instruct" -> instruct = true; case "--gpu" -> useGpu = true; - case "--opencl" -> backend = Backend.OPENCL; - case "--ptx" -> backend = Backend.PTX; - case "--metal" -> backend = Backend.METAL; + case "--opencl", "--ptx", "--cuda", "--metal" -> { + System.err.println(args[i] + " has been removed - the TornadoVM backend is now " + + "auto-detected from TORNADOVM_HOME/etc/tornado.backend"); + System.exit(1); + } case "--gpu-memory" -> gpuMemory = args[++i]; case "--heap-min" -> heapMin = args[++i]; case "--heap-max" -> heapMax = args[++i]; @@ -149,9 +154,7 @@ void printUsage() { Hardware: --gpu Enable GPU acceleration - --opencl Use OpenCL backend (default) - --ptx Use PTX/CUDA backend - --metal Use Metal backend (macOS) + (backend is auto-detected from TORNADOVM_HOME/etc/tornado.backend) --gpu-memory GPU memory allocation (default: 14GB) --heap-min Min JVM heap (default: 20g) --heap-max Max JVM heap (default: 20g) @@ -203,6 +206,60 @@ String modulePath(String tornadoSdk) { return "." + sep + tornadoSdk + "/share/java/tornado"; } +Backend detectBackend(String tornadoSdk) { + var backendFile = Path.of(tornadoSdk, "etc", "tornado.backend"); + if (!Files.exists(backendFile)) { + System.err.println("Error: Could not detect TornadoVM backend - " + backendFile + " not found"); + System.err.println("Note: this file is written by the TornadoVM installer; make sure TORNADOVM_HOME points at a valid SDK"); + System.exit(1); + } + + var nameMap = Map.of( + "opencl-backend", Backend.OPENCL, + "ptx-backend", Backend.PTX, + "cuda-backend", Backend.CUDA, + "metal-backend", Backend.METAL + ); + + var installedNames = new ArrayList(); + try { + for (var line : Files.readAllLines(backendFile)) { + line = line.strip(); + if (line.startsWith("tornado.backends=")) { + for (var n : line.substring("tornado.backends=".length()).split(",")) { + n = n.strip(); + if (!n.isEmpty()) installedNames.add(n); + } + break; + } + } + } catch (IOException e) { + System.err.println("Error reading " + backendFile + ": " + e.getMessage()); + System.exit(1); + } + + if (installedNames.isEmpty()) { + System.err.println("Error: No backends declared in " + backendFile); + System.exit(1); + } + + var detected = new ArrayList(); + for (var n : installedNames) { + var b = nameMap.get(n); + if (b != null) detected.add(b); + } + + if (detected.isEmpty()) { + System.err.println("Error: Unsupported backend(s) in " + backendFile + ": " + String.join(", ", installedNames)); + System.exit(1); + } + + for (var b : BACKEND_PRIORITY) { + if (detected.contains(b)) return b; + } + return detected.getFirst(); +} + List buildCommand(Config cfg, String javaHome, String tornadoSdk, String llamaRoot) { var cmd = new ArrayList(); @@ -274,6 +331,11 @@ List buildCommand(Config cfg, String javaHome, String tornadoSdk, String cmd.addAll(List.of("--add-modules", "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.ptx")); } + case CUDA -> { + cmd.add("@" + tornadoSdk + "/etc/exportLists/cuda-exports"); + cmd.addAll(List.of("--add-modules", + "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.cuda")); + } case METAL -> { cmd.add("@" + tornadoSdk + "/etc/exportLists/metal-exports"); cmd.addAll(List.of("--add-modules", @@ -347,7 +409,10 @@ void main(String... args) { var tornadoSdk = requireEnv("TORNADOVM_HOME"); var llamaRoot = resolveLlamaRoot(); - var cfg = parseArgs(args); + var backend = detectBackend(tornadoSdk); + IO.println("Detected TornadoVM backend: " + backend.toString().toLowerCase() + " (from " + tornadoSdk + "/etc/tornado.backend)"); + + var cfg = parseArgs(args, backend); var cmd = buildCommand(cfg, javaHome, tornadoSdk, llamaRoot); if (cfg.showCommand()) { From aab9e9c5ad2a91c93f61a1c8487bcff1c2ee51ba Mon Sep 17 00:00:00 2001 From: Thanos Stratikopoulos Date: Wed, 22 Jul 2026 13:13:00 +0300 Subject: [PATCH 2/8] [ci] Drop explicit backend flags from llama-tornado invocations --- .github/actions/run-inference/action.yml | 3 ++- .github/workflows/build-and-run.yml | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/actions/run-inference/action.yml b/.github/actions/run-inference/action.yml index fe3d574b..cbb90495 100644 --- a/.github/actions/run-inference/action.yml +++ b/.github/actions/run-inference/action.yml @@ -43,7 +43,8 @@ runs: METRICS_FILE: ${{ inputs.metrics_file }} run: | # Run inference and emit raw metrics JSON via JAVA_TOOL_OPTIONS - ./llama-tornado --gpu --${{ inputs.backend }} \ + # Backend (opencl/ptx/cuda) is auto-detected from $TORNADOVM_HOME/etc/tornado.backend + ./llama-tornado --gpu \ --model $MODELS_DIR/${{ inputs.model_file }} \ --prompt "${{ inputs.prompt }}" \ ${{ inputs.flags }} diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml index 81ceed98..af61e397 100644 --- a/.github/workflows/build-and-run.yml +++ b/.github/workflows/build-and-run.yml @@ -312,7 +312,7 @@ jobs: run: | cd ${{ github.workspace }} export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ + ./llama-tornado --gpu \ --model $MODELS_DIR/Llama-3.2-1B-Instruct-Q8_0.gguf \ --prompt "Say hello" \ --with-prefill-decode @@ -336,7 +336,7 @@ jobs: run: | cd ${{ github.workspace }} export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ + ./llama-tornado --gpu \ --model $MODELS_DIR/Llama-3.2-1B-Instruct-Q8_0.gguf \ --prompt "Say hello" \ --with-prefill-decode --batch-prefill-size 32 @@ -362,7 +362,7 @@ jobs: run: | cd ${{ github.workspace }} export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --ptx \ + ./llama-tornado --gpu \ --model $MODELS_DIR/Llama-3.2-1B-Instruct-Q8_0.gguf \ --prompt "Say hello" \ --with-prefill-decode \ @@ -388,7 +388,7 @@ jobs: run: | cd ${{ github.workspace }} export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --ptx \ + ./llama-tornado --gpu \ --model $MODELS_DIR/Llama-3.2-1B-Instruct-Q8_0.gguf \ --prompt "Say hello" \ --with-prefill-decode --batch-prefill-size 32 \ From d34e429f70c1be05952b44c3e35b7ec5974755f4 Mon Sep 17 00:00:00 2001 From: Thanos Stratikopoulos Date: Wed, 22 Jul 2026 13:13:36 +0300 Subject: [PATCH 3/8] [chore] Update benchmark scripts for backend auto-detection --- scripts/all.sh | 2 +- scripts/benchmark_backends.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/all.sh b/scripts/all.sh index e7c05c65..e945987a 100644 --- a/scripts/all.sh +++ b/scripts/all.sh @@ -51,6 +51,6 @@ for model in "${models[@]}"; do #java @argfile -cp /home/devoxx2025-demo/java-ai-demos/GPULlama3.java/target/gpu-llama3-0.2.2.jar org.beehive.gpullama3.LlamaApp --model "$model" --stream true --echo false -p "Who are you?" --instruct - #./llama-tornado --gpu --opencl --model "$model" --prompt "Who are you?" + #./llama-tornado --gpu --model "$model" --prompt "Who are you?" done diff --git a/scripts/benchmark_backends.sh b/scripts/benchmark_backends.sh index cf76679a..80aa0f1d 100755 --- a/scripts/benchmark_backends.sh +++ b/scripts/benchmark_backends.sh @@ -117,7 +117,7 @@ run_inference() { export JAVA_TOOL_OPTIONS="-Dllama.metrics.format=json -Dllama.metrics.output=file -Dllama.metrics.file=$metrics_file" # shellcheck disable=SC2086 ( cd "$LLAMA_ROOT_DIR" && \ - ./llama-tornado --gpu --"$backend" \ + ./llama-tornado --gpu \ --model "$MODELS_DIR/$model_file" \ --prompt "$PROMPT" \ --max-tokens "$MAX_TOKENS" \ From 108b88ad8b5fd341a2f7e11b0188cac7a9a7bf4d Mon Sep 17 00:00:00 2001 From: Thanos Stratikopoulos Date: Wed, 22 Jul 2026 13:14:23 +0300 Subject: [PATCH 4/8] [docs] Reflect llama-tornado backend auto-detection --- .../gpullama-benchmarking-specialist.md | 7 +- .claude/skills/build-n-run-engine/SKILL.md | 26 +++---- README.md | 70 ++++++++++++------- 3 files changed, 62 insertions(+), 41 deletions(-) diff --git a/.claude/agents/gpullama-benchmarking-specialist.md b/.claude/agents/gpullama-benchmarking-specialist.md index f6074b8a..f6fb45b1 100644 --- a/.claude/agents/gpullama-benchmarking-specialist.md +++ b/.claude/agents/gpullama-benchmarking-specialist.md @@ -76,10 +76,15 @@ Results land under `perf-results//`. ```bash export JAVA_TOOL_OPTIONS="-Dllama.metrics.format=json -Dllama.metrics.output=file -Dllama.metrics.file=" -./llama-tornado --model --prompt --verbose-init --max-tokens 2048 --seed +./llama-tornado --gpu --model --prompt --verbose-init --max-tokens 2048 --seed unset JAVA_TOOL_OPTIONS ``` +`llama-tornado` auto-detects the backend from `$TORNADOVM_HOME/etc/tornado.backend` — there is no +`--opencl`/`--ptx`/`--cuda`/`--metal` flag. To benchmark a specific backend, point `TORNADOVM_HOME` +at an SDK built for that backend (e.g. via `scripts/benchmark_backends.sh`, which rebuilds +TornadoVM per backend and re-sources its env before each `llama-tornado` call). + When testing a feature flag, change one variable at a time and keep the baseline command structurally identical to the treatment command — differences beyond the flag under test invalidate the comparison. diff --git a/.claude/skills/build-n-run-engine/SKILL.md b/.claude/skills/build-n-run-engine/SKILL.md index 62a6fc98..e5915285 100644 --- a/.claude/skills/build-n-run-engine/SKILL.md +++ b/.claude/skills/build-n-run-engine/SKILL.md @@ -63,9 +63,10 @@ If this prints usage instead of erroring, the build succeeded and the launcher i ### Step 6: (Optional) Smoke-test a run -Only if a GGUF model path is available. Prefer `--cuda` (CUDA backend) on NVIDIA GPUs: +Only if a GGUF model path is available. The backend (OpenCL/PTX/CUDA/Metal) is +auto-detected from `$TORNADOVM_HOME/etc/tornado.backend` — no backend flag needed: ```bash -./llama-tornado --gpu --cuda --verbose-init --model --prompt "write a matmul in Java" --max-tokens 2048 +./llama-tornado --gpu --verbose-init --model --prompt "write a matmul in Java" --max-tokens 2048 ``` ## MANDATORY: Use --help When Uncertain About Flags @@ -74,7 +75,9 @@ Before using any `llama-tornado` flag you are not 100% certain about, run `--hel ## Running GPULlama3.java -`./llama-tornado --model [options]`. On NVIDIA GPUs, prefer `--cuda` (CUDA backend) over `--opencl` (default) unless the user asks for OpenCL specifically or is on non-NVIDIA hardware. +`./llama-tornado --model [options]`. The TornadoVM backend is auto-detected from the +installed SDK (`$TORNADOVM_HOME/etc/tornado.backend`) — there is no `--opencl`/`--ptx`/`--cuda`/`--metal` +flag to pass; to run on a different backend, point `TORNADOVM_HOME` at an SDK built for it. ### Core options @@ -84,9 +87,6 @@ Before using any `llama-tornado` flag you are not 100% certain about, run `--hel | `--prompt "..."` | single-shot generation | | `-i` / `--interactive` | chat loop instead of one-shot | | `--gpu` | required for GPU acceleration; omit to run CPU-only | -| `--ptx` | CUDA backend on NVIDIA — **prefer this by default** | -| `--opencl` | cross-vendor fallback (default if no backend flag given) | -| `--metal` | Apple Silicon only, TornadoVM 4.0+ | | `--gpu-memory 15GB`/`20GB` | bump from default 14GB for 3B/8B models — avoids OOM | | `--temperature`, `--top-p`, `--seed`, `-n` | standard sampling knobs | | `-sp/--system-prompt` | instruct-mode framing | @@ -111,15 +111,15 @@ Before using any `llama-tornado` flag you are not 100% certain about, run `--hel ### Typical invocations ```bash -# quick single-shot GPU test, CUDA backend (preferred) -./llama-tornado --gpu --ptx --model model.gguf --prompt "..." +# quick single-shot GPU test (backend auto-detected from TORNADOVM_HOME) +./llama-tornado --gpu --model model.gguf --prompt "..." -# interactive chat, CUDA backend -./llama-tornado --gpu --ptx -i --model model.gguf +# interactive chat +./llama-tornado --gpu -i --model model.gguf # bigger model, needs more GPU mem -./llama-tornado --gpu --ptx --model llama-3.2-8b-instruct-fp16.gguf --gpu-memory 20GB --prompt "..." +./llama-tornado --gpu --model llama-3.2-8b-instruct-fp16.gguf --gpu-memory 20GB --prompt "..." -# benchmarking prefill/decode split with CUDA graphs -./llama-tornado --gpu --ptx --model model.gguf --with-prefill-decode --cuda-graphs --profiler --profiler-dump-dir ./perf-results --prompt "..." +# benchmarking prefill/decode split with CUDA graphs (needs a PTX-backend TORNADOVM_HOME) +./llama-tornado --gpu --model model.gguf --with-prefill-decode --cuda-graphs --profiler --profiler-dump-dir ./perf-results --prompt "..." ``` diff --git a/README.md b/README.md index e7a99102..2ebb03c5 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,8 @@ GPULlama3ChatModel model = GPULlama3ChatModel.builder() Ensure you have the following installed and configured: - **Java 21**: Required for Vector API support & TornadoVM. -- [TornadoVM](https://github.com/beehive-lab/TornadoVM) with OpenCL, PTX, or CUDA backends. - - The `--cuda` backend requires a TornadoVM build that includes the CUDA backend from [TornadoVM PR #861](https://github.com/beehive-lab/TornadoVM/pull/861). This project currently builds against TornadoVM `5.0.0-jdk21-dev`. +- [TornadoVM](https://github.com/beehive-lab/TornadoVM) with an OpenCL, PTX, CUDA, or Metal backend. `llama-tornado`/`llamaTornado` auto-detect whichever backend your installed SDK was built with. + - The CUDA backend requires a TornadoVM build that includes the CUDA backend from [TornadoVM PR #861](https://github.com/beehive-lab/TornadoVM/pull/861). This project currently builds against TornadoVM `5.0.0-jdk21-dev`. - GCC/G++ 13 or newer: Required to build and run TornadoVM native components. ### Install, Build, and Run @@ -297,20 +297,20 @@ jbang LlamaTornadoCli.java -m beehive-llama-3.2-1b-instruct-fp16.gguf \ To execute Llama3, or Mistral models with TornadoVM on GPUs use the `llama-tornado` script with the `--gpu` flag. +The TornadoVM backend (OpenCL, PTX, CUDA, or Metal) is auto-detected from your installed +TornadoVM SDK (`TORNADOVM_HOME/etc/tornado.backend`) - no need to select it manually. + ### Usage Examples #### Basic Inference Run a model with a text prompt: ```bash -./llama-tornado --gpu --verbose-init --opencl --model beehive-llama-3.2-1b-instruct-fp16.gguf --prompt "Explain the benefits of GPU acceleration." +./llama-tornado --gpu --verbose-init --model beehive-llama-3.2-1b-instruct-fp16.gguf --prompt "Explain the benefits of GPU acceleration." ``` -Select a backend explicitly with `--opencl`, `--ptx`, or `--cuda` (NVIDIA), or `--metal` (Apple Silicon). For example, to run on the CUDA backend: - -```bash -./llama-tornado --gpu --cuda --model beehive-llama-3.2-1b-instruct-fp16.gguf --prompt "Explain the benefits of GPU acceleration." -``` +The script prints which backend it detected, e.g. `Detected TornadoVM backend: cuda (from .../etc/tornado.backend)`. +To run against a different backend, point `TORNADOVM_HOME` at an SDK built for that backend instead. #### GPU Execution (FP16 Model) Enable GPU acceleration with Q8_0 quantization: @@ -323,7 +323,7 @@ Enable GPU acceleration with Q8_0 quantization: `llamaTornado` is a zero-dependency Java 25 single-file script that replaces the Python launcher. It requires `java 25+` on your PATH: ```bash -./llamaTornado --gpu --verbose-init --metal --model /Users/abien/work/workspaces/llms/Mistral-7B-Instruct-v0.3.Q8_0.gguf --prompt "what is java" +./llamaTornado --gpu --verbose-init --model /Users/abien/work/workspaces/llms/Mistral-7B-Instruct-v0.3.Q8_0.gguf --prompt "what is java" ``` ----------- @@ -348,7 +348,6 @@ docker run --rm -it --gpus all \ beehivelab/gpullama3.java-nvidia-openjdk-opencl \ /gpullama3/GPULlama3.java/llama-tornado \ --gpu --verbose-init \ - --opencl \ --model /data/Llama-3.2-1B-Instruct.FP16.gguf \ --prompt "Tell me a joke" ``` @@ -399,30 +398,42 @@ Supported command-line options include: ```bash cmd ➜ llama-tornado --help -usage: llama-tornado [-h] --model MODEL_PATH [--prompt PROMPT] [-sp SYSTEM_PROMPT] [--temperature TEMPERATURE] [--top-p TOP_P] [--seed SEED] [-n MAX_TOKENS] - [--stream STREAM] [--echo ECHO] [-i] [--instruct] [--gpu] [--opencl] [--ptx] [--cuda] [--metal] [--gpu-memory GPU_MEMORY] [--heap-min HEAP_MIN] [--heap-max HEAP_MAX] - [--debug] [--profiler] [--profiler-dump-dir PROFILER_DUMP_DIR] [--print-bytecodes] [--print-threads] [--print-kernel] [--full-dump] - [--show-command] [--execute-after-show] [--opencl-flags OPENCL_FLAGS] [--max-wait-events MAX_WAIT_EVENTS] [--verbose] - -GPU-accelerated LLaMA.java model runner using TornadoVM +usage: llama-tornado [-h] --model MODEL_PATH [--prompt PROMPT] + [-sp SYSTEM_PROMPT] [--temperature TEMPERATURE] + [--top-p TOP_P] [--seed SEED] [-n MAX_TOKENS] + [--stream STREAM] [--echo ECHO] [--suffix SUFFIX] [-i] + [--instruct] [--gpu] [--gpu-memory GPU_MEMORY] + [--heap-min HEAP_MIN] [--heap-max HEAP_MAX] [--debug] + [--profiler] [--profiler-dump-dir PROFILER_DUMP_DIR] + [--print-bytecodes] [--print-threads] [--print-kernel] + [--full-dump] [--verbose-init] [--show-command] + [--execute-after-show] [--with-prefill-decode] + [--batch-prefill-size N] [--cuda-graphs] + [--opencl-flags OPENCL_FLAGS] + [--max-wait-events MAX_WAIT_EVENTS] [--verbose] + +GPU-accelerated LLM runner using TornadoVM +(the TornadoVM backend is auto-detected from TORNADOVM_HOME/etc/tornado.backend) options: -h, --help show this help message and exit - --model MODEL_PATH Path to the LLaMA model file (e.g., beehive-llama-3.2-8b-instruct-fp16.gguf) (default: None) + --model MODEL_PATH Path to the LLM gguf file (e.g., + Llama-3.2-1B-Instruct-Q8_0.gguf) LLaMA Configuration: --prompt PROMPT Input prompt for the model (default: None) - -sp SYSTEM_PROMPT, --system-prompt SYSTEM_PROMPT + -sp, --system-prompt SYSTEM_PROMPT System prompt for the model (default: None) --temperature TEMPERATURE Sampling temperature (0.0 to 2.0) (default: 0.1) --top-p TOP_P Top-p sampling parameter (default: 0.95) --seed SEED Random seed (default: current timestamp) (default: None) - -n MAX_TOKENS, --max-tokens MAX_TOKENS + -n, --max-tokens MAX_TOKENS Maximum number of tokens to generate (default: 512) --stream STREAM Enable streaming output (default: True) --echo ECHO Echo the input prompt (default: False) - --suffix SUFFIX Suffix for fill-in-the-middle request (Codestral) (default: None) + --suffix SUFFIX Suffix for fill-in-the-middle request (Codestral) + (default: None) Mode Selection: -i, --interactive Run in interactive/chat mode (default: False) @@ -430,12 +441,8 @@ Mode Selection: Hardware Configuration: --gpu Enable GPU acceleration (default: False) - --opencl Use OpenCL backend (default) (default: None) - --ptx Use PTX backend (default: None) - --cuda Use CUDA backend (requires TornadoVM built with the CUDA backend) (default: None) - --metal Use Apple Metal backend (macOS only) (default: None) --gpu-memory GPU_MEMORY - GPU memory allocation (default: 7GB) + GPU memory allocation (default: 14GB) --heap-min HEAP_MIN Minimum JVM heap size (default: 20g) --heap-max HEAP_MAX Maximum JVM heap size (default: 20g) @@ -443,7 +450,7 @@ Debug and Profiling: --debug Enable debug output (default: False) --profiler Enable TornadoVM profiler (default: False) --profiler-dump-dir PROFILER_DUMP_DIR - Directory for profiler output (default: /home/mikepapadim/repos/gpu-llama3.java/prof.json) + Directory for profiler output (default: None) TornadoVM Execution Verbose: --print-bytecodes Print bytecodes (tornado.print.bytecodes=true) (default: False) @@ -456,6 +463,15 @@ Command Display Options: --show-command Display the full Java command that will be executed (default: False) --execute-after-show Execute the command after showing it (use with --show-command) (default: False) +Prefill-Decode Optimizations: + --with-prefill-decode + Enable single-token prefill decode (default: False) + --batch-prefill-size N + Enable batching in prefill when --with-prefill-decode is active and N>1. (default: None) + +Advanced CUDA Features: + --cuda-graphs Enable CUDA graph capture/replay (llama.cudaGraphs=true); PTX backend only. (default: False) + Advanced Options: --opencl-flags OPENCL_FLAGS OpenCL compiler flags (default: -cl-denorms-are-zero -cl-no-signed-zeros -cl-finite-math-only) @@ -489,7 +505,7 @@ View TornadoVM's internal behavior: - **Support for GGUF format models** with full FP16 and partial support for Q8_0 and Q4_0 quantization. - **Instruction-following and chat modes** for various use cases. - **Interactive CLI** with `--interactive` and `--instruct` modes. - - **Flexible backend switching** - choose OpenCL, PTX, or CUDA at runtime (need to build TornadoVM with the chosen backends enabled). + - **Automatic backend detection** - `llama-tornado`/`llamaTornado` detect and use whichever backend (OpenCL, PTX, CUDA, or Metal) your installed TornadoVM SDK was built with. - **Cross-platform compatibility**: - ✅ NVIDIA GPUs (OpenCL, PTX & CUDA) - ✅ Intel GPUs (OpenCL) From 8cf5a5fdc4fad77ae229417bb8e2ffc0f8cb886e Mon Sep 17 00:00:00 2001 From: Thanos Stratikopoulos Date: Wed, 22 Jul 2026 13:40:27 +0300 Subject: [PATCH 5/8] [docs] Update of README --- README.md | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 56576704..76e87afa 100644 --- a/README.md +++ b/README.md @@ -402,9 +402,11 @@ usage: llama-tornado [-h] --model MODEL_PATH [--prompt PROMPT] [-sp SYSTEM_PROMPT] [--temperature TEMPERATURE] [--top-p TOP_P] [--seed SEED] [-n MAX_TOKENS] [--stream STREAM] [--echo ECHO] [--suffix SUFFIX] [-i] - [--instruct] [--gpu] [--gpu-memory GPU_MEMORY] - [--heap-min HEAP_MIN] [--heap-max HEAP_MAX] [--debug] - [--profiler] [--profiler-dump-dir PROFILER_DUMP_DIR] + [--instruct] [--server] [--port PORT] [--bench] + [--bench-args BENCH_ARGS] [--gpu] + [--gpu-memory GPU_MEMORY] [--heap-min HEAP_MIN] + [--heap-max HEAP_MAX] [--debug] [--profiler] + [--profiler-dump-dir PROFILER_DUMP_DIR] [--print-bytecodes] [--print-threads] [--print-kernel] [--full-dump] [--verbose-init] [--show-command] [--execute-after-show] [--with-prefill-decode] @@ -439,6 +441,15 @@ Mode Selection: -i, --interactive Run in interactive/chat mode (default: False) --instruct Run in instruction mode (default) (default: True) +OpenAI-compatible server: + --server Run the OpenAI-compatible HTTP server instead of inference (default: False) + --port PORT Server port (default 8080) (default: 8080) + +Benchmark (llama-bench style): + --bench Run the llama-bench-style benchmark (bench.LlamaBench) instead of inference (default: False) + --bench-args BENCH_ARGS + Extra benchmark options (use --bench-args="..."), e.g. "-p 512 -n 128 -d 0,4096 -r 5 -o md" (see LlamaBench) (default: ) + Hardware Configuration: --gpu Enable GPU acceleration (default: False) --gpu-memory GPU_MEMORY @@ -487,7 +498,7 @@ Serve the model behind the HTTP API OpenAI clients already speak — no external (JDK `HttpServer`), streaming (SSE) and non-streaming. ```bash -llama-tornado --gpu --cuda --model model.gguf --server --port 8080 +llama-tornado --gpu --model model.gguf --server --port 8080 # or directly: java ... org.beehive.gpullama3.server.OpenAIServer --model model.gguf --port 8080 --gpu ``` @@ -519,10 +530,10 @@ the forward pass only (no tokenization, no sampling), matching llama-bench metho ```bash # defaults: -p 512 -n 128 -r 5, markdown output -llama-tornado --gpu --cuda --model model1.gguf --bench +llama-tornado --gpu --model model1.gguf --bench # multiple models, custom matrix, CSV -llama-tornado --gpu --cuda --model model1.gguf --bench \ +llama-tornado --gpu --model model1.gguf --bench \ --bench-args="-m model2.gguf -p 256,512 -n 64,128 -pg 512,128 -d 0,4096 -r 5 -o csv" ``` From b247c09d4e932f088c8808172766b6b3bed170cd Mon Sep 17 00:00:00 2001 From: Thanos Stratikopoulos Date: Wed, 22 Jul 2026 16:52:14 +0300 Subject: [PATCH 6/8] [fix] Support multi-backend TornadoVM SDKs in llama-tornado/llamaTornado --- llama-tornado | 122 +++++++++++++++++++++++++++++++------------------- llamaTornado | 108 ++++++++++++++++++++++++++++++-------------- 2 files changed, 151 insertions(+), 79 deletions(-) diff --git a/llama-tornado b/llama-tornado index 18050b2a..00fb6253 100755 --- a/llama-tornado +++ b/llama-tornado @@ -34,7 +34,35 @@ BACKEND_NAME_MAP: Dict[str, Backend] = { "metal-backend": Backend.METAL, } -# When an SDK is built with more than one backend, prefer in this order. +BACKEND_EXPORTS_FILE: Dict[Backend, str] = { + Backend.OPENCL: "opencl-exports", + Backend.PTX: "ptx-exports", + Backend.CUDA: "cuda-exports", + Backend.METAL: "metal-exports", +} + +BACKEND_MODULE_NAME: Dict[Backend, str] = { + Backend.OPENCL: "tornado.drivers.opencl", + Backend.PTX: "tornado.drivers.ptx", + Backend.CUDA: "tornado.drivers.cuda", + Backend.METAL: "tornado.drivers.metal", +} + +# TornadoVM system property that ranks each backend for "driver 0" (default +# device) selection when more than one backend is loaded - see +# TornadoOptions.*_BACKEND_PRIORITY. Defaults there favor OpenCL (10) over +# everything else (0), which doesn't match BACKEND_PRIORITY below, so we set +# these explicitly whenever more than one backend is installed. +BACKEND_PRIORITY_PROPERTY: Dict[Backend, str] = { + Backend.OPENCL: "tornado.opencl.priority", + Backend.PTX: "tornado.ptx.priority", + Backend.CUDA: "tornado.cuda.priority", + Backend.METAL: "tornado.metal.priority", +} + +# When an SDK is built with more than one backend, prefer this one for +# properties that need a single "primary" backend (e.g. the printed message, +# and which backend becomes TornadoVM's default device). BACKEND_PRIORITY: List[Backend] = [Backend.CUDA, Backend.PTX, Backend.OPENCL, Backend.METAL] @@ -65,8 +93,8 @@ class LlamaRunner: print(f"Error: {name} path does not exist: {path}") sys.exit(1) - def detect_backend(self) -> Backend: - """Detect the TornadoVM backend from the installed SDK's tornado.backend file.""" + def detect_installed_backends(self) -> List[Backend]: + """Return every backend the installed TornadoVM SDK was built with (tornado.backend order).""" backend_file = Path(self.tornado_sdk) / "etc" / "tornado.backend" if not backend_file.exists(): print(f"Error: Could not detect TornadoVM backend - {backend_file} not found") @@ -84,15 +112,20 @@ class LlamaRunner: print(f"Error: No backends declared in {backend_file}") sys.exit(1) - detected = [BACKEND_NAME_MAP[n] for n in installed_names if n in BACKEND_NAME_MAP] - if not detected: + installed = [BACKEND_NAME_MAP[n] for n in installed_names if n in BACKEND_NAME_MAP] + if not installed: print(f"Error: Unsupported backend(s) in {backend_file}: {', '.join(installed_names)}") sys.exit(1) + return installed + + @staticmethod + def select_primary_backend(installed: List[Backend]) -> Backend: + """Pick which installed backend to report/default to when the SDK has more than one.""" for backend in BACKEND_PRIORITY: - if backend in detected: + if backend in installed: return backend - return detected[0] + return installed[0] @staticmethod def module_path_colon_sep(paths: List[str]) -> str: @@ -196,49 +229,39 @@ class LlamaRunner: cmd.extend(tornado_runtime_config) # Backend-specific configuration - if args.backend == Backend.OPENCL: + if Backend.OPENCL in args.installed_backends: # OpenCL specific flags cmd.append(f"-Dtornado.opencl.compiler.flags={args.opencl_flags}") - # Module configuration - varies by backend + if len(args.installed_backends) > 1: + # More than one backend is loaded, so pin TornadoVM's own driver-0 + # (default device) selection to match args.backend - otherwise it + # falls back to its own defaults (OpenCL beats everything else). + n = len(BACKEND_PRIORITY) + for rank, backend in enumerate(BACKEND_PRIORITY): + if backend in args.installed_backends: + cmd.append(f"-D{BACKEND_PRIORITY_PROPERTY[backend]}={n - rank}") + + # Module configuration - TornadoCoreRuntime tries to initialize every + # backend listed in tornado.backend at startup (not just the one we + # picked as primary), so every installed backend needs its exports and + # driver module here or it fails with an IllegalAccessError. module_config = [ f"--upgrade-module-path", f"{self.tornado_sdk}/share/java/graalJars", f"@{self.tornado_sdk}/etc/exportLists/common-exports", ] - # Add backend-specific exports and modules - if args.backend == Backend.OPENCL: - module_config.extend( - [ - f"@{self.tornado_sdk}/etc/exportLists/opencl-exports", - "--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.opencl", - ] - ) - elif args.backend == Backend.PTX: - module_config.extend( - [ - f"@{self.tornado_sdk}/etc/exportLists/ptx-exports", - "--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.ptx", - ] - ) - elif args.backend == Backend.CUDA: - module_config.extend( - [ - f"@{self.tornado_sdk}/etc/exportLists/cuda-exports", - "--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.cuda", - ] - ) - elif args.backend == Backend.METAL: - module_config.extend( - [ - f"@{self.tornado_sdk}/etc/exportLists/metal-exports", - "--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.metal", - ] - ) + add_modules = [ + "ALL-SYSTEM", + "jdk.incubator.vector", + "tornado.runtime", + "tornado.annotation", + "tornado.drivers.common", + ] + for backend in args.installed_backends: + module_config.append(f"@{self.tornado_sdk}/etc/exportLists/{BACKEND_EXPORTS_FILE[backend]}") + add_modules.append(BACKEND_MODULE_NAME[backend]) + module_config.extend(["--add-modules", ",".join(add_modules)]) if getattr(args, "server", False): main_class = "org.beehive.gpullama3.server.OpenAIServer" @@ -651,10 +674,19 @@ def main(): if args.interactive: args.instruct = False - # Create the LLaMA runner and auto-detect the TornadoVM backend + # Create the LLaMA runner and auto-detect the TornadoVM backend(s) runner = LlamaRunner() - args.backend = runner.detect_backend() - print(f"Detected TornadoVM backend: {args.backend.value} (from {runner.tornado_sdk}/etc/tornado.backend)") + installed_backends = runner.detect_installed_backends() + args.installed_backends = installed_backends + args.backend = runner.select_primary_backend(installed_backends) + if len(installed_backends) > 1: + names = ", ".join(b.value for b in installed_backends) + print( + f"Detected TornadoVM backends: {names} (from {runner.tornado_sdk}/etc/tornado.backend) " + f"- using {args.backend.value}" + ) + else: + print(f"Detected TornadoVM backend: {args.backend.value} (from {runner.tornado_sdk}/etc/tornado.backend)") return runner.run(args) diff --git a/llamaTornado b/llamaTornado index c8457736..bfd1134a 100755 --- a/llamaTornado +++ b/llamaTornado @@ -7,9 +7,37 @@ String version = "2026-04-11.1"; enum Backend { OPENCL, PTX, CUDA, METAL } -// When an SDK is built with more than one backend, prefer in this order. +// When an SDK is built with more than one backend, prefer in this order. Also +// determines which backend becomes TornadoVM's default device (driver 0) - see +// BACKEND_PRIORITY_PROPERTY below. List BACKEND_PRIORITY = List.of(Backend.CUDA, Backend.PTX, Backend.OPENCL, Backend.METAL); +Map BACKEND_EXPORTS_FILE = Map.of( + Backend.OPENCL, "opencl-exports", + Backend.PTX, "ptx-exports", + Backend.CUDA, "cuda-exports", + Backend.METAL, "metal-exports" +); + +Map BACKEND_MODULE_NAME = Map.of( + Backend.OPENCL, "tornado.drivers.opencl", + Backend.PTX, "tornado.drivers.ptx", + Backend.CUDA, "tornado.drivers.cuda", + Backend.METAL, "tornado.drivers.metal" +); + +// TornadoVM system property that ranks each backend for "driver 0" (default +// device) selection when more than one backend is loaded - see +// TornadoOptions.*_BACKEND_PRIORITY. Defaults there favor OpenCL (10) over +// everything else (0), which doesn't match BACKEND_PRIORITY above, so we set +// these explicitly whenever more than one backend is installed. +Map BACKEND_PRIORITY_PROPERTY = Map.of( + Backend.OPENCL, "tornado.opencl.priority", + Backend.PTX, "tornado.ptx.priority", + Backend.CUDA, "tornado.cuda.priority", + Backend.METAL, "tornado.metal.priority" +); + record Config( String modelPath, String prompt, String systemPrompt, double temperature, double topP, long seed, int maxTokens, @@ -216,7 +244,7 @@ String modulePath(String tornadoSdk) { return "." + sep + tornadoSdk + "/share/java/tornado"; } -Backend detectBackend(String tornadoSdk) { +List detectInstalledBackends(String tornadoSdk) { var backendFile = Path.of(tornadoSdk, "etc", "tornado.backend"); if (!Files.exists(backendFile)) { System.err.println("Error: Could not detect TornadoVM backend - " + backendFile + " not found"); @@ -253,24 +281,28 @@ Backend detectBackend(String tornadoSdk) { System.exit(1); } - var detected = new ArrayList(); + var installed = new ArrayList(); for (var n : installedNames) { var b = nameMap.get(n); - if (b != null) detected.add(b); + if (b != null) installed.add(b); } - if (detected.isEmpty()) { + if (installed.isEmpty()) { System.err.println("Error: Unsupported backend(s) in " + backendFile + ": " + String.join(", ", installedNames)); System.exit(1); } + return installed; +} + +Backend selectPrimaryBackend(List installed) { for (var b : BACKEND_PRIORITY) { - if (detected.contains(b)) return b; + if (installed.contains(b)) return b; } - return detected.getFirst(); + return installed.getFirst(); } -List buildCommand(Config cfg, String javaHome, String tornadoSdk, String llamaRoot) { +List buildCommand(Config cfg, String javaHome, String tornadoSdk, String llamaRoot, List installedBackends) { var cmd = new ArrayList(); cmd.addAll(List.of( @@ -320,38 +352,40 @@ List buildCommand(Config cfg, String javaHome, String tornadoSdk, String "-Dtornado.eventpool.maxwaitevents=" + cfg.maxWaitEvents() )); - if (cfg.backend() == Backend.OPENCL) { + if (installedBackends.contains(Backend.OPENCL)) { cmd.add("-Dtornado.opencl.compiler.flags=" + cfg.openclFlags()); } - // Module configuration + if (installedBackends.size() > 1) { + // More than one backend is loaded, so pin TornadoVM's own driver-0 + // (default device) selection to match cfg.backend() - otherwise it + // falls back to its own defaults (OpenCL beats everything else). + int n = BACKEND_PRIORITY.size(); + for (int rank = 0; rank < n; rank++) { + var b = BACKEND_PRIORITY.get(rank); + if (installedBackends.contains(b)) { + cmd.add("-D" + BACKEND_PRIORITY_PROPERTY.get(b) + "=" + (n - rank)); + } + } + } + + // Module configuration - TornadoCoreRuntime tries to initialize every + // backend listed in tornado.backend at startup (not just cfg.backend()), + // so every installed backend needs its exports and driver module here or + // it fails with an IllegalAccessError. cmd.addAll(List.of( "--upgrade-module-path", tornadoSdk + "/share/java/graalJars", "@" + tornadoSdk + "/etc/exportLists/common-exports" )); - switch (cfg.backend()) { - case OPENCL -> { - cmd.add("@" + tornadoSdk + "/etc/exportLists/opencl-exports"); - cmd.addAll(List.of("--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.opencl")); - } - case PTX -> { - cmd.add("@" + tornadoSdk + "/etc/exportLists/ptx-exports"); - cmd.addAll(List.of("--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.ptx")); - } - case CUDA -> { - cmd.add("@" + tornadoSdk + "/etc/exportLists/cuda-exports"); - cmd.addAll(List.of("--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.cuda")); - } - case METAL -> { - cmd.add("@" + tornadoSdk + "/etc/exportLists/metal-exports"); - cmd.addAll(List.of("--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.metal")); - } + var addModules = new ArrayList<>(List.of( + "ALL-SYSTEM", "jdk.incubator.vector", "tornado.runtime", "tornado.annotation", "tornado.drivers.common" + )); + for (var b : installedBackends) { + cmd.add("@" + tornadoSdk + "/etc/exportLists/" + BACKEND_EXPORTS_FILE.get(b)); + addModules.add(BACKEND_MODULE_NAME.get(b)); } + cmd.addAll(List.of("--add-modules", String.join(",", addModules))); String mainClass = cfg.bench() ? "org.beehive.gpullama3.bench.LlamaBench" @@ -437,11 +471,17 @@ void main(String... args) { var tornadoSdk = requireEnv("TORNADOVM_HOME"); var llamaRoot = resolveLlamaRoot(); - var backend = detectBackend(tornadoSdk); - IO.println("Detected TornadoVM backend: " + backend.toString().toLowerCase() + " (from " + tornadoSdk + "/etc/tornado.backend)"); + var installedBackends = detectInstalledBackends(tornadoSdk); + var backend = selectPrimaryBackend(installedBackends); + if (installedBackends.size() > 1) { + var names = String.join(", ", installedBackends.stream().map(b -> b.toString().toLowerCase()).toList()); + IO.println("Detected TornadoVM backends: " + names + " (from " + tornadoSdk + "/etc/tornado.backend) - using " + backend.toString().toLowerCase()); + } else { + IO.println("Detected TornadoVM backend: " + backend.toString().toLowerCase() + " (from " + tornadoSdk + "/etc/tornado.backend)"); + } var cfg = parseArgs(args, backend); - var cmd = buildCommand(cfg, javaHome, tornadoSdk, llamaRoot); + var cmd = buildCommand(cfg, javaHome, tornadoSdk, llamaRoot, installedBackends); if (cfg.showCommand()) { IO.println("Full Java command:"); From c915a6d2feb8d98b05a076c836839ddc972d6361 Mon Sep 17 00:00:00 2001 From: Thanos Stratikopoulos Date: Wed, 22 Jul 2026 17:33:44 +0300 Subject: [PATCH 7/8] [feat] Restore --opencl/--ptx/--cuda/--metal as optional backend overrides --- llama-tornado | 57 ++++++++++++++++++++++++++++++++------------------- llamaTornado | 56 +++++++++++++++++++++++++++++++++++++------------- 2 files changed, 78 insertions(+), 35 deletions(-) diff --git a/llama-tornado b/llama-tornado index 00fb6253..cf152139 100755 --- a/llama-tornado +++ b/llama-tornado @@ -237,8 +237,11 @@ class LlamaRunner: # More than one backend is loaded, so pin TornadoVM's own driver-0 # (default device) selection to match args.backend - otherwise it # falls back to its own defaults (OpenCL beats everything else). - n = len(BACKEND_PRIORITY) - for rank, backend in enumerate(BACKEND_PRIORITY): + # args.backend goes first (whether auto-picked or --backend + # override), then the rest of BACKEND_PRIORITY as a tiebreak. + effective_priority = [args.backend] + [b for b in BACKEND_PRIORITY if b != args.backend] + n = len(effective_priority) + for rank, backend in enumerate(effective_priority): if backend in args.installed_backends: cmd.append(f"-D{BACKEND_PRIORITY_PROPERTY[backend]}={n - rank}") @@ -440,21 +443,6 @@ def load_env_from_script(): sys.exit(1) -class _RemovedBackendFlag(argparse.Action): - """Gives a clear error for the removed --opencl/--ptx/--cuda/--metal flags - instead of silently abbreviating to an unrelated option (e.g. --cuda -> --cuda-graphs).""" - - def __init__(self, option_strings, dest, **kwargs): - kwargs["nargs"] = 0 - super().__init__(option_strings, dest, **kwargs) - - def __call__(self, parser, namespace, values, option_string=None): - parser.error( - f"{option_string} has been removed - the TornadoVM backend is now " - f"auto-detected from TORNADOVM_HOME/etc/tornado.backend" - ) - - def create_parser() -> argparse.ArgumentParser: """Create and configure the argument parser.""" parser = argparse.ArgumentParser( @@ -543,8 +531,22 @@ def create_parser() -> argparse.ArgumentParser: hw_group.add_argument( "--gpu", dest="use_gpu", action="store_true", help="Enable GPU acceleration" ) - for flag in ("--opencl", "--ptx", "--cuda", "--metal"): - hw_group.add_argument(flag, action=_RemovedBackendFlag, help=argparse.SUPPRESS) + hw_group.add_argument( + "--opencl", dest="backend_override", action="store_const", const=Backend.OPENCL, + help="Force the OpenCL backend when the installed TornadoVM SDK has more than one (default: auto-detected)", + ) + hw_group.add_argument( + "--ptx", dest="backend_override", action="store_const", const=Backend.PTX, + help="Force the PTX backend when the installed TornadoVM SDK has more than one (default: auto-detected)", + ) + hw_group.add_argument( + "--cuda", dest="backend_override", action="store_const", const=Backend.CUDA, + help="Force the CUDA backend when the installed TornadoVM SDK has more than one (default: auto-detected)", + ) + hw_group.add_argument( + "--metal", dest="backend_override", action="store_const", const=Backend.METAL, + help="Force the Metal backend when the installed TornadoVM SDK has more than one (default: auto-detected)", + ) hw_group.add_argument("--gpu-memory", default="14GB", help="GPU memory allocation") hw_group.add_argument("--heap-min", default="20g", help="Minimum JVM heap size") hw_group.add_argument("--heap-max", default="20g", help="Maximum JVM heap size") @@ -678,12 +680,25 @@ def main(): runner = LlamaRunner() installed_backends = runner.detect_installed_backends() args.installed_backends = installed_backends - args.backend = runner.select_primary_backend(installed_backends) + + overridden = False + if args.backend_override is not None: + requested = args.backend_override + if requested not in installed_backends: + installed_names = ", ".join(b.value for b in installed_backends) + print(f"Error: --{requested.value} requested, but the installed TornadoVM SDK only has: {installed_names}") + sys.exit(1) + args.backend = requested + overridden = True + else: + args.backend = runner.select_primary_backend(installed_backends) + if len(installed_backends) > 1: names = ", ".join(b.value for b in installed_backends) + reason = f"requested via --{args.backend.value}" if overridden else "auto-selected" print( f"Detected TornadoVM backends: {names} (from {runner.tornado_sdk}/etc/tornado.backend) " - f"- using {args.backend.value}" + f"- using {args.backend.value} ({reason})" ) else: print(f"Detected TornadoVM backend: {args.backend.value} (from {runner.tornado_sdk}/etc/tornado.backend)") diff --git a/llamaTornado b/llamaTornado index bfd1134a..f3d4f92d 100755 --- a/llamaTornado +++ b/llamaTornado @@ -42,7 +42,7 @@ record Config( String modelPath, String prompt, String systemPrompt, double temperature, double topP, long seed, int maxTokens, boolean stream, boolean echo, boolean interactive, boolean instruct, - boolean useGpu, Backend backend, String gpuMemory, + boolean useGpu, Backend backend, boolean backendOverridden, String gpuMemory, String heapMin, String heapMax, String directMemory, boolean debug, boolean profiler, String profilerDumpDir, boolean printBytecodes, boolean threads, boolean printKernel, @@ -52,7 +52,7 @@ record Config( boolean bench, String benchArgs ) {} -Config parseArgs(String[] args, Backend detectedBackend) { +Config parseArgs(String[] args, List installedBackends) { String modelPath = null; String prompt = null; String systemPrompt = null; @@ -65,7 +65,7 @@ Config parseArgs(String[] args, Backend detectedBackend) { boolean interactive = false; boolean instruct = true; boolean useGpu = false; - Backend backend = detectedBackend; + Backend backendOverride = null; String gpuMemory = "14GB"; String heapMin = "20g"; String heapMax = "20g"; @@ -100,11 +100,10 @@ Config parseArgs(String[] args, Backend detectedBackend) { case "-i", "--interactive" -> { interactive = true; instruct = false; } case "--instruct" -> instruct = true; case "--gpu" -> useGpu = true; - case "--opencl", "--ptx", "--cuda", "--metal" -> { - System.err.println(args[i] + " has been removed - the TornadoVM backend is now " - + "auto-detected from TORNADOVM_HOME/etc/tornado.backend"); - System.exit(1); - } + case "--opencl" -> backendOverride = Backend.OPENCL; + case "--ptx" -> backendOverride = Backend.PTX; + case "--cuda" -> backendOverride = Backend.CUDA; + case "--metal" -> backendOverride = Backend.METAL; case "--gpu-memory" -> gpuMemory = args[++i]; case "--heap-min" -> heapMin = args[++i]; case "--heap-max" -> heapMax = args[++i]; @@ -146,8 +145,24 @@ Config parseArgs(String[] args, Backend detectedBackend) { directMemory = parseAndScale(heapMax, 3); } + Backend backend; + boolean backendOverridden; + if (backendOverride != null) { + if (!installedBackends.contains(backendOverride)) { + var names = String.join(", ", installedBackends.stream().map(b -> b.toString().toLowerCase()).toList()); + System.err.println("Error: --" + backendOverride.toString().toLowerCase() + + " requested, but the installed TornadoVM SDK only has: " + names); + System.exit(1); + } + backend = backendOverride; + backendOverridden = true; + } else { + backend = selectPrimaryBackend(installedBackends); + backendOverridden = false; + } + return new Config(modelPath, prompt, systemPrompt, temperature, topP, seed, maxTokens, - stream, echo, interactive, instruct, useGpu, backend, gpuMemory, heapMin, heapMax, directMemory, + stream, echo, interactive, instruct, useGpu, backend, backendOverridden, gpuMemory, heapMin, heapMax, directMemory, debug, profiler, profilerDumpDir, printBytecodes, threads, printKernel, fullDump, verboseInit, showCommand, executeAfterShow, openclFlags, maxWaitEvents, verbose, bench, benchArgs); @@ -189,6 +204,10 @@ void printUsage() { Hardware: --gpu Enable GPU acceleration (backend is auto-detected from TORNADOVM_HOME/etc/tornado.backend) + --opencl Force the OpenCL backend if the SDK has more than one + --ptx Force the PTX backend if the SDK has more than one + --cuda Force the CUDA backend if the SDK has more than one + --metal Force the Metal backend if the SDK has more than one --gpu-memory GPU memory allocation (default: 14GB) --heap-min Min JVM heap (default: 20g) --heap-max Max JVM heap (default: 20g) @@ -360,9 +379,17 @@ List buildCommand(Config cfg, String javaHome, String tornadoSdk, String // More than one backend is loaded, so pin TornadoVM's own driver-0 // (default device) selection to match cfg.backend() - otherwise it // falls back to its own defaults (OpenCL beats everything else). - int n = BACKEND_PRIORITY.size(); + // cfg.backend() goes first (whether auto-picked or --opencl/--ptx/ + // --cuda/--metal override), then the rest of BACKEND_PRIORITY as a + // tiebreak. + var effectivePriority = new ArrayList(); + effectivePriority.add(cfg.backend()); + for (var b : BACKEND_PRIORITY) { + if (b != cfg.backend()) effectivePriority.add(b); + } + int n = effectivePriority.size(); for (int rank = 0; rank < n; rank++) { - var b = BACKEND_PRIORITY.get(rank); + var b = effectivePriority.get(rank); if (installedBackends.contains(b)) { cmd.add("-D" + BACKEND_PRIORITY_PROPERTY.get(b) + "=" + (n - rank)); } @@ -472,15 +499,16 @@ void main(String... args) { var llamaRoot = resolveLlamaRoot(); var installedBackends = detectInstalledBackends(tornadoSdk); - var backend = selectPrimaryBackend(installedBackends); + var cfg = parseArgs(args, installedBackends); + var backend = cfg.backend(); if (installedBackends.size() > 1) { var names = String.join(", ", installedBackends.stream().map(b -> b.toString().toLowerCase()).toList()); - IO.println("Detected TornadoVM backends: " + names + " (from " + tornadoSdk + "/etc/tornado.backend) - using " + backend.toString().toLowerCase()); + var reason = cfg.backendOverridden() ? "requested via --" + backend.toString().toLowerCase() : "auto-selected"; + IO.println("Detected TornadoVM backends: " + names + " (from " + tornadoSdk + "/etc/tornado.backend) - using " + backend.toString().toLowerCase() + " (" + reason + ")"); } else { IO.println("Detected TornadoVM backend: " + backend.toString().toLowerCase() + " (from " + tornadoSdk + "/etc/tornado.backend)"); } - var cfg = parseArgs(args, backend); var cmd = buildCommand(cfg, javaHome, tornadoSdk, llamaRoot, installedBackends); if (cfg.showCommand()) { From 3d6be02655589aeddeaa90604acb05028f9332c1 Mon Sep 17 00:00:00 2001 From: Thanos Stratikopoulos Date: Wed, 22 Jul 2026 17:34:16 +0300 Subject: [PATCH 8/8] [docs] Document the --opencl/--ptx/--cuda/--metal override flags --- .../gpullama-benchmarking-specialist.md | 9 +++--- .claude/skills/build-n-run-engine/SKILL.md | 8 +++-- README.md | 29 +++++++++++++++---- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/.claude/agents/gpullama-benchmarking-specialist.md b/.claude/agents/gpullama-benchmarking-specialist.md index f6fb45b1..b94fe16d 100644 --- a/.claude/agents/gpullama-benchmarking-specialist.md +++ b/.claude/agents/gpullama-benchmarking-specialist.md @@ -80,10 +80,11 @@ export JAVA_TOOL_OPTIONS="-Dllama.metrics.format=json -Dllama.metrics.output=fil unset JAVA_TOOL_OPTIONS ``` -`llama-tornado` auto-detects the backend from `$TORNADOVM_HOME/etc/tornado.backend` — there is no -`--opencl`/`--ptx`/`--cuda`/`--metal` flag. To benchmark a specific backend, point `TORNADOVM_HOME` -at an SDK built for that backend (e.g. via `scripts/benchmark_backends.sh`, which rebuilds -TornadoVM per backend and re-sources its env before each `llama-tornado` call). +`llama-tornado` auto-detects the backend from `$TORNADOVM_HOME/etc/tornado.backend`. To benchmark +a specific backend, either point `TORNADOVM_HOME` at an SDK built for that backend (e.g. via +`scripts/benchmark_backends.sh`, which rebuilds TornadoVM per backend and re-sources its env +before each `llama-tornado` call), or, if the current SDK has more than one backend installed, +pass `--opencl`/`--ptx`/`--cuda`/`--metal` to pin one without a separate SDK/rebuild. When testing a feature flag, change one variable at a time and keep the baseline command structurally identical to the treatment command — differences beyond the flag under test diff --git a/.claude/skills/build-n-run-engine/SKILL.md b/.claude/skills/build-n-run-engine/SKILL.md index e5915285..d6f46fa4 100644 --- a/.claude/skills/build-n-run-engine/SKILL.md +++ b/.claude/skills/build-n-run-engine/SKILL.md @@ -76,8 +76,11 @@ Before using any `llama-tornado` flag you are not 100% certain about, run `--hel ## Running GPULlama3.java `./llama-tornado --model [options]`. The TornadoVM backend is auto-detected from the -installed SDK (`$TORNADOVM_HOME/etc/tornado.backend`) — there is no `--opencl`/`--ptx`/`--cuda`/`--metal` -flag to pass; to run on a different backend, point `TORNADOVM_HOME` at an SDK built for it. +installed SDK (`$TORNADOVM_HOME/etc/tornado.backend`); to run on a different backend, point +`TORNADOVM_HOME` at an SDK built for it. If the current SDK was built with more than one backend +(e.g. a `cuda-opencl` build), pass `--opencl`/`--ptx`/`--cuda`/`--metal` to force one of the +installed ones instead — these error out if the requested backend isn't part of the SDK, and +are a no-op (redundant but harmless) on a single-backend SDK. ### Core options @@ -87,6 +90,7 @@ flag to pass; to run on a different backend, point `TORNADOVM_HOME` at an SDK bu | `--prompt "..."` | single-shot generation | | `-i` / `--interactive` | chat loop instead of one-shot | | `--gpu` | required for GPU acceleration; omit to run CPU-only | +| `--opencl`/`--ptx`/`--cuda`/`--metal` | rarely needed — only to force a backend when the SDK has more than one installed | | `--gpu-memory 15GB`/`20GB` | bump from default 14GB for 3B/8B models — avoids OOM | | `--temperature`, `--top-p`, `--seed`, `-n` | standard sampling knobs | | `-sp/--system-prompt` | instruct-mode framing | diff --git a/README.md b/README.md index 76e87afa..575754ae 100644 --- a/README.md +++ b/README.md @@ -310,7 +310,15 @@ Run a model with a text prompt: ``` The script prints which backend it detected, e.g. `Detected TornadoVM backend: cuda (from .../etc/tornado.backend)`. -To run against a different backend, point `TORNADOVM_HOME` at an SDK built for that backend instead. +To run against a different backend, point `TORNADOVM_HOME` at an SDK built for that backend instead - or, +if your SDK was built with more than one backend (e.g. `cuda-opencl`), pass `--opencl`/`--ptx`/`--cuda`/`--metal` +to force one of the installed backends without needing a separate SDK: + +```bash +./llama-tornado --gpu --opencl --model beehive-llama-3.2-1b-instruct-fp16.gguf --prompt "Explain the benefits of GPU acceleration." +``` + +These flags error out if the requested backend isn't part of the installed SDK. #### GPU Execution (FP16 Model) Enable GPU acceleration with Q8_0 quantization: @@ -326,6 +334,13 @@ Enable GPU acceleration with Q8_0 quantization: ./llamaTornado --gpu --verbose-init --model /Users/abien/work/workspaces/llms/Mistral-7B-Instruct-v0.3.Q8_0.gguf --prompt "what is java" ``` +Same backend auto-detection as `llama-tornado`, including the `--opencl`/`--ptx`/`--cuda`/`--metal` +override for SDKs built with more than one backend: + +```bash +./llamaTornado --gpu --opencl --model /Users/abien/work/workspaces/llms/Mistral-7B-Instruct-v0.3.Q8_0.gguf --prompt "what is java" +``` + ----------- ## 🐳 Docker @@ -403,10 +418,10 @@ usage: llama-tornado [-h] --model MODEL_PATH [--prompt PROMPT] [--top-p TOP_P] [--seed SEED] [-n MAX_TOKENS] [--stream STREAM] [--echo ECHO] [--suffix SUFFIX] [-i] [--instruct] [--server] [--port PORT] [--bench] - [--bench-args BENCH_ARGS] [--gpu] - [--gpu-memory GPU_MEMORY] [--heap-min HEAP_MIN] - [--heap-max HEAP_MAX] [--debug] [--profiler] - [--profiler-dump-dir PROFILER_DUMP_DIR] + [--bench-args BENCH_ARGS] [--gpu] [--opencl] [--ptx] + [--cuda] [--metal] [--gpu-memory GPU_MEMORY] + [--heap-min HEAP_MIN] [--heap-max HEAP_MAX] [--debug] + [--profiler] [--profiler-dump-dir PROFILER_DUMP_DIR] [--print-bytecodes] [--print-threads] [--print-kernel] [--full-dump] [--verbose-init] [--show-command] [--execute-after-show] [--with-prefill-decode] @@ -452,6 +467,10 @@ Benchmark (llama-bench style): Hardware Configuration: --gpu Enable GPU acceleration (default: False) + --opencl Force the OpenCL backend when the installed TornadoVM SDK has more than one (default: auto-detected) (default: None) + --ptx Force the PTX backend when the installed TornadoVM SDK has more than one (default: auto-detected) (default: None) + --cuda Force the CUDA backend when the installed TornadoVM SDK has more than one (default: auto-detected) (default: None) + --metal Force the Metal backend when the installed TornadoVM SDK has more than one (default: auto-detected) (default: None) --gpu-memory GPU_MEMORY GPU memory allocation (default: 14GB) --heap-min HEAP_MIN Minimum JVM heap size (default: 20g)