diff --git a/.claude/agents/gpullama-benchmarking-specialist.md b/.claude/agents/gpullama-benchmarking-specialist.md index f6074b8a..b94fe16d 100644 --- a/.claude/agents/gpullama-benchmarking-specialist.md +++ b/.claude/agents/gpullama-benchmarking-specialist.md @@ -76,10 +76,16 @@ Results land under `perf-results//`. ```bash export JAVA_TOOL_OPTIONS="-Dllama.metrics.format=json -Dllama.metrics.output=file -Dllama.metrics.file=" -./llama-tornado --model --prompt --verbose-init --max-tokens 2048 --seed +./llama-tornado --gpu --model --prompt --verbose-init --max-tokens 2048 --seed unset JAVA_TOOL_OPTIONS ``` +`llama-tornado` auto-detects the backend from `$TORNADOVM_HOME/etc/tornado.backend`. To benchmark +a specific backend, either point `TORNADOVM_HOME` at an SDK built for that backend (e.g. via +`scripts/benchmark_backends.sh`, which rebuilds TornadoVM per backend and re-sources its env +before each `llama-tornado` call), or, if the current SDK has more than one backend installed, +pass `--opencl`/`--ptx`/`--cuda`/`--metal` to pin one without a separate SDK/rebuild. + When testing a feature flag, change one variable at a time and keep the baseline command structurally identical to the treatment command — differences beyond the flag under test invalidate the comparison. diff --git a/.claude/skills/build-n-run-engine/SKILL.md b/.claude/skills/build-n-run-engine/SKILL.md index 62a6fc98..d6f46fa4 100644 --- a/.claude/skills/build-n-run-engine/SKILL.md +++ b/.claude/skills/build-n-run-engine/SKILL.md @@ -63,9 +63,10 @@ If this prints usage instead of erroring, the build succeeded and the launcher i ### Step 6: (Optional) Smoke-test a run -Only if a GGUF model path is available. Prefer `--cuda` (CUDA backend) on NVIDIA GPUs: +Only if a GGUF model path is available. The backend (OpenCL/PTX/CUDA/Metal) is +auto-detected from `$TORNADOVM_HOME/etc/tornado.backend` — no backend flag needed: ```bash -./llama-tornado --gpu --cuda --verbose-init --model --prompt "write a matmul in Java" --max-tokens 2048 +./llama-tornado --gpu --verbose-init --model --prompt "write a matmul in Java" --max-tokens 2048 ``` ## MANDATORY: Use --help When Uncertain About Flags @@ -74,7 +75,12 @@ Before using any `llama-tornado` flag you are not 100% certain about, run `--hel ## Running GPULlama3.java -`./llama-tornado --model [options]`. On NVIDIA GPUs, prefer `--cuda` (CUDA backend) over `--opencl` (default) unless the user asks for OpenCL specifically or is on non-NVIDIA hardware. +`./llama-tornado --model [options]`. The TornadoVM backend is auto-detected from the +installed SDK (`$TORNADOVM_HOME/etc/tornado.backend`); to run on a different backend, point +`TORNADOVM_HOME` at an SDK built for it. If the current SDK was built with more than one backend +(e.g. a `cuda-opencl` build), pass `--opencl`/`--ptx`/`--cuda`/`--metal` to force one of the +installed ones instead — these error out if the requested backend isn't part of the SDK, and +are a no-op (redundant but harmless) on a single-backend SDK. ### Core options @@ -84,9 +90,7 @@ Before using any `llama-tornado` flag you are not 100% certain about, run `--hel | `--prompt "..."` | single-shot generation | | `-i` / `--interactive` | chat loop instead of one-shot | | `--gpu` | required for GPU acceleration; omit to run CPU-only | -| `--ptx` | CUDA backend on NVIDIA — **prefer this by default** | -| `--opencl` | cross-vendor fallback (default if no backend flag given) | -| `--metal` | Apple Silicon only, TornadoVM 4.0+ | +| `--opencl`/`--ptx`/`--cuda`/`--metal` | rarely needed — only to force a backend when the SDK has more than one installed | | `--gpu-memory 15GB`/`20GB` | bump from default 14GB for 3B/8B models — avoids OOM | | `--temperature`, `--top-p`, `--seed`, `-n` | standard sampling knobs | | `-sp/--system-prompt` | instruct-mode framing | @@ -111,15 +115,15 @@ Before using any `llama-tornado` flag you are not 100% certain about, run `--hel ### Typical invocations ```bash -# quick single-shot GPU test, CUDA backend (preferred) -./llama-tornado --gpu --ptx --model model.gguf --prompt "..." +# quick single-shot GPU test (backend auto-detected from TORNADOVM_HOME) +./llama-tornado --gpu --model model.gguf --prompt "..." -# interactive chat, CUDA backend -./llama-tornado --gpu --ptx -i --model model.gguf +# interactive chat +./llama-tornado --gpu -i --model model.gguf # bigger model, needs more GPU mem -./llama-tornado --gpu --ptx --model llama-3.2-8b-instruct-fp16.gguf --gpu-memory 20GB --prompt "..." +./llama-tornado --gpu --model llama-3.2-8b-instruct-fp16.gguf --gpu-memory 20GB --prompt "..." -# benchmarking prefill/decode split with CUDA graphs -./llama-tornado --gpu --ptx --model model.gguf --with-prefill-decode --cuda-graphs --profiler --profiler-dump-dir ./perf-results --prompt "..." +# benchmarking prefill/decode split with CUDA graphs (needs a PTX-backend TORNADOVM_HOME) +./llama-tornado --gpu --model model.gguf --with-prefill-decode --cuda-graphs --profiler --profiler-dump-dir ./perf-results --prompt "..." ``` diff --git a/.github/actions/run-inference/action.yml b/.github/actions/run-inference/action.yml index fe3d574b..cbb90495 100644 --- a/.github/actions/run-inference/action.yml +++ b/.github/actions/run-inference/action.yml @@ -43,7 +43,8 @@ runs: METRICS_FILE: ${{ inputs.metrics_file }} run: | # Run inference and emit raw metrics JSON via JAVA_TOOL_OPTIONS - ./llama-tornado --gpu --${{ inputs.backend }} \ + # Backend (opencl/ptx/cuda) is auto-detected from $TORNADOVM_HOME/etc/tornado.backend + ./llama-tornado --gpu \ --model $MODELS_DIR/${{ inputs.model_file }} \ --prompt "${{ inputs.prompt }}" \ ${{ inputs.flags }} diff --git a/.github/workflows/build-and-run.yml b/.github/workflows/build-and-run.yml index 81ceed98..af61e397 100644 --- a/.github/workflows/build-and-run.yml +++ b/.github/workflows/build-and-run.yml @@ -312,7 +312,7 @@ jobs: run: | cd ${{ github.workspace }} export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ + ./llama-tornado --gpu \ --model $MODELS_DIR/Llama-3.2-1B-Instruct-Q8_0.gguf \ --prompt "Say hello" \ --with-prefill-decode @@ -336,7 +336,7 @@ jobs: run: | cd ${{ github.workspace }} export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --${{ matrix.backend.name }} \ + ./llama-tornado --gpu \ --model $MODELS_DIR/Llama-3.2-1B-Instruct-Q8_0.gguf \ --prompt "Say hello" \ --with-prefill-decode --batch-prefill-size 32 @@ -362,7 +362,7 @@ jobs: run: | cd ${{ github.workspace }} export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --ptx \ + ./llama-tornado --gpu \ --model $MODELS_DIR/Llama-3.2-1B-Instruct-Q8_0.gguf \ --prompt "Say hello" \ --with-prefill-decode \ @@ -388,7 +388,7 @@ jobs: run: | cd ${{ github.workspace }} export PATH="$TORNADOVM_HOME/bin:$JAVA_HOME/bin:$PATH" - ./llama-tornado --gpu --ptx \ + ./llama-tornado --gpu \ --model $MODELS_DIR/Llama-3.2-1B-Instruct-Q8_0.gguf \ --prompt "Say hello" \ --with-prefill-decode --batch-prefill-size 32 \ diff --git a/README.md b/README.md index d9bb49c6..575754ae 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,8 @@ GPULlama3ChatModel model = GPULlama3ChatModel.builder() Ensure you have the following installed and configured: - **Java 21**: Required for Vector API support & TornadoVM. -- [TornadoVM](https://github.com/beehive-lab/TornadoVM) with OpenCL, PTX, or CUDA backends. - - The `--cuda` backend requires a TornadoVM build that includes the CUDA backend from [TornadoVM PR #861](https://github.com/beehive-lab/TornadoVM/pull/861). This project currently builds against TornadoVM `5.0.0-jdk21-dev`. +- [TornadoVM](https://github.com/beehive-lab/TornadoVM) with an OpenCL, PTX, CUDA, or Metal backend. `llama-tornado`/`llamaTornado` auto-detect whichever backend your installed SDK was built with. + - The CUDA backend requires a TornadoVM build that includes the CUDA backend from [TornadoVM PR #861](https://github.com/beehive-lab/TornadoVM/pull/861). This project currently builds against TornadoVM `5.0.0-jdk21-dev`. - GCC/G++ 13 or newer: Required to build and run TornadoVM native components. ### Install, Build, and Run @@ -297,21 +297,29 @@ jbang LlamaTornadoCli.java -m beehive-llama-3.2-1b-instruct-fp16.gguf \ To execute Llama3, or Mistral models with TornadoVM on GPUs use the `llama-tornado` script with the `--gpu` flag. +The TornadoVM backend (OpenCL, PTX, CUDA, or Metal) is auto-detected from your installed +TornadoVM SDK (`TORNADOVM_HOME/etc/tornado.backend`) - no need to select it manually. + ### Usage Examples #### Basic Inference Run a model with a text prompt: ```bash -./llama-tornado --gpu --verbose-init --opencl --model beehive-llama-3.2-1b-instruct-fp16.gguf --prompt "Explain the benefits of GPU acceleration." +./llama-tornado --gpu --verbose-init --model beehive-llama-3.2-1b-instruct-fp16.gguf --prompt "Explain the benefits of GPU acceleration." ``` -Select a backend explicitly with `--opencl`, `--ptx`, or `--cuda` (NVIDIA), or `--metal` (Apple Silicon). For example, to run on the CUDA backend: +The script prints which backend it detected, e.g. `Detected TornadoVM backend: cuda (from .../etc/tornado.backend)`. +To run against a different backend, point `TORNADOVM_HOME` at an SDK built for that backend instead - or, +if your SDK was built with more than one backend (e.g. `cuda-opencl`), pass `--opencl`/`--ptx`/`--cuda`/`--metal` +to force one of the installed backends without needing a separate SDK: ```bash -./llama-tornado --gpu --cuda --model beehive-llama-3.2-1b-instruct-fp16.gguf --prompt "Explain the benefits of GPU acceleration." +./llama-tornado --gpu --opencl --model beehive-llama-3.2-1b-instruct-fp16.gguf --prompt "Explain the benefits of GPU acceleration." ``` +These flags error out if the requested backend isn't part of the installed SDK. + #### GPU Execution (FP16 Model) Enable GPU acceleration with Q8_0 quantization: ```bash @@ -323,7 +331,14 @@ Enable GPU acceleration with Q8_0 quantization: `llamaTornado` is a zero-dependency Java 25 single-file script that replaces the Python launcher. It requires `java 25+` on your PATH: ```bash -./llamaTornado --gpu --verbose-init --metal --model /Users/abien/work/workspaces/llms/Mistral-7B-Instruct-v0.3.Q8_0.gguf --prompt "what is java" +./llamaTornado --gpu --verbose-init --model /Users/abien/work/workspaces/llms/Mistral-7B-Instruct-v0.3.Q8_0.gguf --prompt "what is java" +``` + +Same backend auto-detection as `llama-tornado`, including the `--opencl`/`--ptx`/`--cuda`/`--metal` +override for SDKs built with more than one backend: + +```bash +./llamaTornado --gpu --opencl --model /Users/abien/work/workspaces/llms/Mistral-7B-Instruct-v0.3.Q8_0.gguf --prompt "what is java" ``` ----------- @@ -348,7 +363,6 @@ docker run --rm -it --gpus all \ beehivelab/gpullama3.java-nvidia-openjdk-opencl \ /gpullama3/GPULlama3.java/llama-tornado \ --gpu --verbose-init \ - --opencl \ --model /data/Llama-3.2-1B-Instruct.FP16.gguf \ --prompt "Tell me a joke" ``` @@ -399,43 +413,66 @@ Supported command-line options include: ```bash cmd ➜ llama-tornado --help -usage: llama-tornado [-h] --model MODEL_PATH [--prompt PROMPT] [-sp SYSTEM_PROMPT] [--temperature TEMPERATURE] [--top-p TOP_P] [--seed SEED] [-n MAX_TOKENS] - [--stream STREAM] [--echo ECHO] [-i] [--instruct] [--gpu] [--opencl] [--ptx] [--cuda] [--metal] [--gpu-memory GPU_MEMORY] [--heap-min HEAP_MIN] [--heap-max HEAP_MAX] - [--debug] [--profiler] [--profiler-dump-dir PROFILER_DUMP_DIR] [--print-bytecodes] [--print-threads] [--print-kernel] [--full-dump] - [--show-command] [--execute-after-show] [--opencl-flags OPENCL_FLAGS] [--max-wait-events MAX_WAIT_EVENTS] [--verbose] - -GPU-accelerated LLaMA.java model runner using TornadoVM +usage: llama-tornado [-h] --model MODEL_PATH [--prompt PROMPT] + [-sp SYSTEM_PROMPT] [--temperature TEMPERATURE] + [--top-p TOP_P] [--seed SEED] [-n MAX_TOKENS] + [--stream STREAM] [--echo ECHO] [--suffix SUFFIX] [-i] + [--instruct] [--server] [--port PORT] [--bench] + [--bench-args BENCH_ARGS] [--gpu] [--opencl] [--ptx] + [--cuda] [--metal] [--gpu-memory GPU_MEMORY] + [--heap-min HEAP_MIN] [--heap-max HEAP_MAX] [--debug] + [--profiler] [--profiler-dump-dir PROFILER_DUMP_DIR] + [--print-bytecodes] [--print-threads] [--print-kernel] + [--full-dump] [--verbose-init] [--show-command] + [--execute-after-show] [--with-prefill-decode] + [--batch-prefill-size N] [--cuda-graphs] + [--opencl-flags OPENCL_FLAGS] + [--max-wait-events MAX_WAIT_EVENTS] [--verbose] + +GPU-accelerated LLM runner using TornadoVM +(the TornadoVM backend is auto-detected from TORNADOVM_HOME/etc/tornado.backend) options: -h, --help show this help message and exit - --model MODEL_PATH Path to the LLaMA model file (e.g., beehive-llama-3.2-8b-instruct-fp16.gguf) (default: None) + --model MODEL_PATH Path to the LLM gguf file (e.g., + Llama-3.2-1B-Instruct-Q8_0.gguf) LLaMA Configuration: --prompt PROMPT Input prompt for the model (default: None) - -sp SYSTEM_PROMPT, --system-prompt SYSTEM_PROMPT + -sp, --system-prompt SYSTEM_PROMPT System prompt for the model (default: None) --temperature TEMPERATURE Sampling temperature (0.0 to 2.0) (default: 0.1) --top-p TOP_P Top-p sampling parameter (default: 0.95) --seed SEED Random seed (default: current timestamp) (default: None) - -n MAX_TOKENS, --max-tokens MAX_TOKENS + -n, --max-tokens MAX_TOKENS Maximum number of tokens to generate (default: 512) --stream STREAM Enable streaming output (default: True) --echo ECHO Echo the input prompt (default: False) - --suffix SUFFIX Suffix for fill-in-the-middle request (Codestral) (default: None) + --suffix SUFFIX Suffix for fill-in-the-middle request (Codestral) + (default: None) Mode Selection: -i, --interactive Run in interactive/chat mode (default: False) --instruct Run in instruction mode (default) (default: True) +OpenAI-compatible server: + --server Run the OpenAI-compatible HTTP server instead of inference (default: False) + --port PORT Server port (default 8080) (default: 8080) + +Benchmark (llama-bench style): + --bench Run the llama-bench-style benchmark (bench.LlamaBench) instead of inference (default: False) + --bench-args BENCH_ARGS + Extra benchmark options (use --bench-args="..."), e.g. "-p 512 -n 128 -d 0,4096 -r 5 -o md" (see LlamaBench) (default: ) + Hardware Configuration: --gpu Enable GPU acceleration (default: False) - --opencl Use OpenCL backend (default) (default: None) - --ptx Use PTX backend (default: None) - --cuda Use CUDA backend (requires TornadoVM built with the CUDA backend) (default: None) - --metal Use Apple Metal backend (macOS only) (default: None) + --opencl Force the OpenCL backend when the installed TornadoVM SDK has more than one (default: auto-detected) (default: None) + --ptx Force the PTX backend when the installed TornadoVM SDK has more than one (default: auto-detected) (default: None) + --cuda Force the CUDA backend when the installed TornadoVM SDK has more than one (default: auto-detected) (default: None) + --metal Force the Metal backend when the installed TornadoVM SDK has more than one (default: auto-detected) (default: None) --gpu-memory GPU_MEMORY - GPU memory allocation (default: 7GB) + GPU memory allocation (default: 14GB) --heap-min HEAP_MIN Minimum JVM heap size (default: 20g) --heap-max HEAP_MAX Maximum JVM heap size (default: 20g) @@ -443,7 +480,7 @@ Debug and Profiling: --debug Enable debug output (default: False) --profiler Enable TornadoVM profiler (default: False) --profiler-dump-dir PROFILER_DUMP_DIR - Directory for profiler output (default: /home/mikepapadim/repos/gpu-llama3.java/prof.json) + Directory for profiler output (default: None) TornadoVM Execution Verbose: --print-bytecodes Print bytecodes (tornado.print.bytecodes=true) (default: False) @@ -456,6 +493,15 @@ Command Display Options: --show-command Display the full Java command that will be executed (default: False) --execute-after-show Execute the command after showing it (use with --show-command) (default: False) +Prefill-Decode Optimizations: + --with-prefill-decode + Enable single-token prefill decode (default: False) + --batch-prefill-size N + Enable batching in prefill when --with-prefill-decode is active and N>1. (default: None) + +Advanced CUDA Features: + --cuda-graphs Enable CUDA graph capture/replay (llama.cudaGraphs=true); PTX backend only. (default: False) + Advanced Options: --opencl-flags OPENCL_FLAGS OpenCL compiler flags (default: -cl-denorms-are-zero -cl-no-signed-zeros -cl-finite-math-only) @@ -471,7 +517,7 @@ Serve the model behind the HTTP API OpenAI clients already speak — no external (JDK `HttpServer`), streaming (SSE) and non-streaming. ```bash -llama-tornado --gpu --cuda --model model.gguf --server --port 8080 +llama-tornado --gpu --model model.gguf --server --port 8080 # or directly: java ... org.beehive.gpullama3.server.OpenAIServer --model model.gguf --port 8080 --gpu ``` @@ -503,10 +549,10 @@ the forward pass only (no tokenization, no sampling), matching llama-bench metho ```bash # defaults: -p 512 -n 128 -r 5, markdown output -llama-tornado --gpu --cuda --model model1.gguf --bench +llama-tornado --gpu --model model1.gguf --bench # multiple models, custom matrix, CSV -llama-tornado --gpu --cuda --model model1.gguf --bench \ +llama-tornado --gpu --model model1.gguf --bench \ --bench-args="-m model2.gguf -p 256,512 -n 64,128 -pg 512,128 -d 0,4096 -r 5 -o csv" ``` @@ -552,7 +598,7 @@ View TornadoVM's internal behavior: - **Support for GGUF format models** with full FP16 and partial support for Q8_0 and Q4_0 quantization. - **Instruction-following and chat modes** for various use cases. - **Interactive CLI** with `--interactive` and `--instruct` modes. - - **Flexible backend switching** - choose OpenCL, PTX, or CUDA at runtime (need to build TornadoVM with the chosen backends enabled). + - **Automatic backend detection** - `llama-tornado`/`llamaTornado` detect and use whichever backend (OpenCL, PTX, CUDA, or Metal) your installed TornadoVM SDK was built with. - **Cross-platform compatibility**: - ✅ NVIDIA GPUs (OpenCL, PTX & CUDA) - ✅ Intel GPUs (OpenCL) diff --git a/llama-tornado b/llama-tornado index 9f5367c6..cf152139 100755 --- a/llama-tornado +++ b/llama-tornado @@ -2,6 +2,9 @@ """ llama-tornado: GPU-accelerated Java LLM runner with TornadoVM Run LLM models using OpenCL, PTX, CUDA, or Metal backends. + +The backend is auto-detected from the installed TornadoVM SDK +(TORNADOVM_HOME/etc/tornado.backend) - no need to select it manually. """ import argparse @@ -23,6 +26,46 @@ class Backend(Enum): METAL = "metal" +# Maps the names TornadoVM writes to etc/tornado.backend to our Backend enum. +BACKEND_NAME_MAP: Dict[str, Backend] = { + "opencl-backend": Backend.OPENCL, + "ptx-backend": Backend.PTX, + "cuda-backend": Backend.CUDA, + "metal-backend": Backend.METAL, +} + +BACKEND_EXPORTS_FILE: Dict[Backend, str] = { + Backend.OPENCL: "opencl-exports", + Backend.PTX: "ptx-exports", + Backend.CUDA: "cuda-exports", + Backend.METAL: "metal-exports", +} + +BACKEND_MODULE_NAME: Dict[Backend, str] = { + Backend.OPENCL: "tornado.drivers.opencl", + Backend.PTX: "tornado.drivers.ptx", + Backend.CUDA: "tornado.drivers.cuda", + Backend.METAL: "tornado.drivers.metal", +} + +# TornadoVM system property that ranks each backend for "driver 0" (default +# device) selection when more than one backend is loaded - see +# TornadoOptions.*_BACKEND_PRIORITY. Defaults there favor OpenCL (10) over +# everything else (0), which doesn't match BACKEND_PRIORITY below, so we set +# these explicitly whenever more than one backend is installed. +BACKEND_PRIORITY_PROPERTY: Dict[Backend, str] = { + Backend.OPENCL: "tornado.opencl.priority", + Backend.PTX: "tornado.ptx.priority", + Backend.CUDA: "tornado.cuda.priority", + Backend.METAL: "tornado.metal.priority", +} + +# When an SDK is built with more than one backend, prefer this one for +# properties that need a single "primary" backend (e.g. the printed message, +# and which backend becomes TornadoVM's default device). +BACKEND_PRIORITY: List[Backend] = [Backend.CUDA, Backend.PTX, Backend.OPENCL, Backend.METAL] + + class LlamaRunner: """Main class for managing LLM execution with GPU acceleration.""" @@ -50,6 +93,40 @@ class LlamaRunner: print(f"Error: {name} path does not exist: {path}") sys.exit(1) + def detect_installed_backends(self) -> List[Backend]: + """Return every backend the installed TornadoVM SDK was built with (tornado.backend order).""" + backend_file = Path(self.tornado_sdk) / "etc" / "tornado.backend" + if not backend_file.exists(): + print(f"Error: Could not detect TornadoVM backend - {backend_file} not found") + print("Note: this file is written by the TornadoVM installer; make sure TORNADOVM_HOME points at a valid SDK") + sys.exit(1) + + installed_names: List[str] = [] + for line in backend_file.read_text().splitlines(): + line = line.strip() + if line.startswith("tornado.backends="): + installed_names = [b.strip() for b in line.split("=", 1)[1].split(",") if b.strip()] + break + + if not installed_names: + print(f"Error: No backends declared in {backend_file}") + sys.exit(1) + + installed = [BACKEND_NAME_MAP[n] for n in installed_names if n in BACKEND_NAME_MAP] + if not installed: + print(f"Error: Unsupported backend(s) in {backend_file}: {', '.join(installed_names)}") + sys.exit(1) + + return installed + + @staticmethod + def select_primary_backend(installed: List[Backend]) -> Backend: + """Pick which installed backend to report/default to when the SDK has more than one.""" + for backend in BACKEND_PRIORITY: + if backend in installed: + return backend + return installed[0] + @staticmethod def module_path_colon_sep(paths: List[str]) -> str: """Return OS-specific separator for Java module paths.""" @@ -152,49 +229,42 @@ class LlamaRunner: cmd.extend(tornado_runtime_config) # Backend-specific configuration - if args.backend == Backend.OPENCL: + if Backend.OPENCL in args.installed_backends: # OpenCL specific flags cmd.append(f"-Dtornado.opencl.compiler.flags={args.opencl_flags}") - # Module configuration - varies by backend + if len(args.installed_backends) > 1: + # More than one backend is loaded, so pin TornadoVM's own driver-0 + # (default device) selection to match args.backend - otherwise it + # falls back to its own defaults (OpenCL beats everything else). + # args.backend goes first (whether auto-picked or --backend + # override), then the rest of BACKEND_PRIORITY as a tiebreak. + effective_priority = [args.backend] + [b for b in BACKEND_PRIORITY if b != args.backend] + n = len(effective_priority) + for rank, backend in enumerate(effective_priority): + if backend in args.installed_backends: + cmd.append(f"-D{BACKEND_PRIORITY_PROPERTY[backend]}={n - rank}") + + # Module configuration - TornadoCoreRuntime tries to initialize every + # backend listed in tornado.backend at startup (not just the one we + # picked as primary), so every installed backend needs its exports and + # driver module here or it fails with an IllegalAccessError. module_config = [ f"--upgrade-module-path", f"{self.tornado_sdk}/share/java/graalJars", f"@{self.tornado_sdk}/etc/exportLists/common-exports", ] - # Add backend-specific exports and modules - if args.backend == Backend.OPENCL: - module_config.extend( - [ - f"@{self.tornado_sdk}/etc/exportLists/opencl-exports", - "--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.opencl", - ] - ) - elif args.backend == Backend.PTX: - module_config.extend( - [ - f"@{self.tornado_sdk}/etc/exportLists/ptx-exports", - "--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.ptx", - ] - ) - elif args.backend == Backend.CUDA: - module_config.extend( - [ - f"@{self.tornado_sdk}/etc/exportLists/cuda-exports", - "--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.cuda", - ] - ) - elif args.backend == Backend.METAL: - module_config.extend( - [ - f"@{self.tornado_sdk}/etc/exportLists/metal-exports", - "--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.metal", - ] - ) + add_modules = [ + "ALL-SYSTEM", + "jdk.incubator.vector", + "tornado.runtime", + "tornado.annotation", + "tornado.drivers.common", + ] + for backend in args.installed_backends: + module_config.append(f"@{self.tornado_sdk}/etc/exportLists/{BACKEND_EXPORTS_FILE[backend]}") + add_modules.append(BACKEND_MODULE_NAME[backend]) + module_config.extend(["--add-modules", ",".join(add_modules)]) if getattr(args, "server", False): main_class = "org.beehive.gpullama3.server.OpenAIServer" @@ -379,6 +449,7 @@ def create_parser() -> argparse.ArgumentParser: prog="llama-tornado", description="GPU-accelerated LLM runner using TornadoVM", formatter_class=argparse.ArgumentDefaultsHelpFormatter, + allow_abbrev=False, ) # Required arguments @@ -461,32 +532,20 @@ def create_parser() -> argparse.ArgumentParser: "--gpu", dest="use_gpu", action="store_true", help="Enable GPU acceleration" ) hw_group.add_argument( - "--opencl", - dest="backend", - action="store_const", - const=Backend.OPENCL, - help="Use OpenCL backend (default)", + "--opencl", dest="backend_override", action="store_const", const=Backend.OPENCL, + help="Force the OpenCL backend when the installed TornadoVM SDK has more than one (default: auto-detected)", ) hw_group.add_argument( - "--ptx", - dest="backend", - action="store_const", - const=Backend.PTX, - help="Use PTX backend", + "--ptx", dest="backend_override", action="store_const", const=Backend.PTX, + help="Force the PTX backend when the installed TornadoVM SDK has more than one (default: auto-detected)", ) hw_group.add_argument( - "--cuda", - dest="backend", - action="store_const", - const=Backend.CUDA, - help="Use CUDA backend (requires TornadoVM built with the CUDA backend)", + "--cuda", dest="backend_override", action="store_const", const=Backend.CUDA, + help="Force the CUDA backend when the installed TornadoVM SDK has more than one (default: auto-detected)", ) hw_group.add_argument( - "--metal", - dest="backend", - action="store_const", - const=Backend.METAL, - help="Use Apple Metal backend (macOS only, requires TornadoVM 4.0+)", + "--metal", dest="backend_override", action="store_const", const=Backend.METAL, + help="Force the Metal backend when the installed TornadoVM SDK has more than one (default: auto-detected)", ) hw_group.add_argument("--gpu-memory", default="14GB", help="GPU memory allocation") hw_group.add_argument("--heap-min", default="20g", help="Minimum JVM heap size") @@ -613,16 +672,37 @@ def main(): if args.seed is None: args.seed = int(time.time()) - # Set default backend to OpenCL if not specified - if not hasattr(args, "backend") or args.backend is None: - args.backend = Backend.OPENCL - # Handle mode selection logic if args.interactive: args.instruct = False - # Create and run the LLaMA runner + # Create the LLaMA runner and auto-detect the TornadoVM backend(s) runner = LlamaRunner() + installed_backends = runner.detect_installed_backends() + args.installed_backends = installed_backends + + overridden = False + if args.backend_override is not None: + requested = args.backend_override + if requested not in installed_backends: + installed_names = ", ".join(b.value for b in installed_backends) + print(f"Error: --{requested.value} requested, but the installed TornadoVM SDK only has: {installed_names}") + sys.exit(1) + args.backend = requested + overridden = True + else: + args.backend = runner.select_primary_backend(installed_backends) + + if len(installed_backends) > 1: + names = ", ".join(b.value for b in installed_backends) + reason = f"requested via --{args.backend.value}" if overridden else "auto-selected" + print( + f"Detected TornadoVM backends: {names} (from {runner.tornado_sdk}/etc/tornado.backend) " + f"- using {args.backend.value} ({reason})" + ) + else: + print(f"Detected TornadoVM backend: {args.backend.value} (from {runner.tornado_sdk}/etc/tornado.backend)") + return runner.run(args) diff --git a/llamaTornado b/llamaTornado index c1c28df6..f3d4f92d 100755 --- a/llamaTornado +++ b/llamaTornado @@ -5,13 +5,44 @@ import module java.logging; String name = MethodHandles.lookup().lookupClass().getName(); String version = "2026-04-11.1"; -enum Backend { OPENCL, PTX, METAL } +enum Backend { OPENCL, PTX, CUDA, METAL } + +// When an SDK is built with more than one backend, prefer in this order. Also +// determines which backend becomes TornadoVM's default device (driver 0) - see +// BACKEND_PRIORITY_PROPERTY below. +List BACKEND_PRIORITY = List.of(Backend.CUDA, Backend.PTX, Backend.OPENCL, Backend.METAL); + +Map BACKEND_EXPORTS_FILE = Map.of( + Backend.OPENCL, "opencl-exports", + Backend.PTX, "ptx-exports", + Backend.CUDA, "cuda-exports", + Backend.METAL, "metal-exports" +); + +Map BACKEND_MODULE_NAME = Map.of( + Backend.OPENCL, "tornado.drivers.opencl", + Backend.PTX, "tornado.drivers.ptx", + Backend.CUDA, "tornado.drivers.cuda", + Backend.METAL, "tornado.drivers.metal" +); + +// TornadoVM system property that ranks each backend for "driver 0" (default +// device) selection when more than one backend is loaded - see +// TornadoOptions.*_BACKEND_PRIORITY. Defaults there favor OpenCL (10) over +// everything else (0), which doesn't match BACKEND_PRIORITY above, so we set +// these explicitly whenever more than one backend is installed. +Map BACKEND_PRIORITY_PROPERTY = Map.of( + Backend.OPENCL, "tornado.opencl.priority", + Backend.PTX, "tornado.ptx.priority", + Backend.CUDA, "tornado.cuda.priority", + Backend.METAL, "tornado.metal.priority" +); record Config( String modelPath, String prompt, String systemPrompt, double temperature, double topP, long seed, int maxTokens, boolean stream, boolean echo, boolean interactive, boolean instruct, - boolean useGpu, Backend backend, String gpuMemory, + boolean useGpu, Backend backend, boolean backendOverridden, String gpuMemory, String heapMin, String heapMax, String directMemory, boolean debug, boolean profiler, String profilerDumpDir, boolean printBytecodes, boolean threads, boolean printKernel, @@ -21,7 +52,7 @@ record Config( boolean bench, String benchArgs ) {} -Config parseArgs(String[] args) { +Config parseArgs(String[] args, List installedBackends) { String modelPath = null; String prompt = null; String systemPrompt = null; @@ -34,7 +65,7 @@ Config parseArgs(String[] args) { boolean interactive = false; boolean instruct = true; boolean useGpu = false; - Backend backend = Backend.OPENCL; + Backend backendOverride = null; String gpuMemory = "14GB"; String heapMin = "20g"; String heapMax = "20g"; @@ -69,9 +100,10 @@ Config parseArgs(String[] args) { case "-i", "--interactive" -> { interactive = true; instruct = false; } case "--instruct" -> instruct = true; case "--gpu" -> useGpu = true; - case "--opencl" -> backend = Backend.OPENCL; - case "--ptx" -> backend = Backend.PTX; - case "--metal" -> backend = Backend.METAL; + case "--opencl" -> backendOverride = Backend.OPENCL; + case "--ptx" -> backendOverride = Backend.PTX; + case "--cuda" -> backendOverride = Backend.CUDA; + case "--metal" -> backendOverride = Backend.METAL; case "--gpu-memory" -> gpuMemory = args[++i]; case "--heap-min" -> heapMin = args[++i]; case "--heap-max" -> heapMax = args[++i]; @@ -113,8 +145,24 @@ Config parseArgs(String[] args) { directMemory = parseAndScale(heapMax, 3); } + Backend backend; + boolean backendOverridden; + if (backendOverride != null) { + if (!installedBackends.contains(backendOverride)) { + var names = String.join(", ", installedBackends.stream().map(b -> b.toString().toLowerCase()).toList()); + System.err.println("Error: --" + backendOverride.toString().toLowerCase() + + " requested, but the installed TornadoVM SDK only has: " + names); + System.exit(1); + } + backend = backendOverride; + backendOverridden = true; + } else { + backend = selectPrimaryBackend(installedBackends); + backendOverridden = false; + } + return new Config(modelPath, prompt, systemPrompt, temperature, topP, seed, maxTokens, - stream, echo, interactive, instruct, useGpu, backend, gpuMemory, heapMin, heapMax, directMemory, + stream, echo, interactive, instruct, useGpu, backend, backendOverridden, gpuMemory, heapMin, heapMax, directMemory, debug, profiler, profilerDumpDir, printBytecodes, threads, printKernel, fullDump, verboseInit, showCommand, executeAfterShow, openclFlags, maxWaitEvents, verbose, bench, benchArgs); @@ -155,9 +203,11 @@ void printUsage() { Hardware: --gpu Enable GPU acceleration - --opencl Use OpenCL backend (default) - --ptx Use PTX/CUDA backend - --metal Use Metal backend (macOS) + (backend is auto-detected from TORNADOVM_HOME/etc/tornado.backend) + --opencl Force the OpenCL backend if the SDK has more than one + --ptx Force the PTX backend if the SDK has more than one + --cuda Force the CUDA backend if the SDK has more than one + --metal Force the Metal backend if the SDK has more than one --gpu-memory GPU memory allocation (default: 14GB) --heap-min Min JVM heap (default: 20g) --heap-max Max JVM heap (default: 20g) @@ -213,7 +263,65 @@ String modulePath(String tornadoSdk) { return "." + sep + tornadoSdk + "/share/java/tornado"; } -List buildCommand(Config cfg, String javaHome, String tornadoSdk, String llamaRoot) { +List detectInstalledBackends(String tornadoSdk) { + var backendFile = Path.of(tornadoSdk, "etc", "tornado.backend"); + if (!Files.exists(backendFile)) { + System.err.println("Error: Could not detect TornadoVM backend - " + backendFile + " not found"); + System.err.println("Note: this file is written by the TornadoVM installer; make sure TORNADOVM_HOME points at a valid SDK"); + System.exit(1); + } + + var nameMap = Map.of( + "opencl-backend", Backend.OPENCL, + "ptx-backend", Backend.PTX, + "cuda-backend", Backend.CUDA, + "metal-backend", Backend.METAL + ); + + var installedNames = new ArrayList(); + try { + for (var line : Files.readAllLines(backendFile)) { + line = line.strip(); + if (line.startsWith("tornado.backends=")) { + for (var n : line.substring("tornado.backends=".length()).split(",")) { + n = n.strip(); + if (!n.isEmpty()) installedNames.add(n); + } + break; + } + } + } catch (IOException e) { + System.err.println("Error reading " + backendFile + ": " + e.getMessage()); + System.exit(1); + } + + if (installedNames.isEmpty()) { + System.err.println("Error: No backends declared in " + backendFile); + System.exit(1); + } + + var installed = new ArrayList(); + for (var n : installedNames) { + var b = nameMap.get(n); + if (b != null) installed.add(b); + } + + if (installed.isEmpty()) { + System.err.println("Error: Unsupported backend(s) in " + backendFile + ": " + String.join(", ", installedNames)); + System.exit(1); + } + + return installed; +} + +Backend selectPrimaryBackend(List installed) { + for (var b : BACKEND_PRIORITY) { + if (installed.contains(b)) return b; + } + return installed.getFirst(); +} + +List buildCommand(Config cfg, String javaHome, String tornadoSdk, String llamaRoot, List installedBackends) { var cmd = new ArrayList(); cmd.addAll(List.of( @@ -263,33 +371,48 @@ List buildCommand(Config cfg, String javaHome, String tornadoSdk, String "-Dtornado.eventpool.maxwaitevents=" + cfg.maxWaitEvents() )); - if (cfg.backend() == Backend.OPENCL) { + if (installedBackends.contains(Backend.OPENCL)) { cmd.add("-Dtornado.opencl.compiler.flags=" + cfg.openclFlags()); } - // Module configuration + if (installedBackends.size() > 1) { + // More than one backend is loaded, so pin TornadoVM's own driver-0 + // (default device) selection to match cfg.backend() - otherwise it + // falls back to its own defaults (OpenCL beats everything else). + // cfg.backend() goes first (whether auto-picked or --opencl/--ptx/ + // --cuda/--metal override), then the rest of BACKEND_PRIORITY as a + // tiebreak. + var effectivePriority = new ArrayList(); + effectivePriority.add(cfg.backend()); + for (var b : BACKEND_PRIORITY) { + if (b != cfg.backend()) effectivePriority.add(b); + } + int n = effectivePriority.size(); + for (int rank = 0; rank < n; rank++) { + var b = effectivePriority.get(rank); + if (installedBackends.contains(b)) { + cmd.add("-D" + BACKEND_PRIORITY_PROPERTY.get(b) + "=" + (n - rank)); + } + } + } + + // Module configuration - TornadoCoreRuntime tries to initialize every + // backend listed in tornado.backend at startup (not just cfg.backend()), + // so every installed backend needs its exports and driver module here or + // it fails with an IllegalAccessError. cmd.addAll(List.of( "--upgrade-module-path", tornadoSdk + "/share/java/graalJars", "@" + tornadoSdk + "/etc/exportLists/common-exports" )); - switch (cfg.backend()) { - case OPENCL -> { - cmd.add("@" + tornadoSdk + "/etc/exportLists/opencl-exports"); - cmd.addAll(List.of("--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.opencl")); - } - case PTX -> { - cmd.add("@" + tornadoSdk + "/etc/exportLists/ptx-exports"); - cmd.addAll(List.of("--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.ptx")); - } - case METAL -> { - cmd.add("@" + tornadoSdk + "/etc/exportLists/metal-exports"); - cmd.addAll(List.of("--add-modules", - "ALL-SYSTEM,jdk.incubator.vector,tornado.runtime,tornado.annotation,tornado.drivers.common,tornado.drivers.metal")); - } + var addModules = new ArrayList<>(List.of( + "ALL-SYSTEM", "jdk.incubator.vector", "tornado.runtime", "tornado.annotation", "tornado.drivers.common" + )); + for (var b : installedBackends) { + cmd.add("@" + tornadoSdk + "/etc/exportLists/" + BACKEND_EXPORTS_FILE.get(b)); + addModules.add(BACKEND_MODULE_NAME.get(b)); } + cmd.addAll(List.of("--add-modules", String.join(",", addModules))); String mainClass = cfg.bench() ? "org.beehive.gpullama3.bench.LlamaBench" @@ -375,8 +498,18 @@ void main(String... args) { var tornadoSdk = requireEnv("TORNADOVM_HOME"); var llamaRoot = resolveLlamaRoot(); - var cfg = parseArgs(args); - var cmd = buildCommand(cfg, javaHome, tornadoSdk, llamaRoot); + var installedBackends = detectInstalledBackends(tornadoSdk); + var cfg = parseArgs(args, installedBackends); + var backend = cfg.backend(); + if (installedBackends.size() > 1) { + var names = String.join(", ", installedBackends.stream().map(b -> b.toString().toLowerCase()).toList()); + var reason = cfg.backendOverridden() ? "requested via --" + backend.toString().toLowerCase() : "auto-selected"; + IO.println("Detected TornadoVM backends: " + names + " (from " + tornadoSdk + "/etc/tornado.backend) - using " + backend.toString().toLowerCase() + " (" + reason + ")"); + } else { + IO.println("Detected TornadoVM backend: " + backend.toString().toLowerCase() + " (from " + tornadoSdk + "/etc/tornado.backend)"); + } + + var cmd = buildCommand(cfg, javaHome, tornadoSdk, llamaRoot, installedBackends); if (cfg.showCommand()) { IO.println("Full Java command:"); diff --git a/scripts/all.sh b/scripts/all.sh index e7c05c65..e945987a 100644 --- a/scripts/all.sh +++ b/scripts/all.sh @@ -51,6 +51,6 @@ for model in "${models[@]}"; do #java @argfile -cp /home/devoxx2025-demo/java-ai-demos/GPULlama3.java/target/gpu-llama3-0.2.2.jar org.beehive.gpullama3.LlamaApp --model "$model" --stream true --echo false -p "Who are you?" --instruct - #./llama-tornado --gpu --opencl --model "$model" --prompt "Who are you?" + #./llama-tornado --gpu --model "$model" --prompt "Who are you?" done diff --git a/scripts/benchmark_backends.sh b/scripts/benchmark_backends.sh index cf76679a..80aa0f1d 100755 --- a/scripts/benchmark_backends.sh +++ b/scripts/benchmark_backends.sh @@ -117,7 +117,7 @@ run_inference() { export JAVA_TOOL_OPTIONS="-Dllama.metrics.format=json -Dllama.metrics.output=file -Dllama.metrics.file=$metrics_file" # shellcheck disable=SC2086 ( cd "$LLAMA_ROOT_DIR" && \ - ./llama-tornado --gpu --"$backend" \ + ./llama-tornado --gpu \ --model "$MODELS_DIR/$model_file" \ --prompt "$PROMPT" \ --max-tokens "$MAX_TOKENS" \