diff --git a/.github/models.csv b/.github/models.csv index e792b2d9..3fe2b67d 100644 --- a/.github/models.csv +++ b/.github/models.csv @@ -17,5 +17,5 @@ Qwen2.5-1.5B-Instruct-Q4_K_M.gguf,https://huggingface.co/bartowski/Qwen2.5-1.5B- nomic-embed-text-v1.5.f16.gguf,https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.f16.gguf SmolVLM-500M-Instruct-Q8_0.gguf,https://huggingface.co/ggml-org/SmolVLM-500M-Instruct-GGUF/resolve/main/SmolVLM-500M-Instruct-Q8_0.gguf mmproj-SmolVLM-500M-Instruct-Q8_0.gguf,https://huggingface.co/ggml-org/SmolVLM-500M-Instruct-GGUF/resolve/main/mmproj-SmolVLM-500M-Instruct-Q8_0.gguf -OuteTTS-0.2-500M-Q4_K_M.gguf,https://huggingface.co/second-state/OuteTTS-0.2-500M-GGUF/resolve/main/OuteTTS-0.2-500M-Q4_K_M.gguf -WavTokenizer-Large-75-F16.gguf,https://huggingface.co/ggml-org/WavTokenizer/resolve/main/WavTokenizer-Large-75-F16.gguf +Qwen3-TTS-12Hz-1.7B-Base-Q4_K_M.gguf,https://huggingface.co/ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF/resolve/main/Qwen3-TTS-12Hz-1.7B-Base-Q4_K_M.gguf +mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf,https://huggingface.co/ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF/resolve/main/mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 094ede94..68632791 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -36,9 +36,12 @@ env: # under the upstream mtmd pipeline. VISION_MODEL_NAME: "SmolVLM-500M-Instruct-Q8_0.gguf" VISION_MMPROJ_NAME: "mmproj-SmolVLM-500M-Instruct-Q8_0.gguf" - # Text-to-speech models for AudioInputIntegrationTest's sibling TtsIntegrationTest (OuteTTS pipeline). - TTS_MODEL_NAME: "OuteTTS-0.2-500M-Q4_K_M.gguf" - TTS_VOCODER_NAME: "WavTokenizer-Large-75-F16.gguf" + # Qwen3-TTS backbone + mmproj for TtsIntegrationTest (the OuteTTS+WavTokenizer pair this replaced + # was retired when upstream #26254 replaced the whole TTS pipeline — see + # docs/history/llama-cpp-breaking-changes.md, b10269-b10270). Smallest available quants: + # Q4_K_M backbone (0.96 GiB), Q8_0 mmproj (0.42 GiB; no smaller mmproj quant is published). + TTS_MODEL_NAME: "Qwen3-TTS-12Hz-1.7B-Base-Q4_K_M.gguf" + TTS_MMPROJ_NAME: "mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf" # Test image used by MultimodalIntegrationTest is committed to the repo # at src/test/resources/images/test-image.jpg (see the README in that # directory for licensing). No download step is needed; CI just points @@ -2241,8 +2244,8 @@ jobs: -Dnet.ladenthin.llama.vision.model=models/${VISION_MODEL_NAME} \ -Dnet.ladenthin.llama.vision.mmproj=models/${VISION_MMPROJ_NAME} \ -Dnet.ladenthin.llama.vision.image=${VISION_IMAGE_PATH} \ - -Dnet.ladenthin.llama.tts.ttc.model=models/${TTS_MODEL_NAME} \ - -Dnet.ladenthin.llama.tts.vocoder.model=models/${TTS_VOCODER_NAME} + -Dnet.ladenthin.llama.tts.model=models/${TTS_MODEL_NAME} \ + -Dnet.ladenthin.llama.tts.mmproj=models/${TTS_MMPROJ_NAME} - uses: actions/upload-artifact@v7 if: success() with: @@ -2362,8 +2365,8 @@ jobs: -Dnet.ladenthin.llama.vision.model=models/${VISION_MODEL_NAME} \ -Dnet.ladenthin.llama.vision.mmproj=models/${VISION_MMPROJ_NAME} \ -Dnet.ladenthin.llama.vision.image=${VISION_IMAGE_PATH} \ - -Dnet.ladenthin.llama.tts.ttc.model=models/${TTS_MODEL_NAME} \ - -Dnet.ladenthin.llama.tts.vocoder.model=models/${TTS_VOCODER_NAME} + -Dnet.ladenthin.llama.tts.model=models/${TTS_MODEL_NAME} \ + -Dnet.ladenthin.llama.tts.mmproj=models/${TTS_MMPROJ_NAME} - name: Memory after tests if: always() run: vm_stat && sysctl hw.memsize hw.physmem @@ -2431,8 +2434,8 @@ jobs: -Dnet.ladenthin.llama.vision.model=models/${VISION_MODEL_NAME} \ -Dnet.ladenthin.llama.vision.mmproj=models/${VISION_MMPROJ_NAME} \ -Dnet.ladenthin.llama.vision.image=${VISION_IMAGE_PATH} \ - -Dnet.ladenthin.llama.tts.ttc.model=models/${TTS_MODEL_NAME} \ - -Dnet.ladenthin.llama.tts.vocoder.model=models/${TTS_VOCODER_NAME} + -Dnet.ladenthin.llama.tts.model=models/${TTS_MODEL_NAME} \ + -Dnet.ladenthin.llama.tts.mmproj=models/${TTS_MMPROJ_NAME} - name: Memory after tests if: always() run: vm_stat && sysctl hw.memsize hw.physmem @@ -2500,8 +2503,8 @@ jobs: -Dnet.ladenthin.llama.vision.model=models/${VISION_MODEL_NAME} \ -Dnet.ladenthin.llama.vision.mmproj=models/${VISION_MMPROJ_NAME} \ -Dnet.ladenthin.llama.vision.image=${VISION_IMAGE_PATH} \ - -Dnet.ladenthin.llama.tts.ttc.model=models/${TTS_MODEL_NAME} \ - -Dnet.ladenthin.llama.tts.vocoder.model=models/${TTS_VOCODER_NAME} + -Dnet.ladenthin.llama.tts.model=models/${TTS_MODEL_NAME} \ + -Dnet.ladenthin.llama.tts.mmproj=models/${TTS_MMPROJ_NAME} - name: Memory after tests if: always() run: vm_stat && sysctl hw.memsize hw.physmem @@ -2587,8 +2590,8 @@ jobs: "-Dnet.ladenthin.llama.vision.model=models/$env:VISION_MODEL_NAME" ` "-Dnet.ladenthin.llama.vision.mmproj=models/$env:VISION_MMPROJ_NAME" ` "-Dnet.ladenthin.llama.vision.image=$env:VISION_IMAGE_PATH" ` - "-Dnet.ladenthin.llama.tts.ttc.model=models/$env:TTS_MODEL_NAME" ` - "-Dnet.ladenthin.llama.tts.vocoder.model=models/$env:TTS_VOCODER_NAME" + "-Dnet.ladenthin.llama.tts.model=models/$env:TTS_MODEL_NAME" ` + "-Dnet.ladenthin.llama.tts.mmproj=models/$env:TTS_MMPROJ_NAME" - name: Memory after tests if: always() run: Get-CimInstance Win32_OperatingSystem | Select-Object FreePhysicalMemory,TotalVisibleMemorySize | Format-List @@ -2681,8 +2684,8 @@ jobs: "-Dnet.ladenthin.llama.vision.model=models/$env:VISION_MODEL_NAME" ` "-Dnet.ladenthin.llama.vision.mmproj=models/$env:VISION_MMPROJ_NAME" ` "-Dnet.ladenthin.llama.vision.image=$env:VISION_IMAGE_PATH" ` - "-Dnet.ladenthin.llama.tts.ttc.model=models/$env:TTS_MODEL_NAME" ` - "-Dnet.ladenthin.llama.tts.vocoder.model=models/$env:TTS_VOCODER_NAME" + "-Dnet.ladenthin.llama.tts.model=models/$env:TTS_MODEL_NAME" ` + "-Dnet.ladenthin.llama.tts.mmproj=models/$env:TTS_MMPROJ_NAME" - name: Memory after tests if: always() run: Get-CimInstance Win32_OperatingSystem | Select-Object FreePhysicalMemory,TotalVisibleMemorySize | Format-List diff --git a/CLAUDE.md b/CLAUDE.md index b1a59295..63180378 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Java bindings for [llama.cpp](https://github.com/ggerganov/llama.cpp) via JNI, providing a high-level API for LLM inference in Java. The Java layer communicates with a native C++ library through JNI. -Current llama.cpp pinned version: **b10217** +Current llama.cpp pinned version: **b10319** ## Upgrading CUDA Version @@ -429,7 +429,7 @@ needs no extra step here, `build-webui` re-reads the tag and rebuilds the matchi ships no UI): ```bash # needs node/npm + network; embed.cpp is plain C++17 (no npm) -git clone --depth 1 --branch b10217 https://github.com/ggml-org/llama.cpp /tmp/lc +git clone --depth 1 --branch b10319 https://github.com/ggml-org/llama.cpp /tmp/lc ( cd /tmp/lc/tools/ui && npm ci && npm run build \ && ( cd dist && find . -type f -not -path './_gzip/*' \ | while read -r f; do mkdir -p "_gzip/$(dirname "$f")"; gzip -9 -c "$f" > "_gzip/$f"; done ) \ @@ -469,7 +469,7 @@ cache lives in **Depot Cache** over sccache's **WebDAV** backend: - `SCCACHE_WEBDAV_TOKEN: ${{ secrets.DEPOT_TOKEN }}` — a Depot **organization** token, stored as the repo secret **`DEPOT_TOKEN`**. -Because `sccache` is **content-addressed** and llama.cpp is pinned (`GIT_TAG b10217`), the +Because `sccache` is **content-addressed** and llama.cpp is pinned (`GIT_TAG b10319`), the ~280 upstream object files are byte-identical every run, so a warm cache recompiles only the *changed* files. Depot's cache is **shared across all branches** (unlike GitHub's per-branch `actions/cache`), so every branch builds incrementally; a `b` version bump @@ -583,7 +583,23 @@ Current patches: | `0007-server-attach-http-frontend.patch` | **Adds `llama_server_attach(argc, argv, server_context&)`** so the `NativeServer` *attach mode* can serve an **already-loaded `LlamaModel`** over the upstream HTTP frontend — no second model load, no `start_loop()`; the LlamaModel's worker keeps driving the shared `server_context` and the HTTP routes post tasks to its queue (the queue is the synchronization point). Mechanically: (1) extracts the **pure core route table** (`health` … `slots`) out of `llama_server()` into `static void llama_server_register_common_routes(ctx_http, routes)` (shared, so the two entry points cannot drift on the core endpoint set). **Scope note (narrowed at the b10154 bump):** the helper deliberately carries **only** the stable, state-independent route table — **not** the resumable-streaming routes (their handlers differ between router / non-router), the GCP-compat shim, or the experimental **CORS-proxy / MCP-server / built-in-tools** wiring. b10154 (upstream MCP-server support) moved the streaming routes into the middle of that block and coupled tools/CORS to a per-call `server_mcp mcp_mgr` lifecycle, so the earlier contiguous "route-table + CORS-proxy + tools" extraction is no longer possible; `llama_server()` keeps all of that inline, **byte-identical to upstream b10154** (only the route-table block is factored out). (2) adds `llama_server_attach`, which parses only the HTTP-side argv via `common_params_parse`, starts the stream-session GC + `server_http_context`, registers the common route table, the **non-router** resumable-streaming handlers (upstream b10154 paths `/v1/stream` GET/DEL + `/v1/streams/lookup` POST), the GCP-compat shim, and **403 "disabled" stubs for `/cors-proxy` + `/tools`** (attach mode does not wire the experimental CORS-proxy / MCP / built-in-tools host — those belong to a full `llama-server`, not an embedded model), marks ready immediately (model already loaded), and blocks on the HTTP thread until `llama_server_request_shutdown()` — never calling `common_init()`, backend init, `ctx_server.terminate()` or `llama_backend_free()` (the embedding caller owns those). Applies after `0001`+`0006` (same file); closes the "NativeServer — reuse an already-loaded LlamaModel" TODO. Upstream-submittable ("server: let embedding callers attach the HTTP frontend to an existing server_context"). | | `0008-server-models-worker-cmd-override.patch` | **Makes router mode usable in-JVM.** The router (`server-models.cpp`) spawns each model worker by re-executing its own binary (`get_server_exec_path()` = `/proc/self/exe` & friends) — inside a JVM that binary is `java`, not a llama-server, so embedded router workers could never start. The patch adds env `LLAMA_SERVER_WORKER_CMD` (whitespace-split; read in `server_model_meta::update_args`) which replaces only the leading binary-path token of the rendered worker args, letting an embedding host relaunch workers through its own bootstrap — e.g. `java -cp app.jar net.ladenthin.llama.server.NativeServer` (each worker is then a fresh JVM running the classic single-model `NativeServer`). Exposed in Java as `NativeServer.setWorkerCommand(String...)` (JNI `setenv`); exercised by `RouterModeIntegrationTest` (Linux CI). Upstream-submittable (also useful for containerized/wrapped deployments). | | `0006-server-embed-native-server-jni.patch` | **Makes `server.cpp`'s `llama_server` embeddable in the JVM** so the `NativeServer` JNI bridge can run the full upstream HTTP server (WebUI included) inside `libjllama` — see "Two server modes" below. b9870 already exposes `int llama_server(int, char**)` (non-static; no `main` in the file), so the patch only adds embedded-mode support: (1) a `g_llama_server_embedded` flag + `llama_server_set_embedded()` / `llama_server_request_shutdown()` (declared in the committed `src/main/cpp/native_server_bridge.h`); (2) skips installing the process-wide SIGINT/SIGTERM handlers when embedded (they would hijack the JVM's); (3) in embedded mode parses the **forwarded** argv via `common_params_parse` instead of `common_params_parse_main` (whose `GetCommandLineW` recovery would pick up `java.exe`'s command line — the same Windows class of bug `0001` fixes). `llama_server_request_shutdown()` mirrors the SIGTERM path (invokes the installed `shutdown_handler` → `ctx_server.terminate()` unblocks `start_loop()`), giving JNI an out-of-band stop since `ctx_server` is loop-local. Applies **after `0001`** (which flips this call site to `common_params_parse_main`), so its context is the post-`0001` tree; regenerate against `0001`+source on a bump. Only touches `tools/server/server.cpp`. | -| `0009-subprocess-guard-addchdir-np-old-glibc.patch` | **Fixes the b10154 cross-compile break on old glibc.** b10154 bumped the vendored `vendor/sheredom/subprocess.h` to a version that calls `posix_spawn_file_actions_addchdir_np` (a non-portable extension: glibc **≥ 2.29**, bionic API ≥ 34, macOS ≥ 10.15) to honor a spawn `process_cwd`, and added `common/subproc.cpp` (both pulled in via the new MCP-server support). Upstream guards that call **only for macOS**, so on **manylinux2014 (glibc 2.17)** the declaration is absent and `subprocess.h` (via `subproc.cpp` + `mtmd-helper.cpp` + `server-mcp.cpp`) fails to compile (`'posix_spawn_file_actions_addchdir_np' was not declared`). Android is unaffected — `__ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__` already makes the declaration visible (weak). The patch adds a `SUBPROCESS_HAVE_CWD` compile-time probe (`__GLIBC_PREREQ(2, 29)`, nested under `defined(__GLIBC__)` so non-glibc platforms never evaluate it, and skipped entirely when the macro is already defined so unrecognized platforms can override it) next to the POSIX includes and, when unavailable, reports a requested cwd as `ENOSYS` instead of failing to compile — this build never spawns with a cwd. **Not reproducible on a modern-glibc dev box** (the `addchdir_np` branch is taken there); the manylinux CI job is the gate. Also affects `manylinux_2_28` (glibc 2.28). **Submitted upstream as [sheredom/subprocess.h#104](https://github.com/sheredom/subprocess.h/pull/104)** and byte-identical to its head `620ce44`, so the applier will report "already applied" and skip once llama.cpp bumps the vendored pin. Two side findings from that work were filed separately and do not affect this project: [#105](https://github.com/sheredom/subprocess.h/pull/105) (`-std=c++20` unknown to GCC 8) and [#106](https://github.com/sheredom/subprocess.h/pull/106) (`posix_spawn` not reporting exec failures before glibc 2.24). Only touches `vendor/sheredom/subprocess.h`. | + +**`0009` was dropped at the b10280 bump.** Upstream merged +[sheredom/subprocess.h#104](https://github.com/sheredom/subprocess.h/pull/104) — the exact fix this +patch submitted — via [ggml-org/llama.cpp#26606](https://github.com/ggml-org/llama.cpp/pull/26606) +("vendor: apply patches for subprocess.h"): `vendor/sheredom/subprocess.h` at b10280 already defines +the `SUBPROCESS_HAVE_CWD` compile-time probe and the `ENOSYS` fallback our patch added, so the +old-glibc build break `0009` fixed no longer exists upstream. The applier's idempotent +`git apply --reverse --check` could **not** auto-skip this one (unlike a byte-identical carry): the +same PR also rewrote the neighboring Windows argv-quoting logic and added a `__NetBSD__` branch next +to the `process_cwd` guard, shifting the patch's second hunk's context, so it failed loud with "does +not apply cleanly" at configure time exactly as designed — confirming the patch needed to be dropped, +not refreshed. Also folded into that PR: a fix for `-std=c++20` unknown to GCC 8 +([#105](https://github.com/sheredom/subprocess.h/pull/105)) and a `posix_spawn` exec-failure +reporting fix pre-glibc-2.24 ([#106](https://github.com/sheredom/subprocess.h/pull/106)) — neither +affects this project. If a regression surfaces on old-glibc builds (manylinux2014/manylinux_2_28), +re-check `vendor/sheredom/subprocess.h`'s `SUBPROCESS_HAVE_CWD` guard against this description before +reintroducing a local patch. **`0005` was dropped at the b9981 bump.** Upstream's own `server-context.cpp` picked up an equivalent — and broader — fix for the same checkpoint-starvation problem: `create_checkpoint` @@ -604,40 +620,36 @@ prefill behavior shows up, re-check `tools/server/server-context.cpp`'s `create_ (`test_reasoning_budget_tokens_per_request` / `test_reasoning_budget_message_per_request`, byte-identical body). No local patch needed — the tree already matches what `0004` used to add. -## OuteTTS build-time extraction (`llama/cmake/generate-tts-upstream.cmake`) - -The `TextToSpeech` native pipeline reuses llama.cpp's OuteTTS helpers (`tools/tts/tts.cpp`) -**without hand-copying them**. A verbatim copy would be a DRY/maintenance hazard that silently -diverges on every upgrade, and `tts.cpp` cannot simply be added to `target_sources` — it defines its -own `main()`, which would clash at link time (the same reason `tools/server/server.cpp` is excluded -while `server-*.cpp` are compiled in), and all its helpers are `static` (internal linkage), so they -are unreachable from another TU even if it were linked. - -Instead the helpers are **DERIVED mechanically at configure time** from the pinned upstream source: - -- **`llama/cmake/generate-tts-upstream.cmake`** — reads `${llama.cpp_SOURCE_DIR}/tools/tts/tts.cpp`, keeps - the pre-`main()` span (the DSP `fill_hann_window`/`irfft`/`fold`/`embd_to_audio`, the prompt/text - helpers incl. `process_text`'s number-to-words, the `outetts_version` enum), strips `static` from - the handful the JNI engine calls (giving them external linkage), and extracts the two hard-coded - default-speaker literals out of `main()` into `extern const` strings. Writes - `build/tts_generated/tts_upstream_gen.cpp`. -- **`CMakeLists.txt`** — runs the generator via `execute_process` right after - `FetchContent_MakeAvailable(llama.cpp)`, then compiles the generated TU into `jllama`. The file is - **never committed** (build artifact, like the native libs / WebUI assets); it is regenerated from - whatever `tts.cpp` the pinned `GIT_TAG` resolves to, so a version bump is picked up automatically. -- **`src/main/cpp/tts_upstream.h`** — committed, hand-written declarations of the extracted symbols - (interface facts, not the implementation). `tts_engine.cpp` includes it and links against the - generated definitions. The in-memory WAV writer (`tts_wav.hpp`) is ours, not extracted. - -**Fail-loud on drift (same contract as `patches/`):** the generator asserts every anchor — the -`int main(` split point, each `static ` it de-statics, the `outetts_version` enum -(enumerators + order, kept ODR-identical to the hand-written copy in `tts_upstream.h`), both -`prompt_add` overloads the header declares (the bare `void prompt_add(` prefix de-statics all three -upstream overloads, so the two the header relies on are pinned individually), and both speaker -literals. If an upgrade renames a helper, reorders the enum, or moves a literal, the **configure step -aborts** with a pointer to the generator; if upstream changes a *type*, `tts_upstream.h` stops -matching and the **link fails**. Either way a silent divergence is impossible. On a llama.cpp bump, -re-verify the generator the same way you re-verify `patches/`. +## Qwen3-TTS via `mtmd_helper::gen_audio` (was: OuteTTS build-time extraction) + +The `TextToSpeech` native pipeline (`tts_engine.{h,cpp}`) drives llama.cpp's upstream Qwen3-TTS +audio-generation pipeline directly through its public C++ API — `mtmd_helper::gen_audio` +(`tools/mtmd/mtmd-helper.h`) — rather than deriving/extracting anything from upstream source. `mtmd` +is already a `target_link_libraries(jllama ...)` dependency (vision/audio-input support), so this +needed no new CMake wiring at all: no generator, no build-time extraction, no hand-written interface +header to keep in sync. Loads a backbone text GGUF (a normal `llama_model`) plus an mmproj GGUF +(speaker encoder + code predictor + code2wav decoder, all bundled in one file by upstream's +`conversion/qwen3tts.py`), and drives the streaming API: `mtmd_helper_gen_audio_set_input()` (prompt ++ optional speaker-reference audio + language) → a `step_prompt()` loop → a `step_gen()` loop (the +engine owns semantic-token sampling via a `common_sampler`, feeding each sampled token + the +backbone's hidden state into `step_gen()` and receiving the next hidden state back — the same pattern +upstream's own `tools/tts/tts.cpp` `main()` uses) → `get_output()` for raw PCM, which the engine +encodes to WAV itself via `tts_wav.hpp` (not upstream's own WAV writer) so that already-tested code +stays in the loop. + +**Why this replaced OuteTTS, not extended it.** Upstream #26254 ("mtmd: support Qwen3-TTS") deleted +the entire OuteTTS implementation from `tools/tts/tts.cpp` (it shrank from ~1450 to 205 lines) and +replaced the two-model OuteTTS-(text-to-codes)-+-WavTokenizer-(vocoder) design with the single +backbone+mmproj design above — there is no upstream code path for OuteTTS left at all past b10269 +(`enum mtmd_gen_audio_type` has exactly `MTMD_GEN_AUDIO_TYPE_NONE` and `MTMD_GEN_AUDIO_TYPE_QWEN3TTS`). +See `docs/history/llama-cpp-breaking-changes.md`'s `b10269–b10270` row for the full investigation; +this was a breaking **public API** change (`TextToSpeech`'s constructor and `synthesize()` overloads +all changed shape) done deliberately — the project does not carry OuteTTS-compatibility shims. + +**Nothing to re-verify on a llama.cpp bump.** Because there is no generator or extracted header +anymore, a version bump cannot silently break the TTS surface the way `patches/` or the old +extraction could — `mtmd_helper::gen_audio`'s API surface is upstream's own committed public header, +covered by the normal priority-8 API-compat review (`tools/mtmd/mtmd-helper.h` is on that list). ## Upgrading/Downgrading llama.cpp Version @@ -743,14 +755,15 @@ Also review the project `CMakeLists.txt` for build-system-level breaks (e.g. ren | `common/chat.cpp` | Chat parsing implementation | | `common/sampling.h` | Sampler API, `common_sampler_*` functions | | `common/log.h` | Log macro signatures | -| `tools/mtmd/mtmd-helper.h` | Multimodal helper functions | +| `tools/mtmd/mtmd-helper.h` | `mtmd_helper::gen_audio` (used directly by `tts_engine.cpp` since the Qwen3-TTS rework — no longer safe to skip), `mtmd_helper_bitmap_init_from_file` | +| `tools/server/server-schema.h` | `eval_llama_cmpl_schema` signature (called directly by `jllama.cpp`'s `populate_completion_task`) — **b10275 dropped its `n_ctx_slot` parameter** and broke a full build without any diff review catching it, because this file is a same-repo header `jllama.cpp` includes directly rather than one pulled in transitively through `common.h`/`llama.h`; it was outside this table until that incident (see `docs/history/llama-cpp-breaking-changes.md`'s b10270–b10275 row). Also watch `server-common.h`/`server-chat.h`/`server-task.h` the same way — anything under `tools/server/*.h` that `jllama.cpp`/`jni_helpers.hpp`/`json_helpers.hpp` `#include`s directly is in scope here, not just headers reachable from the dependency graph above. | | `common/json-schema-to-grammar.h` | Grammar API | | `ggml/include/ggml.h` | `ggml_type` enum values (e.g. `GGML_TYPE_F16`), tensor primitives | | `ggml/include/ggml-backend.h` | Backend/device abstraction types | | `ggml/include/ggml-opt.h` | Optimizer params pulled in via `common.h` | **Safe to skip** (have never caused a break; not used directly by project code): -`common/sampling.h`, `common/log.h`, `tools/mtmd/mtmd-helper.h`, `common/json-schema-to-grammar.h`, +`common/sampling.h`, `common/log.h`, `common/json-schema-to-grammar.h`, `ggml/include/ggml.h`, `ggml/include/ggml-backend.h`, `ggml/include/ggml-opt.h`, `ggml-alloc.h`, `ggml-cpu.h`, `peg-parser.h`, `base64.hpp` @@ -864,8 +877,8 @@ the README. The summary below covers only the optional-model bindings: | `net.ladenthin.llama.audio.model` | `AudioInputIntegrationTest` (llama.cpp discussion #13759) | audio-input model GGUF, e.g. `ultravox-v0_5-llama-3_2-1b.gguf` | | `net.ladenthin.llama.audio.mmproj` | `AudioInputIntegrationTest` | matching audio mmproj/encoder, e.g. `mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf` | | `net.ladenthin.llama.audio.input` | `AudioInputIntegrationTest` | committed default `src/test/resources/audios/sample.wav`; override to any `.wav`/`.mp3` on disk | -| `net.ladenthin.llama.tts.ttc.model` | `TtsIntegrationTest` | OuteTTS text-to-codes model, e.g. `OuteTTS-0.2-500M-Q4_K_M.gguf` | -| `net.ladenthin.llama.tts.vocoder.model` | `TtsIntegrationTest` | matching codes-to-speech vocoder, e.g. `WavTokenizer-Large-75-F16.gguf` | +| `net.ladenthin.llama.tts.model` | `TtsIntegrationTest` | Qwen3-TTS backbone GGUF (any Qwen3-TTS-family model works) | +| `net.ladenthin.llama.tts.mmproj` | `TtsIntegrationTest` | matching mmproj GGUF (speaker encoder + code predictor + code2wav) | Run those tests by setting the property: ```bash @@ -884,8 +897,8 @@ mvn test -Dtest=AudioInputIntegrationTest \ -Dnet.ladenthin.llama.audio.mmproj=models/mmproj-ultravox-v0_5-llama-3_2-1b-f16.gguf \ -Dnet.ladenthin.llama.audio.input=/path/to/speech.wav # optional: defaults to the committed src/test/resources/audios/sample.wav mvn test -Dtest=TtsIntegrationTest \ - -Dnet.ladenthin.llama.tts.ttc.model=models/OuteTTS-0.2-500M-Q4_K_M.gguf \ - -Dnet.ladenthin.llama.tts.vocoder.model=models/WavTokenizer-Large-75-F16.gguf + -Dnet.ladenthin.llama.tts.model=models/qwen3-tts-backbone.gguf \ + -Dnet.ladenthin.llama.tts.mmproj=models/qwen3-tts-mmproj.gguf ``` `MultimodalIntegrationTest` self-skips when any of the three vision properties @@ -1020,7 +1033,7 @@ If the local check passes (`BUILD SUCCESS`), the `mvn package` job in **Java layer** (`src/main/java/net/ladenthin/llama/`): - `LlamaModel` — Main API class (AutoCloseable). Wraps native context for inference, embeddings, re-ranking, and tokenization. -- `TextToSpeech` — Separate AutoCloseable native type for speech synthesis over the two-model OuteTTS (text-to-codes) + WavTokenizer (codes-to-speech vocoder) pipeline; `synthesize(text)` returns a 24 kHz mono 16-bit WAV byte stream. Native orchestration in `tts_engine.{h,cpp}`; the OuteTTS DSP / prompt / text helpers + default speaker are **derived at build time from upstream `tts.cpp`** (see "OuteTTS build-time extraction" below), not hand-copied; the in-memory WAV writer is `tts_wav.hpp`. +- `TextToSpeech` — Separate AutoCloseable native type for speech synthesis over llama.cpp's upstream Qwen3-TTS pipeline (a backbone text GGUF + an mmproj GGUF bundling the speaker encoder, code predictor, and code2wav decoder); `synthesize(text)` returns a 24 kHz mono 16-bit WAV byte stream, with overloads for a cloned-voice speaker-reference clip and language. Native orchestration in `tts_engine.{h,cpp}` drives upstream's `mtmd_helper::gen_audio` streaming API directly (see "Qwen3-TTS via `mtmd_helper::gen_audio`" below) — there is nothing extracted or hand-copied from llama.cpp source; the in-memory WAV writer is `tts_wav.hpp`. - `ModelParameters` / `InferenceParameters` — Builder-pattern parameter classes that serialize to JSON (extend `JsonParameters`) for passing to native code. - `LlamaIterator` / `LlamaIterable` — Streaming generation via Java `Iterator`/`Iterable`. - `LlamaLoader` — Extracts the platform-specific native library from the JAR to a temp directory, or finds it on `java.library.path`. @@ -1189,12 +1202,14 @@ Require a model file. The CI downloads models from HuggingFace: **CI model policy (publish.yml): the full model set is downloaded and exercised on EVERY Java test job** — Linux x86_64, all three macOS arm64 jobs (Metal / no-Metal / Metal-15), and both Windows jobs (MSVC + Ninja). That includes the nomic embedding model, the SmolVLM vision -model + mmproj, and the OuteTTS + WavTokenizer TTS pair, with their `-Dnet.ladenthin.llama.*` -properties set, so `LlamaEmbeddingsTest`, `MultimodalIntegrationTest`, and `TtsIntegrationTest` -**run on every platform** rather than self-skipping. `validate-models.{sh,bat}` treats all of -these as **required** (a missing model hard-fails the job before tests run, so a download -regression can never silently downgrade to a skip). The only model still self-skipping is the -audio-input model (`AudioInputIntegrationTest`) — the prompt clip is committed +model + mmproj, and the Qwen3-TTS backbone + mmproj (`ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF`, +smallest available quants: `Qwen3-TTS-12Hz-1.7B-Base-Q4_K_M.gguf` backbone + +`mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf` mmproj — no smaller mmproj quant is published), with +their `-Dnet.ladenthin.llama.*` properties set, so `LlamaEmbeddingsTest`, `MultimodalIntegrationTest`, +and `TtsIntegrationTest` **run on every platform** rather than self-skipping. `validate-models.{sh,bat}` +treats all of these as **required** (a missing model hard-fails the job before tests run, so a +download regression can never silently downgrade to a skip). Only the audio-input model +(`AudioInputIntegrationTest`) still self-skips — the prompt clip is committed (`src/test/resources/audios/sample.wav`) but the audio model + mmproj have no CI download. The model set has a **single source of truth: `.github/models.csv`** (one `filename,url` row per model; `#` comments). Everything derives from it: the **`download-models`** job (ubuntu, @@ -1257,13 +1272,13 @@ ctest --test-dir build --output-on-failure -R "ResultsToJson" | `src/test/cpp/test_json_helpers.cpp` | 50 | All functions in `json_helpers.hpp`: `get_result_error_message`, `results_to_json`, `rerank_results_to_json` (incl. missing/out-of-range `index` rejection), `parse_encoding_format`, `extract_embedding_prompt`, `is_infill_request`, `parse_slot_prompt_similarity`, `parse_positive_int_config`, `wrap_stream_chunk` | | `src/test/cpp/test_log_helpers.cpp` | 13 | All functions in `log_helpers.hpp`: `log_level_name`, `format_log_as_json` | | `src/test/cpp/test_jni_helpers.cpp` | 54 | All functions in `jni_helpers.hpp` using a zero-filled `JNINativeInterface_` mock (incl. the `utf8_to_jstring_impl` byte-array string path: emoji byte-preservation, truncated-UTF-8 replace-not-throw) | -| `src/test/cpp/test_tts_wav.cpp` | 5 | The in-memory WAV writer `pcm_to_wav16_bytes` in `tts_wav.hpp` (WAV header/payload + little-endian clamping) plus the pure OuteTTS codec-window filter `filter_outetts_codec_tokens` (`tts_engine.h`: boundary keep/drop + rebase). The OuteTTS DSP it pairs with is derived from upstream `tts.cpp` and covered end-to-end by the Java `TtsIntegrationTest`, not unit-tested here. | +| `src/test/cpp/test_tts_wav.cpp` | 2 | The in-memory WAV writer `pcm_to_wav16_bytes` in `tts_wav.hpp` (WAV header/payload + little-endian clamping) — our own code, not upstream. The Qwen3-TTS pipeline it pairs with (`mtmd_helper::gen_audio`) is entirely upstream-owned (no project-side DSP to unit-test here) and covered end-to-end by the Java `TtsIntegrationTest`. | -**Current total: 485 tests (all passing).** +**Current total: 482 tests (all passing).** #### Upstream source location (in CMake build tree) -llama.cpp is fetched via CMake FetchContent, pinned to `GIT_TAG b10217`. +llama.cpp is fetched via CMake FetchContent, pinned to `GIT_TAG b10319`. **GoogleTest** is a separate `BUILD_TESTING`-only FetchContent (`GIT_TAG v1.17.0`), used solely by the `jllama_test` C++ unit-test binary — not by the shipped library, and not coupled to the diff --git a/README.md b/README.md index 94a88be8..8a2d7de2 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ **Build:** ![Java 8+](https://img.shields.io/badge/Java-8%2B-informational) ![Platform](https://img.shields.io/badge/Platform-Linux%20%7C%20macOS%20%7C%20Windows%20%7C%20Android-lightgrey) -[![llama.cpp b10217](https://img.shields.io/badge/llama.cpp-%23b10217-informational)](https://github.com/ggml-org/llama.cpp/releases/tag/b10217) +[![llama.cpp b10319](https://img.shields.io/badge/llama.cpp-%23b10319-informational)](https://github.com/ggml-org/llama.cpp/releases/tag/b10319) [![JPMS](https://img.shields.io/badge/JPMS-modular%20JAR-25A162)](https://openjdk.org/projects/jigsaw/) ![JUnit](https://img.shields.io/badge/tested%20with-JUnit6-25A162) [![JSpecify](https://img.shields.io/badge/JSpecify-1.0.0%20%40NullMarked-25A162)](https://jspecify.dev) @@ -108,7 +108,7 @@ Inference of Meta's LLaMA model (and others) in pure C/C++. - OpenAI-compatible **chat completion** with automatic chat-template application, including streaming and tool/function calling support via the upstream server. - **Embeddings** (single and native-batched via `embed(Collection)`) and **reranking** for retrieval pipelines. - **Runtime LoRA adapter control** — list the loaded adapters and change their scales at runtime without reloading the model (`getLoraAdapters()` / `setLoraAdapters(Map)`), the typed counterpart of the upstream `GET`/`POST /lora-adapters` endpoints. -- **Text-to-speech** (`TextToSpeech`) over the two-model OuteTTS + WavTokenizer pipeline, returning WAV audio. +- **Text-to-speech** (`TextToSpeech`) over llama.cpp's Qwen3-TTS pipeline (`mtmd_helper::gen_audio`), returning WAV audio. - **In-JVM GGUF quantization** (`LlamaQuantizer`) over llama.cpp's `llama_model_quantize` — convert a GGUF to another quantization scheme without shelling out to `llama-quantize`. - **Infilling** (fill-in-the-middle) for code models. - **Tokenize / detokenize** and **JSON-schema → grammar** conversion. @@ -356,10 +356,10 @@ Every `net.ladenthin.llama.*` system property recognised by the library, deep-sc | `net.ladenthin.llama.audio.model` | unset (test self-skips) | test | `AudioInputIntegrationTest` (llama.cpp discussion #13759) | Path to an audio-input model GGUF (e.g. Ultravox, Qwen2.5-Omni). | | `net.ladenthin.llama.audio.mmproj` | unset (test self-skips) | test | `AudioInputIntegrationTest` | Matching audio mmproj (encoder) GGUF. | | `net.ladenthin.llama.audio.input` | `src/test/resources/audios/sample.wav` (committed) | test | `AudioInputIntegrationTest` | `.wav`/`.mp3` audio prompt clip; the extension drives format detection. | -| `net.ladenthin.llama.tts.ttc.model` | unset (test self-skips) | test | `TtsIntegrationTest` | Path to the OuteTTS text-to-codes GGUF. CI default is `OuteTTS-0.2-500M-Q4_K_M.gguf`. | -| `net.ladenthin.llama.tts.vocoder.model` | unset (test self-skips) | test | `TtsIntegrationTest` | Path to the matching codes-to-speech vocoder GGUF. CI default is `WavTokenizer-Large-75-F16.gguf`. | +| `net.ladenthin.llama.tts.model` | unset (test self-skips) | test | `TtsIntegrationTest` | Path to the Qwen3-TTS backbone (text) GGUF. Any Qwen3-TTS-family model works; CI default is `Qwen3-TTS-12Hz-1.7B-Base-Q4_K_M.gguf`. | +| `net.ladenthin.llama.tts.mmproj` | unset (test self-skips) | test | `TtsIntegrationTest` | Path to the matching Qwen3-TTS mmproj GGUF (speaker encoder + code predictor + code2wav decoder); CI default is `mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf`. | -`MultimodalIntegrationTest` self-skips when any of the three `vision.*` properties points at a missing path, so a partial setup (just the vision model + the committed image, no mmproj) lets the test class load without erroring. `AudioInputIntegrationTest` self-skips the same way over the three `audio.*` properties. `TtsIntegrationTest` likewise self-skips unless both `tts.ttc.model` and `tts.vocoder.model` point at existing files. +`MultimodalIntegrationTest` self-skips when any of the three `vision.*` properties points at a missing path, so a partial setup (just the vision model + the committed image, no mmproj) lets the test class load without erroring. `AudioInputIntegrationTest` self-skips the same way over the three `audio.*` properties. `TtsIntegrationTest` likewise self-skips unless both `tts.model` and `tts.mmproj` point at existing files. ## Documentation @@ -590,29 +590,29 @@ effective adapter set changes. ### Text-to-Speech -`TextToSpeech` synthesizes audio from text over llama.cpp's OuteTTS pipeline. It is a separate -`AutoCloseable` native type (not a `LlamaModel`) because TTS is a **two-model** pipeline: a -text-to-codes model (OuteTTS) and a codes-to-speech vocoder (WavTokenizer). `synthesize(String)` -returns a 24 kHz mono 16-bit WAV byte stream. +`TextToSpeech` synthesizes audio from text over llama.cpp's upstream Qwen3-TTS pipeline +(`mtmd_helper::gen_audio`). It is a separate `AutoCloseable` native type (not a `LlamaModel`) +because TTS loads its own **model pair**: a backbone (text) GGUF and an mmproj GGUF bundling the +speaker encoder, code predictor, and code2wav decoder. `synthesize(String)` returns a +24 kHz mono 16-bit WAV byte stream. ```java try (TextToSpeech tts = new TextToSpeech( - "models/OuteTTS-0.2-500M-Q4_K_M.gguf", - "models/WavTokenizer-Large-75-F16.gguf")) { + "models/qwen3-tts-backbone.gguf", "models/qwen3-tts-mmproj.gguf")) { byte[] wav = tts.synthesize("Hello from llama dot c p p."); Files.write(Paths.get("out.wav"), wav); } ``` -Add `(ttcPath, vocoderPath, gpuLayers, threads)` to offload to the GPU, or -`synthesize(text, maxCodeTokens, topK, seed)` for explicit sampling. As with `LlamaModel`, native -memory is not GC-managed — use try-with-resources or call `close()`. Synthesis uses the built-in -default speaker profile; English number words are expanded for speech (`3` → "three"), and -non-English text is not romanized. +Add `(modelPath, mmprojPath, gpuLayers, threads)` to offload to the GPU, or +`synthesize(text, maxFrames, topK, seed)` for explicit sampling, or the full +`synthesize(text, speakerReferenceAudioPath, language, maxFrames, topK, seed)` overload for +voice cloning from a reference clip. As with `LlamaModel`, native memory is not GC-managed — use +try-with-resources or call `close()`. -Compatible GGUFs (the CI test defaults): OuteTTS -[`OuteTTS-0.2-500M-GGUF`](https://huggingface.co/second-state/OuteTTS-0.2-500M-GGUF) + -[`WavTokenizer`](https://huggingface.co/ggml-org/WavTokenizer). +This replaces the project's earlier two-model OuteTTS + WavTokenizer pipeline, which upstream +[#26254](https://github.com/ggml-org/llama.cpp/pull/26254) deleted entirely in favor of Qwen3-TTS +(llama.cpp `b10270`); there is no backward-compatible path for the old model pair. ### GGUF Quantization diff --git a/docs/history/llama-cpp-breaking-changes.md b/docs/history/llama-cpp-breaking-changes.md index 445583ad..fe7a8142 100644 --- a/docs/history/llama-cpp-breaking-changes.md +++ b/docs/history/llama-cpp-breaking-changes.md @@ -532,3 +532,50 @@ Used during `llama.cpp` version bumps: when upgrading, scan this file from the r | b10207–b10216 | upstream verification (sandbox) | All **7** patches re-verified against a clean b10216 checkout (ggml/llama.cpp commit `876a43211`): fail-loud `PATCH_COMMAND` clean; `0003`'s `server-context.cpp` hunk still applied cleanly via fuzzy context matching despite upstream inserting lines earlier in the file (the speculative-replay accounting change). OuteTTS generator anchors hold (`tts.cpp @ b10216`). **Full local verification:** fresh configure + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **485/485 passing** + `NativeLibraryLoadSmokeTest` green. Per-platform build + `ctest` confirmation deferred to the CI pipeline. | | b10216–b10217 | `common/chat.cpp` (**DeepSeek-V4 tool-call-in-thinking parsing**) | **No project-source change (1 commit, ~15 KiB — final chunk; reaches the b10217 target).** Upstream #26269 reworks the DeepSeek-V4 chat-format parser (`common/chat.cpp`) so a tool call can be emitted while still inside a `` block, plus two new `tests/test-chat.cpp` regression cases — pure parsing-logic internals inside the upstream-compiled `llama-common` TU; `common/chat.h`'s public surface (`common_chat_parser_params`, `to_json_oaicompat`, etc.) is unchanged, so no project source (`jllama.cpp`, `json_helpers.hpp`) needs updating. No patch-target file touched, no OuteTTS generator anchor touched. | | b10216–b10217 | upstream verification (sandbox) | All **7** patches re-verified against a clean b10217 checkout (ggml/llama.cpp commit `ddd4ec142`): fail-loud `PATCH_COMMAND` clean, no patch-target file changed in the range. OuteTTS generator anchors hold (`tts.cpp @ b10217`). **Full local verification:** fresh configure + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **485/485 passing** + `NativeLibraryLoadSmokeTest` green. **b10217 is the topmost release; the b10197→b10217 bump is complete** (4 chunks: b10197→b10204→b10207→b10216→b10217). Per-platform build + `ctest` confirmation deferred to the CI pipeline. | +| b10217–b10221 | `tools/mtmd/{clip.cpp,mtmd-image.{h,cpp},mtmd.cpp}` + `tools/cli/cli-context.cpp` + `vendor/cpp-httplib/CMakeLists.txt` (**BoringSSL bump**) | **No project-source change (4 commits, ~9 KiB — first step of a fixed 5-tag-max-per-step walk toward the b10310 target).** `tools/mtmd/mtmd-image.h`/`clip.cpp` add a new `mtmd_image_preprocessor_minicpmv` slicing subclass (MiniCPM-V-4.6 downsample support, upstream #25993) and widen `get_best_resize` from `private` to `protected` so it can; internal to the mtmd image-preprocessing hierarchy, no signature project code calls changed (`tools/mtmd/mtmd.h`'s public API is untouched). `vendor/cpp-httplib/CMakeLists.txt` only bumps the `BORINGSSL_VERSION` string used when `LLAMA_BUILD_BORINGSSL` is enabled — this project never sets that option (plain-HTTP `cpp-httplib`, `CPPHTTPLIB_OPENSSL_SUPPORT` left undefined), so it is inert here. `tools/cli/cli-context.cpp` (persist `reasoning_content` in cli chat history) is standalone-tool-only, not compiled into `jllama`. None of the priority-8 headers or patch-target files changed. | +| b10217–b10221 | upstream verification (sandbox) | All **7** patches re-verified against a clean b10221 checkout (ggml/llama.cpp commit `815a2a591`): fresh `cmake -B build` configure succeeded (fail-loud `PATCH_COMMAND` clean; no patch-target file changed in the range). OuteTTS generator anchors hold (`tts.cpp @ b10221`, confirmed via the configure-time `generate-tts-upstream: wrote ...` log line). **Configure-only verification this step** (fixed-step walk to b10310 — full `cmake --build`+`ctest` is checkpointed every ~5 steps and at the final b10310 target, not after every intermediate tag, to keep the walk tractable). | +| b10221–b10226 | `ggml/src/{ggml-sycl/**,ggml-vulkan/**,ggml-webgpu/**}` + `src/models/mimo2.cpp` + `tools/ui/CMakeLists.txt` | **No project-source change (5 commits, ~15 KiB).** Entirely GPU-backend-internal: SYCL iGPU device-classification fix, Vulkan `topk_moe` fusion extended to `sqrt(softplus)`, WebGPU f16 `repeat` support. `src/models/mimo2.cpp` only skips loading MTP tensors when unused (mirrors the `bool load_mtp` field added at b10207–b10216, no new public `llama.h` surface). `tools/ui/CMakeLists.txt` is the auto-followed WebUI build config. None of the priority-8 headers or patch-target files changed. | +| b10221–b10226 | upstream verification (sandbox) | All **7** patches re-verified against a clean b10226 checkout (ggml/llama.cpp commit `272700b36`): fresh `cmake -B build` configure succeeded (fail-loud `PATCH_COMMAND` clean; no patch-target file changed in the range). OuteTTS generator anchors hold (`tts.cpp @ b10226`). **Configure-only verification this step.** | +| b10226–b10231 | `common/{arg.cpp,download.{h,cpp},chat.cpp,chat-peg-parser.{h,cpp}}` + `src/{llama-context.cpp,llama-graph.{h,cpp},llama-kv-cache-dsv4.{h,cpp},llama-model.cpp,llama-arch.cpp,models/{deepseek4.cpp,dflash.cpp,models.h}}` (**DeepSeek-V4 MTP + DSpark speculative sidecar, upstream #25784**) | **No project-source change (5 commits, ~55 KiB).** New `COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK` speculative type + `common_download_opts::download_dspark` / `common_download_hf_plan::dspark` fields (both purely additive struct members in `common/download.h`) and matching `common_models_handler_init`/`_apply` HF-download-plan wiring in `common/arg.cpp` — this is the `--hf-repo` auto-download-a-model CLI path, nowhere near patch `0001`'s `common_params_parse`/`common_params_parse_main` win32-argv hunks (confirmed no line overlap). `common/speculative.cpp` gets a one-line assertion fix (`llama_model_n_embd`→`llama_model_n_embd_out` for the MTP target-width check). `common/chat.cpp` adds a Qwen3 specialized tool-call parser via the new (already-present) PEG parser infra; `common/chat.h`'s public surface is unchanged. Everything else (`llama-graph`, `llama-kv-cache-dsv4`, `models/deepseek4.cpp`, new `models/dflash.cpp`) is DeepSeek-V4 architecture internals in `src/` — not part of the public `include/llama.h` API. None of the priority-8 headers changed. | +| b10226–b10231 | upstream verification (sandbox) | All **7** patches re-verified against a clean b10231 checkout (ggml/llama.cpp commit `bb4e0e1b3`): fresh `cmake -B build` configure succeeded (fail-loud `PATCH_COMMAND` clean — `0001`'s `common/arg.cpp` hunks unaffected by this range's unrelated HF-download-plan additions). OuteTTS generator anchors hold (`tts.cpp @ b10231`). **Configure-only verification this step.** | +| b10231–b10236 | `ggml/src/ggml-metal/**` + `ggml/src/ggml-opencl/ggml-opencl.cpp` | **No project-source change (5 commits, ~24 KiB).** Metal backend gains DeepSeek-V4 Lightning Indexer + hyper-connections kernels, `SILU_BACK`, and F16 binary-op support; OpenCL limits its GLU-op local workgroup size. Entirely GPU-backend-internal (`ggml-metal-*`/`ggml-opencl.cpp`), no priority-8 header or patch-target file touched. | +| b10231–b10236 | upstream verification (sandbox) | All **7** patches re-verified against a clean b10236 checkout (ggml/llama.cpp commit `1464c62d8`): fresh `cmake -B build` configure succeeded (fail-loud `PATCH_COMMAND` clean; no patch-target file changed in the range). OuteTTS generator anchors hold (`tts.cpp @ b10236`). **Configure-only verification this step.** | +| b10236–b10241 | `tools/server/{server.cpp,server-tools.cpp}` + `src/models/{deepseek32.cpp,qwen3next.cpp}` + `ggml/src/ggml-cuda/{common.cuh,norm.cu,softmax.cu}` (**DeepSeek V3.2 + Qwen3-Next MTP**) | **No project-source change (5 commits, ~78 KiB).** `tools/server/server.cpp` gains a one-line startup log notice about a future default-port change (8080→9931, upstream #26508) — pure logging, no signature change, well clear of `patches/0006`/`0007`'s embedded-mode hunks. `tools/server/server-tools.cpp` adds an optional `cwd` parameter to `tools_io_basic` (new `x-tool-cwd` HTTP header, upstream #26420) plus a `get_info` built-in tool — additive, not a file `patches/` touches. Everything else is DeepSeek V3.2/Qwen3-Next MTP model internals (`src/models/`) and CUDA SMEM-race fixes — none of the priority-8 headers changed. | +| b10236–b10241 | upstream verification (sandbox) | All **7** patches re-verified against a clean b10241 checkout (ggml/llama.cpp commit `9bd4c09ea`): fail-loud `PATCH_COMMAND` clean, no patch-target file changed in the range. OuteTTS generator anchors hold (`tts.cpp @ b10241`). **Full local verification (checkpoint):** fresh configure + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **485/485 passing**. Per-platform build + `ctest` confirmation deferred to the CI pipeline. | +| b10241–b10246 | `include/llama.h` + `common/{arg.cpp,sampling.{h,cpp}}` + `tools/server/server-context.cpp` (**CUDA penalties sampler backend, upstream #25262**) | **No project-source change (5 commits, ~52 KiB).** `include/llama.h`: `llama_sampler_init_penalties()` doc comments tightened (validity constraints noted, e.g. "must be finite") but the **signature itself is unchanged** (still 3 floats after `penalty_last_n`); a new `int64_t n_vocab` field was added to an internal `llama_sampler_data`-adjacent struct — not one this project constructs. `common/sampling.h`/`.cpp`: `common_sampler_init()` gains a 3rd defaulted `int32_t n_ctx = 0` param (upstream-internal call site in `server-context.cpp` updated); project never calls `common_sampler_init` directly. `common/arg.cpp`: `--repeat-penalty`/`--presence-penalty`/`--frequency-penalty` CLI flags now reject non-finite values, and `--repeat-penalty` additionally rejects `<= 0.0` (previously `1.0 = disabled` was documented but `0`/negative/non-finite were silently accepted). **Worth knowing, no fix needed:** `ModelParameters.setRepeatPenalty/setPresencePenalty/setFrequencyPenalty` (`ModelParameters.java`) emit these as `--repeat-penalty`/`--presence-penalty`/`--frequency-penalty` CLI flags for `NativeServer`'s argv — a caller passing `0.0f`/`NaN`/`Infinity` to `setRepeatPenalty` will now hit a native-side parse error at server start instead of silently misbehaving (this is a validation *hardening*, not a removed feature; the per-request JSON path via `InferenceParameters`'s `repeat_penalty` field is unaffected — it goes through `server-schema.cpp`'s `eval_llama_cmpl_schema`, not CLI arg parsing). | +| b10241–b10246 | upstream verification (sandbox) | All **7** patches re-verified against a clean b10246 checkout (ggml/llama.cpp commit `39eab74a0`): fresh `cmake -B build` configure succeeded (fail-loud `PATCH_COMMAND` clean — `0002`/`0003`'s `server-context.cpp` hunks unaffected by the unrelated `common_sampler_init` call-site update). OuteTTS generator anchors hold (`tts.cpp @ b10246`). **Configure-only verification this step.** | +| b10246–b10251 | `common/{arg.cpp,common.{h,cpp}}` + `src/{llama-vocab.cpp,models/deepseek2.cpp}` + `ggml/src/ggml-backend.cpp` + `tools/server/server-tools.cpp` (**GLM-4.7-Flash MTP, dynamic split-graph allocation**) | **No project-source change (5 commits, ~30 KiB).** `common/common.h` adds `common_params::is_gen_docs` (bool, default `false`, purely additive — not constructed positionally by project code) and free functions `common_get_env`/`common_set_env` (new, not shadowing anything project-defined). `common/arg.cpp`'s only change gates a doc-gen-only `--rpc` option branch and updates the `--tools` help string (new `get_info` tool name) — nowhere near patch `0001`'s argv-parsing hunks. `tools/server/server-tools.cpp` adds a `get_info` built-in tool (additive, not a patch-target file). `ggml-backend.cpp`'s dynamic split-graph-input allocation and the vocab/model internals are upstream-only. None of the priority-8 headers' *function signatures* changed (only `common.h` struct additions). | +| b10246–b10251 | upstream verification (sandbox) | All **7** patches re-verified against a clean b10251 checkout (ggml/llama.cpp commit `57c092139`): fresh `cmake -B build` configure succeeded (fail-loud `PATCH_COMMAND` clean; no patch-target file changed in the range). OuteTTS generator anchors hold (`tts.cpp @ b10251`). **Configure-only verification this step.** | +| b10251–b10256 | `vendor/cpp-httplib/{httplib.h,httplib.cpp,CMakeLists.txt}` (**cpp-httplib bumped 0.51.x → 0.52.0**) + `common/chat.cpp` (new DeepSeek V4 Flash 0731 template) | **No project-source change, but elevated review depth (5 commits, ~40 KiB).** `httplib.h`/`httplib.cpp` are compiled directly into `jllama` (`server-http.cpp`'s `#include ` + `target_sources` on `httplib.cpp`) and are the largest single-file diff of this bump so far (+914/−215 net). Reviewed for signature drops on anything `server-http.cpp` calls — none found; the upstream server TU itself is unmodified by any local patch. **Because a vendor-library major bump carries more compile risk than a configure-only check can see, this step additionally did a *targeted* build** (not the full checkpoint): `make -f CMakeFiles/jllama.dir/build.make .../vendor/cpp-httplib/httplib.cpp.o` and `.../tools/server/server-http.cpp.o` both compiled clean against 0.52.0. `common/chat.cpp`'s new template addition doesn't touch `common/chat.h`'s public surface. | +| b10251–b10256 | upstream verification (sandbox) | All **7** patches re-verified against a clean b10256 checkout (ggml/llama.cpp commit `6c8dcaa7a`): fresh `cmake -B build` configure succeeded (fail-loud `PATCH_COMMAND` clean; no patch-target file changed in the range). OuteTTS generator anchors hold (`tts.cpp @ b10256`). **Configure + targeted-object-compile verification this step** (see above); full checkpoint build deferred to b10265. | +| b10256–b10261 | `include/llama.h` (`llama_sampler_init_penalties` gains `n_vocab` param) + `src/{llama-sampler.cpp,llama-model-loader.{h,cpp}}` (**move `n_vocab` from a struct field to a function param, upstream #26520**) | **No project-source change (5 commits, ~18 KiB).** `llama_sampler_init_penalties()` gains a new leading `int32_t n_vocab` parameter (the `int64_t n_vocab` struct field added at b10241–b10246 was removed again — same field, relocated to the call site instead) — a **real signature change**, but `jllama.cpp`/`jni_helpers.hpp` never call `llama_sampler_init_penalties` directly (confirmed via grep); it is reached only through `common_sampler_init` (upstream-compiled, updated internally). `src/llama-model-loader.{h,cpp}` gains reshape-during-load support (upstream #26531) — internal loader logic, no `include/llama.h` model-loading API changed. | +| b10256–b10261 | upstream verification (sandbox) | All **7** patches re-verified against a clean b10261 checkout (ggml/llama.cpp commit `cf06ad7df`): fresh `cmake -B build` configure succeeded (fail-loud `PATCH_COMMAND` clean; no patch-target file changed in the range). OuteTTS generator anchors hold (`tts.cpp @ b10261`). **Configure-only verification this step.** | +| b10261–b10265 | `ggml/{CMakeLists.txt,src/ggml-vulkan/**}` (**ggml bumped to 0.18.1, Vulkan GATED_LINEAR_ATTN op**) | **No project-source change (4 commits, ~157 KiB — dominated by an auto-generated `docs/ops/Vulkan.csv` op-support matrix).** Cosmetic `GGML_VERSION_PATCH` bump + a new Vulkan backend op (`gla.comp` shader) for gated linear attention; internal to the Vulkan backend. No priority-8 header or patch-target file touched. | +| b10261–b10265 | upstream verification (sandbox) | All **7** patches re-verified against a clean b10265 checkout (ggml/llama.cpp commit `2e17f69ef`): fail-loud `PATCH_COMMAND` clean, no patch-target file changed in the range. OuteTTS generator anchors hold (`tts.cpp @ b10265` — **last tag in this walk where the anchor-based OuteTTS extraction still works**; see the next row). **Full local verification (checkpoint):** fresh configure + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **485/485 passing**. Per-platform build + `ctest` confirmation deferred to the CI pipeline. | +| b10265–b10269 | `common/speculative.cpp` (**dedup `common_speculative_init` config-list building via a lambda**) | **No project-source change (4 commits, ~1 KiB code + a large auto-generated `docs/ops/Vulkan.csv`/`.github/workflows/*.yml` refresh).** Pure internal refactor of `common_speculative_init`'s enabled-config list-building (`common/speculative.h`'s public surface unchanged); a `gguf-py` reader-validation fix and a `dflash` model-loading tweak are also upstream/Python-only. **This is the last tag before the Qwen3-TTS architecture replacement lands at b10270 (see the next row) — chosen deliberately to stop one tag short of that break** rather than at the originally-planned b10270. | +| b10265–b10269 | upstream verification (sandbox) | All **7** patches re-verified against a clean b10269 checkout (ggml/llama.cpp commit `1c3c9674d`): fail-loud `PATCH_COMMAND` clean, no patch-target file changed in the range. OuteTTS generator anchors hold (`tts.cpp @ b10269`, confirmed via the configure-time `generate-tts-upstream: wrote ...` log line — the **last tag where they do**, see the next row). **Full local verification (checkpoint):** fresh configure + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **485/485 passing**. Per-platform build + `ctest` confirmation deferred to the CI pipeline. | +| b10269–b10270 | `tools/tts/tts.cpp` + `tools/mtmd/{mtmd.h,mtmd-helper.h,mtmd-helper-gen.cpp,models/qwen3tts-gen.cpp,models/qwen3tts-spkenc.cpp}` (**upstream #26254, "mtmd: support Qwen3-TTS (note: breaking change to llama-tts binary)"**) | **RESOLVED — full rework onto Qwen3-TTS, deliberate breaking public-API change, no compat shim.** Upstream's own commit message calls this a breaking change to the `llama-tts` binary; `tools/tts/tts.cpp` — the file `llama/cmake/generate-tts-upstream.cmake` used to mechanically extract our OuteTTS pipeline from — shrank from ~1450 to 205 lines, and every anchor the old generator asserted is gone. The two-model OuteTTS-(text-to-codes)-+-WavTokenizer-(vocoder) pipeline is replaced wholesale by a **single-model** (backbone + `--mmproj`) design driven by the new `mtmd_helper::gen_audio` streaming API; `enum mtmd_gen_audio_type` at b10270 has exactly two values, `MTMD_GEN_AUDIO_TYPE_NONE` and `MTMD_GEN_AUDIO_TYPE_QWEN3TTS` — Qwen3-TTS is the *only* model family the new pipeline supports, and there is no OuteTTS path left to extract from anywhere in the upstream tree at this tag. Decision (explicit user direction — no backward compatibility needed, "we are more or less forwarding just to llama.cpp from java side"): `TextToSpeech`/`tts_engine.{h,cpp}` were fully reworked onto `mtmd_helper::gen_audio` rather than vendor-freezing the old pipeline or dropping the feature. `mtmd` was already linked into `jllama`/`jllama_test` (pre-existing vision/audio-input wiring), so no new CMake target wiring was needed; the OuteTTS build-time extraction (`generate-tts-upstream.cmake` + `tts_upstream.h`) was deleted outright. Public API changed (constructor now takes `(modelPath, mmprojPath, ...)` instead of `(ttcPath, vocoderPath, ...)`; `synthesize` gained optional speaker-reference/language voice-cloning parameters) — this is a breaking change for any consumer, accepted per the "clean API for the use case, no compat" directive. `test_tts_wav.cpp`'s WAV-writer tests (`TtsWav.*`) are untouched (our own code); the dead `OuteTtsCodecFilter.*` tests (3) were deleted since `filter_outetts_codec_tokens` no longer exists. `TtsIntegrationTest`'s WAV-format assertions are unchanged (still 24 kHz mono 16-bit PCM) — only its setup/constructor call and the `PROP_TTS_MODEL`/`PROP_TTS_MMPROJ` property names changed. CI had no Qwen3-TTS model download at first (`.github/models.csv`'s stale `OuteTTS-0.2-500M-Q4_K_M.gguf`/`WavTokenizer-Large-75-F16.gguf` rows were removed, not replaced — huggingface.co was fully network-blocked in the sandbox that did this rework, so exact Qwen3-TTS GGUF/mmproj filenames could not be verified). **Follow-up (same branch, separate session with real HF access):** exact repo contents of `ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF` were verified externally (6 files, no gating/auth) and `.github/models.csv` gained two rows — `Qwen3-TTS-12Hz-1.7B-Base-Q4_K_M.gguf` (0.96 GiB, smallest backbone quant) + `mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf` (0.42 GiB, smallest available mmproj quant — no Q4_K_M mmproj is published); `TTS_MODEL_NAME`/`TTS_MMPROJ_NAME` env vars and the matching `-Dnet.ladenthin.llama.tts.{model,mmproj}` properties were wired into every Java test job in `publish.yml`, the same way as the vision/nomic models — so `TtsIntegrationTest` now **runs on every platform** instead of self-skipping. The emulator jobs' free-disk steps needed no change (they already delete-by-exclusion, keeping only the draft model). **Full local verification (checkpoint):** fresh configure + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link against the real b10270 `mtmd_helper::gen_audio`/`mtmd_gen_audio_get_info`/`mtmd_helper_bitmap_init_from_file` symbols, `-O3`, no undefined references) + `ctest` **482/482 passing** (485 minus the 3 removed dead tests). Per-platform build + `ctest` confirmation deferred to the CI pipeline. | +| b10270–b10275 | `common/{common.h,common.cpp}`, `include/llama.h` (**`common_sampler_init` drops its `n_ctx` param; `llama_sampler_init_dry` drops `n_ctx_train`; `penalty_last_n`/`dry_penalty_last_n` no longer accept `-1` = context-size sentinel**), and — missed by the review scope below — `tools/server/server-schema.{h,cpp}` (**`eval_llama_cmpl_schema` drops its `n_ctx_slot` parameter; the internal `-1`→ctx-size expansion for `penalty_last_n`/`dry_penalty_last_n` is deleted along with it, and their hard lower limit moves from `-1` to `0`**) | **CORRECTED (see the b10280 checkpoint row below) — this step's original "no project-source change" verdict was wrong.** `tools/server/server-schema.h` is compiled directly as a project source dependency (`jllama.cpp` calls `eval_llama_cmpl_schema` directly) but is not on the priority-ordered header-diff review list (that list covers headers *pulled in transitively*; `server-schema.h` is a same-repo file `jllama.cpp` includes directly and was overlooked). This step was verified **configure-only** (per the walk's process), so the resulting link/compile failure was invisible until the next full build — at the b10280 checkpoint. The actual fix (both applied together, see that row): `jllama.cpp`'s `populate_completion_task` drops its now-unused `n_ctx_slot` parameter (both call sites simplified accordingly), and `test_server.cpp`'s `parse_params` test helper drops its `n_ctx` parameter; the two `*_ExpandsToNCtxSlot` tests become `*_MinusOne_Throws` (a request-supplied `-1` is now out-of-range instead of expanding). **Lesson for future steps in this walk:** treat any `tools/server/*.h` file `jllama.cpp` `#include`s directly as in-scope for diff review, not just the header dependency-graph list. `tts_engine.cpp`'s own `common_sampler_init(engine->model, sparams)` call already used the 2-argument form and needed no change. Rest of the range (5 commits) is a WebUI `tools/ui/**` refresh, no other C++ surface touched. | +| b10270–b10275 | upstream verification (sandbox) | All **9** patches re-verified against a clean b10275 checkout (ggml/llama.cpp commit `4308a4f0`): fail-loud `PATCH_COMMAND` clean (fresh `cmake -B build` succeeded, no `FATAL_ERROR`), no patch-target file changed in the range. **Configure-only verification per this step's process** (full build deferred to the next checkpoint) — this is exactly what let the `eval_llama_cmpl_schema` break above slip through undetected until the b10280 checkpoint's full build. | +| b10275–b10280 | `vendor/sheredom/subprocess.h` (**upstream merged our own submitted fix, #26606 "vendor: apply patches for subprocess.h"**) | **Local patch dropped, not refreshed — `0009-subprocess-guard-addchdir-np-old-glibc.patch` removed.** Upstream merged [sheredom/subprocess.h#104](https://github.com/sheredom/subprocess.h/pull/104) (the exact `SUBPROCESS_HAVE_CWD`/`ENOSYS` fix `0009` carried) verbatim, so the old-glibc build break it fixed no longer exists at this tag. The same PR also rewrote the neighboring Windows argv-quoting logic and added a `__NetBSD__` branch next to the `process_cwd` guard, shifting `0009`'s second hunk's context enough that the applier's idempotent `git apply --reverse --check` could not auto-detect "already applied" — confirmed by actually attempting the bump with `0009` still present: configure failed loud with "does not apply cleanly" exactly as the fail-loud contract promises, which is what triggered dropping (not refreshing) the patch. Rest of the range (4 commits) is WebUI/build-script only (`ui: show generation statistics...`, `build: remove GGML_METAL_USE_BF16...`, `ui: update vulnerable packages...`, `npm ci` CI hygiene) — no other C++ surface touched. See CLAUDE.md's patches table ("`0009` was dropped at the b10280 bump") for the full note. | +| b10275–b10280 | upstream verification (sandbox, checkpoint) | Remaining **6** patches (`0001`,`0002`,`0003`,`0006`,`0007`,`0008`) re-verified against a clean b10280 checkout (ggml/llama.cpp commit `61881b1f7`): fail-loud `PATCH_COMMAND` clean after dropping `0009`; spot-checked `server.cpp` for `0006`/`0007` markers (`g_llama_server_embedded`, `llama_server_attach`) and `subprocess.h` for the native `SUBPROCESS_HAVE_CWD` guard, both present as expected. This checkpoint's full build is also what surfaced the `eval_llama_cmpl_schema`/`n_ctx_slot` break carried over (unbuilt) from the b10270–b10275 step above — fixed here alongside the patch drop (`jllama.cpp`'s `populate_completion_task`, `test_server.cpp`'s `parse_params` + the two renamed `*_MinusOne_Throws` tests). **Full local verification (checkpoint, patch-drop warrants it):** fresh configure + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **482/482 passing**. Per-platform build + `ctest` confirmation deferred to the CI pipeline. | +| b10280–b10285 | `tools/server/{server-task.h,server-task.cpp,server-context.cpp}` (**purely additive speculative-decoding metrics** — `n_draft_tokens_total`/`n_draft_accepted_total`/`n_draft_verif_steps_total`/`n_accepted_per_pos_total` fields + Prometheus counters), `common/fit.cpp` (internal) | **No project-source change.** New fields on `server_task_result_metrics`/`server_metrics` and new `/metrics` Prometheus series; no existing field renamed or removed, no signature changed on anything project code calls. `tools/server/server-task.h` (a directly-included header per the new review-list entry from the last step) checked explicitly — additive only. | +| b10280–b10285 | upstream verification (sandbox) | Remaining **6** patches re-verified against a clean b10285 checkout (ggml/llama.cpp commit `717dad5c`): fail-loud `PATCH_COMMAND` clean. **Configure-only verification per this step's process** (full build deferred to the next checkpoint or the b10310 target). | +| b10285–b10290 | `tools/server/server-tools.cpp` (**hardens `file_glob_search`'s directory walk, #26626**) | **No project-source change.** Internal hardening fix to the built-in-tools file-search helper; `server-tools.h` unchanged (confirmed via diff — zero lines touched), so nothing project code or `server.cpp`/`patches/0006`/`0007` reference shifted. Rest of the range is WebUI (`ChatFormWorkingDirectory.svelte`) and a Python test addition (`test_tools_builtin.py`), no other C++ surface touched. | +| b10285–b10290 | upstream verification (sandbox) | Remaining **6** patches re-verified against a clean b10290 checkout (ggml/llama.cpp commit `c8e03ce8`): fail-loud `PATCH_COMMAND` clean. **Configure-only verification per this step's process** (full build deferred to the next checkpoint or the b10310 target). | +| b10290–b10295 | `ggml/src/{ggml-cuda/ggml-cuda.cu,ggml-vulkan/ggml-vulkan.cpp}`, `src/llama-model-loader.cpp` (**GPU backend internals + model-loader tweaks**) | **No project-source change.** All changes are internal to the CUDA/Vulkan backends and the model loader; no header this project includes directly was touched (confirmed: zero diff under `common/`, `include/`, `tools/mtmd/mtmd.h`, `tools/mtmd/mtmd-helper.h`, `tools/server/*.h`, `tools/server/*.cpp`, `CMakeLists.txt`, `vendor/sheredom/subprocess.h`). Rest of the range is Python conversion scripts (`conversion/exaone.py` etc.) and upstream's own CI/test files, no C++ surface this project touches. | +| b10290–b10295 | upstream verification (sandbox) | Remaining **6** patches re-verified against a clean b10295 checkout (ggml/llama.cpp commit `3db4ff87`): fail-loud `PATCH_COMMAND` clean. **Configure-only verification per this step's process**, followed by a supplementary full local verification after 3 consecutive configure-only steps: full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **482/482 passing** — nothing surfaced across the b10280→b10295 range beyond the `n_ctx_slot` fix already applied at the b10280 checkpoint. | +| b10295–b10299 | `tools/mtmd/mtmd.h` (**purely additive: `MTMD_INPUT_CHUNK_TYPE_COUNT` enum sentinel appended, new `mtmd_input_chunk_save`/`mtmd_input_chunk_load` functions**), `tools/server/server-models.cpp` (**internal proxy fix: forward headers for bodyless statuses like 204 that `httplib` skips `response_handler` for**) | **No project-source change.** `mtmd.h`'s new enum value is appended last (does not renumber `MTMD_INPUT_CHUNK_TYPE_{TEXT,IMAGE,AUDIO}`) and the two new save/load functions are unrelated to `tts_engine.cpp`'s `mtmd_helper::gen_audio`/`mtmd_helper_bitmap_init_from_file`/`mtmd_context_params_default`/`mtmd_init_from_file`/`mtmd_gen_audio_get_info` usage — none of those touched. `server-models.cpp`'s router-proxy fix is internal to `server_http_proxy`, not called by project code. Rest of the range (2 commits) is a Python router-proxy test addition, no other C++ surface touched. | +| b10295–b10299 | upstream verification (sandbox) | Remaining **6** patches re-verified against a clean b10299 checkout (ggml/llama.cpp commit `e40bf886`): fail-loud `PATCH_COMMAND` clean. **Configure-only verification per this step's process** (full build deferred to the next checkpoint or the b10310 target). | +| b10299–b10303 | `ggml/src/{ggml-cuda/{ggml-cuda.cu,quantize.cu},ggml-sycl/{fattn-vec.hpp,ggml-sycl.cpp,set_rows.cpp}}` (**GPU backend internals**) | **No project-source change.** All changes are internal to the CUDA/SYCL backends; zero diff under `common/`, `include/`, `tools/mtmd/mtmd.h`, `tools/mtmd/mtmd-helper.h`, `tools/server/*.h`, `tools/server/*.cpp`, `CMakeLists.txt`, `vendor/sheredom/subprocess.h`. Rest of the range is a `ci/run.sh` tweak, no C++ surface this project touches. | +| b10299–b10303 | upstream verification (sandbox) | Remaining **6** patches re-verified against a clean b10303 checkout (ggml/llama.cpp commit `eef5f3e3`): fail-loud `PATCH_COMMAND` clean. **Configure-only verification per this step's process** (full build deferred to the next checkpoint or the b10310 target). | +| b10303–b10308 | `ggml/src/ggml-sycl/{common.hpp,dsv4-hc.{cpp,hpp},element_wise.cpp,ggml-sycl.cpp,lightning-indexer.{cpp,hpp},presets.hpp}` (**new SYCL kernels: DSv4 host-conjugate + lightning-indexer ops**), `docs/ops/SYCL.csv` (**auto-generated op-support matrix, dominates the diff size at 23541 of 23605 inserted lines**) | **No project-source change.** All new/changed code is internal to the SYCL backend (two new `.cpp`/`.hpp` kernel pairs, `element_wise.cpp`/`ggml-sycl.cpp` tweaks); zero diff under `common/`, `include/`, `tools/mtmd/mtmd.h`, `tools/mtmd/mtmd-helper.h`, `tools/server/*.h`, `tools/server/*.cpp`, `CMakeLists.txt`, `vendor/sheredom/subprocess.h`. Rest of the range is docs/examples-script tweaks and upstream's own `tests/test-backend-ops.cpp`/`test-model-resolution.cpp`, no C++ surface this project touches. | +| b10303–b10308 | upstream verification (sandbox, checkpoint) | Remaining **6** patches re-verified against a clean b10308 checkout (ggml/llama.cpp commit `42e98813`): fail-loud `PATCH_COMMAND` clean. **Full local verification (checkpoint, penultimate step before the b10310 target):** fresh configure + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **482/482 passing**. | +| b10308–b10310 | `ggml/src/ggml-cpu/arch/arm/cpu-feats.cpp` (**aarch64 HWCAP fallbacks + fp16 variant detection fix, #25554**), `tools/ui/**` (**router model-list modality reading, #26709**) | **No project-source change (final step of the b10217→b10310 walk).** ARM CPU-feature-detection fix internal to the ggml-cpu backend; WebUI-only TypeScript change. Zero diff under `common/`, `include/`, `tools/mtmd/mtmd.h`, `tools/mtmd/mtmd-helper.h`, `tools/server/*.h`, `tools/server/*.cpp`, `CMakeLists.txt`, `vendor/sheredom/subprocess.h` — the smallest and safest step of the entire walk (2 commits, 52 inserted / 4 removed lines). | +| b10308–b10310 | upstream verification (sandbox, final target) | Remaining **6** patches re-verified against a clean b10310 checkout (ggml/llama.cpp commit `cb26014d`): fail-loud `PATCH_COMMAND` clean. **Full local verification (mandatory for the final target):** fresh configure + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **482/482 passing**. This closes the b10217→b10310 walk. | +| b10310–b10313 | `tools/server/{server-models.h,server-models.cpp}` (**router-mode LRU scheduling queue for `models_max`, `ensure_model_ready` gains an optional `should_stop` predicate**), `tools/mtmd/mtmd-helper-gen.cpp` (**internal `mtmd_helper::gen_audio` overlay-application bugfix**) | **No project-source change.** `ensure_model_ready(const std::string&, const std::function& should_stop = nullptr)` — the new param has a default, so any caller (there are none in project source) stays source-compatible; `server_models` gains a destructor and `server_http_proxy` gains internal `cleanup_pipes` handling, both additive/internal. `patches/0008` (adds `LLAMA_SERVER_WORKER_CMD` env-var override) touches the exact function this range also touches (`server_model_meta::update_args`) — verified it still applies cleanly and its content (the `std::getenv("LLAMA_SERVER_WORKER_CMD")` block) is present in the checked-out source. `mtmd-helper-gen.cpp`'s overlay-application rewrite (per-position lookup table → a single constant `tts_pad` overlay added every frame) is entirely internal to the `gen_audio` implementation; the public API (`set_input`/`step_prompt`/`step_gen`/`get_output`) `tts_engine.cpp` calls is unchanged — likely a TTS-quality bugfix that benefits this project's rework for free. | +| b10310–b10313 | upstream verification (sandbox, checkpoint) | Remaining **6** patches re-verified against a clean b10313 checkout (ggml/llama.cpp commit `dff15d4a`): fail-loud `PATCH_COMMAND` clean, `patches/0008`'s content confirmed present in `server-models.cpp` despite the large diff to that file. **Full local verification (extra-safety checkpoint given the size/target overlap of this step's diff):** fresh configure + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **482/482 passing**. | +| b10313–b10318 | `tools/server/server-schema.cpp` (**comment typo fix only — "can still them" → "can still override them"**), `ggml/CMakeLists.txt` + `scripts/sync-ggml.last` (**cosmetic ggml-version sync**) | **No project-source change.** The only change to a file this project includes directly is a doc-comment fix with zero code impact. Rest of the range (5 commits) is WebUI-only (`tools/ui/**`), no other C++ surface touched. | +| b10313–b10318 | upstream verification (sandbox) | Remaining **6** patches re-verified against a clean b10318 checkout (ggml/llama.cpp commit `4cf5cab6`): fail-loud `PATCH_COMMAND` clean. **Configure-only verification per this step's process** (full build deferred to the b10319 final target). | +| b10318–b10319 | `tools/mtmd/{clip-model.h,clip.cpp,mtmd-image.cpp}` (**vision-encoder (clip) image-token-limit preprocessing tweak — new `clip_hparams::set_limit_image_tokens()`**) | **No project-source change (final step of this b10310→b10319 follow-up walk).** Internal to the `clip` vision-encoder preprocessing path; not included directly by any project source file and unrelated to the Qwen3-TTS pipeline (`tts_engine.cpp` uses `mtmd_helper::gen_audio`/`mtmd_helper_bitmap_init_from_file`, not `clip`/`mtmd-image.cpp` directly). Zero diff under `common/`, `include/`, `tools/mtmd/mtmd.h`, `tools/mtmd/mtmd-helper.h`, `tools/server/*.h`, `tools/server/*.cpp`, `CMakeLists.txt`, `vendor/sheredom/subprocess.h`. | +| b10318–b10319 | upstream verification (sandbox, final target) | Remaining **6** patches re-verified against a clean b10319 checkout (ggml/llama.cpp commit `4cb22cd5`): fail-loud `PATCH_COMMAND` clean. **Full local verification (mandatory for the final target):** fresh configure + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **482/482 passing**. Closes the b10310→b10319 follow-up walk (3 steps: b10310→b10313→b10318→b10319). | diff --git a/llama/CMakeLists.txt b/llama/CMakeLists.txt index 69fdff20..b5050fa2 100644 --- a/llama/CMakeLists.txt +++ b/llama/CMakeLists.txt @@ -173,7 +173,7 @@ set(LLAMA_BUILD_APP OFF CACHE BOOL "" FORCE) FetchContent_Declare( llama.cpp GIT_REPOSITORY https://github.com/ggerganov/llama.cpp.git - GIT_TAG b10217 + GIT_TAG b10319 PATCH_COMMAND ${CMAKE_COMMAND} -DPATCH_DIR=${CMAKE_CURRENT_SOURCE_DIR}/patches -DLLAMA_SRC= @@ -181,29 +181,6 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(llama.cpp) -# OuteTTS native pipeline: DERIVE the upstream tts.cpp helpers (DSP + prompt + text + the default -# speaker profile) into a compilable translation unit at configure time, rather than hand-copying -# them — a hand copy is a DRY/maintenance hazard that silently diverges on every llama.cpp upgrade. -# tts.cpp cannot simply be added to target_sources because it defines its own main(); the generator -# drops main() and gives the helpers external linkage. See cmake/generate-tts-upstream.cmake. The -# generated file is never committed; it is regenerated from whatever tts.cpp the pinned GIT_TAG -# resolves to, so a version bump is picked up automatically. The tag below is cosmetic provenance in -# the generated banner — keep it in sync with the llama.cpp GIT_TAG above. -set(JLLAMA_TTS_GEN_DIR ${CMAKE_BINARY_DIR}/tts_generated) -set(JLLAMA_TTS_GEN_CPP ${JLLAMA_TTS_GEN_DIR}/tts_upstream_gen.cpp) -file(MAKE_DIRECTORY ${JLLAMA_TTS_GEN_DIR}) -execute_process( - COMMAND ${CMAKE_COMMAND} - -DTTS_SRC=${llama.cpp_SOURCE_DIR}/tools/tts/tts.cpp - -DOUT_CPP=${JLLAMA_TTS_GEN_CPP} - -DLLAMA_TAG=b10217 - -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/generate-tts-upstream.cmake - RESULT_VARIABLE JLLAMA_TTS_GEN_RESULT -) -if(NOT JLLAMA_TTS_GEN_RESULT EQUAL 0) - message(FATAL_ERROR "OuteTTS extraction failed; see cmake/generate-tts-upstream.cmake") -endif() - # b8831 added ggml_graph_next_uid() which calls _InterlockedIncrement64 via # on x86. The intrinsic only exists on x64; provide the # implementation in a compat TU so the linker resolves __InterlockedIncrement64. @@ -382,18 +359,10 @@ add_library(jllama SHARED src/main/cpp/jllama.cpp src/main/cpp/tts_engine.cpp src/main/cpp/train_engine.cpp - ${JLLAMA_TTS_GEN_CPP} src/main/cpp/utils.hpp ${llama.cpp_SOURCE_DIR}/tools/server/server-common.cpp ${llama.cpp_SOURCE_DIR}/tools/server/server-chat.cpp) -# The generated TU keeps the whole pre-main() span of tts.cpp, so a few upstream CLI-only -# helpers (print_usage, save_wav16, xterm colour) come along unused. Silence the resulting -# unused-function warning on that one file (non-MSVC; MSVC's C4505 is off by default). -if(NOT MSVC) - set_source_files_properties(${JLLAMA_TTS_GEN_CPP} PROPERTIES COMPILE_FLAGS "-Wno-unused-function") -endif() - # Phase 1 refactoring: compile upstream server library units directly into jllama # server.hpp has been replaced by direct upstream includes in jllama.cpp. # server-context.cpp, server-queue.cpp, server-task.cpp compile on all platforms diff --git a/llama/cmake/generate-tts-upstream.cmake b/llama/cmake/generate-tts-upstream.cmake deleted file mode 100644 index 4232c462..00000000 --- a/llama/cmake/generate-tts-upstream.cmake +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Bernard Ladenthin -# -# SPDX-License-Identifier: MIT -# -# Build-time extractor for the OuteTTS native pipeline. -# -# Rather than hand-copying functions out of llama.cpp's tools/tts/tts.cpp (a maintenance -# burden that silently diverges on every upgrade), this script DERIVES a compilable -# translation unit MECHANICALLY from the pinned upstream source at configure time. The -# generated file is never committed (it lives in the build tree) and is regenerated from -# whatever tts.cpp the pinned GIT_TAG resolves to, so divergence is impossible and an -# upstream bump is picked up automatically. -# -# What it does: -# 1. Keeps everything in tts.cpp BEFORE `int main(` (includes, the outetts_version enum, -# the number-to-words tables, and the DSP + prompt + text helpers). main() itself — -# the standalone CLI entry point — is dropped (that is why tts.cpp cannot simply be -# added to target_sources: its main() would clash at link time). -# 2. Strips `static` from exactly the helpers the JNI engine calls, giving them external -# linkage so tts_engine.cpp can link against them (they are `static`/internal upstream). -# 3. Extracts the two hard-coded default-speaker literals (audio_text / audio_data), which -# upstream embeds as locals inside main(), into two external constants. -# -# Every anchor is asserted: if upstream renames a function or moves the literals, the -# configure step FAILS LOUDLY with a pointer here, the same fail-on-drift contract as -# patches/. Inputs (via -D): TTS_SRC, OUT_CPP, LLAMA_TAG. - -if(NOT EXISTS "${TTS_SRC}") - message(FATAL_ERROR "generate-tts-upstream: upstream tts.cpp not found at '${TTS_SRC}'") -endif() - -file(READ "${TTS_SRC}" SRC) - -# --- 1. keep the pre-main() portion (main() is the unbuildable-as-library CLI entry point) --- -string(FIND "${SRC}" "\nint main(" MAIN_POS) -if(MAIN_POS EQUAL -1) - message(FATAL_ERROR "generate-tts-upstream: 'int main(' anchor not found in tts.cpp — upstream layout changed; update cmake/generate-tts-upstream.cmake") -endif() -string(SUBSTRING "${SRC}" 0 ${MAIN_POS} PREMAIN) - -# --- 2. give external linkage to the helpers the JNI engine calls --- -# Each entry is asserted present as `static ` before stripping, so an upstream rename -# fails the configure instead of silently dropping the symbol (caught later only at link). -set(JLLAMA_TTS_DESTATIC - "std::vector embd_to_audio(" - "std::string process_text(" - "void prompt_add(" - "void prompt_init(" - "std::vector prepare_guide_tokens(" - "outetts_version get_tts_version(") -foreach(sig IN LISTS JLLAMA_TTS_DESTATIC) - string(FIND "${PREMAIN}" "static ${sig}" _pos) - if(_pos EQUAL -1) - message(FATAL_ERROR "generate-tts-upstream: expected 'static ${sig}' in upstream tts.cpp but it is absent — upstream changed; update the de-static list in cmake/generate-tts-upstream.cmake") - endif() - string(REPLACE "static ${sig}" "${sig}" PREMAIN "${PREMAIN}") -endforeach() - -# --- 2a. pin the outetts_version enum against the hand-written copy in tts_upstream.h --- -# src/main/cpp/tts_upstream.h re-declares `enum outetts_version { OUTETTS_V0_2, OUTETTS_V0_3 }` because -# it cannot include the generated TU. The two definitions live in different translation units and must -# stay token-identical: if upstream reorders/renames/extends the enum, the generated TU and the header -# would bind the same name to different integer values (a silent miscompile). Capture the upstream enum -# body and compare its enumerator list so a drift fails the configure with a pointer to update the header. -string(REGEX MATCH "enum[ \t\r\n]+outetts_version[ \t\r\n]*{([^}]*)}" _enum_match "${PREMAIN}") -if(_enum_match STREQUAL "") - message(FATAL_ERROR "generate-tts-upstream: 'enum outetts_version' not found in tts.cpp — upstream changed; update cmake/generate-tts-upstream.cmake and src/main/cpp/tts_upstream.h") -endif() -set(_enum_body "${CMAKE_MATCH_1}") -string(REGEX REPLACE "//[^\n]*" "" _enum_body "${_enum_body}") # strip any line comments -string(REGEX REPLACE "[ \t\r\n]+" "" _enum_body "${_enum_body}") # strip all whitespace -string(REGEX REPLACE ",+$" "" _enum_body "${_enum_body}") # strip a trailing comma -if(NOT _enum_body STREQUAL "OUTETTS_V0_2,OUTETTS_V0_3") - message(FATAL_ERROR "generate-tts-upstream: upstream 'enum outetts_version' enumerators are now '${_enum_body}' (expected 'OUTETTS_V0_2,OUTETTS_V0_3'). Update the matching enum in src/main/cpp/tts_upstream.h to keep the two definitions ODR-identical, then update this assertion in cmake/generate-tts-upstream.cmake") -endif() - -# --- 2b. verify BOTH prompt_add overloads that tts_upstream.h declares are present --- -# `void prompt_add(` is shared by three upstream overloads; the de-static REPLACE above (correctly) gives -# all of them external linkage, but the single string(FIND) only proves >=1 exists. tts_upstream.h -# declares exactly two — (llama_tokens&, const llama_tokens&) and the (vocab, txt, add_special, -# parse_special) builder — and tts_engine.cpp links against them. Pin both here (whitespace-tolerant) so -# dropping or renaming either fails the configure with a clear pointer instead of a cryptic link error. -if(NOT PREMAIN MATCHES "void[ \t]+prompt_add[ \t]*\\([^)]*const[ \t]+llama_tokens[ \t]*&[ \t]*tokens[ \t]*\\)") - message(FATAL_ERROR "generate-tts-upstream: the prompt_add(llama_tokens&, const llama_tokens&) overload declared in src/main/cpp/tts_upstream.h was not found in tts.cpp — upstream changed; update the de-static list and src/main/cpp/tts_upstream.h") -endif() -if(NOT PREMAIN MATCHES "void[ \t]+prompt_add[ \t]*\\([^)]*vocab[^)]*add_special[^)]*parse_special[^)]*\\)") - message(FATAL_ERROR "generate-tts-upstream: the prompt_add(llama_tokens&, const llama_vocab*, const std::string&, bool, bool) overload declared in src/main/cpp/tts_upstream.h was not found in tts.cpp — upstream changed; update the de-static list and src/main/cpp/tts_upstream.h") -endif() - -# --- 3. extract the two default-speaker literals from inside main() --- -# audio_text: a single-line std::string audio_text = "<|text_start|>the<|text_sep|>..."; -# The leading "<|text_start|>the<|text_sep|>" disambiguates it from the empty-seed literal -# in audio_text_from_speaker(). Content runs to the next double-quote (it embeds none). -set(_AT_DECL "std::string audio_text = \"") -string(FIND "${SRC}" "${_AT_DECL}<|text_start|>the<|text_sep|>" _at_at) -if(_at_at EQUAL -1) - message(FATAL_ERROR "generate-tts-upstream: default audio_text literal not found in tts.cpp main() — upstream changed; update cmake/generate-tts-upstream.cmake") -endif() -string(LENGTH "${_AT_DECL}" _at_decl_len) -math(EXPR _at_content "${_at_at} + ${_at_decl_len}") -string(SUBSTRING "${SRC}" ${_at_content} -1 _at_rest) -string(FIND "${_at_rest}" "\"" _at_len) -string(SUBSTRING "${_at_rest}" 0 ${_at_len} AUDIO_TEXT) - -# audio_data: a multi-line raw string std::string audio_data = R"(...)"; -# The R"( form disambiguates it from the empty-seed "..." literal in audio_data_from_speaker(). -# Content runs to the first )" (the body embeds none — only <|...|> tokens). -set(_AD_DECL "std::string audio_data = R\"(") -string(FIND "${SRC}" "${_AD_DECL}" _ad_at) -if(_ad_at EQUAL -1) - message(FATAL_ERROR "generate-tts-upstream: default audio_data raw-string literal not found in tts.cpp main() — upstream changed; update cmake/generate-tts-upstream.cmake") -endif() -string(LENGTH "${_AD_DECL}" _ad_decl_len) -math(EXPR _ad_content "${_ad_at} + ${_ad_decl_len}") -string(SUBSTRING "${SRC}" ${_ad_content} -1 _ad_rest) -string(FIND "${_ad_rest}" ")\"" _ad_len) -string(SUBSTRING "${_ad_rest}" 0 ${_ad_len} AUDIO_DATA) - -# --- 4. emit the derived translation unit --- -set(BANNER -"// AUTO-GENERATED — DO NOT EDIT, DO NOT COMMIT. -// Derived mechanically at build time by cmake/generate-tts-upstream.cmake from -// llama.cpp tools/tts/tts.cpp @ ${LLAMA_TAG} (MIT-licensed, the llama.cpp authors). -// Regenerated from the pinned upstream source on every configure; see CLAUDE.md. - -") -set(SPEAKER -" -// --- default OuteTTS speaker profile (en_male_1), extracted from upstream main() --- -// `extern const` forces external linkage (a namespace-scope `const` is internal by default), -// so tts_engine.cpp links against these via the `extern` declarations in tts_upstream.h. -extern const std::string jllama_tts_default_audio_text = \"${AUDIO_TEXT}\"; -extern const std::string jllama_tts_default_audio_data = R\"(${AUDIO_DATA})\"; -") -file(WRITE "${OUT_CPP}" "${BANNER}${PREMAIN}${SPEAKER}") -message(STATUS "generate-tts-upstream: wrote ${OUT_CPP} (from tts.cpp @ ${LLAMA_TAG})") diff --git a/llama/patches/0009-subprocess-guard-addchdir-np-old-glibc.patch b/llama/patches/0009-subprocess-guard-addchdir-np-old-glibc.patch deleted file mode 100644 index fd862be1..00000000 --- a/llama/patches/0009-subprocess-guard-addchdir-np-old-glibc.patch +++ /dev/null @@ -1,35 +0,0 @@ -diff --git a/vendor/sheredom/subprocess.h b/vendor/sheredom/subprocess.h -index 5e80902..dc0153c 100644 ---- a/vendor/sheredom/subprocess.h -+++ b/vendor/sheredom/subprocess.h -@@ -274,6 +274,21 @@ subprocess_weak int subprocess_alive(struct subprocess_s *const process); - #include - #endif - -+/* Whether subprocess_create_ex can honour process_cwd. glibc only gained -+ posix_spawn_file_actions_addchdir_np in 2.29. Define this yourself to -+ override the detection, for instance on musl older than 1.1.24. */ -+#if !defined(SUBPROCESS_HAVE_CWD) -+#if defined(__GLIBC__) -+#if __GLIBC_PREREQ(2, 29) -+#define SUBPROCESS_HAVE_CWD 1 -+#else -+#define SUBPROCESS_HAVE_CWD 0 -+#endif -+#else -+#define SUBPROCESS_HAVE_CWD 1 -+#endif -+#endif -+ - #if defined(_WIN32) - - #include -@@ -1207,6 +1222,8 @@ cleanup: - if (process_cwd) { - #if defined(__APPLE__) && MAC_OS_X_VERSION_MIN_REQUIRED >= 260000 - posix_error = posix_spawn_file_actions_addchdir(&actions, process_cwd); -+#elif !SUBPROCESS_HAVE_CWD -+ posix_error = ENOSYS; - #else - #if defined(__APPLE__) && defined(__clang__) - #pragma clang diagnostic push diff --git a/llama/src/main/cpp/jllama.cpp b/llama/src/main/cpp/jllama.cpp index e287520b..d86f1c69 100644 --- a/llama/src/main/cpp/jllama.cpp +++ b/llama/src/main/cpp/jllama.cpp @@ -243,7 +243,7 @@ static void throw_invalid_request(JNIEnv *env, const std::exception &e) { // Tokenise the prompt in `data` and fill task.tokens + task.params. // Callers must wrap this in try/catch (params_from_json_cmpl can throw). -static void populate_completion_task(server_task &task, jllama_context *jctx, int n_ctx_slot, +static void populate_completion_task(server_task &task, jllama_context *jctx, const std::vector &logit_bias_eog, const json &data, bool has_mtmd, std::vector files = {}) { if (!configure_multimodal_task_impl(task, has_mtmd, data, std::move(files))) { @@ -252,7 +252,7 @@ static void populate_completion_task(server_task &task, jllama_context *jctx, in task.tokens = std::move(tokenized_prompts[0]); } } - task.params = server_schema::eval_llama_cmpl_schema(jctx->vocab, jctx->params, n_ctx_slot, logit_bias_eog, data); + task.params = server_schema::eval_llama_cmpl_schema(jctx->vocab, jctx->params, logit_bias_eog, data); configure_task_slot_impl(task, data); } @@ -266,8 +266,7 @@ static void populate_completion_task(server_task &task, jllama_context *jctx, in try { server_task task(task_type); task.id = tid; - populate_completion_task(task, jctx, meta.slot_n_ctx, meta.logit_bias_eog, data, meta.has_mtmd, - std::move(files)); + populate_completion_task(task, jctx, meta.logit_bias_eog, data, meta.has_mtmd, std::move(files)); task.params.res_type = res_type; rd->post_task(std::move(task)); } catch (const std::exception &e) { @@ -296,8 +295,7 @@ static void populate_completion_task(server_task &task, jllama_context *jctx, in server_task task(task_type); task.id = rd.get_new_id(); try { - populate_completion_task(task, jctx, meta.slot_n_ctx, meta.logit_bias_eog, data, meta.has_mtmd, - std::move(files)); + populate_completion_task(task, jctx, meta.logit_bias_eog, data, meta.has_mtmd, std::move(files)); } catch (const std::exception &e) { throw_invalid_request(env, e); return nullptr; @@ -1750,21 +1748,22 @@ JNIEXPORT jboolean JNICALL Java_net_ladenthin_llama_LlamaModel_configureParallel } // --------------------------------------------------------------------------- -// TextToSpeech (OuteTTS) — native methods for the two-model TTS pipeline. -// Separate Java type (net.ladenthin.llama.TextToSpeech); implemented here so it -// can reuse parse_jstring / c_llama_error / c_error_oom from this TU. +// TextToSpeech (upstream Qwen3-TTS via mtmd_helper::gen_audio) — native methods for the +// backbone+mmproj TTS pipeline. Separate Java type (net.ladenthin.llama.TextToSpeech); +// implemented here so it can reuse parse_jstring / c_llama_error / c_error_oom from this TU. // --------------------------------------------------------------------------- extern "C" { -JNIEXPORT jlong JNICALL Java_net_ladenthin_llama_TextToSpeech_loadNative(JNIEnv *env, jclass clazz, jstring jttc, - jstring jcts, jint gpu_layers, jint threads) { +JNIEXPORT jlong JNICALL Java_net_ladenthin_llama_TextToSpeech_loadNative(JNIEnv *env, jclass clazz, jstring jmodel, + jstring jmmproj, jint gpu_layers, + jint threads) { (void)clazz; try { - const std::string ttc = parse_jstring(env, jttc); - const std::string cts = parse_jstring(env, jcts); + const std::string model = parse_jstring(env, jmodel); + const std::string mmproj = parse_jstring(env, jmmproj); std::string err; jllama_tts::tts_engine *engine = - jllama_tts::engine_init(ttc, cts, static_cast(gpu_layers), static_cast(threads), err); + jllama_tts::engine_init(model, mmproj, static_cast(gpu_layers), static_cast(threads), err); if (engine == nullptr) { env->ThrowNew(c_llama_error, err.c_str()); return 0; @@ -1778,8 +1777,9 @@ JNIEXPORT jlong JNICALL Java_net_ladenthin_llama_TextToSpeech_loadNative(JNIEnv JNIEXPORT jbyteArray JNICALL Java_net_ladenthin_llama_TextToSpeech_synthesizeNative(JNIEnv *env, jclass clazz, jlong handle, jstring jtext, - jint max_codes, jint top_k, - jint seed) { + jstring jspeaker_reference_path, + jstring jlang, jint max_frames, + jint top_k, jint seed) { (void)clazz; try { auto *engine = reinterpret_cast(handle); @@ -1788,10 +1788,13 @@ JNIEXPORT jbyteArray JNICALL Java_net_ladenthin_llama_TextToSpeech_synthesizeNat return nullptr; } const std::string text = parse_jstring(env, jtext); + const std::string speaker_reference_path = + jspeaker_reference_path == nullptr ? std::string() : parse_jstring(env, jspeaker_reference_path); + const std::string lang = jlang == nullptr ? std::string() : parse_jstring(env, jlang); std::vector wav; std::string err; - if (!jllama_tts::engine_synthesize(engine, text, static_cast(max_codes), static_cast(top_k), - static_cast(seed), wav, err)) { + if (!jllama_tts::engine_synthesize(engine, text, speaker_reference_path, lang, static_cast(max_frames), + static_cast(top_k), static_cast(seed), wav, err)) { env->ThrowNew(c_llama_error, err.c_str()); return nullptr; } diff --git a/llama/src/main/cpp/tts_engine.cpp b/llama/src/main/cpp/tts_engine.cpp index 5efc0643..95e7c4dd 100644 --- a/llama/src/main/cpp/tts_engine.cpp +++ b/llama/src/main/cpp/tts_engine.cpp @@ -2,56 +2,38 @@ // // SPDX-License-Identifier: MIT // -// Native OuteTTS text-to-speech pipeline, single-stream (n_parallel = 1) with the built-in default -// speaker. Loads the TTC (OuteTTS) and CTS (WavTokenizer vocoder) models, builds the OuteTTS prompt, -// generates audio codes, runs the vocoder, and turns the result into a 16-bit WAV. -// -// The OuteTTS DSP / prompt / text helpers and the default-speaker profile are NOT reimplemented here: -// they are derived at build time from the pinned upstream llama.cpp tools/tts/tts.cpp (see -// tts_upstream.h + cmake/generate-tts-upstream.cmake) and called directly. This file is only the -// orchestration that drives those helpers across the two models. The in-memory WAV writer is ours -// (tts_wav.hpp). +// See tts_engine.h for the design summary. #include "tts_engine.h" -#include "tts_upstream.h" // embd_to_audio, process_text, prompt_*, prepare_guide_tokens, get_tts_version, default speaker -#include "tts_wav.hpp" // pcm_to_wav16_bytes +#include "tts_wav.hpp" // pcm_to_wav16_bytes #include "common.h" #include "llama.h" +#include "mtmd-helper.h" +#include "mtmd.h" #include "sampling.h" -// Full json definition: tts_upstream.h only forward-declares nlohmann::ordered_json (keeping the heavy -// header out of the shared interface), but this TU constructs the empty-object speaker argument for -// get_tts_version(), which needs the complete type. -#include - -#include -#include #include -#include #include #include namespace jllama_tts { struct tts_engine { - common_init_result_ptr init_ttc; - common_init_result_ptr init_cts; - llama_model *model_ttc = nullptr; - llama_context *ctx_ttc = nullptr; - llama_model *model_cts = nullptr; - llama_context *ctx_cts = nullptr; - const llama_vocab *vocab = nullptr; - outetts_version tts_version = OUTETTS_V0_2; + common_init_result_ptr init; + llama_model *model = nullptr; + llama_context *ctx = nullptr; + mtmd::context_ptr mctx; int n_threads = 4; - // Serializes engine_synthesize: it drives llama_decode/llama_encode on the shared - // ctx_ttc/ctx_cts contexts, so two threads on one engine would race. + int n_batch = 2048; + // Serializes engine_synthesize: it drives llama_decode (via mtmd_helper::gen_audio) on the + // shared ctx/mctx, so two threads on one engine would race. std::mutex synthesize_mutex; }; -tts_engine *engine_init(const std::string &ttc_model_path, const std::string &cts_model_path, int n_gpu_layers, - int n_threads, std::string &err) { +tts_engine *engine_init(const std::string &model_path, const std::string &mmproj_path, int n_gpu_layers, int n_threads, + std::string &err) { llama_backend_init(); auto engine = new tts_engine(); @@ -59,36 +41,33 @@ tts_engine *engine_init(const std::string &ttc_model_path, const std::string &ct common_params params; params.n_ctx = 8192; - params.n_batch = 8192; + params.n_batch = engine->n_batch; params.n_gpu_layers = n_gpu_layers; params.cpuparams.n_threads = engine->n_threads; - params.sampling.top_k = 4; - params.sampling.samplers = {COMMON_SAMPLER_TYPE_TOP_K}; - - // Text-to-codes (TTC) model. - params.model.path = ttc_model_path; - engine->init_ttc = common_init_from_params(params); - engine->model_ttc = engine->init_ttc ? engine->init_ttc->model() : nullptr; - engine->ctx_ttc = engine->init_ttc ? engine->init_ttc->context() : nullptr; - if (engine->model_ttc == nullptr || engine->ctx_ttc == nullptr) { - err = "failed to load TTC (text-to-codes) model: " + ttc_model_path; + params.model.path = model_path; + // Always enable embd so the backbone's hidden state can be handed to the audio-generation + // helper between frames (mirrors upstream tools/tts/tts.cpp main()). + params.embedding = true; + + engine->init = common_init_from_params(params); + engine->model = engine->init ? engine->init->model() : nullptr; + engine->ctx = engine->init ? engine->init->context() : nullptr; + if (engine->model == nullptr || engine->ctx == nullptr) { + err = "failed to load TTS backbone model: " + model_path; engine_free(engine); return nullptr; } - engine->vocab = llama_model_get_vocab(engine->model_ttc); - // Explicit empty-object speaker: tts_upstream.h declares no default (it forward-declares json), so - // the default lives only in the generated TU. We always use the built-in default speaker profile. - engine->tts_version = get_tts_version(engine->model_ttc, nlohmann::ordered_json::object()); - // Codes-to-speech (CTS) vocoder, loaded in embedding mode. - params.model.path = cts_model_path; - params.embedding = true; - params.n_ubatch = params.n_batch; - engine->init_cts = common_init_from_params(params); - engine->model_cts = engine->init_cts ? engine->init_cts->model() : nullptr; - engine->ctx_cts = engine->init_cts ? engine->init_cts->context() : nullptr; - if (engine->model_cts == nullptr || engine->ctx_cts == nullptr) { - err = "failed to load CTS (vocoder) model: " + cts_model_path; + mtmd_context_params mtmd_params = mtmd_context_params_default(); + mtmd_params.use_gpu = n_gpu_layers != 0; + engine->mctx.reset(mtmd_init_from_file(mmproj_path.c_str(), engine->model, mtmd_params)); + if (!engine->mctx) { + err = "failed to load TTS mmproj: " + mmproj_path; + engine_free(engine); + return nullptr; + } + if (mtmd_gen_audio_get_info(engine->mctx.get()).type == MTMD_GEN_AUDIO_TYPE_NONE) { + err = "mmproj does not support audio generation: " + mmproj_path; engine_free(engine); return nullptr; } @@ -96,141 +75,110 @@ tts_engine *engine_init(const std::string &ttc_model_path, const std::string &ct return engine; } -bool engine_synthesize(tts_engine *engine, const std::string &text, int n_predict, int top_k, uint32_t seed, - std::vector &out_wav, std::string &err) { +bool engine_synthesize(tts_engine *engine, const std::string &text, const std::string &speaker_reference_path, + const std::string &lang, int n_predict, int top_k, uint32_t seed, std::vector &out_wav, + std::string &err) { if (engine == nullptr) { err = "engine is null"; return false; } // Serialize against concurrent calls on the same engine. std::lock_guard engine_lock(engine->synthesize_mutex); - const llama_vocab *vocab = engine->vocab; common_params_sampling sparams; sparams.top_k = top_k > 0 ? top_k : 4; sparams.seed = seed; sparams.samplers = {COMMON_SAMPLER_TYPE_TOP_K}; - common_sampler *smpl = common_sampler_init(engine->model_ttc, sparams); + common_sampler *smpl = common_sampler_init(engine->model, sparams); if (smpl == nullptr) { err = "failed to init sampler"; return false; } - // Build the OuteTTS prompt: speaker text + processed input + speaker audio codes. - std::string audio_text = jllama_tts_default_audio_text; - std::string audio_data = jllama_tts_default_audio_data; - if (engine->tts_version == OUTETTS_V0_3) { - audio_text = std::regex_replace(audio_text, std::regex(R"(<\|text_sep\|>)"), "<|space|>"); - audio_data = std::regex_replace(audio_data, std::regex(R"(<\|code_start\|>)"), ""); - audio_data = std::regex_replace(audio_data, std::regex(R"(<\|code_end\|>)"), "<|space|>"); + mtmd::bitmap_ptr speaker_bitmap; + if (!speaker_reference_path.empty()) { + auto wrapper = mtmd_helper_bitmap_init_from_file(engine->mctx.get(), speaker_reference_path.c_str(), false); + if (!wrapper.bitmap) { + common_sampler_free(smpl); + err = "failed to load speaker reference audio: " + speaker_reference_path; + return false; + } + speaker_bitmap.reset(wrapper.bitmap); } - llama_tokens prompt_inp; - prompt_init(prompt_inp, vocab); - prompt_add(prompt_inp, vocab, audio_text, false, true); - - std::string prompt_clean = process_text(text, engine->tts_version); - std::vector guide_tokens = prepare_guide_tokens(vocab, prompt_clean, engine->tts_version); - prompt_add(prompt_inp, vocab, prompt_clean, false, true); - prompt_add(prompt_inp, vocab, "<|text_end|>\n", false, true); - prompt_add(prompt_inp, vocab, audio_data, false, true); - - std::vector codes; - - // Decode the prompt (logits for the last token only). - llama_batch batch = llama_batch_init((int32_t)prompt_inp.size(), 0, 1); - for (size_t i = 0; i < prompt_inp.size(); ++i) { - common_batch_add(batch, prompt_inp[i], (llama_pos)i, {0}, false); - } - batch.logits[batch.n_tokens - 1] = true; - if (llama_decode(engine->ctx_ttc, batch) != 0) { - llama_batch_free(batch); + mtmd_helper::gen_audio gen(engine->ctx, engine->mctx.get()); + mtmd_helper_gen_audio_inp inp{}; + inp.seq_id = 0; + inp.prompt = text.c_str(); + inp.prompt_len = text.size(); + inp.speaker_ref = speaker_bitmap.get(); + inp.lang = lang.empty() ? "english" : lang.c_str(); + inp.top_k = top_k > 0 ? top_k : 50; + inp.top_p = 1.0f; + // We do our own WAV framing (pcm_to_wav16_bytes) rather than trust upstream's own writer, so + // that already-tested code stays in the loop; ask for raw PCM. + inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM; + + if (gen.set_input(&inp) != 0) { common_sampler_free(smpl); - err = "llama_decode failed on the TTS prompt"; + err = "failed to set TTS input (prompt/speaker-reference/lang rejected)"; return false; } - llama_synchronize(engine->ctx_ttc); - - // Generate audio codec tokens. - int n_past = batch.n_tokens; - int n_decode = 0; - int i_batch = batch.n_tokens - 1; - bool next_token_uses_guide_token = true; - const int predict = n_predict > 0 ? n_predict : 4096; - - while (n_decode <= predict) { - common_batch_clear(batch); - - llama_token new_token_id = common_sampler_sample(smpl, engine->ctx_ttc, i_batch); - if (!guide_tokens.empty() && next_token_uses_guide_token && !llama_vocab_is_control(vocab, new_token_id) && - !llama_vocab_is_eog(vocab, new_token_id)) { - new_token_id = guide_tokens.front(); - guide_tokens.erase(guide_tokens.begin()); - } - next_token_uses_guide_token = (new_token_id == 198); - - common_sampler_accept(smpl, new_token_id, true); - codes.push_back(new_token_id); - if (llama_vocab_is_eog(vocab, new_token_id) || n_decode == predict) { + for (;;) { + int32_t remaining = gen.step_prompt(engine->n_batch); + if (remaining < 0) { + common_sampler_free(smpl); + err = "prompt processing failed"; + return false; + } + if (remaining == 0) { break; } + } - i_batch = batch.n_tokens; - common_batch_add(batch, new_token_id, n_past, {0}, true); - - n_decode += 1; - n_past += 1; - if (llama_decode(engine->ctx_ttc, batch) != 0) { - llama_batch_free(batch); + const llama_vocab *vocab = llama_model_get_vocab(engine->model); + auto sample_semantic_code = [&]() -> llama_token { + llama_token t = common_sampler_sample(smpl, engine->ctx, -1); + common_sampler_accept(smpl, t, true); + return t; + }; + + const int max_new = n_predict > 0 ? n_predict : 512; + int n_frames = 0; + llama_token sampled = sample_semantic_code(); + const float *h_state = llama_get_embeddings_ith(engine->ctx, -1); + + for (; n_frames < max_new && !llama_vocab_is_eog(vocab, sampled); n_frames++) { + const float *h_next = nullptr; + if (gen.step_gen(sampled, h_state, &h_next) != 0) { common_sampler_free(smpl); - err = "llama_decode failed during code generation"; + err = "audio-frame generation failed at frame " + std::to_string(n_frames); return false; } + h_state = h_next; + sampled = sample_semantic_code(); } - llama_batch_free(batch); common_sampler_free(smpl); - // Keep only audio codec tokens and rebase to 0-based codec ids. The window is identical - // for OuteTTS V0_2 and V0_3 (verified against upstream tools/tts/tts.cpp) — delegated to - // the pure filter_outetts_codec_tokens helper. - filter_outetts_codec_tokens(codes); - if (codes.empty()) { - err = "no audio codes were generated"; + int32_t sample_rate = 0; + const char *data = nullptr; + size_t data_len = 0; + if (gen.get_output(&sample_rate, &data, &data_len) != 0) { + err = "failed to read generated audio output"; return false; } - - // Run the vocoder over the codes and read the output embeddings. - const int n_codes = (int)codes.size(); - llama_batch cts_batch = llama_batch_init(n_codes, 0, 1); - for (size_t i = 0; i < codes.size(); ++i) { - common_batch_add(cts_batch, codes[i], (llama_pos)i, {0}, true); - } - if (llama_encode(engine->ctx_cts, cts_batch) != 0) { - llama_batch_free(cts_batch); - err = "llama_encode (vocoder) failed"; + if (n_frames == 0 || data_len == 0) { + err = "no audio was generated"; return false; } - llama_synchronize(engine->ctx_cts); - - // llama_model_n_embd_out (not llama_model_n_embd): read the vocoder's OUTPUT embedding width, which - // is what llama_get_embeddings returns here. This matches upstream tts.cpp, which also queries - // llama_model_n_embd_out at this step. - const int n_embd = llama_model_n_embd_out(engine->model_cts); - const float *embd = llama_get_embeddings(engine->ctx_cts); - std::vector audio = embd_to_audio(embd, n_codes, n_embd, engine->n_threads); - llama_batch_free(cts_batch); - - // 24 kHz mono — the OuteTTS / WavTokenizer output rate. - const int n_sr = 24000; - // Zero the first 0.25 s, mirroring upstream tts.cpp's post-vocoder cleanup (it suppresses a leading - // click). The `&& i < audio.size()` guard is ours: it keeps the loop in-bounds for clips shorter - // than 0.25 s, where upstream's fixed 24000/4 bound would read past the buffer. - for (int i = 0; i < n_sr / 4 && i < (int)audio.size(); ++i) { - audio[i] = 0.0f; - } - out_wav = pcm_to_wav16_bytes(audio, n_sr); + // out_type=PCM: raw F32LE samples. + const auto *samples = reinterpret_cast(data); + const size_t n_samples = data_len / sizeof(float); + std::vector audio(samples, samples + n_samples); + + out_wav = pcm_to_wav16_bytes(audio, sample_rate); return true; } @@ -238,7 +186,8 @@ void engine_free(tts_engine *engine) { if (engine == nullptr) { return; } - // init_ttc / init_cts own the models + contexts and free them on destruction. + // init owns the model + context and frees them on destruction; mctx (mtmd::context_ptr) frees + // the mmproj context in its own destructor. delete engine; } diff --git a/llama/src/main/cpp/tts_engine.h b/llama/src/main/cpp/tts_engine.h index b17644ef..4bb10430 100644 --- a/llama/src/main/cpp/tts_engine.h +++ b/llama/src/main/cpp/tts_engine.h @@ -2,56 +2,51 @@ // // SPDX-License-Identifier: MIT // -// Native text-to-speech engine: a self-contained orchestration of the two-model OuteTTS pipeline -// (TTC OuteTTS LLM -> audio codec tokens; CTS WavTokenizer vocoder -> embeddings -> embd_to_audio -> -// 16-bit WAV), single-stream (n_parallel = 1). Kept out of jllama.cpp so the JNI layer stays thin. +// Native text-to-speech engine: a thin orchestration over llama.cpp's upstream Qwen3-TTS +// audio-generation pipeline (tools/mtmd/mtmd-helper.h's mtmd_helper::gen_audio), single-stream +// (n_parallel = 1). Kept out of jllama.cpp so the JNI layer stays thin. +// +// Loads a backbone text model plus an mmproj (speaker encoder + code predictor + code2wav +// decoder, all in one GGUF) and drives mtmd_helper::gen_audio's streaming API: set_input() -> +// step_prompt() -> a step_gen() loop (we own the semantic-token sampling, same pattern upstream's +// own tools/tts/tts.cpp main() uses) -> get_output(). Upstream owns the DSP/prompt-building/code2wav +// internals entirely — there is nothing left here to extract or hand-copy from llama.cpp source, +// unlike the OuteTTS pipeline this replaced (see docs/history/llama-cpp-breaking-changes.md for the +// b10269->b10270 upstream architecture replacement this followed). The in-memory WAV writer is ours +// (tts_wav.hpp): the engine asks upstream for raw PCM and encodes it locally, keeping that already +// -tested code in the loop rather than trusting upstream's own WAV framing. #ifndef JLLAMA_TTS_ENGINE_H #define JLLAMA_TTS_ENGINE_H -#include #include #include #include namespace jllama_tts { -// Opaque handle owning both loaded models / contexts. Created by engine_init, freed by engine_free. +// Opaque handle owning the loaded backbone model/context and the mmproj mtmd context. Created by +// engine_init, freed by engine_free. struct tts_engine; -// Load the TTC (OuteTTS text-to-codes) and CTS (codes-to-speech vocoder) models. Returns nullptr and -// sets `err` on failure. -tts_engine *engine_init(const std::string &ttc_model_path, const std::string &cts_model_path, int n_gpu_layers, - int n_threads, std::string &err); - -// Synthesize `text` to a 24 kHz mono 16-bit WAV byte stream in `out_wav`. Returns false and sets `err` -// on failure. Thread-compatible but not re-entrant on the same engine instance. -bool engine_synthesize(tts_engine *engine, const std::string &text, int n_predict, int top_k, uint32_t seed, - std::vector &out_wav, std::string &err); - -// Release both models / contexts. Safe on nullptr. +// Load the backbone (text) model and the mmproj (speaker encoder + code predictor + code2wav) +// GGUF. Returns nullptr and sets `err` on failure, including when the mmproj does not support +// audio generation (e.g. a vision/audio-input mmproj passed by mistake). +tts_engine *engine_init(const std::string &model_path, const std::string &mmproj_path, int n_gpu_layers, int n_threads, + std::string &err); + +// Synthesize `text` to a 24 kHz mono 16-bit WAV byte stream in `out_wav`. `speaker_reference_path` +// may be empty (uses the model's default voice); `lang` selects the codec_language special token +// (e.g. "english", "chinese" — see upstream tools/tts/README.md for the supported set) and may be +// empty (defaults to "english"). Returns false and sets `err` on failure. Thread-compatible but not +// re-entrant on the same engine instance. +bool engine_synthesize(tts_engine *engine, const std::string &text, const std::string &speaker_reference_path, + const std::string &lang, int n_predict, int top_k, uint32_t seed, std::vector &out_wav, + std::string &err); + +// Release the loaded model/context. Safe on nullptr. void engine_free(tts_engine *engine); -// --------------------------------------------------------------------------- -// OuteTTS audio-codec token filtering (pure transform, testable without a model). -// -// The TTC model emits codec tokens in a high id window; the CTS vocoder only -// understands the 0-based codec ids inside [k_outetts_codec_lo, k_outetts_codec_hi]. -// This window is identical for OuteTTS V0_2 and V0_3 (verified against upstream -// tools/tts/tts.cpp), so it is NOT gated on tts_version. Keep only tokens in the -// window and rebase them to 0-based ids. -constexpr int32_t k_outetts_codec_lo = 151672; -constexpr int32_t k_outetts_codec_hi = 155772; - -inline void filter_outetts_codec_tokens(std::vector &tokens) { - tokens.erase(std::remove_if(tokens.begin(), tokens.end(), - [](int32_t t) { return t < k_outetts_codec_lo || t > k_outetts_codec_hi; }), - tokens.end()); - for (auto &t : tokens) { - t -= k_outetts_codec_lo; - } -} - } // namespace jllama_tts #endif // JLLAMA_TTS_ENGINE_H diff --git a/llama/src/main/cpp/tts_upstream.h b/llama/src/main/cpp/tts_upstream.h deleted file mode 100644 index 0ef82e79..00000000 --- a/llama/src/main/cpp/tts_upstream.h +++ /dev/null @@ -1,62 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Bernard Ladenthin -// -// SPDX-License-Identifier: MIT -// -// Interface to the OuteTTS helpers that the JNI engine links against. The definitions are NOT -// here and NOT hand-copied: cmake/generate-tts-upstream.cmake derives them at build time from the -// pinned llama.cpp tools/tts/tts.cpp (which declares them `static`), giving them external linkage. -// This header only declares the call surface — interface facts, not the upstream implementation — -// so tts_engine.cpp can call them. If upstream changes a signature, the generator's de-static -// assert fails the configure; if it changes a type, this header stops matching and the link fails. - -#ifndef JLLAMA_TTS_UPSTREAM_H -#define JLLAMA_TTS_UPSTREAM_H - -#include -#include - -// Forward declarations only. This shared interface header names nlohmann::ordered_json once (the -// get_tts_version() speaker parameter) but never instantiates it, so it must not pull the full -// ~25k-line into every translation unit that includes it. The single caller that -// constructs the empty-object default (tts_engine.cpp) includes the full itself. -#include - -#include "common.h" // llama_tokens -#include "llama.h" // llama_model, llama_vocab, llama_token - -// Mirrors the upstream enum (identical definition; ODR-compatible across translation units). The -// generated TU carries upstream's own copy, so these enumerators and their order MUST stay -// token-identical to upstream — otherwise the two definitions assign different integer values to the -// same name (a silent miscompile). cmake/generate-tts-upstream.cmake asserts the upstream enum still -// reads `{ OUTETTS_V0_2, OUTETTS_V0_3 }` at configure time and fails loud (pointing here) if a -// llama.cpp bump changes it. -enum outetts_version { OUTETTS_V0_2, OUTETTS_V0_3 }; - -// --- derived from upstream tts.cpp (defined in the generated translation unit) --- - -// Spectral synthesis: codec embeddings -> float PCM. The number-to-words / prompt formatting that -// the upstream CLI applies is preserved (process_text calls replace_numbers_with_words), so digits -// are spoken rather than dropped. -std::vector embd_to_audio(const float *embd, const int n_codes, const int n_embd, const int n_thread); - -std::string process_text(const std::string &text, outetts_version tts_version); - -void prompt_add(llama_tokens &prompt, const llama_tokens &tokens); -void prompt_add(llama_tokens &prompt, const llama_vocab *vocab, const std::string &txt, bool add_special, - bool parse_special); -void prompt_init(llama_tokens &prompt, const llama_vocab *vocab); - -std::vector prepare_guide_tokens(const llama_vocab *vocab, const std::string &str, - outetts_version tts_version); - -// No default argument here on purpose: constructing nlohmann::ordered_json::object() needs the full -// json definition, which this header deliberately does not include (see the json_fwd note above). The -// sole caller (tts_engine.cpp) passes an explicit empty object; the generated TU keeps upstream's own -// default, so its internal calls are unaffected. -outetts_version get_tts_version(llama_model *model, nlohmann::ordered_json speaker); - -// Default OuteTTS speaker profile, extracted from upstream main() into the generated TU. -extern const std::string jllama_tts_default_audio_text; -extern const std::string jllama_tts_default_audio_data; - -#endif // JLLAMA_TTS_UPSTREAM_H diff --git a/llama/src/main/java/net/ladenthin/llama/TextToSpeech.java b/llama/src/main/java/net/ladenthin/llama/TextToSpeech.java index d789bb67..7c95542a 100644 --- a/llama/src/main/java/net/ladenthin/llama/TextToSpeech.java +++ b/llama/src/main/java/net/ladenthin/llama/TextToSpeech.java @@ -5,26 +5,24 @@ package net.ladenthin.llama; import net.ladenthin.llama.loader.LlamaLoader; +import org.jspecify.annotations.Nullable; /** - * Text-to-speech synthesis over llama.cpp's OuteTTS pipeline. Loads two models — a text-to-codes - * (OuteTTS) model and a codes-to-speech (WavTokenizer) vocoder — and turns text into a 24 kHz - * mono 16-bit WAV byte stream. + * Text-to-speech synthesis over llama.cpp's upstream Qwen3-TTS audio-generation pipeline. Loads a + * backbone text model plus an mmproj GGUF (speaker encoder + code predictor + code2wav decoder, + * all bundled in one file) and turns text into a 24 kHz mono 16-bit WAV byte stream. * - *

This is a separate native type from {@link LlamaModel} because TTS is a two-model pipeline that - * does not use the chat/completion server path. Native memory is not GC-managed: use + *

This is a separate native type from {@link LlamaModel} because TTS uses its own model pair + * and does not go through the chat/completion server path. Native memory is not GC-managed: use * try-with-resources or call {@link #close()} explicitly. * * 

{@code
  * try (TextToSpeech tts = new TextToSpeech(
- *         "models/OuteTTS-0.2-500M.gguf", "models/WavTokenizer.gguf")) {
+ *         "models/qwen3-tts-backbone.gguf", "models/qwen3-tts-mmproj.gguf")) {
  *     byte[] wav = tts.synthesize("Hello from llama dot c p p.");
  *     Files.write(Paths.get("out.wav"), wav);
  * }
  * }
- * - *

Synthesis uses the built-in default speaker profile. English number words are expanded for - * speech (e.g. {@code 3} is spoken as "three"); non-English text is not romanized. */ public final class TextToSpeech implements AutoCloseable { @@ -37,49 +35,74 @@ public final class TextToSpeech implements AutoCloseable { /** * Load the TTS pipeline CPU-only. * - * @param ttcModelPath path to the text-to-codes (OuteTTS) GGUF - * @param vocoderModelPath path to the codes-to-speech (WavTokenizer) vocoder GGUF + * @param modelPath path to the backbone (text) GGUF + * @param mmprojPath path to the mmproj GGUF (speaker encoder + code predictor + code2wav) */ - public TextToSpeech(String ttcModelPath, String vocoderModelPath) { - this(ttcModelPath, vocoderModelPath, 0, 0); + public TextToSpeech(String modelPath, String mmprojPath) { + this(modelPath, mmprojPath, 0, 0); } /** * Load the TTS pipeline. * - * @param ttcModelPath path to the text-to-codes (OuteTTS) GGUF - * @param vocoderModelPath path to the codes-to-speech (WavTokenizer) vocoder GGUF + * @param modelPath path to the backbone (text) GGUF + * @param mmprojPath path to the mmproj GGUF (speaker encoder + code predictor + code2wav) * @param gpuLayers number of layers to offload to the GPU (0 = CPU only) - * @param threads CPU threads for the spectral DSP (0 = a small default) + * @param threads CPU threads for the backbone (0 = a small default) */ - public TextToSpeech(String ttcModelPath, String vocoderModelPath, int gpuLayers, int threads) { - this.handle = loadNative(ttcModelPath, vocoderModelPath, gpuLayers, threads); + public TextToSpeech(String modelPath, String mmprojPath, int gpuLayers, int threads) { + this.handle = loadNative(modelPath, mmprojPath, gpuLayers, threads); } /** - * Synthesize speech with default sampling (top-k 4, seed 0, up to 4096 code tokens). + * Synthesize speech with the model's default voice and default sampling (top-k 4, seed 0, up + * to 512 audio frames). * * @param text the text to speak * @return a 24 kHz mono 16-bit WAV byte stream */ public byte[] synthesize(String text) { - return synthesize(text, 4096, 4, 0); + return synthesize(text, 512, 4, 0); + } + + /** + * Synthesize speech with the model's default voice and explicit sampling parameters. + * + * @param text the text to speak + * @param maxFrames cap on generated audio frames (longer = longer audio) + * @param topK top-k sampling cutoff for the semantic (backbone) token stream + * @param seed sampler seed + * @return a 24 kHz mono 16-bit WAV byte stream + */ + public byte[] synthesize(String text, int maxFrames, int topK, int seed) { + return synthesize(text, null, null, maxFrames, topK, seed); } /** - * Synthesize speech with explicit sampling parameters. + * Synthesize speech with an optional cloned voice and language. * * @param text the text to speak - * @param maxCodeTokens cap on generated audio-code tokens (longer = longer audio) - * @param topK top-k sampling cutoff for the code model + * @param speakerReferenceAudioPath path to a reference audio clip (wav/mp3) whose voice is + * cloned, or {@code null}/empty for the model's default voice + * @param language ISO 639-1-ish language name understood by the model (e.g. {@code "english"}, + * {@code "chinese"} — see the model's own documentation for the supported set), or + * {@code null}/empty for the model's default + * @param maxFrames cap on generated audio frames (longer = longer audio) + * @param topK top-k sampling cutoff for the semantic (backbone) token stream * @param seed sampler seed * @return a 24 kHz mono 16-bit WAV byte stream */ - public byte[] synthesize(String text, int maxCodeTokens, int topK, int seed) { + public byte[] synthesize( + String text, + @Nullable String speakerReferenceAudioPath, + @Nullable String language, + int maxFrames, + int topK, + int seed) { if (handle == 0L) { throw new IllegalStateException("TextToSpeech is closed"); } - return synthesizeNative(handle, text, maxCodeTokens, topK, seed); + return synthesizeNative(handle, text, speakerReferenceAudioPath, language, maxFrames, topK, seed); } @Override @@ -90,9 +113,16 @@ public synchronized void close() { } } - private static native long loadNative(String ttcModelPath, String vocoderModelPath, int gpuLayers, int threads); + private static native long loadNative(String modelPath, String mmprojPath, int gpuLayers, int threads); - private static native byte[] synthesizeNative(long handle, String text, int maxCodeTokens, int topK, int seed); + private static native byte[] synthesizeNative( + long handle, + String text, + @Nullable String speakerReferenceAudioPath, + @Nullable String language, + int maxFrames, + int topK, + int seed); private static native void deleteNative(long handle); } diff --git a/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java b/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java index d1fa8db6..ff841256 100644 --- a/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java +++ b/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java @@ -10,13 +10,13 @@ * library was compiled against, exposed as a compile-time constant so callers can render a badge or * emit a startup log line without loading the native library. * - *

{@link #LLAMA_CPP_VERSION} is a pure-Java string ({@code "b10217"}) that mirrors the + *

{@link #LLAMA_CPP_VERSION} is a pure-Java string ({@code "b10319"}) that mirrors the * {@code GIT_TAG} in {@code llama/CMakeLists.txt}. It is available even when {@code libjllama} is * absent (pure-Java checkout, before {@code System.load}), which is what makes it suitable for a * lightweight version badge in Android or other UIs.

* *

For the authoritative value that is baked into the native binary — the build number - * plus the resolved upstream commit, e.g. {@code "b10217-"} — call + * plus the resolved upstream commit, e.g. {@code "b10319-"} — call * {@link net.ladenthin.llama.LlamaModel#getLlamaCppBuildInfo()} instead; that reads llama.cpp's own * {@code build-info} through JNI and therefore cannot drift from the compiled library (but requires * the native library to be loaded).

@@ -24,14 +24,14 @@ public final class LlamaCppVersion { /** - * The pinned llama.cpp release tag this library was built against, e.g. {@code "b10217"}. + * The pinned llama.cpp release tag this library was built against, e.g. {@code "b10319"}. * *

Kept in lockstep with {@code GIT_TAG} in {@code llama/CMakeLists.txt} — see the * "Upgrading/Downgrading llama.cpp Version" checklist in {@code CLAUDE.md}. This is the * compile-time pin; use {@link net.ladenthin.llama.LlamaModel#getLlamaCppBuildInfo()} for the * value actually linked into the native binary.

*/ - public static final String LLAMA_CPP_VERSION = "b10217"; + public static final String LLAMA_CPP_VERSION = "b10319"; // Constants holder — not instantiable. private LlamaCppVersion() {} diff --git a/llama/src/test/cpp/test_server.cpp b/llama/src/test/cpp/test_server.cpp index 404c7add..6cd99ab0 100644 --- a/llama/src/test/cpp/test_server.cpp +++ b/llama/src/test/cpp/test_server.cpp @@ -1773,20 +1773,19 @@ TEST(CmplFinalChatStream, IncludeUsageTrue_TrailingChunkHasEmptyChoicesAndUsage) // Called with nullptr vocab when the JSON does not exercise // grammar/preserved_tokens tokenisation. Tests verify: // - simple field round-trip (temperature, seed, n_predict) -// - repeat_last_n=-1 is expanded to n_ctx_slot -// - dry_penalty_last_n=-1 is expanded to n_ctx_slot +// - repeat_last_n/dry_penalty_last_n reject negative values (b10275 dropped the -1=ctx-size +// sentinel; the hard lower limit moved from -1 to 0) // - dry_base < 1.0 is reset to default // - n_discard negative throws std::invalid_argument (b9739: range-checked, no longer clamped) // - empty dry_sequence_breakers throws std::invalid_argument // - lora field not an array throws std::invalid_argument -// - repeat_last_n < -1 throws std::invalid_argument // ============================================================ namespace { -task_params parse_params(const json &data, int n_ctx = 512) { +task_params parse_params(const json &data) { common_params params_base; std::vector no_bias; - return server_schema::eval_llama_cmpl_schema(nullptr, params_base, n_ctx, no_bias, data); + return server_schema::eval_llama_cmpl_schema(nullptr, params_base, no_bias, data); } } // namespace @@ -1817,14 +1816,15 @@ TEST(ParamsFromJsonCmpl, SsePingInterval_Absent_InheritsServerSetting) { EXPECT_EQ(parse_params({}).sse_ping_interval, defaults.sse_ping_interval); } -TEST(ParamsFromJsonCmpl, RepeatLastN_MinusOne_ExpandsToNCtxSlot) { - const auto p = parse_params({{"repeat_last_n", -1}}, /*n_ctx=*/256); - EXPECT_EQ(p.sampling.penalty_last_n, 256); +// b10275: the -1=ctx-size sentinel was dropped from repeat_last_n; the hard lower limit is now 0, +// so a request-supplied -1 is out of range and throws instead of expanding to the slot context size. +TEST(ParamsFromJsonCmpl, RepeatLastN_MinusOne_Throws) { + EXPECT_THROW(parse_params({{"repeat_last_n", -1}}), std::invalid_argument); } -TEST(ParamsFromJsonCmpl, DryPenaltyLastN_MinusOne_ExpandsToNCtxSlot) { - const auto p = parse_params({{"dry_penalty_last_n", -1}}, /*n_ctx=*/128); - EXPECT_EQ(p.sampling.dry_penalty_last_n, 128); +// b10275: same sentinel removal for dry_penalty_last_n. +TEST(ParamsFromJsonCmpl, DryPenaltyLastN_MinusOne_Throws) { + EXPECT_THROW(parse_params({{"dry_penalty_last_n", -1}}), std::invalid_argument); } TEST(ParamsFromJsonCmpl, DryBase_BelowOne_ResetToDefault) { @@ -1868,7 +1868,7 @@ TEST(ParamsFromJsonCmpl, DryAllowedLength_RoundTrip) { } TEST(ParamsFromJsonCmpl, DryPenaltyLastN_Positive_RoundTrip) { - // a positive value is kept verbatim (only -1 expands to n_ctx_slot, covered above) + // a positive value is kept verbatim (negative values throw, covered above) const auto p = parse_params({{"dry_penalty_last_n", 64}}); EXPECT_EQ(p.sampling.dry_penalty_last_n, 64); } @@ -1887,7 +1887,7 @@ TEST(ParamsFromJsonCmpl, LoraNotArray_Throws) { EXPECT_THROW(parse_params({{"lora", "not-an-array"}}), std::invalid_argument); } -TEST(ParamsFromJsonCmpl, RepeatLastN_BelowMinusOne_Throws) { +TEST(ParamsFromJsonCmpl, RepeatLastN_BelowZero_Throws) { EXPECT_THROW(parse_params({{"repeat_last_n", -2}}), std::invalid_argument); } diff --git a/llama/src/test/cpp/test_tts_wav.cpp b/llama/src/test/cpp/test_tts_wav.cpp index 75cb68e3..efb5d9a7 100644 --- a/llama/src/test/cpp/test_tts_wav.cpp +++ b/llama/src/test/cpp/test_tts_wav.cpp @@ -3,8 +3,8 @@ // SPDX-License-Identifier: MIT // // Unit tests for the in-memory WAV writer (src/main/cpp/tts_wav.hpp) — our own code, not upstream. -// The OuteTTS DSP it pairs with (embd_to_audio etc.) is derived from upstream tts.cpp at build time -// and exercised end-to-end by the Java TtsIntegrationTest, not unit-tested here. +// The Qwen3-TTS pipeline it pairs with (mtmd_helper::gen_audio) is entirely upstream-owned (no +// project-side DSP to unit-test here) and exercised end-to-end by the Java TtsIntegrationTest. #include "tts_wav.hpp" @@ -12,8 +12,6 @@ #include #include -#include "tts_engine.h" - using namespace jllama_tts; namespace { @@ -53,34 +51,3 @@ TEST(TtsWav, ClampsAndEncodesSamplesLittleEndian) { EXPECT_EQ(sample(2), -32767); EXPECT_EQ(sample(3), -32768); } - -// ============================================================ -// filter_outetts_codec_tokens (OuteTTS V0_2 / V0_3 codec window) -// ============================================================ - -// The OuteTTS codec window is identical for V0_2 and V0_3 (verified against upstream -// tools/tts/tts.cpp). In-range tokens — including both window boundaries — are kept -// and rebased to 0-based codec ids. -TEST(OuteTtsCodecFilter, InRangeTokensAreRebased) { - std::vector codes = {k_outetts_codec_lo, k_outetts_codec_lo + 1, k_outetts_codec_hi}; - filter_outetts_codec_tokens(codes); - ASSERT_EQ(codes.size(), 3u); - EXPECT_EQ(codes[0], 0); - EXPECT_EQ(codes[1], 1); - EXPECT_EQ(codes[2], k_outetts_codec_hi - k_outetts_codec_lo); -} - -// Tokens just below / just above the window (e.g. text, control, or speaker tokens) -// are dropped — the off-by-one boundary cases on both ends. -TEST(OuteTtsCodecFilter, OutOfRangeTokensAreDropped) { - std::vector codes = {k_outetts_codec_lo - 1, 198 /*newline*/, k_outetts_codec_hi + 1, k_outetts_codec_lo}; - filter_outetts_codec_tokens(codes); - ASSERT_EQ(codes.size(), 1u); - EXPECT_EQ(codes[0], 0); -} - -TEST(OuteTtsCodecFilter, EmptyInput_StaysEmpty) { - std::vector codes; - filter_outetts_codec_tokens(codes); - EXPECT_TRUE(codes.empty()); -} diff --git a/llama/src/test/java/net/ladenthin/llama/TestConstants.java b/llama/src/test/java/net/ladenthin/llama/TestConstants.java index 261b396f..753f7eef 100644 --- a/llama/src/test/java/net/ladenthin/llama/TestConstants.java +++ b/llama/src/test/java/net/ladenthin/llama/TestConstants.java @@ -99,11 +99,14 @@ public class TestConstants { public static final String DEFAULT_AUDIO_INPUT_PATH = "src/test/resources/audios/sample.wav"; /** - * System property holding a path to the text-to-codes (OuteTTS) GGUF used by - * {@code TtsIntegrationTest}. The test self-skips when this or the vocoder is unset/missing. + * System property holding a path to the Qwen3-TTS backbone GGUF used by + * {@code TtsIntegrationTest}. The test self-skips when this or the mmproj is unset/missing. */ - public static final String PROP_TTS_TTC_MODEL = LlamaSystemProperties.PREFIX + ".tts.ttc.model"; + public static final String PROP_TTS_MODEL = LlamaSystemProperties.PREFIX + ".tts.model"; - /** System property holding a path to the codes-to-speech (WavTokenizer) vocoder GGUF. */ - public static final String PROP_TTS_VOCODER_MODEL = LlamaSystemProperties.PREFIX + ".tts.vocoder.model"; + /** + * System property holding a path to the Qwen3-TTS mmproj GGUF (speaker encoder + code + * predictor + code2wav decoder). + */ + public static final String PROP_TTS_MMPROJ = LlamaSystemProperties.PREFIX + ".tts.mmproj"; } diff --git a/llama/src/test/java/net/ladenthin/llama/TtsIntegrationTest.java b/llama/src/test/java/net/ladenthin/llama/TtsIntegrationTest.java index 2516bbd8..4288f41f 100644 --- a/llama/src/test/java/net/ladenthin/llama/TtsIntegrationTest.java +++ b/llama/src/test/java/net/ladenthin/llama/TtsIntegrationTest.java @@ -19,13 +19,13 @@ import org.junit.jupiter.api.Timeout; /** - * Real-model coverage for {@link TextToSpeech} (OuteTTS audio output, llama.cpp {@code llama-tts} - * pipeline). Loads the two-model TTS pipeline and synthesizes a short clip, checking the WAV - * container is well-formed. + * Real-model coverage for {@link TextToSpeech} (Qwen3-TTS audio output, llama.cpp's upstream + * {@code mtmd_helper::gen_audio} pipeline). Loads the backbone+mmproj TTS pipeline and synthesizes + * a short clip, checking the WAV container is well-formed. * - *

Self-skips when {@link TestConstants#PROP_TTS_TTC_MODEL} or - * {@link TestConstants#PROP_TTS_VOCODER_MODEL} is unset or its file is missing, so it runs only where - * the (large) OuteTTS + WavTokenizer GGUFs have been staged. + *

Self-skips when {@link TestConstants#PROP_TTS_MODEL} or {@link TestConstants#PROP_TTS_MMPROJ} + * is unset or its file is missing, so it runs only where the (large) Qwen3-TTS backbone + mmproj + * GGUFs have been staged. */ public class TtsIntegrationTest { @@ -36,18 +36,18 @@ public class TtsIntegrationTest { @DisplayName("synthesize() returns a well-formed, non-silent 24 kHz mono 16-bit WAV") @Timeout(value = 300_000, unit = TimeUnit.MILLISECONDS) public void synthesizesWellFormedWav() { - String ttc = System.getProperty(TestConstants.PROP_TTS_TTC_MODEL); - String vocoder = System.getProperty(TestConstants.PROP_TTS_VOCODER_MODEL); + String model = System.getProperty(TestConstants.PROP_TTS_MODEL); + String mmproj = System.getProperty(TestConstants.PROP_TTS_MMPROJ); Assumptions.assumeTrue( - ttc != null && !ttc.isEmpty(), "TTS model not set (-D" + TestConstants.PROP_TTS_TTC_MODEL + "=...)"); + model != null && !model.isEmpty(), "TTS model not set (-D" + TestConstants.PROP_TTS_MODEL + "=...)"); Assumptions.assumeTrue( - vocoder != null && !vocoder.isEmpty(), - "TTS vocoder not set (-D" + TestConstants.PROP_TTS_VOCODER_MODEL + "=...)"); - Assumptions.assumeTrue(new File(ttc).exists(), "TTS model file missing: " + ttc); - Assumptions.assumeTrue(new File(vocoder).exists(), "TTS vocoder file missing: " + vocoder); + mmproj != null && !mmproj.isEmpty(), + "TTS mmproj not set (-D" + TestConstants.PROP_TTS_MMPROJ + "=...)"); + Assumptions.assumeTrue(new File(model).exists(), "TTS model file missing: " + model); + Assumptions.assumeTrue(new File(mmproj).exists(), "TTS mmproj file missing: " + mmproj); int gpuLayers = Integer.getInteger(TestConstants.PROP_TEST_NGL, 0); - try (TextToSpeech tts = new TextToSpeech(ttc, vocoder, gpuLayers, 0)) { + try (TextToSpeech tts = new TextToSpeech(model, mmproj, gpuLayers, 0)) { byte[] wav = tts.synthesize("hello from llama"); assertNotNull(wav, "WAV bytes must not be null");