diff --git a/CLAUDE.md b/CLAUDE.md index 4c57fed0..1637c551 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Java bindings for [llama.cpp](https://github.com/ggerganov/llama.cpp) via JNI, providing a high-level API for LLM inference in Java. The Java layer communicates with a native C++ library through JNI. -Current llama.cpp pinned version: **b10360** +Current llama.cpp pinned version: **b10423** ## Upgrading CUDA Version @@ -429,7 +429,7 @@ needs no extra step here, `build-webui` re-reads the tag and rebuilds the matchi ships no UI): ```bash # needs node/npm + network; embed.cpp is plain C++17 (no npm) -git clone --depth 1 --branch b10360 https://github.com/ggml-org/llama.cpp /tmp/lc +git clone --depth 1 --branch b10423 https://github.com/ggml-org/llama.cpp /tmp/lc ( cd /tmp/lc/tools/ui && npm ci && npm run build \ && ( cd dist && find . -type f -not -path './_gzip/*' \ | while read -r f; do mkdir -p "_gzip/$(dirname "$f")"; gzip -9 -c "$f" > "_gzip/$f"; done ) \ @@ -469,7 +469,7 @@ cache lives in **Depot Cache** over sccache's **WebDAV** backend: - `SCCACHE_WEBDAV_TOKEN: ${{ secrets.DEPOT_TOKEN }}` — a Depot **organization** token, stored as the repo secret **`DEPOT_TOKEN`**. -Because `sccache` is **content-addressed** and llama.cpp is pinned (`GIT_TAG b10360`), the +Because `sccache` is **content-addressed** and llama.cpp is pinned (`GIT_TAG b10423`), the ~280 upstream object files are byte-identical every run, so a warm cache recompiles only the *changed* files. Depot's cache is **shared across all branches** (unlike GitHub's per-branch `actions/cache`), so every branch builds incrementally; a `b` version bump @@ -1286,17 +1286,17 @@ ctest --test-dir build --output-on-failure -R "ResultsToJson" | File | Tests | Scope | |------|-------|-------| | `src/test/cpp/test_utils.cpp` | 162 | Upstream helpers: `server_tokens`, `server_grammar_trigger`, `gen_tool_call_id`, `json_value`, `json_get_nested_values`, UTF-8 helpers, `format_response_rerank`, `format_embeddings_response_oaicompat`, `oaicompat_completion_params_parse`, `oaicompat_chat_params_parse`, `are_lora_equal`, `strip_flag_from_argv`, `token_piece_value`, `json_is_array_and_contains_numbers`, `format_oai_sse`, `format_oai_resp_sse`, `format_anthropic_sse`, `parse_lora_request` | -| `src/test/cpp/test_server.cpp` | 201 | Upstream result types: `result_timings`, `task_params::to_json()` (incl. `dry_sequence_breakers`, `preserved_tokens`, `timings_per_token`), `completion_token_output`, `server_task_result_cmpl_partial` (non-oaicompat + `to_json_oaicompat` + logprobs + `to_json_oaicompat_chat` + `to_json_anthropic` + dispatcher), `server_task_result_cmpl_final` (non-oaicompat + `to_json_oaicompat` + `to_json_oaicompat_chat` + `to_json_oaicompat_chat_stream` + `to_json_anthropic` + `to_json_anthropic_stream` + tool_calls + dispatcher), `server_task_result_embd`, `server_task_result_rerank`, `server_task_result_metrics`, `server_task_result_slot_save_load`, `server_task_result_slot_erase`, `server_task_result_apply_lora`, `server_task_result_get_lora`, `server_task_result_error`, `format_error_response`, `server_task::need_sampling()`, `server_task::n_tokens()`, `server_schema::eval_llama_cmpl_schema()` (parsing pipeline + grammar routing + error paths + per-request `dry_*` and `sse_ping_interval` field round-trips incl. hard-limit + server-default inheritance), `response_fields` projection | +| `src/test/cpp/test_server.cpp` | 204 | Upstream result types: `server_slot_stats` (the `timings` JSON payload; replaced `result_timings` in b10408), `task_params::to_json()` (incl. `dry_sequence_breakers`, `preserved_tokens`, `timings_per_token`), `completion_token_output`, `server_task_result_cmpl_partial` (non-oaicompat + `to_json_oaicompat` + logprobs + `to_json_oaicompat_chat` + `to_json_anthropic` + dispatcher), `server_task_result_cmpl_final` (non-oaicompat + `to_json_oaicompat` + `to_json_oaicompat_chat` + `to_json_oaicompat_chat_stream` + `to_json_anthropic` + `to_json_anthropic_stream` + tool_calls + dispatcher), `server_task_result_embd`, `server_task_result_rerank`, `server_task_result_metrics` (`to_json()` = the `/slots` array + `to_metrics()` = the `/metrics` Prometheus text), `server_task_result_slot_save_load`, `server_task_result_slot_erase`, `server_task_result_apply_lora`, `server_task_result_get_lora`, `server_task_result_error`, `format_error_response`, `server_task::need_sampling()`, `server_task::n_tokens()`, `server_schema::eval_llama_cmpl_schema()` (parsing pipeline + grammar routing + error paths + per-request `dry_*` and `sse_ping_interval` field round-trips incl. hard-limit + server-default inheritance), `response_fields` projection | | `src/test/cpp/test_json_helpers.cpp` | 50 | All functions in `json_helpers.hpp`: `get_result_error_message`, `results_to_json`, `rerank_results_to_json` (incl. missing/out-of-range `index` rejection), `parse_encoding_format`, `extract_embedding_prompt`, `is_infill_request`, `parse_slot_prompt_similarity`, `parse_positive_int_config`, `wrap_stream_chunk` | | `src/test/cpp/test_log_helpers.cpp` | 13 | All functions in `log_helpers.hpp`: `log_level_name`, `format_log_as_json` | | `src/test/cpp/test_jni_helpers.cpp` | 54 | All functions in `jni_helpers.hpp` using a zero-filled `JNINativeInterface_` mock (incl. the `utf8_to_jstring_impl` byte-array string path: emoji byte-preservation, truncated-UTF-8 replace-not-throw) | | `src/test/cpp/test_tts_wav.cpp` | 2 | The in-memory WAV writer `pcm_to_wav16_bytes` in `tts_wav.hpp` (WAV header/payload + little-endian clamping) — our own code, not upstream. The Qwen3-TTS pipeline it pairs with (`mtmd_helper::gen_audio`) is entirely upstream-owned (no project-side DSP to unit-test here) and covered end-to-end by the Java `TtsIntegrationTest`. | -**Current total: 482 tests (all passing).** +**Current total: 485 tests (all passing).** #### Upstream source location (in CMake build tree) -llama.cpp is fetched via CMake FetchContent, pinned to `GIT_TAG b10360`. +llama.cpp is fetched via CMake FetchContent, pinned to `GIT_TAG b10423`. **GoogleTest** is a separate `BUILD_TESTING`-only FetchContent (`GIT_TAG v1.17.0`), used solely by the `jllama_test` C++ unit-test binary — not by the shipped library, and not coupled to the diff --git a/README.md b/README.md index 32428f4c..f4edbee3 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ **Build:** ![Java 8+](https://img.shields.io/badge/Java-8%2B-informational) ![Platform](https://img.shields.io/badge/Platform-Linux%20%7C%20macOS%20%7C%20Windows%20%7C%20Android-lightgrey) -[![llama.cpp b10360](https://img.shields.io/badge/llama.cpp-%23b10360-informational)](https://github.com/ggml-org/llama.cpp/releases/tag/b10360) +[![llama.cpp b10423](https://img.shields.io/badge/llama.cpp-%23b10423-informational)](https://github.com/ggml-org/llama.cpp/releases/tag/b10423) [![JPMS](https://img.shields.io/badge/JPMS-modular%20JAR-25A162)](https://openjdk.org/projects/jigsaw/) ![JUnit](https://img.shields.io/badge/tested%20with-JUnit6-25A162) [![JSpecify](https://img.shields.io/badge/JSpecify-1.0.0%20%40NullMarked-25A162)](https://jspecify.dev) diff --git a/docs/history/llama-cpp-breaking-changes.md b/docs/history/llama-cpp-breaking-changes.md index 4fa8ac39..482f0b12 100644 --- a/docs/history/llama-cpp-breaking-changes.md +++ b/docs/history/llama-cpp-breaking-changes.md @@ -599,3 +599,14 @@ Used during `llama.cpp` version bumps: when upgrading, scan this file from the r | b10355–b10359 | `include/llama.h` (**two comment lines removed only, `llama_sampler_copy` signature unchanged**), `ggml/src/ggml-opencl/**` (**flash-attn kernel work — compiled but no header API called directly**), `ggml/src/ggml-webgpu/**` (**unused backend**) | **No project-source change (final step of the b10333→b10359 walk).** Rest of the range is upstream CI workflow files, no C++ surface this project touches. | | b10355–b10359 | upstream verification (sandbox, final target) | All **6** patches re-verified against a clean b10359 checkout (ggml/llama.cpp commit `84f712946`): fail-loud `PATCH_COMMAND` clean. **Full local verification (mandatory for the final target):** fresh configure + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **482/482 passing**. Closes the b10333→b10359 walk (5 steps: b10333→b10338→b10342→b10344→b10353→b10354→b10355→b10359), on branch `claude/java-llama-cpp-b10359`. | | b10359–b10360 | `common/peg-parser.cpp` (**suppress incomplete escape sequences in the JSON PEG grammar, #26780**) | **No project-source change (single-step, final target).** Internal bugfix; `common/peg-parser.h` itself has zero diff (also on the "safe to skip" list). All **6** patches re-verified against a clean b10360 checkout (ggml/llama.cpp commit `48d22e295`): fail-loud `PATCH_COMMAND` clean. **Full local verification (mandatory for the final target):** fresh configure + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **482/482 passing**. Same branch as the b10333→b10359 walk (`claude/java-llama-cpp-b10359`). | +| b10360–b10362 | `src/models/exaone4.cpp` (**SWA not enabled for EXAONE 4.5, #26848**), `tests/test-backend-sampler.cpp` + `ci/run.sh` (**upstream CI only**) | **No project-source change.** Model-internal fix; zero diff under `common/`, `include/`, `tools/mtmd/`, `tools/server/`, `CMakeLists.txt`. First step of the b10360→b10423 walk. | +| b10362–b10369 | `tools/mtmd/{mtmd.h,mtmd-helper.h,mtmd-helper-gen.cpp,clip*,models/pockettts-*}` (**pocket-tts support, #26871**), `include/llama.h` + `common/common.h` (**new `LLAMA_LOAD_MODE_AUTO`, now the `common_params::load_mode` default, #26081**) | **Project-source change: `src/main/cpp/tts_engine.cpp`.** `mtmd_helper_gen_audio_step_gen()` gained a `bool * out_stop` output parameter and now reports end-of-speech itself (also signalled by a null `h_state_out`); `mtmd_helper_gen_audio_inp` gained a `seed` field. The C++ wrapper `mtmd_helper::gen_audio::step_gen()` defaults `out_stop` to `nullptr`, so the old 3-arg call still *compiled* — the break was **semantic**, not a compile error: upstream's own `tools/tts/tts.cpp` dropped its `llama_vocab_is_eog(vocab, sampled)` loop guard in the same commit because pocket-tts has no discrete backbone token to test. `tts_engine.cpp` now mirrors upstream: loop on `out_stop` / null `h_next` instead of the EOG token (for Qwen3-TTS this is equivalent — upstream sets `out_stop` for exactly the `codec_eos`/EOG tokens the old check tested, verified in `mtmd-helper-gen.cpp`), and forwards the caller-supplied seed to `inp.seed` as well as to the backbone `common_sampler`. `mtmd.h` additions (`MTMD_GEN_AUDIO_TYPE_POCKETTTS`, `mtmd_gen_inp_default()`, `feats`/`is_eos`/`temp` fields) are appended, not renumbered. | +| b10369–b10375 | `common/chat.cpp` (**Qwen bare-function tool-call parsing, #26793**), `src/models/{dflash,nemotron-h}.cpp`, `tools/imatrix/imatrix.cpp`, conversion scripts | **No project-source change.** `common/chat.h` itself has zero diff; the parser change is internal to an upstream-compiled TU. | +| b10375–b10380 | `common/speculative.h` (**removes `common_speculative_need_embd()` / `common_speculative_need_embd_nextn()`, #26904**), `common/chat.cpp` (**muse-glimmer tool calls after EOM, #26879**), `ggml/src/ggml-cuda/**` | **No project-source change.** Confirmed via grep that no project source calls any `common_speculative_*` function, so the two removed declarations are confined to upstream translation units (`server-context.cpp` and `examples/speculative-simple` are their only callers). | +| b10380–b10398 | **`CMakeLists.txt` + `common/CMakeLists.txt` + `src/CMakeLists.txt` + `tools/mtmd/CMakeLists.txt` (semantic versioning, #26839 — `LLAMA_INSTALL_VERSION` replaced by `LLAMA_VERSION_BASE`/`LLAMA_VERSION_MAJOR`)**, `tools/server/{server-common.h,server-common.cpp,server-context.cpp}` (**slot save/restore with media, #26640 — `server_tokens` loses `has_media()`, gains `serialize()`/`deserialize()`**), `common/{arg.cpp,common.h}` (**system/user `config.ini`, #26118 — new `fs_get_config_directory()`**), `common/build-info.{h,cpp.in}` (**`llama_print_build_info()` takes a version string**), `include/llama.h` (**additive `llama_version()`**) | **Project-source change: `llama/CMakeLists.txt` (build-system break).** This project adds `tools/mtmd` via `add_subdirectory()` from *its own* directory scope, where llama.cpp's variables are not visible, and `tools/mtmd/CMakeLists.txt` passes them to `set_target_properties(mtmd PROPERTIES VERSION … SOVERSION …)` — an undefined variable makes that argument list odd-length and aborts the configure. The pre-existing `LLAMA_INSTALL_VERSION` fallback (added for exactly this reason) no longer covers it, so the guard now defines `LLAMA_VERSION_BASE`/`LLAMA_VERSION_MAJOR` instead. Both stay neutral placeholders: `BUILD_SHARED_LIBS` is OFF, so `mtmd` is a static library and VERSION/SOVERSION are no-ops. The other API changes need no project change — `jllama.cpp` calls only `llama_build_info()` (unchanged), not `llama_print_build_info()`, and no project source calls `server_tokens::has_media()`. | +| b10398–b10405 | `ggml/src/ggml-cpu/**` (**arm unused-variable fix**), `ggml/src/ggml-hip/CMakeLists.txt` (**drops `-funsafe-math-optimizations`**), `tools/ui/**` (**WebUI refactors**) | **No project-source change.** The WebUI is auto-followed by CI's `build-webui` job from the pinned tag, so its refactors need no per-bump review. | +| b10405–b10408 | **`tools/server/{server-task.h,server-task.cpp,server-common.h,server-common.cpp,server-context.cpp}` (metrics refactor + correctness fixes, #26920)** | **Project-source change: `src/test/cpp/test_server.cpp`.** `result_timings` was **removed** and replaced by `server_slot_stats` (raw counters + absolute microsecond timestamps, with every duration/rate derived in accessors); `server_task_result_cmpl_{partial,final}::timings` became `::stats`, and the `"timings"` JSON block is now gated on `stats.is_set()` (`t_start > 0`) instead of a non-negative `prompt_n`. The emitted JSON keys are unchanged, so the **client-facing contract and the Java layer are unaffected** — no `src/main/cpp` file references any of these types (verified via grep). `server_task_result_metrics` lost its flat counter fields to an embedded `server_metrics`, its `to_json()` now returns the `/slots` array verbatim, and the new `to_metrics()` renders the `/metrics` Prometheus exposition text. The C++ regression suite was rewritten to match: `ResultTimings` → `ServerSlotStats` (same JSON assertions, plus `is_set()` and unstarted-generation coverage), and the metrics suite now parses `to_metrics()` samples (including that the per-draft-position labeled series appears only when populated). Suite total 482 → **485**. | +| b10408–b10418 | `common/speculative.{h,cpp}` + `common/arg.cpp` (**additive `common_speculative_types_from_gguf()` for draft-type auto-detection, #26814**), `tools/server/server-http.cpp` (**`index.html` served `no-cache`, hashed assets stay `immutable`, #27006**), `ggml/src/ggml-{sycl,metal,cpu}/**` | **No project-source change.** The speculative addition is a new free function, no existing signature touched; the `server-http.cpp` caching change is internal to upstream's static-asset routes (and inert here unless the WebUI assets are compiled in). | +| b10418–b10419 | `ggml/src/ggml-openvino/**` (**Qwen3.5 support, memory optimization, new ops, #26952**) | **No project-source change.** A single commit entirely inside the OpenVINO backend; no shared header touched, so only the `openvino-{linux,windows}-x86-64` classifier's compiled sources change. | +| b10419–b10423 | `common/{common.h,common.cpp}` + `tools/completion/completion.cpp` (**shared CPU-parameter handling, #27026 — new `common_threadpools` RAII helper, `ggml_threadpool_params_from_cpu_params()` moved next to it**), `tools/ui/**` | **No project-source change (final step).** `ggml_threadpool_params_from_cpu_params()` moved within `common.h` but kept its signature; confirmed via grep that no project source mentions `threadpool` at all. | +| b10360–b10423 | upstream verification (sandbox, final target) | All **6** patches re-verified against a clean b10423 checkout: sequential `git apply` (filename order, mirroring the `PATCH_COMMAND` applier) succeeded at **every** intermediate tag of the walk, and the fail-loud `PATCH_COMMAND` ran clean on a fresh `cmake -B build` at the target — no patch needed refreshing across the whole range. **Full local verification (mandatory for the final target):** fresh configure + full `cmake --build` (`libjllama.so` + `jllama_test` compile and link, `-O3`, no undefined references) + `ctest` **485/485 passing**. Walk of 10 steps (b10360→b10362→b10369→b10375→b10380→b10398→b10405→b10408→b10418→b10419→b10423) on branch `claude/update-b10423-ciih0h`. Four steps exceeded the 100 KiB chunking threshold unavoidably — upstream has **no** intermediate tags there (b10363–b10368, b10381–b10397, b10406–b10407 and b10420–b10422 do not exist), so the tag list itself, not the threshold, bounded the step size. | diff --git a/llama/CMakeLists.txt b/llama/CMakeLists.txt index 8ecef4fe..d7425360 100644 --- a/llama/CMakeLists.txt +++ b/llama/CMakeLists.txt @@ -173,7 +173,7 @@ set(LLAMA_BUILD_APP OFF CACHE BOOL "" FORCE) FetchContent_Declare( llama.cpp GIT_REPOSITORY https://github.com/ggerganov/llama.cpp.git - GIT_TAG b10360 + GIT_TAG b10423 PATCH_COMMAND ${CMAKE_COMMAND} -DPATCH_DIR=${CMAKE_CURRENT_SOURCE_DIR}/patches -DLLAMA_SRC= @@ -193,12 +193,20 @@ endif() # LLAMA_BUILD_TOOLS defaults to LLAMA_STANDALONE, which is OFF when llama.cpp # is consumed via FetchContent. Build mtmd explicitly so the target exists. if(NOT TARGET mtmd) - # LLAMA_INSTALL_VERSION is set inside llama.cpp's directory scope and is not - # visible here. tools/mtmd/CMakeLists.txt uses it in set_target_properties() - # as a VERSION value; if the variable is empty the token list after PROPERTIES - # becomes odd-length and CMake aborts with "incorrect number of arguments". - if(NOT DEFINED LLAMA_INSTALL_VERSION) - set(LLAMA_INSTALL_VERSION "0") + # LLAMA_VERSION_BASE / LLAMA_VERSION_MAJOR are set inside llama.cpp's directory + # scope and are not visible here. tools/mtmd/CMakeLists.txt uses them in + # set_target_properties() as VERSION/SOVERSION values; if a variable is empty the + # token list after PROPERTIES becomes odd-length and CMake aborts with "incorrect + # number of arguments". Both are cosmetic for this build (BUILD_SHARED_LIBS is + # OFF, so mtmd is a static library and VERSION/SOVERSION are no-ops), hence the + # neutral placeholder rather than a mirror of upstream's numbers. + # (b10398 replaced the older LLAMA_INSTALL_VERSION with these two — upstream + # #26839, "cmake: introduce semantic versioning".) + if(NOT DEFINED LLAMA_VERSION_BASE) + set(LLAMA_VERSION_BASE "0") + endif() + if(NOT DEFINED LLAMA_VERSION_MAJOR) + set(LLAMA_VERSION_MAJOR "0") endif() add_subdirectory(${llama.cpp_SOURCE_DIR}/tools/mtmd ${llama.cpp_BINARY_DIR}/tools/mtmd) endif() diff --git a/llama/src/main/cpp/tts_engine.cpp b/llama/src/main/cpp/tts_engine.cpp index 95e7c4dd..67bd20ba 100644 --- a/llama/src/main/cpp/tts_engine.cpp +++ b/llama/src/main/cpp/tts_engine.cpp @@ -115,6 +115,7 @@ bool engine_synthesize(tts_engine *engine, const std::string &text, const std::s inp.lang = lang.empty() ? "english" : lang.c_str(); inp.top_k = top_k > 0 ? top_k : 50; inp.top_p = 1.0f; + inp.seed = seed; // We do our own WAV framing (pcm_to_wav16_bytes) rather than trust upstream's own writer, so // that already-tested code stays in the loop; ask for raw PCM. inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM; @@ -137,7 +138,7 @@ bool engine_synthesize(tts_engine *engine, const std::string &text, const std::s } } - const llama_vocab *vocab = llama_model_get_vocab(engine->model); + // note: some pipelines ignore this token and drive generation from the hidden state instead auto sample_semantic_code = [&]() -> llama_token { llama_token t = common_sampler_sample(smpl, engine->ctx, -1); common_sampler_accept(smpl, t, true); @@ -149,13 +150,21 @@ bool engine_synthesize(tts_engine *engine, const std::string &text, const std::s llama_token sampled = sample_semantic_code(); const float *h_state = llama_get_embeddings_ith(engine->ctx, -1); - for (; n_frames < max_new && !llama_vocab_is_eog(vocab, sampled); n_frames++) { + // End-of-speech is reported by step_gen() itself (out_stop / a null next hidden state) rather + // than by an end-of-generation backbone token, so that pipelines without a discrete backbone + // token (pocket-tts) terminate too. Mirrors upstream tools/tts/tts.cpp. + bool stop = false; + while (!stop && n_frames < max_new) { const float *h_next = nullptr; - if (gen.step_gen(sampled, h_state, &h_next) != 0) { + if (gen.step_gen(sampled, h_state, &h_next, &stop) != 0) { common_sampler_free(smpl); err = "audio-frame generation failed at frame " + std::to_string(n_frames); return false; } + if (h_next == nullptr) { + break; // stopped without generating a frame + } + n_frames++; h_state = h_next; sampled = sample_semantic_code(); } diff --git a/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java b/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java index f231da46..5722caf2 100644 --- a/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java +++ b/llama/src/main/java/net/ladenthin/llama/value/LlamaCppVersion.java @@ -10,13 +10,13 @@ * library was compiled against, exposed as a compile-time constant so callers can render a badge or * emit a startup log line without loading the native library. * - *

{@link #LLAMA_CPP_VERSION} is a pure-Java string ({@code "b10360"}) that mirrors the + *

{@link #LLAMA_CPP_VERSION} is a pure-Java string ({@code "b10423"}) that mirrors the * {@code GIT_TAG} in {@code llama/CMakeLists.txt}. It is available even when {@code libjllama} is * absent (pure-Java checkout, before {@code System.load}), which is what makes it suitable for a * lightweight version badge in Android or other UIs.

* *

For the authoritative value that is baked into the native binary — the build number - * plus the resolved upstream commit, e.g. {@code "b10360-"} — call + * plus the resolved upstream commit, e.g. {@code "b10423-"} — call * {@link net.ladenthin.llama.LlamaModel#getLlamaCppBuildInfo()} instead; that reads llama.cpp's own * {@code build-info} through JNI and therefore cannot drift from the compiled library (but requires * the native library to be loaded).

@@ -24,14 +24,14 @@ public final class LlamaCppVersion { /** - * The pinned llama.cpp release tag this library was built against, e.g. {@code "b10360"}. + * The pinned llama.cpp release tag this library was built against, e.g. {@code "b10423"}. * *

Kept in lockstep with {@code GIT_TAG} in {@code llama/CMakeLists.txt} — see the * "Upgrading/Downgrading llama.cpp Version" checklist in {@code CLAUDE.md}. This is the * compile-time pin; use {@link net.ladenthin.llama.LlamaModel#getLlamaCppBuildInfo()} for the * value actually linked into the native binary.

*/ - public static final String LLAMA_CPP_VERSION = "b10360"; + public static final String LLAMA_CPP_VERSION = "b10423"; // Constants holder — not instantiable. private LlamaCppVersion() {} diff --git a/llama/src/test/cpp/test_server.cpp b/llama/src/test/cpp/test_server.cpp index 6cd99ab0..5b8b9a11 100644 --- a/llama/src/test/cpp/test_server.cpp +++ b/llama/src/test/cpp/test_server.cpp @@ -8,7 +8,7 @@ // the Java integration tests run. // // Covered: -// - result_timings::to_json() — draft_n/draft_n_accepted conditional fields +// - server_slot_stats::to_json() — draft_n/draft_n_accepted conditional fields // - task_params::to_json() — grammar, chat_parser_params, grammar_triggers // - completion_token_output — logarithm edge-case, str_to_bytes, to_json, probs_vector_to_json // - server_task_result_rerank — score / index / tokens_evaluated @@ -29,31 +29,42 @@ #include "utils.hpp" // ============================================================ -// result_timings::to_json -// New fields draft_n / draft_n_accepted added in b8576. -// They must be absent when draft_n == 0 (default) and present -// only when draft_n > 0 (i.e. speculative decoding was active). +// server_slot_stats::to_json +// b10408 (upstream #26920) replaced result_timings — which carried +// pre-computed durations and rates — with server_slot_stats, which stores +// raw counters plus absolute timestamps and derives every duration/rate in +// its accessors. The emitted JSON keys are unchanged, so the client-facing +// "timings" contract is the same; only how the numbers are produced moved. +// +// Fields draft_n / draft_n_accepted must be absent when n_draft_tokens == 0 +// (default) and present only when it is > 0 (speculative decoding active). // ============================================================ namespace { -result_timings make_base_timings() { - result_timings t; - t.prompt_n = 10; - t.prompt_ms = 200.0; - t.prompt_per_token_ms = 20.0; - t.prompt_per_second = 50.0; - t.predicted_n = 5; - t.predicted_ms = 100.0; - t.predicted_per_token_ms = 20.0; - t.predicted_per_second = 50.0; - return t; +// t_start/t_prompt_last/t_gen_last are absolute microsecond timestamps; the +// values below are chosen so the derived numbers are exact: +// t_prompt_ms = (1'200'000 - 1'000'000) / 1000 = 200.0 +// per-token = 200.0 / n_prompt_processed(10) = 20.0 +// prompt tps = 1e3 / 200.0 * 10 = 50.0 +// t_gen_ms = (1'300'000 - 1'200'000) / 1000 = 100.0 +// gen steps = n_gen(5) - 1 (first token is free) = 4 +// per-token = 100.0 / 4 = 25.0 +// gen tps = 1e3 / 100.0 * 4 = 40.0 +server_slot_stats make_base_stats() { + server_slot_stats s; + s.n_prompt_processed = 10; + s.n_gen = 5; + s.t_start = 1000000; + s.t_prompt_last = 1200000; + s.t_gen_last = 1300000; + return s; } } // namespace -TEST(ResultTimings, BaseFields_AlwaysPresent) { - const json j = make_base_timings().to_json(); +TEST(ServerSlotStats, BaseFields_AlwaysPresent) { + const json j = make_base_stats().to_json(); EXPECT_TRUE(j.contains("cache_n")); EXPECT_TRUE(j.contains("prompt_n")); @@ -66,54 +77,77 @@ TEST(ResultTimings, BaseFields_AlwaysPresent) { EXPECT_TRUE(j.contains("predicted_per_second")); } -TEST(ResultTimings, CacheN_ReflectsValue) { - result_timings t = make_base_timings(); - t.cache_n = 7; - const json j = t.to_json(); +TEST(ServerSlotStats, CacheN_ReflectsCachedPromptTokens) { + server_slot_stats s = make_base_stats(); + s.n_prompt_cached = 7; + const json j = s.to_json(); EXPECT_EQ(j.at("cache_n").get(), 7); } -TEST(ResultTimings, BaseFieldValues_MatchInput) { - result_timings t = make_base_timings(); - const json j = t.to_json(); +TEST(ServerSlotStats, BaseFieldValues_DerivedFromCountersAndTimestamps) { + const json j = make_base_stats().to_json(); EXPECT_EQ(j.at("prompt_n").get(), 10); EXPECT_EQ(j.at("predicted_n").get(), 5); EXPECT_DOUBLE_EQ(j.at("prompt_ms").get(), 200.0); - EXPECT_DOUBLE_EQ(j.at("predicted_per_second").get(), 50.0); + EXPECT_DOUBLE_EQ(j.at("prompt_per_token_ms").get(), 20.0); + EXPECT_DOUBLE_EQ(j.at("prompt_per_second").get(), 50.0); + EXPECT_DOUBLE_EQ(j.at("predicted_ms").get(), 100.0); + // The first generated token comes from the prompt batch's logits, so the + // per-token rates divide by n_gen - 1, not n_gen. + EXPECT_DOUBLE_EQ(j.at("predicted_per_token_ms").get(), 25.0); + EXPECT_DOUBLE_EQ(j.at("predicted_per_second").get(), 40.0); +} + +TEST(ServerSlotStats, IsSet_FalseUntilPromptStarts) { + server_slot_stats s; + EXPECT_FALSE(s.is_set()) << "a slot that never ran carries no stats"; + s.t_start = 1; + EXPECT_TRUE(s.is_set()); } -TEST(ResultTimings, WithoutSpeculative_DraftFieldsAbsent) { - // default draft_n = 0 → fields must NOT appear in JSON - result_timings t = make_base_timings(); - // draft_n and draft_n_accepted remain at their default (0) +TEST(ServerSlotStats, UnstartedGeneration_ZeroDurations) { + // Only the prompt was processed: t_gen_last == 0 → generation duration 0. + server_slot_stats s; + s.t_start = 1000000; + s.n_prompt_processed = 4; + const json j = s.to_json(); + EXPECT_DOUBLE_EQ(j.at("prompt_ms").get(), 0.0) << "t_prompt_last == 0 → prompt not finished"; + EXPECT_DOUBLE_EQ(j.at("predicted_ms").get(), 0.0); + EXPECT_DOUBLE_EQ(j.at("predicted_per_second").get(), 0.0); +} + +TEST(ServerSlotStats, WithoutSpeculative_DraftFieldsAbsent) { + // default n_draft_tokens = 0 → fields must NOT appear in JSON + server_slot_stats s = make_base_stats(); + // n_draft_tokens and n_draft_accepted remain at their default (0) - const json j = t.to_json(); + const json j = s.to_json(); - EXPECT_FALSE(j.contains("draft_n")) << "draft_n must be absent when draft_n == 0"; - EXPECT_FALSE(j.contains("draft_n_accepted")) << "draft_n_accepted must be absent when draft_n == 0"; + EXPECT_FALSE(j.contains("draft_n")) << "draft_n must be absent when n_draft_tokens == 0"; + EXPECT_FALSE(j.contains("draft_n_accepted")) << "draft_n_accepted must be absent when n_draft_tokens == 0"; } -TEST(ResultTimings, WithSpeculative_DraftFieldsPresent) { - result_timings t = make_base_timings(); - t.draft_n = 50; - t.draft_n_accepted = 35; +TEST(ServerSlotStats, WithSpeculative_DraftFieldsPresent) { + server_slot_stats s = make_base_stats(); + s.n_draft_tokens = 50; + s.n_draft_accepted = 35; - const json j = t.to_json(); + const json j = s.to_json(); - EXPECT_TRUE(j.contains("draft_n")) << "draft_n must be present when draft_n > 0"; - EXPECT_TRUE(j.contains("draft_n_accepted")) << "draft_n_accepted must be present when draft_n > 0"; + EXPECT_TRUE(j.contains("draft_n")) << "draft_n must be present when n_draft_tokens > 0"; + EXPECT_TRUE(j.contains("draft_n_accepted")) << "draft_n_accepted must be present when n_draft_tokens > 0"; EXPECT_EQ(j.at("draft_n").get(), 50); EXPECT_EQ(j.at("draft_n_accepted").get(), 35); } -TEST(ResultTimings, DraftNOne_FieldsPresent) { +TEST(ServerSlotStats, DraftNOne_FieldsPresent) { // Edge case: even a single speculative token triggers the fields - result_timings t = make_base_timings(); - t.draft_n = 1; - t.draft_n_accepted = 0; + server_slot_stats s = make_base_stats(); + s.n_draft_tokens = 1; + s.n_draft_accepted = 0; - const json j = t.to_json(); + const json j = s.to_json(); EXPECT_TRUE(j.contains("draft_n")); EXPECT_TRUE(j.contains("draft_n_accepted")); @@ -121,12 +155,12 @@ TEST(ResultTimings, DraftNOne_FieldsPresent) { EXPECT_EQ(j.at("draft_n_accepted").get(), 0); } -TEST(ResultTimings, DraftFieldsAbsent_WhenExplicitlyZero) { - result_timings t = make_base_timings(); - t.draft_n = 0; - t.draft_n_accepted = 0; +TEST(ServerSlotStats, DraftFieldsAbsent_WhenExplicitlyZero) { + server_slot_stats s = make_base_stats(); + s.n_draft_tokens = 0; + s.n_draft_accepted = 0; - const json j = t.to_json(); + const json j = s.to_json(); EXPECT_FALSE(j.contains("draft_n")); EXPECT_FALSE(j.contains("draft_n_accepted")); @@ -716,8 +750,13 @@ TEST(ServerTaskNTokens, PopulatedTokens_ReturnsCount) { } // ============================================================ -// server_task_result_metrics::to_json -// Pure struct → JSON; no model needed. +// server_task_result_metrics::to_json / ::to_metrics +// Pure struct → JSON / Prometheus text; no model needed. +// +// b10408 (upstream #26920) split this result in two: to_json() now serves +// /slots and returns the slot array verbatim, while the cumulative counters +// moved into an embedded server_metrics and are rendered as Prometheus +// exposition text by the new to_metrics() for /metrics. // ============================================================ namespace { @@ -726,62 +765,85 @@ server_task_result_metrics make_metrics() { m.n_idle_slots = 2; m.n_processing_slots = 1; m.n_tasks_deferred = 3; - m.t_start = 1234567890LL; - m.n_prompt_tokens_processed_total = 100; - m.t_prompt_processing_total = 50; - m.n_tokens_predicted_total = 200; - m.t_tokens_generation_total = 80; - m.n_prompt_tokens_processed = 10; - m.t_prompt_processing = 5; - m.n_tokens_predicted = 20; - m.t_tokens_generation = 8; - m.n_decode_total = 300; - m.n_busy_slots_total = 4; + m.metrics.t_start = 1234567890LL; + m.metrics.prompt.add(/*n=*/100, /*n_steps=*/100, /*t_us=*/50); + m.metrics.predict.add(/*n=*/200, /*n_steps=*/200, /*t_us=*/80); + m.metrics.n_prompt_cached = 10; + m.metrics.n_decode = 300; + // 600 busy-slot observations over 300 decode calls → exactly 2.0 per decode. + // Prometheus samples are rendered with the default 6-significant-digit + // ostream precision, so the fixture keeps every derived value exact. + m.metrics.n_busy_slots = 600; return m; } -} // namespace -TEST(ServerTaskResultMetrics, ToJson_SlotCountFields) { - const json j = make_metrics().to_json(); - EXPECT_EQ(j.at("idle").get(), 2); - EXPECT_EQ(j.at("processing").get(), 1); - EXPECT_EQ(j.at("deferred").get(), 3); - EXPECT_EQ(j.at("t_start").get(), 1234567890LL); +// Reads the value of a `llamacpp: ` sample line out of Prometheus +// exposition text (the "# HELP"/"# TYPE" lines for the same name are skipped). +double prometheus_value(const std::string &text, const std::string &name) { + const std::string prefix = "llamacpp:" + name + " "; + for (size_t pos = 0; pos < text.size();) { + const size_t eol = text.find('\n', pos); + const std::string line = text.substr(pos, eol == std::string::npos ? std::string::npos : eol - pos); + if (line.rfind(prefix, 0) == 0) { + return std::stod(line.substr(prefix.size())); + } + if (eol == std::string::npos) { + break; + } + pos = eol + 1; + } + ADD_FAILURE() << "no sample line for llamacpp:" << name; + return -1.0; } +} // namespace -TEST(ServerTaskResultMetrics, ToJson_NTokensMax) { +TEST(ServerTaskResultMetrics, ToJson_ReturnsSlotsArrayVerbatim) { server_task_result_metrics m = make_metrics(); - m.n_tokens_max = 4096; + m.slots_data = json::array({{{"id", 0}}, {{"id", 1}}}); const json j = m.to_json(); - EXPECT_EQ(j.at("n_tokens_max").get(), 4096); + ASSERT_TRUE(j.is_array()); + EXPECT_EQ(j.size(), 2u); + EXPECT_EQ(j.at(0).at("id").get(), 0); } -TEST(ServerTaskResultMetrics, ToJson_TokenCountFields) { - const json j = make_metrics().to_json(); - EXPECT_EQ(j.at("n_prompt_tokens_processed_total").get(), 100u); - EXPECT_EQ(j.at("n_tokens_predicted_total").get(), 200u); - EXPECT_EQ(j.at("n_decode_total").get(), 300u); - EXPECT_EQ(j.at("n_busy_slots_total").get(), 4u); +TEST(ServerTaskResultMetrics, ToMetrics_SlotGauges) { + const std::string text = make_metrics().to_metrics(); + EXPECT_DOUBLE_EQ(prometheus_value(text, "requests_processing"), 1.0); + EXPECT_DOUBLE_EQ(prometheus_value(text, "requests_deferred"), 3.0); +} + +TEST(ServerTaskResultMetrics, ToMetrics_NTokensMax) { + server_task_result_metrics m = make_metrics(); + m.metrics.n_tokens_max = 4096; + EXPECT_DOUBLE_EQ(prometheus_value(m.to_metrics(), "n_tokens_max"), 4096.0); } -TEST(ServerTaskResultMetrics, ToJson_TimingAndWindowFields) { - const json j = make_metrics().to_json(); - // Timing totals - EXPECT_EQ(j.at("t_prompt_processing_total").get(), 50u); - EXPECT_EQ(j.at("t_tokens_generation_total").get(), 80u); - // Current-window counts (not the _total variants) - EXPECT_EQ(j.at("n_prompt_tokens_processed").get(), 10u); - EXPECT_EQ(j.at("t_prompt_processing").get(), 5u); - EXPECT_EQ(j.at("n_tokens_predicted").get(), 20u); - EXPECT_EQ(j.at("t_tokens_generation").get(), 8u); +TEST(ServerTaskResultMetrics, ToMetrics_TokenCountCounters) { + const std::string text = make_metrics().to_metrics(); + EXPECT_DOUBLE_EQ(prometheus_value(text, "prompt_tokens_total"), 100.0); + EXPECT_DOUBLE_EQ(prometheus_value(text, "prompt_tokens_cached_total"), 10.0); + EXPECT_DOUBLE_EQ(prometheus_value(text, "tokens_predicted_total"), 200.0); + EXPECT_DOUBLE_EQ(prometheus_value(text, "n_decode_total"), 300.0); + // n_busy_slots is exposed as an average per llama_decode() call: 600 / 300. + EXPECT_DOUBLE_EQ(prometheus_value(text, "n_busy_slots_per_decode"), 2.0); } -TEST(ServerTaskResultMetrics, ToJson_SlotDataIsArray) { +TEST(ServerTaskResultMetrics, ToMetrics_TimeCountersAreSeconds) { + const std::string text = make_metrics().to_metrics(); + // The struct stores microseconds; Prometheus counters are in seconds. + EXPECT_DOUBLE_EQ(prometheus_value(text, "prompt_seconds_total"), 50.0 / 1e6); + EXPECT_DOUBLE_EQ(prometheus_value(text, "tokens_predicted_seconds_total"), 80.0 / 1e6); +} + +TEST(ServerTaskResultMetrics, ToMetrics_SpeculativePerPositionSeriesOnlyWhenPresent) { server_task_result_metrics m = make_metrics(); - m.slots_data = json::array({{{"id", 0}}, {{"id", 1}}}); - const json j = m.to_json(); - ASSERT_TRUE(j.at("slots").is_array()); - EXPECT_EQ(j.at("slots").size(), 2u); + EXPECT_EQ(m.to_metrics().find("spec_decode_num_accepted_tokens_per_pos_total"), std::string::npos) + << "the labeled series must be omitted when no per-position data was collected"; + + m.metrics.n_accepted_per_pos = {7, 3}; + const std::string text = m.to_metrics(); + EXPECT_NE(text.find("llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\"0\"} 7"), std::string::npos); + EXPECT_NE(text.find("llamacpp:spec_decode_num_accepted_tokens_per_pos_total{position=\"1\"} 3"), std::string::npos); } // ============================================================ @@ -1019,16 +1081,16 @@ TEST(ServerTaskResultCmplPartial, NonOaicompat_TimingsAbsentByDefault) { server_task_result_cmpl_partial p; p.is_updated = true; p.res_type = TASK_RESPONSE_TYPE_NONE; - // timings.prompt_n == 0 by default → timings should be absent + // stats.t_start == 0 by default (slot never started) → timings should be absent const json j = p.to_json_non_oaicompat(); EXPECT_FALSE(j.contains("timings")); } -TEST(ServerTaskResultCmplPartial, NonOaicompat_TimingsPresentWhenPromptNNonzero) { +TEST(ServerTaskResultCmplPartial, NonOaicompat_TimingsPresentWhenStatsSet) { server_task_result_cmpl_partial p; p.is_updated = true; p.res_type = TASK_RESPONSE_TYPE_NONE; - p.timings.prompt_n = 5; + p.stats.t_start = 1000000; // is_set() → timings are emitted const json j = p.to_json_non_oaicompat(); EXPECT_TRUE(j.contains("timings")); } @@ -1686,14 +1748,14 @@ TEST(CmplFinalVerboseFlag, OaicompatChat_VerboseTrue_DebugKeyPresent) { TEST(CmplFinalVerboseFlag, Oaicompat_TimingsAbsentByDefault) { auto f = make_oai_final(); - // timings.prompt_n is default-constructed to a value < 0 — absent + // stats.t_start is default-constructed to 0 (slot never started) — absent const json j = f.to_json_oaicompat(); EXPECT_FALSE(j.contains("timings")); } -TEST(CmplFinalVerboseFlag, Oaicompat_TimingsPresentWhenPromptNNonNeg) { +TEST(CmplFinalVerboseFlag, Oaicompat_TimingsPresentWhenStatsSet) { auto f = make_oai_final(); - f.timings.prompt_n = 0; // >= 0 triggers inclusion + f.stats.t_start = 1000000; // is_set() triggers inclusion const json j = f.to_json_oaicompat(); EXPECT_TRUE(j.contains("timings")); }