From eee5ab5b9b6bc6fa9481c0bac0417eb4ee1e1864 Mon Sep 17 00:00:00 2001 From: Deano Date: Thu, 13 Aug 2026 12:20:44 +0000 Subject: [PATCH 1/2] fix(ds4): surface the DSpark AR fallback and the fused-verify preconditions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A DSpark server prints 'spec-decode ENABLED' at startup and then decodes pure AR for any request the greedy-only verifier cannot serve, saying nothing. The sampler case is a live trap in this tree: when a request OMITS temperature the HTTP layer falls back to the model card's sampling defaults, and share/model_cards/deepseek-v4-flash-0731-rocmfpx.json sets temperature 1.0 — so a temperature-less benchmark measures autoregressive decode while the log still claims speculation. Measured, same binary and flags, same prompt: no temperature field gives 290 tokens in 289 steps (one step per token, i.e. pure AR); "temperature": 0.0 gives 268 tokens at accept_rate 0.69. This cost a day of chasing a spec-decode 'regression' between two builds that differed only in whether they shipped that card. The decode gate now logs one line naming which condition routed the request to AR — force_ar / budget stop tokens / sampling — with the effective sampler values. The fused-verify gate reports every precondition, including verify_hooks, out_logits and expert_runtime; the previous version gated its own warning on two of the pointers whose absence it was meant to report, so a null there stayed invisible. Both fire once. The fused-verify one has never fired on gfx1151 — it is preventive, and the comment says so rather than implying it caught something. --- server/src/deepseek4/deepseek4_backend.cpp | 19 ++++++++++++++ server/src/deepseek4/deepseek4_graph.cpp | 29 ++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index 47c67a300..0595c364f 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -1841,6 +1841,25 @@ GenerateResult DeepSeek4Backend::generate_from_state( // The DSpark verifier is greedy-only. Route sampling and penalties through // AR so the request's sampler contract is not silently ignored. const bool sampling_requires_ar = sampler_.needs_logit_processing(); + // A drafter was loaded and the operator asked for spec decode, but this + // request routes to AR anyway. Say why, once: the DS4 model card defaults + // temperature to 1.0, so a request that merely OMITS temperature lands + // here — the server then decodes pure AR while the startup log still says + // "spec-decode ENABLED", which reads as a spec-engagement regression. + if (spec_enabled_ && spec_drafter_ && req.n_gen > 0 && + (req.force_ar_decode || budget_requires_ar || sampling_requires_ar)) { + static bool warned = false; + if (!warned) { + warned = true; + std::fprintf(stderr, + "[deepseek4] DSpark spec loaded but this request decodes AR: " + "force_ar=%d stop_tokens=%d sampling=%d (temp=%.2f rep_pen=%.2f " + "freq_pen=%.2f pres_pen=%.2f; greedy needs temperature 0)\n", + req.force_ar_decode ? 1 : 0, budget_requires_ar ? 1 : 0, + sampling_requires_ar ? 1 : 0, sampler_.temp, + sampler_.rep_pen, sampler_.freq_pen, sampler_.pres_pen); + } + } if (spec_enabled_ && spec_drafter_ && req.n_gen > 0 && !req.force_ar_decode && !budget_requires_ar && !sampling_requires_ar) { if (last_logits_.empty()) { diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index 223254902..114fc8213 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -6779,6 +6779,35 @@ bool deepseek4_step_layer_range( q5_verify_candidate) && verify_hooks && layer_begin == 0 && is_last_shard && out_logits && ds4_backend_is_gpu(backend) && ds4_fused_verify_enabled(); + // Fused verify has many preconditions and declining any of them is + // invisible: the request still decodes, still reports a healthy acceptance + // rate, and only the throughput differs. Name the failed condition once so + // a slow DSpark run can be attributed from the log instead of guessed at. + // (No run has yet tripped this on gfx1151 — it is here so that the next + // "spec decode is slow" report starts from evidence.) + if (ds4_fused_verify_enabled() && !fused_verify_candidate && + n_tokens >= 2 && layer_begin == 0 && is_last_shard) { + static bool warned = false; + if (!warned) { + warned = true; + std::fprintf(stderr, + "[deepseek4] DFLASH_DS4_FUSED_VERIFY=1 but fused verify is " + "inactive: n_tokens=%d (cap %d) verify_hooks=%d out_logits=%d " + "backend_gpu=%d moe_hybrid=%d expert_runtime=%d " + "materialized_cold=%d cold_backend_kind_gpu=%d " + "cold_backend_distinct=%d; verify falls back to the dense " + "full-expert path\n", + n_tokens, GGML_CUDA_DS4_MIX_MMV_MAX_TOKENS, + verify_hooks ? 1 : 0, out_logits ? 1 : 0, + ds4_backend_is_gpu(backend) ? 1 : 0, + moe_hybrid ? 1 : 0, expert_runtime ? 1 : 0, + moe_hybrid && moe_hybrid->materialized_cold_experts ? 1 : 0, + moe_hybrid && moe_hybrid->cold_backend_kind == + MoeHybridColdBackend::Gpu ? 1 : 0, + moe_hybrid && moe_hybrid->cold_backend && + moe_hybrid->cold_backend != backend ? 1 : 0); + } + } const bool heterogeneous_sparse_prefill = !fused_verify_candidate && moe_hybrid && cache.prefill_mode == PrefillAttentionMode::Sparse && From 5bc173f14bffb8fed418048a39c9513c9f9ae2f3 Mon Sep 17 00:00:00 2001 From: Deano Date: Thu, 13 Aug 2026 12:20:44 +0000 Subject: [PATCH 2/2] docs(ds4): record the AR-fallback conditions and the acceptance envelope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three things a DSpark measurement needs and DS4.md did not say. The fallback conditions and the tell in the log (decode tokens=N steps=N-1 instead of [ds4-spec] gen=... steps=...), so 'spec decode is broken' can be checked before it is reported. The acceptance envelope, because speculative throughput is acceptance-bound and acceptance is workload-bound: on a gfx1151 host, code/math prompts (accept 0.83-0.90, ~2.4 committed tokens per verify step) measured 24-27 tok/s where open-ended prose (accept 0.64-0.69, ~1.1 committed) measured 16-18 tok/s from the SAME server under the same flags. A spec tok/s figure is not comparable across prompt mixes. And an idle-box warning: Strix Halo decode is unified-memory bandwidth bound, so an unrelated co-tenant compile moved an identical request from 18.2 to 3.8 tok/s with acceptance and step count UNCHANGED — which reads exactly like a model or kernel regression. Record loadavg beside any tok/s figure. --- server/docs/DS4.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/server/docs/DS4.md b/server/docs/DS4.md index 0ad0b31fd..70c7a1033 100644 --- a/server/docs/DS4.md +++ b/server/docs/DS4.md @@ -373,6 +373,44 @@ sampler stay on the main backend while routed target experts execute on their configured owners. `--ds4-expert-top-k 4` remains a separate approximate policy; omit it to retain the model's default six routed experts. +### Silent AR fallback (check this before calling spec decode broken) + +The DSpark verifier is greedy-only, so the server routes a request to plain +autoregressive decode — with the drafter loaded and `DSpark spec-decode +ENABLED` already printed at startup — whenever any of these hold: + +- the effective sampler is non-greedy (`temperature > 0`, repetition or + presence/frequency penalties), +- the request carries budget stop tokens, +- the request forces AR explicitly. + +The sampler trap is the subtle one: when a request **omits** `temperature`, +the HTTP layer falls back to the model card's sampling defaults, and +`share/model_cards/deepseek-v4-flash-0731-rocmfpx.json` sets +`temperature: 1.0`. A temperature-less benchmark request therefore decodes +pure AR on any build that ships the card, while the same request engages +speculation on a tree without it — which looks exactly like a spec-decode +regression between the two binaries. It is not one; send +`"temperature": 0.0` explicitly. The tell in the log is `decode tokens=N +steps=N-1` (one step per token) instead of `[ds4-spec] gen=... steps=...` +lines; the server also now logs one `DSpark spec loaded but this request +decodes AR: ...` line naming the condition the first time it happens. + +Speculative throughput is acceptance-bound, and acceptance is workload-bound: +on the gfx1151 host, code/math prompts (accept 0.83–0.90, ~2.4 committed +tokens per verify step) measured 24–27 tok/s where open-ended prose (accept +0.64–0.69, ~1.1 committed per step) measured 16–18 tok/s from the same server +under the same flags. Judge a spec-decode number against the acceptance it +was measured at, not against the headline figure from a different prompt mix +(see the gfx1151 numbers below for the measured governor/top-k envelope). + +Measure on an otherwise idle box. Strix Halo decode is unified-memory +bandwidth bound, so an unrelated co-tenant compile on the same host dropped +the identical request from 18.2 to 3.8 tok/s — a 5x swing with no change to +the acceptance rate or the step count, which is what makes it easy to misread +as a model or kernel regression. Record `/proc/loadavg` alongside any tok/s +figure, and re-run anything anomalous before believing it. + ### Verifier graph-cache safety Every heterogeneous verifier slot owns a scheduler and its per-backend scratch