diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 92af24238e21..c176a552d854 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -35,6 +35,24 @@ #include #include +static std::set collect_graph_output_names(const ggml_cgraph * cgraph) { + std::set outputs; + for (int node_n = 0; node_n < cgraph->n_nodes; node_n++) { + auto * node = cgraph->nodes[node_n]; + if (node->flags & GGML_TENSOR_FLAG_OUTPUT) { + outputs.insert(node->name); + } + } + return outputs; +} + +bool GgmlOvDecoder::has_same_graph_io(const ggml_cgraph * cgraph) const { + if (m_built_output_names.empty()) { + return true; // no snapshot (naive / test decoders): nothing to compare against + } + return collect_graph_output_names(cgraph) == m_built_output_names; +} + GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, ModelParams & model_params, ComputeParams & compute_params, @@ -64,6 +82,8 @@ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, validate_cgraph(); + m_built_output_names = collect_graph_output_names(cgraph); + set_input_output(); compute_node_dynamic_dims(); compute_model_inputs(); @@ -79,6 +99,7 @@ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, void GgmlOvDecoder::update_io(ggml_cgraph * cgraph) { m_cgraph = cgraph; + m_built_output_names = collect_graph_output_names(cgraph); m_model_inputs.clear(); m_model_outputs.clear(); m_node_info_list.clear(); @@ -983,7 +1004,7 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, } else { input_shape = ov::PartialShape{get_shape(input)}; } - if (dynamic_dim_index != -1 && m_model_is_splitted) { + if (dynamic_dim_index != -1) { input_shape[3 - dynamic_dim_index] = -1; } if (op->op == GGML_OP_SOFT_MAX && op->src[1] != nullptr && op->src[1]->op == GGML_OP_NONE && @@ -1171,6 +1192,30 @@ void GgmlOvDecoder::compute_model_outputs() { if (cur_node->op == GGML_OP_NONE || cur_node->op == GGML_OP_VIEW || cur_node->op == GGML_OP_RESHAPE) { continue; } + // A node explicitly marked by ggml_set_output() must be a model output even when it is + // fully consumed inside the graph, because the host reads it back afterwards. The + // use-count analysis below cannot see that: it only asks whether anything still in the + // graph needs the value. + // + // Mid-graph taps break that assumption. A model that exposes its residual stream at a few + // layers (llm_graph_result marks each such tensor with ggml_set_output()) has tensors that + // are read by the host AND feed the next layer, so use-count logic dropped them, the OV + // model never produced them, and the host read an unwritten buffer -- whatever consumes + // those features saw zeros. This affects any consumer of a mid-graph tensor, not one + // particular feature. + if (cur_node->flags & GGML_TENSOR_FLAG_OUTPUT) { + // in-place ops publish their result through view_src, as in the use_count == 0 path + auto * flagged_node = cur_node; + if (flagged_node->op == GGML_OP_SET_ROWS && flagged_node->view_src != nullptr) { + flagged_node = flagged_node->view_src; + } + std::string flagged_name = get_tensor_ov_name(m_cgraph, flagged_node); + if (m_model_outputs.find(flagged_name) == m_model_outputs.end()) { + m_model_outputs[flagged_name] = flagged_node; + m_model_output_names.push_back(flagged_name); + } + continue; + } auto cur_node_use_count = m_cgraph->use_counts[ggml_hash_find(&m_cgraph->visited_hash_set, cur_node)]; if (cur_node_use_count == 0) { // The output of in-place ops is the view_src tensor, which is updated in place. We should use the view_src name as the output name to make sure it can be correctly matched with the later ops that use the view_src. @@ -1193,8 +1238,10 @@ void GgmlOvDecoder::compute_model_outputs() { } if (cur_node != nullptr) { std::string cur_node_name = get_tensor_ov_name(m_cgraph, cur_node); + if (m_model_outputs.find(cur_node_name) == m_model_outputs.end()) { + m_model_output_names.push_back(cur_node_name); + } m_model_outputs[cur_node_name] = cur_node; - m_model_output_names.insert(cur_node_name); } } } @@ -1899,6 +1946,10 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { m_node_dynamic_dims[src] = 1; continue; } + if (is_inp_embd_2d(src, node)) { + m_node_dynamic_dims[src] = 1; + continue; + } self(self, src); } } @@ -1954,6 +2005,11 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { } break; case GGML_OP_VIEW: { + // A VIEW that does not change the shape is a pass-through for dynamic-dim purposes. + if (is_same_shape(node->src[0], node)) { + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; + break; + } // Use stride-based matching: the stride of a VIEW dimension directly // encodes which source dimension it indexes into, so it uniquely // identifies the dynamic dim even when two dims share the same size. diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 183aa4cd5bd2..3ff029f5f1f1 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -8,10 +8,10 @@ #include #include #include +#include #include #include #include -#include #include #include @@ -250,7 +250,9 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { return m_model_weights; } - virtual std::set get_model_output_names() const override { return m_model_output_names; } + virtual std::set get_model_output_names() const override { + return std::set(m_model_output_names.begin(), m_model_output_names.end()); + } const std::map & get_model_outputs() const { return m_model_outputs; } @@ -349,6 +351,16 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { void update_io(ggml_cgraph * cgraph); + // The cgraph this decoder currently describes. A cached decoder can be handed a DIFFERENT + // cgraph instance with the same graph_key, and every cached ggml_tensor* (and m_cgraph itself) + // then dangles, so callers must compare and refresh. + const ggml_cgraph * get_cgraph() const { return m_cgraph; } + + // Was this decoder built for a graph with the same OUTPUT set as `cgraph`? The cache key is + // only (n_nodes, first/last node name), which cannot see a change in which tensors are marked + // as outputs, and the compiled model's Results are fixed at compile time. + bool has_same_graph_io(const ggml_cgraph * cgraph) const; + inline static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE; } @@ -367,6 +379,16 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { return tensor->op == GGML_OP_GET_ROWS && op->op == GGML_OP_RMS_NORM; } + // A 2-D float graph input feeding a matmul as src[1]. Speculative decoding hands the draft the + // target's hidden features this way ([n_embd, n_tokens], with no GET_ROWS lookup, so none of the + // other seeds match it) and the token count differs between the prompt pass and the shorter + // draft blocks, so its token dim must stay dynamic. + inline static bool is_inp_embd_2d(const ggml_tensor * tensor, const ggml_tensor * op) { + return (tensor->flags & GGML_TENSOR_FLAG_INPUT) && tensor->op == GGML_OP_NONE && + tensor->type == GGML_TYPE_F32 && tensor->ne[2] == 1 && tensor->ne[3] == 1 && + op->op == GGML_OP_MUL_MAT && tensor == op->src[1]; + } + inline static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]) || (op->op == GGML_OP_SOFT_MAX && tensor == op->src[1]); @@ -454,7 +476,9 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { std::map m_model_extra_inputs; std::map> m_model_weights; std::map m_model_outputs; - std::set m_model_output_names; + std::vector m_model_output_names; + // The output set the compiled model was built from, for has_same_graph_io(). + std::set m_built_output_names; std::vector m_node_info_list; std::map m_node_dynamic_dims; diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index bd9e2f7451ce..c2d250fe2bcb 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -57,9 +57,12 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_RELEASE_WEIGHTS", "GGML_OPENVINO_REDUCE_COMPILE_MEM", "GGML_OPENVINO_LOG_UNSUPPORTED_OPS", + "GGML_OPENVINO_ROPE_F32_ANGLE", "GGML_OPENVINO_LOG_SWA_LAYERS", "GGML_OPENVINO_REQUANT_KQUANT", "GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT", + "GGML_OPENVINO_LOG_SUPPORTS_OP", + "GGML_OPENVINO_DIAG_GRAPH_IO", }; for (const char * const & env_var : env_var_names) { diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 9e9d31760bf4..435455b35a47 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -1523,16 +1524,114 @@ static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(gg return {true, ""}; } +// Report what supports_op() accepts and rejects, so a graph that silently falls back to another +// backend can be diagnosed. Every op rejected here becomes a graph split; if nothing is rejected, +// the whole graph runs on OpenVINO. +// +// Gated by GGML_OPENVINO_LOG_SUPPORTS_OP: +// 1 one summary at exit listing every distinct op kind and verdict -- start here +// 2 additionally log every individual call, with the tensor name and shape +// +// This wraps the decision function instead of instrumenting it, so all of its return paths are +// covered -- including the several that reject without logging a reason of their own, and any +// added later. +// +// The summary is reported at exit rather than on each op's first occurrence, because the earliest +// calls happen before the log handler emits anything: with level 2 the first line to actually +// appear already reads "call 1244". A first-occurrence line would therefore be silently dropped +// for exactly the ops that are only ever seen early. Counting is unaffected, so the summary is the +// reliable view; treat the level-2 stream as a sample, not a complete trace. +namespace { + +struct supports_op_tally { + std::mutex mutex; + // key: ":ok" / ":no" -> number of calls + std::map counts; + + // Report totals once, at exit. Registered lazily so a run with the knob off costs nothing. + // atexit runs while the ggml log callback is still valid, which static destruction does not + // guarantee. + void arm_summary() { + static std::once_flag armed; + std::call_once(armed, [] { + std::atexit([] { + auto & self = instance(); + std::lock_guard lock(self.mutex); + + size_t n_accept_kinds = 0; + size_t n_reject_kinds = 0; + GGML_LOG_WARN("ov-supports-op: ---- summary (distinct op kinds, with call counts) ----\n"); + for (const auto & [key, count] : self.counts) { + const bool ok = key.size() > 3 && key.compare(key.size() - 3, 3, ":ok") == 0; + ok ? ++n_accept_kinds : ++n_reject_kinds; + GGML_LOG_WARN("ov-supports-op: %-8s %-24s %zu call(s)\n", ok ? "ACCEPT" : "REJECT", + key.substr(0, key.size() - 3).c_str(), count); + } + if (n_reject_kinds == 0) { + GGML_LOG_WARN("ov-supports-op: %zu op kind(s) accepted, 0 rejected" + " -- every op was supported, so no split originated here\n", + n_accept_kinds); + } else { + GGML_LOG_WARN("ov-supports-op: %zu op kind(s) accepted, %zu REJECTED" + " -- each rejected kind forces a graph split\n", + n_accept_kinds, n_reject_kinds); + } + }); + }); + } + + static supports_op_tally & instance() { + static supports_op_tally tally; + return tally; + } +}; + +} // namespace + static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { auto res = ggml_backend_openvino_device_supports_op_impl(dev, op); - if (!res.is_supported) { + const bool supported = res.is_supported; + if (!supported) { static const bool log_unsupported = ggml_openvino_getenv_int("GGML_OPENVINO_LOG_UNSUPPORTED_OPS") != 0; if (log_unsupported) { GGML_LOG_WARN("OpenVINO op unsupported: op '%s' (%s), type %s: %s\n", op->name, ggml_op_name(op->op), ggml_type_name(op->type), res.reason.c_str()); } } - return res.is_supported; + + static const int log_level = ggml_openvino_getenv_int("GGML_OPENVINO_LOG_SUPPORTS_OP"); + if (log_level <= 0) { + return supported; + } + + // Name the op the way ggml does: UNARY and GLU carry their real identity in a sub-enum, so + // logging only "UNARY" would merge SILU with everything else. + std::string op_desc = ggml_op_name(op->op); + if (op->op == GGML_OP_UNARY) { + op_desc += std::string("/") + ggml_unary_op_name(ggml_get_unary_op(op)); + } else if (op->op == GGML_OP_GLU) { + op_desc += std::string("/") + ggml_glu_op_name(ggml_get_glu_op(op)); + } + + // The scheduler calls supports_op() on every graph build and may do so from several threads, + // so the tally needs a lock. Keyed on op identity *and* verdict, because the same op can be + // accepted at one shape and rejected at another (e.g. a quantized 3-D src). + auto & tally = supports_op_tally::instance(); + tally.arm_summary(); + + size_t count; + { + std::lock_guard lock(tally.mutex); + count = ++tally.counts[op_desc + (supported ? ":ok" : ":no")]; + } + + if (log_level >= 2) { + GGML_LOG_WARN("ov-supports-op: %-6s %-22s name='%s' type=%s ne=[%ld,%ld,%ld,%ld] (call %zu)\n", + supported ? "ACCEPT" : "REJECT", op_desc.c_str(), op->name, ggml_type_name(op->type), + (long) op->ne[0], (long) op->ne[1], (long) op->ne[2], (long) op->ne[3], count); + } + + return supported; } static bool ggml_backend_openvino_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { diff --git a/ggml/src/ggml-openvino/openvino/op/argmax.cpp b/ggml/src/ggml-openvino/openvino/op/argmax.cpp new file mode 100644 index 000000000000..441a6a5b2ad6 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/argmax.cpp @@ -0,0 +1,51 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" +#include "ggml.h" + +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +// GGML_OP_ARGMAX: src0 is a matrix [ne0, ne1]; the result is a 1-D I32 tensor of ne1 entries +// holding, for each row, the index of the maximum along ne0. ggml shapes arrive reversed here, so +// ne0 is the last OV axis and ne1 is axis 2. TopK(k=1, axis=last, MAX) yields exactly that index, +// then reshape to the [1,1,1,ne1] layout the decoder expects for a 1-D ggml output. +// +// Note ggml's argmax returns the FIRST maximum on a tie, while TopK's tie-breaking is not specified +// to match. Ties on real logits are vanishingly rare and none were observed, but the two are not +// proven equivalent. +OutputVector translate_argmax(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + + auto k = ov::op::v0::Constant::create(ov::element::i64, {}, {1}); + auto topk = std::make_shared(input, + k, + 3, + ov::op::v11::TopK::Mode::MAX, + ov::op::v11::TopK::SortType::SORT_VALUES, + context.get_output_type(), + false); + + // ne1 is the dynamic row count, so build the target shape at runtime. + auto leading = ov::op::v0::Constant::create(ov::element::i64, {3}, {1, 1, 1}); + auto rows = get_dimensions(input.get_node_shared_ptr(), {2}); + auto target_shape = std::make_shared(OutputVector{leading, rows}, 0); + auto res = std::make_shared(topk->output(1), target_shape, false); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index d4f5ac307329..46f74b7a980e 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -45,6 +45,7 @@ std::unordered_map get_supported_ops() { {"GGML_OP_SQR", op::translate_sqr }, {"GGML_OP_SQRT", op::translate_sqrt }, {"GGML_OP_SOFT_MAX", op::translate_soft_max }, + {"GGML_OP_ARGMAX", op::translate_argmax }, {"GGML_OP_ARGSORT", op::translate_argsort }, {"GGML_OP_SUB", op::translate_1to1_match_2_inputs}, {"GGML_OP_TRANSPOSE", op::translate_transpose }, diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.h index a0a42bff337d..9314a5284e75 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.h +++ b/ggml/src/ggml-openvino/openvino/op_table.h @@ -42,6 +42,7 @@ GGML_OP_CONVERTER(translate_glu_geglu); GGML_OP_CONVERTER(translate_glu_geglu_quick); GGML_OP_CONVERTER(translate_set_rows); GGML_OP_CONVERTER(translate_cpy); +GGML_OP_CONVERTER(translate_argmax); GGML_OP_CONVERTER(translate_argsort); GGML_OP_CONVERTER(translate_flash_attn_ext); GGML_OP_CONVERTER(translate_clamp); diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp index 8bb7678ee381..9bbdafc95af2 100644 --- a/ggml/src/ggml-openvino/openvino/utils.cpp +++ b/ggml/src/ggml-openvino/openvino/utils.cpp @@ -1,5 +1,6 @@ #include "utils.h" +#include "../ggml-openvino-extra.h" #include "ggml-impl.h" #include @@ -22,6 +23,7 @@ #include #include #include +#include #include namespace ov { @@ -240,14 +242,73 @@ std::pair, ov::Output> make_sin_cos(int32_t * rope_params } } - Output cos_theta = std::make_shared(theta); - Output sin_theta = std::make_shared(theta); + // Keep the ROPE ANGLE in f32 even under INFERENCE_PRECISION_HINT=f16. + // + // Everything above builds theta = pos * freq_factors with a declared f32 type, but a declared + // element type is not an execution precision: under the f16 hint the GPU plugin compresses these + // subgraphs to f16 anyway. theta's magnitude IS the token position, so at position 6525 it lands + // where f16 spacing is 4 and the value is not even representable. Worse, a model whose freq_base + // and head_dim make factor[0] == 1.0 exactly feeds the raw position in as the top-frequency + // angle, so an absolute error of 1-2 becomes an error of 1-2 RADIANS -- up to a third of a cycle. Sin/Cos of an angle wrong by radians is not approximately + // right, it is arbitrary, and trig amplifies the error without bound. + // + // Measured on a large decoder-only model at inp_pos=6525, same device, f16 vs f32 execution: the + // whole Q/K projection chain is clean at rel ~1e-4, then Cos jumps to rel 0.124 and Sin to 0.155, + // and K right after the rotation carries rel 0.081. The error is depth-dependent because f16 + // spacing grows with magnitude: sin rel is 0.006 at position 512 and 0.16 at 6525. + // + // mark_as_precision_sensitive disables fp16 compression of the subgraph feeding an input, which is + // exactly the guarantee needed. Marking Sin/Cos's input covers the whole theta chain back to the + // Convert of inp_pos. The cost is negligible: theta is a few dozen values per token, against the + // matmuls that dominate. The rotation itself and everything downstream stay f16. + const bool pin_angle = ggml_openvino_getenv_int("GGML_OPENVINO_ROPE_F32_ANGLE", 1) != 0; + + auto cos_op = std::make_shared(theta); + auto sin_op = std::make_shared(theta); + if (pin_angle) { + ov::mark_as_precision_sensitive(cos_op->input(0)); + ov::mark_as_precision_sensitive(sin_op->input(0)); + } + Output cos_theta = cos_op; + Output sin_theta = sin_op; if (!imrope) { auto mscale_node = ov::op::v0::Constant::create(ov::element::f32, Shape{}, {mscale}); - cos_theta = std::make_shared(cos_theta, mscale_node); - sin_theta = std::make_shared(sin_theta, mscale_node); + auto cos_scaled = std::make_shared(cos_theta, mscale_node); + auto sin_scaled = std::make_shared(sin_theta, mscale_node); + // Mark this Constant's input too. Marking only Sin/Cos keeps those nodes f32 but leaves the + // sibling mscale Constant to be compressed to f16, and the pass does not reconcile the pair, + // so compilation fails with "Arguments do not have the same element type". + if (pin_angle) { + ov::mark_as_precision_sensitive(cos_scaled->input(1)); + ov::mark_as_precision_sensitive(sin_scaled->input(1)); + } + cos_theta = cos_scaled; + sin_theta = sin_scaled; + } + + // Close the f32 island explicitly with an f16 round trip. + // + // ConvertPrecision does not insert the boundary Convert: with only the marking above, the f32 + // region propagates forward through the shape-only ops of the cos/sin expansion and then collides + // with f16 data in the rotation, so the type mismatch just moves one step down the graph. + // + // Rounding the RESULT of the trig is harmless -- cos/sin are in [-1,1], where f16 spacing is ~6e-4, + // the ordinary f16 cost accepted everywhere else. What must not be rounded is the ANGLE, which the + // marking above protects. So this spends the cheap rounding to buy a type-consistent graph: the + // chain from inp_pos through Sin/Cos stays f32, everything downstream is f16, and the transition is + // a node we control rather than one the pass has to infer. + // + // Declared types are f32 on both sides at build time, so this is a no-op for the frontend and for + // the f32 paths; it only becomes a real boundary once ConvertPrecision runs under the f16 hint. + if (pin_angle) { + auto pin_boundary = [](Output x) -> Output { + auto down = std::make_shared(x, ov::element::f16); + return std::make_shared(down, ov::element::f32); + }; + cos_theta = pin_boundary(cos_theta); + sin_theta = pin_boundary(sin_theta); } return std::make_pair(sin_theta, cos_theta); diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 09f73b53611a..335ae23738d0 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -124,6 +124,12 @@ static std::optional try_make_kv_sliced_tensor(std::shared_ptrdata == nullptr) { + // unallocated input: let the caller build an owning zero tensor rather than slice a null + // pointer + return std::nullopt; + } + ov::Shape sliced_shape = full_shape; sliced_shape[2] = static_cast(n_kv); @@ -187,6 +193,12 @@ ov::Tensor create_ov_output_tensor(std::shared_ptr ggml_decoder, output_shape = ggml_decoder->get_shape(ggml_tensor); } } + + // A graph output can also have no backing storage. Give OV an owning tensor so the infer request + // has somewhere to write; nothing downstream reads it. + if (output_data == nullptr) { + return ov::Tensor(output_type, output_shape); + } ov::Tensor output_tensor(output_type, output_shape, output_data); return output_tensor; } @@ -267,11 +279,12 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< cache_hit = it != r_ctx->decoder_cache.end(); if (cache_hit) { entry = it->second; + r_ctx->touch_locked(key); } else { - r_ctx->clear_caches_locked(); auto mutex = std::make_shared(); entry = std::make_shared(mutex); r_ctx->decoder_cache[key] = entry; + r_ctx->admit_locked(key); } } else { auto mutex = std::make_shared(); @@ -283,10 +296,21 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< if (cache_hit) { ggml_decoder = entry->ptr; + // The cache key is only (n_nodes, first/last node name), so it cannot see a change in + // WHICH tensors are graph outputs -- and the compiled model's Results are fixed when it is + // built. A consumer that starts requesting extra outputs later (speculative decoding marks + // the target's per-layer residuals only once the draft is initialised) would otherwise keep + // reusing a model that never produces them, and the host would read unwritten buffers as + // zeros. Recompile when the output set differs. old_m_params = ggml_decoder->get_model_params(); if (!ggml_decoder->is_splited_model()) { cache_hit = old_m_params.can_reuse_dynamically(m_params); } + // Must come AFTER can_reuse_dynamically(), which ASSIGNS to cache_hit rather than + // and-ing into it and would otherwise silently undo this invalidation. + if (cache_hit && !ggml_decoder->has_same_graph_io(cgraph)) { + cache_hit = false; + } } std::vector ov_input_names; @@ -296,7 +320,14 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< std::map> model_weights; ggml_decoder->set_compute_params(c_params); ggml_decoder->set_model_params(m_params); - if (old_m_params.kv_buffer_changed(m_params)) { + // The decoder caches a ggml_tensor* for every graph input and output, and m_cgraph + // itself. A cached decoder is keyed only on (n_nodes, first/last node name), so a + // DIFFERENT cgraph instance with the same key can hit it -- speculative decoding + // rebuilds its graphs every step with freshly allocated tensors. Those pointers then + // dangle, and walking them segfaults (get_tensor_used_op iterates m_cgraph->nodes). + // Refresh the IO map whenever the cgraph differs; kv_buffer_changed() alone misses + // this because the KV buffer is unchanged across those passes. + if (ggml_decoder->get_cgraph() != cgraph || old_m_params.kv_buffer_changed(m_params)) { ggml_decoder->update_io(cgraph); } ggml_decoder->add_extra_inputs(); @@ -607,7 +638,20 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< for (size_t i = 0; i < ov_input_names.size(); i++) { auto param_name = ov_input_names[i]; auto input_tensor = get_ov_input_tensor(ggml_decoder, param_name); - infer_request->set_input_tensor(i, input_tensor); + try { + infer_request->set_input_tensor(i, input_tensor); + } catch (const std::exception & e) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DIAG_GRAPH_IO")) { + auto port = infer_request->get_compiled_model().input(i); + GGML_LOG_WARN( + "ggml-openvino: DIAG set_input_tensor[%zu] '%s' threw for n_nodes=%d first=%s last=%s: %s\n" + " model_shape=%s bound_shape=%s\n", + i, param_name.c_str(), key.n_nodes, key.first_node_name.c_str(), key.last_node_name.c_str(), + e.what(), port.get_partial_shape().to_string().c_str(), + input_tensor.get_shape().to_string().c_str()); + } + throw; + } if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { print_input_tensor_info(param_name, input_tensor); @@ -630,11 +674,39 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< continue; } auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); - infer_request->set_output_tensor(i, output_tensor); + try { + infer_request->set_output_tensor(i, output_tensor); + } catch (const std::exception & e) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DIAG_GRAPH_IO")) { + auto port = infer_request->get_compiled_model().output(i); + GGML_LOG_WARN( + "ggml-openvino: DIAG set_output_tensor[%zu] '%s' threw for n_nodes=%d first=%s last=%s: %s\n" + " model_shape=%s bound_shape=%s\n", + i, ov_output_names[i].c_str(), key.n_nodes, key.first_node_name.c_str(), + key.last_node_name.c_str(), e.what(), port.get_partial_shape().to_string().c_str(), + output_tensor.get_shape().to_string().c_str()); + } + throw; + } } ov_raw_infer_start = ggml_time_us(); - infer_request->infer(); + try { + infer_request->infer(); + } catch (const std::exception & e) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DIAG_GRAPH_IO")) { + auto cm = infer_request->get_compiled_model(); + GGML_LOG_WARN("ggml-openvino: DIAG infer() threw for n_nodes=%d first=%s last=%s: %s\n", key.n_nodes, + key.first_node_name.c_str(), key.last_node_name.c_str(), e.what()); + for (size_t i = 0; i < ov_input_names.size(); i++) { + auto port = cm.input(i); + GGML_LOG_WARN(" param[%zu] name=%s model_shape=%s bound_shape=%s\n", i, + ov_input_names[i].c_str(), port.get_partial_shape().to_string().c_str(), + infer_request->get_input_tensor(i).get_shape().to_string().c_str()); + } + } + throw; + } infer_end_time = ggml_time_us(); if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || @@ -729,11 +801,12 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptrdecoder_cache.end(); if (cache_hit) { entry = it->second; + r_ctx->touch_locked(key); } else { - r_ctx->clear_caches_locked(); auto mutex = std::make_shared(); entry = std::make_shared(mutex); r_ctx->decoder_cache[key] = entry; + r_ctx->admit_locked(key); } } else { auto mutex = std::make_shared(); @@ -757,7 +830,14 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptrm_is_prefill = is_prefill; ggml_decoder->set_model_params(m_params); ggml_decoder->set_compute_params(c_params); - if (old_m_params.kv_buffer_changed(m_params)) { + // The decoder caches a ggml_tensor* for every graph input and output, and m_cgraph + // itself. A cached decoder is keyed only on (n_nodes, first/last node name), so a + // DIFFERENT cgraph instance with the same key can hit it -- speculative decoding + // rebuilds its graphs every step with freshly allocated tensors. Those pointers then + // dangle, and walking them segfaults (get_tensor_used_op iterates m_cgraph->nodes). + // Refresh the IO map whenever the cgraph differs; kv_buffer_changed() alone misses + // this because the KV buffer is unchanged across those passes. + if (ggml_decoder->get_cgraph() != cgraph || old_m_params.kv_buffer_changed(m_params)) { ggml_decoder->update_io(cgraph); } ggml_decoder->add_extra_inputs(); @@ -1184,6 +1264,20 @@ ov::Tensor convert_ggml_input_to_ov(std::shared_ptr ggml_decoder, return make_contiguous_split_input_tensor(ggml_decoder, ggml_tensor, input_shape); } + // A graph input may legitimately be left unallocated (data == nullptr, no buffer): llama.cpp + // skips filling the attention mask when a graph only stores K/V without attending, which is what + // a speculative KV-injection pass does. Such an input is never read, but OpenVINO still requires + // a backing tensor for the Parameter and constructing an ov::Tensor over a null pointer asserts. + // Hand OV an owning zero-filled tensor; its contents cannot affect the result because no op + // consumes them. + if (input_data == nullptr) { + GGML_LOG_WARN("ggml-openvino: unallocated INPUT '%s' (ggml name '%s', type %s, ne=[%ld,%ld,%ld,%ld]) " + "-> zero tensor\n", + name.c_str(), ggml_tensor->name, ggml_type_name(ggml_tensor->type), + (long) ggml_tensor->ne[0], (long) ggml_tensor->ne[1], (long) ggml_tensor->ne[2], + (long) ggml_tensor->ne[3]); + return ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); + } auto input_tensor = ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape, input_data); return input_tensor; } diff --git a/ggml/src/ggml-openvino/utils.h b/ggml/src/ggml-openvino/utils.h index 5aa74da38d3b..2f0018f4678c 100644 --- a/ggml/src/ggml-openvino/utils.h +++ b/ggml/src/ggml-openvino/utils.h @@ -1,16 +1,18 @@ #include "ggml-decoder.h" #include "ggml-impl.h" -#include #include #include +#include #include +#include #include #include #include #include #include #include +#include #include #include @@ -18,7 +20,7 @@ struct graph_key { int n_nodes; std::string first_node_name; std::string last_node_name; - std::vector input_src_names; + std::vector input_src_buffers; graph_key(const ggml_cgraph * cgraph) : n_nodes(cgraph->n_nodes) { if (n_nodes > 0) { @@ -26,46 +28,33 @@ struct graph_key { last_node_name = cgraph->nodes[n_nodes - 1]->name; } - auto get_input_key_name = [](const ggml_cgraph * graph, const ggml_tensor * tensor) { - std::string name = tensor->name; - const size_t hash_pos = ggml_hash_find(&graph->visited_hash_set, tensor); - if (((tensor->flags & GGML_TENSOR_FLAG_COMPUTE) || GgmlOvDecoder::is_kvcache(tensor, nullptr)) && - hash_pos != GGML_HASHSET_FULL && ggml_bitset_get(graph->visited_hash_set.used, hash_pos)) { - name += "#" + std::to_string(hash_pos); - } - return name; - }; - - std::vector node_names; - node_names.reserve(cgraph->n_nodes); - for (int node_idx = 0; node_idx < cgraph->n_nodes; node_idx++) { - node_names.emplace_back(cgraph->nodes[node_idx]->name); + std::unordered_set node_set; + node_set.reserve(cgraph->n_nodes); + for (int i = 0; i < cgraph->n_nodes; i++) { + node_set.insert(cgraph->nodes[i]); } for (int node_idx = 0; node_idx < cgraph->n_nodes; node_idx++) { const ggml_tensor * node = cgraph->nodes[node_idx]; for (int src_idx = 0; src_idx < GGML_MAX_SRC; src_idx++) { const ggml_tensor * src = node->src[src_idx]; - if (src == nullptr || src->name[0] == '\0') { + if (src == nullptr || src->buffer == nullptr || node_set.count(src) || src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { continue; } - const std::string src_name = get_input_key_name(cgraph, src); - if (std::find(node_names.begin(), node_names.end(), src_name) != node_names.end()) { - continue; - } - if (src_name.find("weight") != std::string::npos) { - continue; - } + const uintptr_t buf_ptr = reinterpret_cast(src->buffer); + const uintptr_t base = reinterpret_cast(ggml_backend_buffer_get_base(src->buffer)); + const uintptr_t offset = reinterpret_cast(src->data) - base; - input_src_names.push_back(std::to_string(node_idx) + ":" + std::to_string(src_idx) + ":" + src_name); + input_src_buffers.push_back(std::to_string(node_idx) + ":" + std::to_string(src_idx) + ":" + + std::to_string(buf_ptr) + "+" + std::to_string(offset)); } } } bool operator==(const graph_key & other) const { return n_nodes == other.n_nodes && first_node_name == other.first_node_name && - last_node_name == other.last_node_name && input_src_names == other.input_src_names; + last_node_name == other.last_node_name && input_src_buffers == other.input_src_buffers; } }; @@ -76,8 +65,8 @@ struct graph_key_hash { hash ^= std::hash{}(key.first_node_name) + 0x9e3779b9 + (hash << 6) + (hash >> 2); hash ^= std::hash{}(key.last_node_name) + 0x9e3779b9 + (hash << 6) + (hash >> 2); } - for (const auto & input_src_name : key.input_src_names) { - hash ^= std::hash{}(input_src_name) + 0x9e3779b9 + (hash << 6) + (hash >> 2); + for (const auto & input_src_buffer : key.input_src_buffers) { + hash ^= std::hash{}(input_src_buffer) + 0x9e3779b9 + (hash << 6) + (hash >> 2); } return hash; } @@ -99,6 +88,12 @@ struct ov_runtime_context { std::unordered_map, graph_key_hash> infer_request_cache_prefill; std::unordered_map, graph_key_hash> ov_input_names_cache; std::unordered_map, graph_key_hash> ov_output_names_cache; + // LRU order of decoder_cache's keys, oldest (least recently used) at the front. A pipeline + // that interleaves distinct graph shapes every call -- e.g. DFlash cycling target/draft/ + // injection graphs every round -- needs a slot per shape open at once; a single-entry cache + // would evict and recompile all of them, every call, forever. + std::list lru_keys; + static constexpr size_t max_cached_graphs = 8; //TODO: Stateful is only supported for single request at a time. // Simultanous stateful inference request support to be added. size_t stateful_kv_size; @@ -107,12 +102,44 @@ struct ov_runtime_context { ov_runtime_context() : device("CPU"), stateful(false), stateful_kv_size(0), backend_count(0) {} + // Mark `key` as the most recently used entry. Call on every cache hit and after inserting a + // new entry, so eviction always picks the graph shape that has gone longest unused. + void touch_locked(const graph_key & key) { + lru_keys.remove(key); + lru_keys.push_back(key); + } + + // Drop the least recently used graph shape's compiled model and its associated per-key + // caches. Leaves every other shape's compiled model intact. + void evict_lru_locked() { + if (lru_keys.empty()) { + return; + } + graph_key victim = lru_keys.front(); + lru_keys.pop_front(); + decoder_cache.erase(victim); + infer_request_cache.erase(victim); + infer_request_cache_prefill.erase(victim); + ov_input_names_cache.erase(victim); + ov_output_names_cache.erase(victim); + } + + // Make room for one more graph shape if the cache is at capacity, then mark `key` as most + // recently used. Call right after inserting `key` into decoder_cache. + void admit_locked(const graph_key & key) { + while (decoder_cache.size() > max_cached_graphs) { + evict_lru_locked(); + } + touch_locked(key); + } + void clear_caches_locked() { decoder_cache.clear(); infer_request_cache.clear(); infer_request_cache_prefill.clear(); ov_input_names_cache.clear(); ov_output_names_cache.clear(); + lru_keys.clear(); kv_state_input_name_map.clear(); stateful_kv_size = 0; }