Npu support mm bert

ggml/src/ggml-openvino/ggml-decoder.cpp

@@ -883,8 +883,41 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
             }
         }
     }
+    if (model_params.n_heads_kv == -1) {
+        for (int i = 0; i < cgraph->n_nodes; i++) {
+            const auto * node = cgraph->nodes[i];
+            const ggml_tensor * mask = nullptr;
+            if (node->op == GGML_OP_SOFT_MAX) {
+                mask = node->src[1];
+            } else if (node->op == GGML_OP_FLASH_ATTN_EXT) {
+                mask = node->src[3];
+            } else {
+                continue;
+            }
+            if (mask == nullptr || mask->op != GGML_OP_NONE || !(mask->flags & GGML_TENSOR_FLAG_INPUT) ||
+                node->src[0] == nullptr) {
+                continue;
+            }
+            model_params.is_cacheless_attn = true;
+            model_params.n_seq = 1;
+            model_params.ctx_per_seq = mask->ne[0];
+            compute_params.input_len = node->src[0]->ne[1];
+            compute_params.token_len_per_seq = compute_params.input_len;
+            break;
+        }
+    }
+
     auto * output_tensor = cgraph->nodes[cgraph->n_nodes - 1];
     compute_params.output_len = output_tensor->ne[1];
+    if (model_params.is_cacheless_attn) {
+        for (int i = 0; i < cgraph->n_nodes; i++) {
+            const auto * node = cgraph->nodes[i];
+            if (node->op == GGML_OP_GET_ROWS && is_output_idx(node->src[1], node)) {
+                compute_params.output_len = node->src[1]->ne[0];
+                break;
+            }
+        }
+    }
     // for NPU, output_len is always 1 except for llama-perplexity
     if (is_static && compute_params.output_len == 0) {
         compute_params.output_len = 1;
@@ -921,6 +954,10 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
         // output index
         input_shape = ov::PartialShape{1, 1, 1, m_is_static ? m_compute_params.output_len : -1};
 
+    } else if (is_inp_mean(input, op)) {
+        input_shape = m_is_static ? ov::PartialShape{1, 1, input->ne[1], m_prefill_chunk_size} :
+                                    ov::PartialShape{1, 1, -1, -1};
+
     } else if (is_inp_mask(input, op)) {
         // mask
         if (m_is_static) {
@@ -979,8 +1016,14 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
     if (op->op == GGML_OP_SOFT_MAX && op->src[1] != nullptr && op->src[1]->op == GGML_OP_NONE &&
         op->src[1]->flags & GGML_TENSOR_FLAG_INPUT && op->src[1] == input) {
         // for softmax input mask, the shape is [1, 1, seq_active, seq_active], where seq_active is determined by the input active sequence length instead of the kv cache sequence length
-        input_shape[2] = -1;
-        input_shape[3] = -1;
+        if (m_is_static) {
+            const int64_t seq_active = m_is_prefill ? m_prefill_chunk_size : 1;
+            input_shape[2] = seq_active;
+            input_shape[3] = seq_active;
+        } else {
+            input_shape[2] = -1;
+            input_shape[3] = -1;
+        }
     }
     return input_shape;
 }

ggml/src/ggml-openvino/ggml-decoder.h

@@ -30,6 +30,7 @@ struct ModelParams {
     int state_size = -1;  // for SSM molels, eg qwen35
     int32_t rope_params[16];
     bool mixed_rope_params = false;
+    bool is_cacheless_attn = false;
     std::vector<int> swa_layers;
     // The sliding-window mask tensor, identified in compute_llm_params() by grouping attention
     // layers on the mask they consume. Only used to tell the two masks apart when naming OV
@@ -372,6 +373,12 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
                (op->op == GGML_OP_SOFT_MAX && tensor == op->src[1]);
     }
 
+    inline static bool is_inp_mean(const ggml_tensor * tensor, const ggml_tensor * op) {
+        return op->op == GGML_OP_MUL_MAT && tensor == op->src[1] && tensor->op == GGML_OP_NONE &&
+               (tensor->flags & GGML_TENSOR_FLAG_INPUT) && tensor->type == GGML_TYPE_F32 &&
+               op->src[0] != nullptr && op->src[0]->op != GGML_OP_NONE;
+    }
+
     inline static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) {
         return op->op == GGML_OP_ROPE && tensor == op->src[2];
     }

ggml/src/ggml-openvino/ggml-openvino.cpp

@@ -992,6 +992,10 @@ static bool is_supported_flash_attn_pattern(const ggml_tensor * op) {
             if (src->src[0] == nullptr || src->src[0]->view_src != nullptr) {
                 return false;
             }
+        } else if (src->op == GGML_OP_CPY) {
+            if (src->src[0] == nullptr || src->src[0]->op != GGML_OP_PERMUTE || src->src[0]->src[0] == nullptr) {
+                return false;
+            }
         } else {
             return false;
         }
@@ -1357,8 +1361,21 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
         if (op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) {
             return {false, "ROPE with type " + std::string(ggml_type_name(op->type)) + " is not supported"};
         }
-        if (op->view_src != nullptr && !ggml_is_contiguous(op->src[0])) {
-            return {false, "ROPE on VIEW / non-contiguous input is not supported"};
+        if (op->src[0]->op == GGML_OP_VIEW) {
+            const struct ggml_tensor * view = op->src[0];
+            const struct ggml_tensor * view_src = view->view_src;
+            const bool same_shape = view_src->ne[1] == view->ne[1] && view_src->ne[2] == view->ne[2] &&
+                                    view_src->ne[3] == view->ne[3];
+            const bool packed_qkv = view_src->ne[1] == view->ne[2] && view_src->ne[2] == view->ne[3];
+            if (!same_shape && !packed_qkv) {
+                return {false, "ROPE with view_src->ne [" + std::to_string(view_src->ne[1]) + ", " +
+                               std::to_string(view_src->ne[2]) + ", " + std::to_string(view_src->ne[3]) +
+                               "] != view->ne [" + std::to_string(view->ne[1]) + ", " +
+                               std::to_string(view->ne[2]) + ", " + std::to_string(view->ne[3]) +
+                               "] is not supported"};
+            }
+                    } else if (!ggml_is_contiguous(op->src[0])) {
+                        return {false, "ROPE on non-contiguous input is not supported"};
         }
         float freq_scale;
         float ext_factor;

ggml/src/ggml-openvino/openvino/node_context.h

@@ -143,6 +143,10 @@ class NodeContext : public frontend::NodeContext {
 
     bool has_input(const std::string & name) const { return m_tensor_map->find(name) != m_tensor_map->end(); }
 
+    void put_shared(const std::string & name, const Output<Node> & value) const {
+        m_tensor_map->insert({name, value});
+    }
+
     const std::string & get_name() const override { return m_decoder->get_op_name(m_node_idx); }
 
     ov::Any get_attribute_as_any(const std::string & name) const override { return m_decoder->get_attribute(name); }

ggml/src/ggml-openvino/openvino/op/norm.cpp

@@ -3,14 +3,8 @@
 #include "../utils.h"
 
 #include <memory>
-#include <openvino/op/add.hpp>
 #include <openvino/op/constant.hpp>
-#include <openvino/op/divide.hpp>
-#include <openvino/op/multiply.hpp>
-#include <openvino/op/power.hpp>
-#include <openvino/op/reduce_mean.hpp>
-#include <openvino/op/sqrt.hpp>
-#include <openvino/op/subtract.hpp>
+#include <openvino/op/mvn.hpp>
 
 namespace ov {
 namespace frontend {
@@ -21,33 +15,11 @@ OutputVector translate_norm(const NodeContext & context) {
     num_inputs_check(context, 1, 1);
 
     auto input_node = process_view_input_new(context, 0);
-
-    // Step 1: Calculate mean along the last dimension
-    // mean = reduce_mean(input, axis=-1, keepdims=true)
-    auto mean = std::make_shared<ov::op::v1::ReduceMean>(
-        input_node, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true);
-
-    // Step 2: Calculate (input - mean)
-    auto centered = std::make_shared<ov::op::v1::Subtract>(input_node, mean);
-
-    // Step 3: Calculate squared differences (input - mean)^2
-    auto squared = std::make_shared<ov::op::v1::Power>(
-        centered, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {2.0f}));
-
-    // Step 4: Calculate variance = mean((input - mean)^2)
-    auto variance = std::make_shared<ov::op::v1::ReduceMean>(
-        squared, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true);
-
-    // Step 5: Get epsilon from op_params
     float eps;
     memcpy(&eps, context.get_output_op_params(), sizeof(float));
 
-    // Step 6: Calculate std = sqrt(variance + eps)
-    auto std_dev = std::make_shared<ov::op::v0::Sqrt>(std::make_shared<ov::op::v1::Add>(
-        variance, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {eps})));
-
-    // Step 7: Normalize: output = (input - mean) / std
-    auto res = std::make_shared<ov::op::v1::Divide>(centered, std_dev);
+    auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1});
+    auto res = std::make_shared<ov::op::v6::MVN>(input_node, axes, true, eps, ov::op::MVNEpsMode::INSIDE_SQRT);
 
     return rename_outputs_with_suffix({res}, context.get_name());
 }

ggml/src/ggml-openvino/openvino/op/rope.cpp

@@ -56,14 +56,28 @@ OutputVector translate_rope(const NodeContext & context) {
         cos_theta_node = context.get_input("rope_cos");
         sin_theta_node = context.get_input("rope_sin");
     } else {
-        auto inp_pos = context.get_input(1).get_node_shared_ptr();
-        std::shared_ptr<ov::Node> rope_freqs_weight;
+        std::string cache_key = "rope_sin_cos";
+        for (int i = 0; i < 15; i++) {
+            cache_key += "_" + std::to_string(op_params[i]);
+        }
         if (context.get_input_size() == 3) {
-            rope_freqs_weight = context.get_input(2).get_node_shared_ptr();
+            cache_key += "_ff_" + context.get_input_names()[2];
+        }
+        if (context.has_input(cache_key + "_cos")) {
+            cos_theta_node = context.get_input(cache_key + "_cos");
+            sin_theta_node = context.get_input(cache_key + "_sin");
+        } else {
+            auto inp_pos = context.get_input(1).get_node_shared_ptr();
+            std::shared_ptr<ov::Node> rope_freqs_weight;
+            if (context.get_input_size() == 3) {
+                rope_freqs_weight = context.get_input(2).get_node_shared_ptr();
+            }
+            auto sin_cos = make_sin_cos(op_params, inp_pos, rope_freqs_weight, mode == TYPE_IMROPE, false);
+            sin_theta_node = sin_cos.first;
+            cos_theta_node = sin_cos.second;
+            context.put_shared(cache_key + "_cos", cos_theta_node);
+            context.put_shared(cache_key + "_sin", sin_theta_node);
         }
-        auto sin_cos = make_sin_cos(op_params, inp_pos, rope_freqs_weight, mode == TYPE_IMROPE, false);
-        sin_theta_node = sin_cos.first;
-        cos_theta_node = sin_cos.second;
     }
 
     if (context.get_view_input_size(0) > 0) {

ggml/src/ggml-openvino/utils.cpp

@@ -675,6 +675,17 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr<
     return GGML_STATUS_SUCCESS;
 }
 
+static ov::AnyMap without_npuw(const ov::AnyMap & config) {
+    ov::AnyMap out;
+    for (const auto & kv : config) {
+        if (kv.first.rfind("NPUW", 0) == 0 || kv.first == "NPU_USE_NPUW") {
+            continue;
+        }
+        out.insert(kv);
+    }
+    return out;
+}
+
 enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<ov_runtime_context> r_ctx) {
     auto & core = ov_singleton_core();
 
@@ -708,7 +719,12 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
     std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static);
 
     const auto * inp_pos = get_inp_pos_tensor(cgraph);
-    const auto is_prefill = get_is_prefill(cgraph, inp_pos);
+    const bool no_kv_cache = m_params.is_cacheless_attn;
+    const auto is_prefill = no_kv_cache ? true : get_is_prefill(cgraph, inp_pos);
+    const ov::AnyMap compile_config = no_kv_cache ? without_npuw(config) : config;
+    if (m_params.n_heads_kv == -1) {
+        prefill_chunk_size = inp_pos->ne[0];
+    }
     graph_key key(cgraph);
     static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE");
     bool cache_hit = false;
@@ -782,20 +798,18 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
         std::shared_ptr<ov::Model> model;
         auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph);
 
-        if (m_params.n_heads_kv == -1) {
-            // graph is not a LLM, e.g. context-shift graph
-            prefill_chunk_size = inp_pos->ne[0];
-        }
         auto ggml_decoder_prefill = std::make_shared<GgmlOvDecoder>(
             cgraph, m_params, c_params, model_weights, is_static, stateful, false, true, prefill_chunk_size);
-        auto ggml_decoder_decode = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static,
-                                                                   stateful, false, false, prefill_chunk_size);
+        auto ggml_decoder_decode =
+            no_kv_cache ? ggml_decoder_prefill :
+                          std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, model_weights, is_static,
+                                                          stateful, false, false, prefill_chunk_size);
         decoder_end_time = ggml_time_us();
 
         const bool dump_ir = ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR");
         const auto dump_ir_timestamp = static_cast<long long>(ggml_time_us());
 
-        auto build_static_model = [&core, &config, dump_ir, dump_ir_timestamp](
+        auto build_static_model = [&core, &compile_config, dump_ir, dump_ir_timestamp](
                           std::shared_ptr<GgmlOvDecoder> decoder,
                           const char * tag,
                           std::shared_ptr<ov::Model> & model,
@@ -815,7 +829,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
                 ov::serialize(model, timestamped_filename);
             }
 
-            compiled_model = core.compile_model(model, device, config);
+            compiled_model = core.compile_model(model, device, compile_config);
             infer_request = std::make_shared<ov::InferRequest>(compiled_model.create_infer_request());
             local_compile_end_time = ggml_time_us();
         };
@@ -829,16 +843,18 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr<o
         int64_t decode_conversion_end_time;
         int64_t prefill_compile_end_time;
         int64_t decode_compile_end_time;
-        auto prefill_future = std::async(std::launch::async, build_static_model, ggml_decoder_prefill, "prefill",
-                                         std::ref(model_prefill), std::ref(compiled_model_prefill),
-                                         std::ref(infer_request_prefill), std::ref(prefill_conversion_end_time),
-                                         std::ref(prefill_compile_end_time));
-        auto decode_future = std::async(std::launch::async, build_static_model, ggml_decoder_decode, "decode",
-                                        std::ref(model_decode), std::ref(compiled_model_decode),
-                                        std::ref(infer_request_decode), std::ref(decode_conversion_end_time),
-                                        std::ref(decode_compile_end_time));
-        prefill_future.get();
-        decode_future.get();
+        build_static_model(ggml_decoder_prefill, "prefill", model_prefill, compiled_model_prefill,
+                   infer_request_prefill, prefill_conversion_end_time, prefill_compile_end_time);
+        if (no_kv_cache) {
+            model_decode = model_prefill;
+            compiled_model_decode = compiled_model_prefill;
+            infer_request_decode = infer_request_prefill;
+            decode_conversion_end_time = prefill_conversion_end_time;
+            decode_compile_end_time = prefill_compile_end_time;
+        } else {
+            build_static_model(ggml_decoder_decode, "decode", model_decode, compiled_model_decode, infer_request_decode,
+                       decode_conversion_end_time, decode_compile_end_time);
+        }
         conversion_end_time = std::max(prefill_conversion_end_time, decode_conversion_end_time);
         compile_end_time = std::max(prefill_compile_end_time, decode_compile_end_time);
 
@@ -1352,6 +1368,20 @@ ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr<GgmlOvDecoder> ggm
         return input_tensor;
     }
 
+    if (GgmlOvDecoder::is_inp_mean(ggml_tensor, op)) {
+        const size_t n_seqs = ggml_tensor->ne[1];
+        const size_t src_stride = ggml_tensor->ne[0];
+        const size_t copy_len = std::min<size_t>(chunk_valid_size, src_stride - chunk_index * chunk_size);
+        ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, n_seqs, chunk_size});
+        auto * dst = input_tensor.data<float>();
+        std::fill(dst, dst + n_seqs * chunk_size, 0.0f);
+        const auto * src = static_cast<const float *>(ggml_tensor->data) + chunk_index * chunk_size;
+        for (size_t s = 0; s < n_seqs; s++) {
+            std::memcpy(dst + s * chunk_size, src + s * src_stride, copy_len * sizeof(float));
+        }
+        return input_tensor;
+    }
+
     if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) {
         size_t cols = ggml_tensor->ne[0];
         size_t rows = ggml_tensor->ne[1];