diff --git a/.devops/openvino.Dockerfile b/.devops/openvino.Dockerfile
index a43e5c4993f8..13301ba287dd 100644
--- a/.devops/openvino.Dockerfile
+++ b/.devops/openvino.Dockerfile
@@ -1,12 +1,12 @@
-ARG OPENVINO_VERSION_MAJOR=2026.3
-ARG OPENVINO_VERSION_FULL=2026.3.0.22451.bd8d6542e3c
+ARG OPENVINO_VERSION_MAJOR=2026.3.1
+ARG OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d
ARG UBUNTU_VERSION=24.04
# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
-ARG IGC_VERSION=v2.38.2
-ARG IGC_VERSION_FULL=2_2.38.2+22051
-ARG COMPUTE_RUNTIME_VERSION=26.27.39122.11
-ARG COMPUTE_RUNTIME_VERSION_FULL=26.27.39122.11-0
+ARG IGC_VERSION=v2.40.13
+ARG IGC_VERSION_FULL=2_2.40.13+22418
+ARG COMPUTE_RUNTIME_VERSION=26.31.39395.13
+ARG COMPUTE_RUNTIME_VERSION_FULL=26.31.39395.13-0
ARG IGDGMM_VERSION=22.10.0
# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml
index 187427a8d4b0..4a23ec2d4d36 100644
--- a/.github/workflows/build-cache.yml
+++ b/.github/workflows/build-cache.yml
@@ -41,8 +41,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3"
- OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
@@ -69,8 +69,8 @@ jobs:
env:
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3"
- OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml
index 0316e7ad97e3..8e0326f4a44f 100644
--- a/.github/workflows/build-openvino.yml
+++ b/.github/workflows/build-openvino.yml
@@ -32,6 +32,8 @@ env:
LLAMA_ARG_LOG_COLORS: 1
LLAMA_ARG_LOG_PREFIX: 1
LLAMA_ARG_LOG_TIMESTAMPS: 1
+ # TODO: fix and re-enable the `test-llama-archs` and `test-recurrent-state-rollback`
+ CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-rollback"
jobs:
ubuntu-24-openvino:
@@ -39,8 +41,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3"
- OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
@@ -78,26 +80,24 @@ jobs:
- name: Test (CPU)
id: cmake_test_cpu
- # TODO: fix and re-enable the `test-llama-archs` test below
run: |
cd ${{ github.workspace }}
- ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" --verbose --timeout 2000
+ ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
- name: Test (GPU)
id: cmake_test_gpu
- # TODO: fix and re-enable the `test-llama-archs` test below
run: |
cd ${{ github.workspace }}
export GGML_OPENVINO_DEVICE=GPU
- ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" --verbose --timeout 3000
+ ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
openvino-windows-2022:
runs-on: windows-2022
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3"
- OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
@@ -159,14 +159,13 @@ jobs:
- name: Test (CPU)
id: cmake_test_cpu
shell: cmd
- # TODO: fix and re-enable the `test-llama-archs` test below
run: |
REM Find extracted OpenVINO folder dynamically
for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i
call "%OPENVINO_ROOT%\setupvars.bat"
cd build
- ctest --test-dir ReleaseOV -L main -E "test-llama-archs|test-recurrent-state-rollback-nemotron-h" -C Release --verbose --timeout 3000
+ ctest --test-dir ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" -C Release --verbose --timeout 3000
- name: ccache-clear
uses: ./.github/actions/ccache-clear
diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml
index fe2ab815473c..ccfe2a604645 100644
--- a/.github/workflows/build-self-hosted.yml
+++ b/.github/workflows/build-self-hosted.yml
@@ -288,8 +288,8 @@ jobs:
env:
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3"
- OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Clone
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 98250c860650..76717d064bd0 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -415,8 +415,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3"
- OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Set OpenVINO version output
@@ -529,8 +529,8 @@ jobs:
env:
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
- OPENVINO_VERSION_MAJOR: "2026.3"
- OPENVINO_VERSION_FULL: "2026.3.0.22451.bd8d6542e3c"
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
steps:
- name: Set OpenVINO version output
diff --git a/ci/run.sh b/ci/run.sh
index 1f1e4bc033c9..1701bc7ed058 100755
--- a/ci/run.sh
+++ b/ci/run.sh
@@ -189,8 +189,8 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
fi
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
- # TODO: fix and re-enable the `test-llama-archs` test below
- CTEST_EXTRA="-E test-llama-archs|test-recurrent-state-rollback-nemotron-h"
+ # TODO: fix and re-enable the `test-llama-archs` and `test-recurrent-state-rollback*`
+ CTEST_EXTRA="-E test-llama-archs|^test-recurrent-state-rollback"
fi
## helpers
diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md
index 3cdf631cebc2..c1e39c5bf153 100644
--- a/docs/backend/OPENVINO.md
+++ b/docs/backend/OPENVINO.md
@@ -22,8 +22,8 @@ The OpenVINO backend is implemented in `ggml/src/ggml-openvino` and provides a t
- [0. Prerequisites](#0-prerequisites)
- [1. Install OpenVINO Runtime](#1-install-openvino-runtime)
- [2. Build llama.cpp with OpenVINO Backend](#2-build-llamacpp-with-openvino-backend)
- - [Automated Ubuntu Build Script](#automated-ubuntu-build-script)
- - [Automated Windows Build Script](#automated-windows-build-script)
+ - [Ubuntu Build Script](#ubuntu-build-script)
+ - [Windows Build Script](#windows-build-script)
- [3. Download Sample Model](#3-download-sample-model)
- [4. Run Inference with OpenVINO Backend](#4-run-inference-with-openvino-backend)
- [5. Docker Build](#5-docker-build)
@@ -96,7 +96,7 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
- **SL** = Stateless (`GGML_OPENVINO_STATEFUL_EXECUTION=0`)
- **SF** = Stateful (`GGML_OPENVINO_STATEFUL_EXECUTION=1`)
- Note: The NPU operates in stateless mode only.
-- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel OpenCL GPU Driver 26.18.38308.1 | Intel NPU Driver 1.33.0.
+- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel OpenCL GPU Driver 26.31.39395.13-0 | Intel NPU Driver 1.35.0.
- See [Known Limitations](#known-limitations) for context on observed failures.
| Model | CPU (SL / SF) | GPU (SL / SF) | NPU (SL) |
@@ -105,27 +105,32 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
| [bartowski/Llama-3.2-3B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [bartowski/Meta-Llama-3.1-8B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
-| [Qwen/qwen2.5-1.5b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [Qwen/qwen2.5-coder-7b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [bartowski/Qwen_Qwen3-0.6B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-0.6B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ / ✓ | ✓ / ✗ | ✓ |
+| [Qwen/qwen2.5-1.5b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [Qwen/qwen2.5-coder-7b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/Qwen_Qwen3-0.6B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-0.6B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/Qwen_Qwen3.5-0.8B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-0.8B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
+| [bartowski/Qwen_Qwen3.5-2B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-2B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
+| [bartowski/Qwen_Qwen3.5-4B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-4B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
+| [lmstudio-community/Qwen3.5-9B-Q4_K_M](https://huggingface.co/lmstudio-community/Qwen3.5-9B-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
| | | | |
-| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
+| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
-| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ |
+| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ |
| | | | |
-| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
+| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/microsoft_Phi-4-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/microsoft_Phi-4-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [bartowski/Mistral-7B-Instruct-v0.3-Q4_K_M](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [QuantFactory/Ministral-3b-instruct.Q4_K_M](https://huggingface.co/QuantFactory/Ministral-3b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| [bartowski/Ministral-8B-Instruct-2410-Q4_K_M](https://huggingface.co/bartowski/Ministral-8B-Instruct-2410-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [bartowski/DeepSeek-R1-Distill-Llama-8B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
-| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
+| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ / ✓ | ✗ / ✗ | ✓ |
| [ibm-granite/granite-4.0-micro-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
@@ -133,10 +138,10 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
| [ibm-research/granite-3.2-8b-instruct-Q4_K_M](https://huggingface.co/ibm-research/granite-3.2-8b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [HuggingFaceTB/smollm2-1.7b-instruct-q4_k_m](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
-| [openbmb/MiniCPM-V-2_6-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [bartowski/tencent_Hunyuan-7B-Instruct-Q4_K_M](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-Q4_K_M](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
-| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ |
+| [openbmb/MiniCPM-V-2_6-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/tencent_Hunyuan-7B-Instruct-Q4_K_M](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-Q4_K_M](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
+| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
| | | | |
| [gpustack/bge-m3-Q4_K_M.gguf](https://huggingface.co/gpustack/bge-m3-GGUF) | ✓ | ✗ | ✗ |
@@ -217,18 +222,18 @@ cmake --build build\ReleaseOV --parallel
> [!NOTE]
> The Windows install path is `C:\Intel\openvino` (no spaces) to avoid quoting problems some CMake/Ninja toolchains have with `C:\Program Files (x86)\...`. Adjust to wherever you installed OpenVINO Runtime. From `cmd`, run `C:\Intel\openvino\setupvars.bat`; from PowerShell, run `& "C:\Intel\openvino\setupvars.ps1"` instead. Once the build is finished you can launch the binaries from any `cmd` or `PowerShell` window after sourcing the matching `setupvars` script for that shell.
-#### Automated Ubuntu Build Script
+#### Ubuntu Build Script
For Ubuntu24 users, the following shell script automates the prerequisite installs (build tools, OpenCL ICD), the OpenVINO Runtime download/extract/setup, and the Ninja-based llama.cpp build.
-Save the following as `ubuntu-llamacpp-ov-install.sh` next to where you want the `llama.cpp` folder to land, then run it:
+Save the following as `build-llamacpp-ov.sh` next to where you want the `llama.cpp` folder to land, then run it:
```bash
-chmod +x ubuntu-llamacpp-ov-install.sh
-./ubuntu-llamacpp-ov-install.sh
+chmod +x build-llamacpp-ov.sh
+./build-llamacpp-ov.sh
```
-Click to expand ubuntu-llamacpp-ov-install.sh
+Click to expand build-llamacpp-ov.sh
```bash
#!/usr/bin/env bash
@@ -237,8 +242,8 @@ chmod +x ubuntu-llamacpp-ov-install.sh
# ============================================
set -euo pipefail
-OPENVINO_VERSION_MAJOR="2026.3"
-OPENVINO_VERSION_FULL="2026.3.0.22451.bd8d6542e3c"
+OPENVINO_VERSION_MAJOR="2026.3.1"
+OPENVINO_VERSION_FULL="2026.3.1.22476.56d9685302d"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
OPENVINO_INSTALL_DIR="/opt/intel/openvino_${OPENVINO_VERSION_MAJOR}"
@@ -313,8 +318,9 @@ fi
echo "============================================"
echo "Configuring with CMake..."
echo "============================================"
-# shellcheck disable=SC1091
+set +u
source "${OPENVINO_ROOT}/setupvars.sh"
+set -u
cmake -B build/ReleaseOV -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
@@ -334,27 +340,27 @@ echo " ./build/ReleaseOV/bin/llama-cli -m model.gguf"
```
> [!NOTE]
-> The script pins OpenVINO `2026.3` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release.
+> The script pins OpenVINO `2026.3.1` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release.
-#### Automated Windows Build Script
+#### Windows Build Script
For Windows users, the following `.bat` script automates the prerequisite installs (Git, Ninja, CMake, Visual Studio 2022 Build Tools, vcpkg + OpenCL), the OpenVINO Runtime download/extract, and the Ninja-based llama.cpp build.
-Save the following as `windows-llamacpp-ov-install.bat` next to where you want the `llama.cpp` to land, then run it from either **Command Prompt** or **PowerShell**:
+Save the following as `build-llamacpp-ov.bat` next to where you want the `llama.cpp` to land, then run it from either **Command Prompt** or **PowerShell**:
```cmd
:: Command Prompt
-windows-llamacpp-ov-install.bat
+build-llamacpp-ov.bat
```
```powershell
# PowerShell
-.\windows-llamacpp-ov-install.bat
+.\build-llamacpp-ov.bat
```
-Click to expand windows-llamacpp-ov-install.bat
+Click to expand build-llamacpp-ov.bat
```bat
@echo off
@@ -364,8 +370,8 @@ REM ============================================
REM llama.cpp OpenVINO Build Script (Ninja)
REM ============================================
-set "OPENVINO_VERSION_MAJOR=2026.3"
-set "OPENVINO_VERSION_FULL=2026.3.0.22451.bd8d6542e3c"
+set "OPENVINO_VERSION_MAJOR=2026.3.1"
+set "OPENVINO_VERSION_FULL=2026.3.1.22476.56d9685302d"
set "SCRIPT_DIR=%~dp0"
set "VCPKG_DIR=C:\vcpkg"
@@ -453,9 +459,6 @@ if exist "%OPENVINO_INSTALL_DIR%\setupvars.bat" (
)
REM Move the single top-level folder contents into the versioned install dir.
- REM NOTE: delayed expansion (!VAR!) is required because the surrounding else( ... )
- REM block is parsed once up-front, so %OPENVINO_EXTRACTED% would expand to "" here
- REM and xcopy would then treat "\*" as C:\* and fail with "Cannot perform a cyclic copy".
set "OPENVINO_EXTRACTED="
for /d %%i in ("%OPENVINO_EXTRACT_TMP%\*") do set "OPENVINO_EXTRACTED=%%i"
if not defined OPENVINO_EXTRACTED (
@@ -547,7 +550,7 @@ endlocal
```
> [!NOTE]
-> The script pins OpenVINO `2026.3` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**.
+> The script pins OpenVINO `2026.3.1` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**.
@@ -712,22 +715,28 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. `
| `GGML_OPENVINO_CACHE_DIR` | String | `not set` | Directory for OpenVINO model caching (recommended: `/tmp/ov_cache`). Enables model caching when set. **Not supported on NPU devices.** |
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` | String | `not set` | Directory for the frontend compiled-model cache. When set, OpenVINO compiled models are exported as blobs and imported on later runs to skip weight requantization, graph conversion, and compilation for matching single-graph models. |
| `GGML_OPENVINO_PREFILL_CHUNK_SIZE`| Integer | `256` | Token chunk size for **NPU** prefill (NPU-only; ignored on CPU/GPU). Must be a positive integer; otherwise the default is used. |
+| `GGML_OPENVINO_NPU_COMPILE_CONFIG` | String | `not set` | NPU-only compiler mode parameters forwarded to OpenVINO as `NPU_COMPILATION_MODE_PARAMS`, for example `optimization-level=3`. |
| `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Enable stateful KV cache for better performance. Recommended on CPU, GPU. |
| `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. |
| `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. |
+| `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT` | Boolean | `0` | Disable the stateful KV-state sequence-axis relayout (relayout is on by default). It moves the KV state sequence axis from dim 1 to dim 2, so the GPU plugin can append new tokens in place instead of copying the whole state every token, and the reader side no longer transposes the whole accumulated state. Set to `1` to disable. |
| `GGML_OPENVINO_MANUAL_GQA_ATTN` | Boolean | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. |
| `GGML_OPENVINO_MEMORY_OPTIMIZE` | Boolean | `0` | Umbrella switch for compile-time memory reductions. Enables `GGML_OPENVINO_REDUCE_COMPILE_MEM` and, on GPU, `GGML_OPENVINO_RELEASE_WEIGHTS` unless those fine-grained variables are explicitly set. |
| `GGML_OPENVINO_REDUCE_COMPILE_MEM`| Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` | Reduce compile-time host memory use by streaming weight requantization and avoiding extra weight-node materialization where possible. Set explicitly to override the umbrella switch. |
| `GGML_OPENVINO_RELEASE_WEIGHTS` | Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` on GPU | GPU-only. Release host weight buffers after the compiled model cache can reuse the device/plugin copy. Requires stable graph shapes; dynamic workloads that need recompilation should leave this disabled. |
+| `GGML_OPENVINO_SPILL_DIR` | String | `not set` | Directory for a disk-backed weight buffer. When set, the repacked weight buffer is mapped from an unlinked file on this path instead of anonymous memory, so its pages are reclaimable under memory pressure instead of staying pinned, cutting the load-time host memory peak. Must point at real storage; a tmpfs mount (e.g. `/tmp` on many systems) backs it with RAM and makes the peak worse. |
+| `GGML_OPENVINO_REQUANT_KQUANT` | String | `not set` | Requantize Q6_K/Q5_K weights (and matching MoE expert weights) to a 4-bit target instead of the default Q8_0_C, trading accuracy for less memory traffic. One of `q4_sym128` (Q6_K/Q5_K only), `q4_sym128_all` (Q4_K too, drops its per-group zero point), `q4_asym64_all` (Q6_K/Q5_K/Q4_K, keeps a real zero point at group 64), or `native` (no requantization). |
| `GGML_OPENVINO_PROFILING` | Boolean | `0` | Enable execution-time profiling. |
| `GGML_OPENVINO_DUMP_CGRAPH` | Boolean | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. |
| `GGML_OPENVINO_DUMP_IR` | Boolean | `0` | Serialize OpenVINO IR files with timestamps. |
| `GGML_OPENVINO_DEBUG_INPUT` | Boolean | `0` | Enable input debugging and print input tensor info. |
| `GGML_OPENVINO_DEBUG_OUTPUT` | Boolean | `0` | Enable output debugging and print output tensor info. |
| `GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS` | Boolean | `0` | Print tensor address map once. |
+| `GGML_OPENVINO_LOG_UNSUPPORTED_OPS`| Boolean | `0` | Log warning messages with tensor details and rejection reasons for any ops not supported by the OpenVINO backend. Emits at `WARN` level (requires `--log-verbosity >= 2`, enabled by default). |
> [!NOTE]
->`GGML_OPENVINO_STATEFUL_EXECUTION` is an **Experimental** feature to allow stateful execution for managing the KV cache internally inside the OpenVINO model, improving performance on CPUs and GPUs. Stateful execution is not effective on NPUs, and not all models currently support this feature. This feature is experimental and has been validated only with the llama-simple, llama-cli, llama-bench, and llama-run applications and is recommended to enable for the best performance. Other applications, such as llama-server and llama-perplexity, are not yet supported.
+> - `GGML_OPENVINO_STATEFUL_EXECUTION` is an **Experimental** feature to allow stateful execution for managing the KV cache internally inside the OpenVINO model, improving performance on CPUs and GPUs. Stateful execution is not effective on NPUs, and not all models currently support this feature. This feature is experimental and has been validated only with the llama-simple, llama-cli, llama-bench, and llama-run applications and is recommended to enable for the best performance. Other applications, such as llama-server and llama-perplexity, are not yet supported.
+> - `GGML_OPENVINO_LOG_UNSUPPORTED_OPS` emits logs at `WARN` level (`GGML_LOG_WARN`), which requires application log verbosity `--log-verbosity >= 2` (or `-lv 2`).
### Example Usage
diff --git a/ggml/src/ggml-openvino/CMakeLists.txt b/ggml/src/ggml-openvino/CMakeLists.txt
index cc089b721fc3..af3e0758ca2f 100644
--- a/ggml/src/ggml-openvino/CMakeLists.txt
+++ b/ggml/src/ggml-openvino/CMakeLists.txt
@@ -1,6 +1,8 @@
find_package(OpenVINO REQUIRED COMPONENTS Runtime Threading)
find_package(OpenCL REQUIRED)
+message(STATUS "Found OpenVINO: ${OpenVINO_DIR} (found version \"${OpenVINO_VERSION}\")")
+
file(GLOB_RECURSE GGML_HEADERS_OPENVINO "*.h" "*.hpp")
file(GLOB_RECURSE GGML_SOURCES_OPENVINO "*.cpp")
diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp
index 599f41aebbdc..3966c5a56e5a 100644
--- a/ggml/src/ggml-openvino/ggml-decoder.cpp
+++ b/ggml/src/ggml-openvino/ggml-decoder.cpp
@@ -128,6 +128,12 @@ bool is_conv_state_writeback(const ggml_tensor * node) {
node->src[1]->view_src == node->view_src;
}
+bool is_full_single_slot_writeback(const ggml_tensor * node) {
+ return node->view_src != nullptr && node->view_src->ne[1] == 1 && node->src[1] != nullptr &&
+ node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src &&
+ node->src[1]->view_offs == 0 && ggml_nbytes(node->src[1]) == ggml_nbytes(node->view_src);
+}
+
// MoE expert aggregation (build_moe_ffn in llama-graph.cpp): each expert plane is
// `ggml_view_2d(experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1])` and the planes
// are summed with a chain of ADDs: moe_out = ((view_0 + view_1) + view_2) + ... + view_{n-1}.
@@ -198,8 +204,20 @@ static std::string get_tensor_graph_input_ov_name(const GgmlOvDecoder * decoder,
if (GgmlOvDecoder::is_inp_emb(tensor, op)) {
return "embd";
}
- if (decoder->is_stateful() && GgmlOvDecoder::is_inp_mask(tensor, op)) {
- return std::string(tensor->name).find("swa") == std::string::npos ? "self_kq_mask" : "self_kq_mask_swa";
+ if (GgmlOvDecoder::is_inp_mask(tensor, op)) {
+ // Give the two attention masks distinct OV parameter names. build_attn_inp_kq_mask()
+ // names the full-attention mask and the sliding-window mask identically, so keying a
+ // parameter off the name alone makes the second mask overwrite the first and both
+ // attention types read one parameter. Tell them apart by tensor identity, using the
+ // SWA classification computed in compute_llm_params(). An empty swa_layers set means
+ // there is only one mask in play and the plain name is correct.
+ const bool is_swa = decoder->is_swa_mask(tensor);
+ if (decoder->is_stateful()) {
+ return is_swa ? "self_kq_mask_swa" : "self_kq_mask";
+ }
+ if (is_swa) {
+ return get_tensor_ov_name(cgraph, tensor) + "_swa";
+ }
}
return get_tensor_ov_name(cgraph, tensor);
}
@@ -330,11 +348,12 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
if (node->src[1]->op == GGML_OP_VIEW) {
// GET_ROWS gathering recurrent state cache rows via the inp->s_copy index list:
// src[0] is a reshape of cache_r/cache_s, src[1] is a view of the s_copy leaf.
- // op_case 3: main view (active sequences, view offset 0)
- // op_case 4: extra view (defrag remainder, nonzero view offset)
+ // op_case 1/2: active/extra rows of a multi-slot cache
+ // op_case 3/4: active/extra rows of a single-slot cache
if (node->src[0]->op == GGML_OP_RESHAPE && node->src[0]->src[0] != nullptr &&
is_kvcache(node->src[0]->src[0], nullptr)) {
- op_case = node->src[1]->view_offs == 0 ? 1 : 2;
+ const bool single_slot = node->src[0]->src[0]->ne[1] == 1;
+ op_case = (node->src[1]->view_offs == 0 ? 1 : 2) + (single_slot ? 2 : 0);
}
}
break;
@@ -357,6 +376,24 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
break;
}
case GGML_OP_VIEW: {
+ if (!m_model_params.has_rs_rollback && node->src[0] != nullptr &&
+ node->src[0]->op == GGML_OP_GATED_DELTA_NET) {
+ // The GDN translator publishes native attention/state outputs under these VIEW names.
+ op_case = 2;
+ break;
+ }
+ if (m_is_static && node->src[0] != nullptr &&
+ (node->src[0]->op == GGML_OP_GATED_DELTA_NET || node->src[0]->op == GGML_OP_CONCAT)) {
+ // VIEW slicing a GATED_DELTA_NET combined [attn|state] output, or the conv_input
+ // CONCAT. The consuming CPY/RMS_NORM op recovers the true window at runtime via
+ // ssm_state_size / the fixed conv kernel width, so this VIEW must stay an identity
+ // pass-through of the full source here too (it already is on the dynamic path);
+ // otherwise the generic static-mode Slice below would bake in the *captured*
+ // cgraph's token count, which is wrong once the compiled static model runs with a
+ // different token count (prefill chunk size or 1).
+ op_case = 1;
+ break;
+ }
if (node->src[0]->op == GGML_OP_VIEW) {
auto * src = node->src[0];
if (ggml_nelements(node) != ggml_nelements(src)) {
@@ -402,18 +439,45 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
if (node->src[0]->op == GGML_OP_VIEW) {
if (is_same_shape(node->src[0]->src[0], node->src[0])) {
op_case = 1;
+ } else if (!m_model_params.has_rs_rollback &&
+ node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) {
+ // GDN attention is routed directly to this VIEW by get_output_names().
+ op_case = 3;
} else if (node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) {
op_case = 2;
}
}
break;
}
+ case GGML_OP_POOL_2D: {
+ const ggml_op_pool pool_mode = static_cast(node->op_params[0]);
+ switch (pool_mode) {
+ case GGML_OP_POOL_MAX: {
+ op_case = 1;
+ break;
+ }
+ case GGML_OP_POOL_AVG: {
+ op_case = 2;
+ break;
+ }
+ default:
+ op_case = 0;
+ break;
+ }
+ break;
+ }
case GGML_OP_CPY: {
if (node->src[0]->op == GGML_OP_VIEW) {
if (node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) {
- op_case = 1;
+ if (!m_model_params.has_rs_rollback) {
+ // op_case 7 replaces a single-slot cache; op_case 10 writes native GDN state
+ // into an active range of a larger non-rollback cache.
+ op_case = is_full_single_slot_writeback(node) ? 7 : 10;
+ } else {
+ op_case = 1;
+ }
} else if (is_conv_state_writeback(node)) {
- op_case = 2;
+ op_case = is_full_single_slot_writeback(node) ? 8 : 2;
break;
} else if (is_conv_states_all_tensor(node->view_src) && node->src[1] != nullptr &&
node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src) {
@@ -424,7 +488,32 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr &&
is_kvcache(node->src[1]->view_src, nullptr)) {
// s_copy defrag remainder writeback: gathered extra state rows copied back into the cache
- op_case = 3;
+ op_case = node->src[1]->view_src->ne[1] == 1 ? 9 : 3;
+ } else if (node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr) {
+ // op_case 5: KV write for decoder self-attention (dynamic write offset)
+ // op_case 6: KV write for encoder self-attn or cross-attn (static offset)
+ const ggml_tensor * kv_buf = node->src[1]->view_src;
+ if (kv_buf->ne[1] == 1 && kv_buf->ne[2] == 1 && kv_buf->ne[3] == 1) {
+ op_case = 6;
+ // Forward-scan the graph for a FLASH_ATTN_EXT that reads from
+ // the same buffer. Having a mask (src[3] != nullptr) implies
+ // decoder self-attention and the write offset is dynamic.
+ for (int i = 0; i < m_cgraph->n_nodes; i++) {
+ const ggml_tensor * n = m_cgraph->nodes[i];
+ if (n->op != GGML_OP_FLASH_ATTN_EXT) {
+ continue;
+ }
+ // K (src[1]) and V (src[2]) are 3-D views whose view_src is
+ // the flat KV buffer we are writing to.
+ if ((n->src[1] != nullptr && n->src[1]->view_src == kv_buf) ||
+ (n->src[2] != nullptr && n->src[2]->view_src == kv_buf)) {
+ if (n->src[3] != nullptr) {
+ op_case = 5; // decoder self-attention: mask present
+ }
+ break;
+ }
+ }
+ }
}
break;
}
@@ -438,7 +527,7 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
}
case GGML_OP_SCALE: {
if (node->view_src && node->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY) {
- op_case = 1;
+ op_case = node->view_src->ne[1] == 1 ? 2 : 1;
}
break;
}
@@ -448,6 +537,15 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
}
break;
}
+ case GGML_OP_FLASH_ATTN_EXT: {
+ if (node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr) {
+ const ggml_tensor * kv_buf = node->src[1]->view_src;
+ if (kv_buf->ne[1] == 1 && kv_buf->ne[2] == 1 && kv_buf->ne[3] == 1) {
+ op_case = (node->src[3] != nullptr) ? 1 : 2;
+ }
+ }
+ break;
+ }
default:
break;
}
@@ -469,6 +567,40 @@ std::optional extract_layer_from_name(const std::string & name) {
return layer;
}
+// Recover the sliding window width from ggml's own SWA mask. llama.cpp never passes n_swa to a
+// backend, but fill_mask() writes it into the mask: a query row keeps exactly the cells inside
+// its window, so the widest row counts min(pos + 1, n_swa) unmasked cells. Counting rather than
+// looking for a contiguous band is what makes this work on the KV-cache mask, where columns are
+// physical cache cells in arbitrary order, not positions.
+// Assumes LLAMA_SWA_TYPE_STANDARD, the only type the caller reconstructs.
+static int get_swa_window_from_mask(const ggml_tensor * mask) {
+ if (mask->data == nullptr || !ggml_backend_buffer_is_host(mask->buffer)) {
+ return -1;
+ }
+ if (mask->type != GGML_TYPE_F16 && mask->type != GGML_TYPE_F32) {
+ return -1;
+ }
+
+ const int64_t n_kv = mask->ne[0];
+ const int64_t n_tokens = mask->ne[1];
+ int64_t window = 0;
+
+ for (int64_t r = 0; r < n_tokens; r++) {
+ int64_t kept = 0;
+ for (int64_t c = 0; c < n_kv; c++) {
+ const size_t i = (size_t) r * n_kv + c;
+ const float v = mask->type == GGML_TYPE_F16 ? ggml_fp16_to_fp32(((const ggml_fp16_t *) mask->data)[i]) :
+ ((const float *) mask->data)[i];
+ if (v > -INFINITY) {
+ kept++;
+ }
+ }
+ window = std::max(window, kept);
+ }
+
+ return window > 0 ? (int) window : -1;
+}
+
std::pair GgmlOvDecoder::compute_llm_params(ggml_cgraph * cgraph, bool is_static) {
ModelParams model_params;
ComputeParams compute_params;
@@ -479,23 +611,35 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr
switch (node->op) {
case GGML_OP_FLASH_ATTN_EXT:
- if (node->src[0] == nullptr || node->src[1] == nullptr || node->src[3] == nullptr) {
+ if (node->src[0] == nullptr || node->src[1] == nullptr) {
return -1;
}
switch (node->src[1]->op) {
case GGML_OP_PERMUTE:
- // case 0: node op is FLASH_ATTN_EXT, src 1 not null & op is PERMUTE & the permuted tensor src is the view of cache k
- if (node->src[1]->src[0] != nullptr && node->src[1]->src[0]->op == GGML_OP_VIEW) {
+ // case 0: src[1] is PERMUTE of a cache VIEW, mask required
+ if (node->src[3] != nullptr && node->src[1]->src[0] != nullptr &&
+ node->src[1]->src[0]->op == GGML_OP_VIEW) {
return 0;
}
break;
case GGML_OP_CPY:
- // case 1: node op is FLASH_ATTN_EXT, src 1 not null & op is CPY & the copied tensor src is PERMUTE & the permuted tensor src is the view of cache k
- if (node->src[1]->src[0] != nullptr && node->src[1]->src[0]->op == GGML_OP_PERMUTE &&
- node->src[1]->src[0]->src[0] != nullptr && node->src[1]->src[0]->src[0]->op == GGML_OP_VIEW) {
+ // case 1: src[1] is CPY of a PERMUTE(VIEW), mask required
+ if (node->src[3] != nullptr && node->src[1]->src[0] != nullptr &&
+ node->src[1]->src[0]->op == GGML_OP_PERMUTE && node->src[1]->src[0]->src[0] != nullptr &&
+ node->src[1]->src[0]->src[0]->op == GGML_OP_VIEW) {
return 1;
}
break;
+ case GGML_OP_VIEW:
+ // cases 4/5/6: whisper - K is a direct non-contiguous VIEW_3D of a KV cache
+ if (node->src[1]->view_src != nullptr) {
+ if (node->src[3] != nullptr) {
+ return 4; // decoder self-attention
+ } else {
+ return 5; // cross-attention or encoder self-attention
+ };
+ }
+ break;
default:
break;
}
@@ -522,6 +666,97 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr
return -1;
};
+ // Resolve the attention mask an attention node consumes, mirroring the src layout that
+ // get_attention_pattern_case() classifies. Used by the SWA pre-pass below.
+ auto get_attention_op_mask = [&get_attention_pattern_case](const ggml_tensor * node) -> const ggml_tensor * {
+ switch (get_attention_pattern_case(node)) {
+ case 0:
+ case 1:
+ return node->src[3];
+ case 2:
+ case 3:
+ return node->src[1];
+ default:
+ return nullptr;
+ }
+ };
+
+ // Pre-pass: classify sliding-window vs full-attention layers.
+ //
+ // An interleaved-SWA model keeps two KV caches and two attention masks, and hands each layer
+ // whichever pair matches its attention type. The mask tensor does not say which is which: both
+ // are named "attn_inp_kq_mask" by build_attn_inp_kq_mask(), and both carry the same n_kv because
+ // llama_kv_cache::get_n_kv() pads occupancy up to a common multiple.
+ //
+ // The KV cache does say. Each cache allocates cache_k_l once at load time with its own cell
+ // count: the windowed cache is sized from the window
+ // (PAD(min(size_base, n_swa*(unified ? n_seq_max : 1) + n_ubatch), 256), see
+ // llama_kv_cache_iswa), the full-attention one spans the whole context. Read the LEAF buffer
+ // behind the VIEW rather than the VIEW itself: the leaf extent is a constant per layer, known
+ // from the first graph onwards, while the view grows with context depth and would invert the
+ // comparison at shallow depth.
+ //
+ // Layers whose leaf is smaller than the largest leaf are the windowed ones. When every layer
+ // reports the same extent there is no distinction to draw -- either the model has no windowed
+ // layers, or the window is at least as large as the context so the two caches coincide, in
+ // which case a windowed layer and a full-attention one compute the same thing.
+ //
+ // Getting this wrong is silent and severe: with the windowed layers classified as
+ // full-attention, permute's KV slicing uses attention_size instead of attention_size_swa. The
+ // two agree while the context is shorter than the window, then diverge, and the mask add fails
+ // shape inference ("Failed to broadcast-merge input shapes") partway into a long prompt.
+ {
+ std::map layer_extent; // layer -> leaf cache_k cell count
+ std::map layer_mask; // layer -> mask it consumes
+ int64_t max_extent = 0;
+
+ for (int i = 0; i < cgraph->n_nodes; i++) {
+ const ggml_tensor * mask = get_attention_op_mask(cgraph->nodes[i]);
+ if (mask == nullptr) {
+ continue;
+ }
+ const ggml_tensor * cache_k_permute = nullptr;
+ switch (get_attention_pattern_case(cgraph->nodes[i])) {
+ case 0: cache_k_permute = cgraph->nodes[i]->src[1]; break;
+ case 1: cache_k_permute = cgraph->nodes[i]->src[1]->src[0]; break;
+ case 2: cache_k_permute = cgraph->nodes[i]->src[0]->src[0]; break;
+ default: cache_k_permute = cgraph->nodes[i]->src[0]->src[0]->src[0]; break;
+ }
+ const ggml_tensor * cache_k_view = cache_k_permute->src[0];
+ if (cache_k_view->op != GGML_OP_VIEW) {
+ continue;
+ }
+ const ggml_tensor * leaf = cache_k_view->src[0];
+ auto layer = extract_layer_from_name(leaf->name);
+ if (!layer.has_value()) {
+ continue;
+ }
+ layer_extent[layer.value()] = leaf->ne[1];
+ layer_mask[layer.value()] = mask;
+ max_extent = std::max(max_extent, leaf->ne[1]);
+ }
+
+ for (const auto & [layer, extent] : layer_extent) {
+ if (extent < max_extent) {
+ model_params.swa_layers.push_back(layer);
+ if (model_params.swa_mask == nullptr) {
+ model_params.swa_mask = layer_mask[layer];
+ }
+ }
+ }
+ std::sort(model_params.swa_layers.begin(), model_params.swa_layers.end());
+
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_LOG_SWA_LAYERS")) {
+ std::string per_layer;
+ for (const auto & [layer, extent] : layer_extent) {
+ per_layer += " " + std::to_string(layer) + ":" + std::to_string(extent) +
+ (extent < max_extent ? "(swa)" : "");
+ }
+ GGML_LOG_WARN("ov-swa: attn_layers=%zu max_extent=%ld swa_layers=%zu |%s\n", layer_extent.size(),
+ (long) max_extent, model_params.swa_layers.size(), per_layer.c_str());
+ }
+ }
+
bool rope_seen = false;
for (int i = 0; i < cgraph->n_nodes; i++) {
auto * node = cgraph->nodes[i];
@@ -548,6 +783,18 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr
cache_k_permute = node->src[0]->src[0]->src[0];
mask = node->src[1];
break;
+ case 4:
+ case 5: {
+ // whisper: K is a direct VIEW_3D of the KV buffer, no PERMUTE node
+ auto * cache_k_view = node->src[1]; // VIEW_3D of kv_self.k or kv_cross.k`
+ compute_params.token_len_per_seq = node->src[0]->ne[1];
+ if (attention_pattern_case == 4) {
+ compute_params.attention_size = cache_k_view->ne[1];
+ } else {
+ compute_params.attention_size_static = cache_k_view->ne[1];
+ }
+ continue;
+ }
default:
break;
}
@@ -567,11 +814,14 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr
ggml_tensor * cache_k = cache_k_view->src[0];
int layer = extract_layer_from_name(cache_k->name).value();
- std::string mask_name(mask->name);
+ // Classified by the pre-pass above, which groups layers by mask tensor identity. The
+ // mask NAME cannot be used: build_attn_inp_kq_mask() gives both masks the same name.
+ const bool layer_is_swa = std::find(model_params.swa_layers.begin(), model_params.swa_layers.end(),
+ layer) != model_params.swa_layers.end();
model_params.kv_buffer_ctx_id = ggml_backend_openvino_buffer_get_ctx_id(cache_k->buffer);
- if (mask_name.find("swa") != std::string::npos) {
- model_params.swa_layers.push_back(layer);
+ model_params.n_heads_kv_per_layer[layer] = cache_k_permute->ne[2];
+ if (layer_is_swa) {
model_params.ctx_per_seq_swa = cache_k->ne[1];
} else {
model_params.ctx_per_seq = cache_k->ne[1];
@@ -584,8 +834,9 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr
memcpy(&offset, cache_k_view->op_params, sizeof(size_t));
compute_params.seq_active_start = offset / seq_size;
- if (mask_name.find("swa") != std::string::npos) {
+ if (layer_is_swa) {
compute_params.attention_size_swa = mask->ne[0];
+ compute_params.swa_window = get_swa_window_from_mask(mask);
} else {
compute_params.attention_size = mask->ne[0];
}
@@ -633,12 +884,17 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr
model_params.state_size = node->src[0]->ne[0];
}
if (node->op == GGML_OP_SCALE && node->view_src != nullptr && is_kvcache(node->view_src, nullptr)) {
+ if (model_params.n_rs_slots == -1) {
+ model_params.n_rs_slots = node->view_src->ne[1];
+ } else {
+ GGML_ASSERT(model_params.n_rs_slots == node->view_src->ne[1]);
+ }
compute_params.cache_rs_reset_len = ggml_nelements(node) / node->view_src->ne[0];
compute_params.cache_rs_reset_idx = node->src[0]->view_offs / node->view_src->ne[0];
}
// Capture the destination slot block of every recurrent state cache writeback, plus the
- // conv_input window the conv state writeback copies. The active sequences occupy a
- // contiguous slot block [begin, begin + n_seqs) of the cache; the block and the window move
+ // source window needed by conv state and packed GDN rollback writes. The active sequences
+ // occupy a contiguous slot block [begin, begin + n_seqs) of the cache; these offsets move
// with the batch, so they are fed to the cached model as runtime inputs.
if (node->op == GGML_OP_CPY && node->view_src != nullptr && is_kvcache(node->view_src, nullptr) &&
node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src) {
@@ -646,23 +902,29 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr
const bool is_gdn = node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr &&
node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET;
const bool is_extra = node->src[0]->op == GGML_OP_GET_ROWS;
+ const bool is_gdn_rollback = is_gdn && is_same_shape(node->src[0], node->src[1]);
const ggml_tensor * dest_view = node->src[1];
const ggml_tensor * cache = node->view_src;
const size_t row_bytes = cache->ne[0] * ggml_type_size(cache->type);
- if (row_bytes > 0 && (is_conv || is_gdn || is_extra)) {
+ if (is_gdn_rollback) {
+ // Rollback GDN exposes an already-flattened [state, seq, snapshot] VIEW and copies
+ // it to an identically-shaped cache VIEW. Non-rollback copies native 4-D state
+ // [value, key, head, seq] into flattened cache rows, so the shapes differ. This
+ // signature is local to the CPY and still works when fallback splits the graph.
+ model_params.has_rs_rollback = true;
+ }
+ if (row_bytes > 0 && (is_conv || is_gdn || is_extra) && !is_full_single_slot_writeback(node)) {
ComputeParams::RsWriteback writeback;
writeback.slot_begin = (int) (dest_view->view_offs / row_bytes);
if (is_conv) {
- // conv_input column the copied window starts at
writeback.src_begin = (int) (node->src[0]->view_offs / node->src[0]->view_src->nb[0]);
- } else if (is_gdn) {
- // first row of the state part of the gated-delta-net output
+ } else if (is_gdn_rollback) {
writeback.src_begin = (int) (node->src[0]->view_offs / node->src[0]->view_src->nb[1]);
}
compute_params.rs_writebacks[get_tensor_ov_name(cgraph, node)] = writeback;
}
- if (is_conv || is_gdn) {
+ if ((is_conv || is_gdn) && !is_full_single_slot_writeback(node)) {
compute_params.s_copy_active_slot_len = (int) dest_view->ne[1];
}
}
@@ -718,18 +980,30 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
} else if (is_kvcache(input, op)) {
// kvcache
input_shape = ov::PartialShape{get_shape(input)};
- if (!m_is_static) {
+ // Whisper.cpp uses a fixed size 1D KV buffer [N, 1, 1, 1] (GGML) or [1, 1, 1, N] (OV).
+ // the token fill level is handled by token_len_per_seq + dynamic mask input.
+ // skip dynamic dim and stateful reshape for this layout.
+ const bool is_flat_kv = (input->ne[1] == 1 && input->ne[2] == 1 && input->ne[3] == 1);
+ if (!m_is_static && !is_flat_kv) {
// do not fix ctx size to make llama-bench work across test params
input_shape[2] = -1;
}
- if (is_stateful()) {
+ if (is_stateful() && !is_flat_kv) {
// Convert stateless KV cache layout [1, 1, seq, n_heads_kv * head_size]
// to stateful layout [1, seq, n_heads_kv, head_size].
+ // NOTE: Gemma4 uses per-layer-type KV shapes, so no single scalar describes every
+ // layer. E2B varies only the head size (sliding 256, full 512); 12B also varies the
+ // head COUNT (sliding 8 x 256, full 1 x 512). Take the head count for this tensor's
+ // own layer type and derive the head size from its own combined dim, so both layer
+ // types get the correct split. Using the model-level count split 12B's sliding
+ // states as 1 x 2048 and decoded garbage.
assert(input_shape.size() == 4 && input_shape[0] == 1 && input_shape[1] == 1 &&
- input_shape[2].is_dynamic() &&
- input_shape[3] == (m_model_params.n_heads_kv * m_model_params.head_size));
- input_shape = {input_shape[0], ov::Dimension::dynamic(), m_model_params.n_heads_kv,
- m_model_params.head_size};
+ input_shape[2].is_dynamic() && input_shape[3].is_static());
+ const int n_heads_kv = get_n_heads_kv_for_tensor(input);
+ assert(n_heads_kv > 0 && input_shape[3].get_length() % n_heads_kv == 0);
+ const int64_t combined_dim = input_shape[3].get_length(); // n_heads_kv * head_size
+ const int64_t head_size = combined_dim / n_heads_kv;
+ input_shape = {input_shape[0], ov::Dimension::dynamic(), n_heads_kv, head_size};
}
} else if (is_kv_idx(input, op)) {
@@ -738,7 +1012,9 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
input_shape = ov::PartialShape{1, 1, 1, len};
} else if (is_inp_s_copy(input, op) || is_s_copy_leaf(input)) {
- input_shape = ov::PartialShape{1, 1, 1, -1};
+ // On NPU the total slot count (n_seq_max) is fixed at translation time, so the s_copy
+ // index list has a static length; on CPU/GPU it may change across compiles (defrag).
+ input_shape = m_is_static ? ov::PartialShape{get_shape(input)} : ov::PartialShape{1, 1, 1, -1};
} else {
input_shape = ov::PartialShape{get_shape(input)};
@@ -790,16 +1066,23 @@ void GgmlOvDecoder::add_extra_inputs() {
// see llama_kv_cache_unified::get_n_kv and llama_kv_cache_unified::get_padding.
// 2. `n_seq_active` and `seq_active_start`, used in FLASH_ATTN_EXT to indicate the active sequences in the batch
- auto create_1d_input = [this](const std::string & name, int64_t value) {
- m_model_extra_inputs[name] = {ov::element::i64, ov::Shape{1}, value, !m_is_static};
+ auto create_1d_input = [this](const std::string & name, int64_t value, bool force_parameter = false) {
+ m_model_extra_inputs[name] = {ov::element::i64, ov::Shape{1}, value, force_parameter || !m_is_static};
};
if (m_compute_params.attention_size != -1) {
create_1d_input("attention_size", m_compute_params.attention_size);
}
+ if (m_compute_params.attention_size_static != -1) {
+ create_1d_input("attention_size_static", m_compute_params.attention_size_static);
+ }
if (m_compute_params.attention_size_swa != -1) {
create_1d_input("attention_size_swa", m_compute_params.attention_size_swa);
}
+ // only the stateful SWA mask consumes this
+ if (is_stateful() && m_compute_params.swa_window != -1) {
+ create_1d_input("swa_window", m_compute_params.swa_window);
+ }
create_1d_input("n_seq_active", m_compute_params.n_seq_active);
create_1d_input("seq_active_start", m_compute_params.seq_active_start);
create_1d_input("seq_active_end", m_compute_params.seq_active_start + m_compute_params.n_seq_active);
@@ -808,18 +1091,33 @@ void GgmlOvDecoder::add_extra_inputs() {
}
// create_1d_input("token_len", m_compute_params.token_len_per_seq * m_compute_params.n_seq_active);
- if (m_compute_params.cache_rs_reset_idx != -1) {
- create_1d_input("cache_rs_reset_idx", m_compute_params.cache_rs_reset_idx);
- create_1d_input("cache_rs_reset_len", m_compute_params.cache_rs_reset_len);
+ if (m_compute_params.cache_rs_reset_idx != -1 && m_model_params.n_rs_slots != 1) {
+ // Whether/which cache slot to reset varies per compute call (e.g. a new sequence starting
+ // vs. continued decoding). can_reuse_statically() does not invalidate the cached static
+ // model on ComputeParams changes, so these must stay runtime Parameters even when static
+ // (scale.cpp op_case 1 only uses them in value comparisons, never as Slice bounds, so this
+ // does not reintroduce dynamic shapes).
+ create_1d_input("cache_rs_reset_idx", m_compute_params.cache_rs_reset_idx, /*force_parameter=*/true);
+ create_1d_input("cache_rs_reset_len", m_compute_params.cache_rs_reset_len, /*force_parameter=*/true);
}
if (m_compute_params.s_copy_active_slot_len != -1) {
create_1d_input("s_copy_active_slot_len", m_compute_params.s_copy_active_slot_len);
+ if (m_is_static) {
+ // Number of real tokens in the current prefill chunk. The last chunk is padded with
+ // fabricated token ids; attention masks them out, but the recurrent (GDN/conv) path
+ // would otherwise fold them into cache_r/cache_s permanently. Varies per chunk, so it
+ // must stay a runtime Parameter; it is only compared against a Range or used as Gather
+ // indices, so it does not make any shape dynamic.
+ create_1d_input("chunk_valid_len", get_static_n_tokens(), /*force_parameter=*/true);
+ }
}
for (const auto & [node_name, writeback] : m_compute_params.rs_writebacks) {
create_1d_input("rs_slot_begin_" + node_name, writeback.slot_begin);
- create_1d_input("rs_src_begin_" + node_name, writeback.src_begin);
+ if (!m_is_static && writeback.src_begin >= 0) {
+ create_1d_input("rs_src_begin_" + node_name, writeback.src_begin);
+ }
}
}
@@ -909,6 +1207,9 @@ void GgmlOvDecoder::compute_model_outputs() {
if (cur_node->op == GGML_OP_NONE || cur_node->op == GGML_OP_VIEW || cur_node->op == GGML_OP_RESHAPE) {
continue;
}
+ if (::is_inplace_op(cur_node) && ggml_nbytes(cur_node) == 0) {
+ continue;
+ }
auto cur_node_use_count = m_cgraph->use_counts[ggml_hash_find(&m_cgraph->visited_hash_set, cur_node)];
if (cur_node_use_count == 0) {
// The output of in-place ops is the view_src tensor, which is updated in place. We should use the view_src name as the output name to make sure it can be correctly matched with the later ops that use the view_src.
@@ -1516,6 +1817,27 @@ std::vector GgmlOvDecoder::get_output_stride(int node_idx) const {
}
std::vector GgmlOvDecoder::get_output_names(int node_idx) const {
+ auto * node = m_node_info_list[node_idx].node;
+ if (node->op == GGML_OP_GATED_DELTA_NET && !m_model_params.has_rs_rollback) {
+ std::string attn_name;
+ std::string state_name;
+ for (int i = node_idx + 1; i < m_cgraph->n_nodes; i++) {
+ auto * consumer = m_cgraph->nodes[i];
+ if (consumer->op != GGML_OP_VIEW || consumer->src[0] != node) {
+ continue;
+ }
+ // GGML packs [attention | state]. The attention VIEW starts at offset 0 and the
+ // state VIEW starts after the token-dependent attention segment.
+ auto & name = consumer->view_offs == 0 ? attn_name : state_name;
+ if (!name.empty()) {
+ return {m_node_info_list[node_idx].node_name};
+ }
+ name = get_tensor_ov_name(m_cgraph, consumer);
+ }
+ if (!attn_name.empty() && !state_name.empty()) {
+ return {attn_name, state_name};
+ }
+ }
return {m_node_info_list[node_idx].node_name};
}
@@ -1785,13 +2107,23 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
auto dynamic_dim_stride = src_logical_nb[dynamic_dim_idx] / ggml_type_size(node->src[0]->type) *
ggml_type_size(node->type);
int matched_dim_count = 0;
+ int first_matched_dim = -1;
for (int i = 0; i < GGML_MAX_DIMS; i++) {
if (node->nb[i] == dynamic_dim_stride && node->ne[i] == node->src[0]->ne[dynamic_dim_idx]) {
+ if (first_matched_dim == -1) {
+ first_matched_dim = i;
+ }
m_node_dynamic_dims[node] = i;
matched_dim_count++;
}
}
- if (matched_dim_count != 1) {
+ if (matched_dim_count > 1 && node->src[0]->ne[dynamic_dim_idx] == 1) {
+ // Single-token capture: every trailing dim is size 1 with the same stride, so
+ // the match is ambiguous. The lowest index is the real axis; the rest are
+ // ggml's size-1 padding. Bailing out here would bake the captured token count
+ // into the static prefill model, which then runs with a different one.
+ m_node_dynamic_dims[node] = first_matched_dim;
+ } else if (matched_dim_count != 1) {
m_node_dynamic_dims[node] = -1;
GGML_LOG_WARN("ggml-openvino: cannot determine dynamic dim for CONT node '%s', src[0]: '%s'\n",
node->name, node->src[0]->name);
diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h
index 8e39a26c8b79..5ee396df8399 100644
--- a/ggml/src/ggml-openvino/ggml-decoder.h
+++ b/ggml/src/ggml-openvino/ggml-decoder.h
@@ -21,11 +21,22 @@ struct ModelParams {
int ctx_per_seq_swa = -1;
int n_seq = 1;
int n_heads_kv = -1;
+ // Per-layer KV head count. gemma-4 12B interleaves 8 x 256 sliding layers with 1 x 512
+ // full-attention layers, so no single scalar describes every layer. Keyed by layer, not by
+ // layer TYPE, because the SWA classification depends on the context size (extents tie at a
+ // small -c) while the head count does not.
+ std::map n_heads_kv_per_layer;
int head_size = -1;
int state_size = -1; // for SSM molels, eg qwen35
- int32_t rope_params[15];
+ int n_rs_slots = -1;
+ bool has_rs_rollback = false;
+ int32_t rope_params[15] = {0};
bool mixed_rope_params = false;
std::vector swa_layers;
+ // The sliding-window mask tensor, identified in compute_llm_params() by grouping attention
+ // layers on the mask they consume. Only used to tell the two masks apart when naming OV
+ // parameters -- both carry the same tensor name. Null when the graph has a single mask.
+ const ggml_tensor * swa_mask = nullptr;
std::vector kv_names;
size_t kv_buffer_ctx_id = 0;
@@ -35,9 +46,15 @@ struct ModelParams {
memcmp(rope_params, other.rope_params, sizeof(int32_t) * 15) == 0;
}
- bool can_reuse_dynamically(const ModelParams & other) const { return same_rope_params(other); }
+ bool can_reuse_dynamically(const ModelParams & other) const {
+ return same_rope_params(other) && n_rs_slots == other.n_rs_slots &&
+ has_rs_rollback == other.has_rs_rollback;
+ }
- bool can_reuse_statically(const ModelParams & other) const { return same_rope_params(other) && ctx == other.ctx; }
+ bool can_reuse_statically(const ModelParams & other) const {
+ return same_rope_params(other) && ctx == other.ctx && n_rs_slots == other.n_rs_slots &&
+ has_rs_rollback == other.has_rs_rollback;
+ }
bool kv_buffer_changed(const ModelParams & other) const { return kv_buffer_ctx_id != other.kv_buffer_ctx_id; }
};
@@ -47,6 +64,12 @@ struct ComputeParams {
int seq_active_start = 0;
int attention_size = -1;
int attention_size_swa = -1;
+ int attention_size_static = -1; // encoder/cross-attn KV fill level (whisper)
+ // Sliding window width, read back from the band of ggml's own SWA mask. ggml never passes
+ // n_swa down to a backend, but fill_mask() bakes it into the mask contents, so the widest
+ // unmasked row recovers it. Shorter than n_swa while the sequence is still short, which is
+ // harmless: every causal pair is inside the window then anyway.
+ int swa_window = -1;
int input_len = -1;
int token_len_per_seq = -1;
int past_kv_len = -1;
@@ -84,16 +107,20 @@ struct ComputeParams {
struct RsWriteback {
int slot_begin = 0; // first cache slot written by the CPY
- int src_begin = 0; // where the copied data starts in the source tensor (in rows of it)
+ int src_begin = -1; // first source column copied by a conv-state CPY
};
std::map rs_writebacks;
- // Offsets of the state cache writeback CPY nodes, keyed by node name. They change with the
- // batch (kv head, active sequence count, token count) and, with rollback enabled
- // (cparams.n_rs_seq > 0), the conv state is written back once per snapshot slot, each snapshot
- // taking a different conv_input window. Passed to the cached model as runtime inputs.
+ // Destination slot offset of each state cache writeback CPY node, keyed by node name. It
+ // changes with the batch (kv head, active sequence count) and, with rollback enabled
+ // (cparams.n_rs_seq > 0), the conv state is written back once per snapshot slot. Passed to the
+ // cached model as a runtime input. Dynamic models also receive the source-side offset; static
+ // models use a fixed end-anchored offset in the translator.
};
+// defined below; declared here because GgmlOvDecoder uses it inline
+std::optional extract_layer_from_name(const std::string & name);
+
class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
public:
struct NodeInfo {
@@ -248,6 +275,21 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
m_model_params.swa_layers.end();
}
+ // KV head count for one layer. Sliding and full layers can differ (gemma-4 12B), so callers
+ // that reinterpret a KV buffer must use this and not the model-level n_heads_kv.
+ int get_n_heads_kv_for_layer(int layer) const {
+ auto it = m_model_params.n_heads_kv_per_layer.find(layer);
+ return it != m_model_params.n_heads_kv_per_layer.end() ? it->second : m_model_params.n_heads_kv;
+ }
+
+ // Same, for a KV cache tensor: its layer comes from the leaf name (cache_k_l).
+ int get_n_heads_kv_for_tensor(const ggml_tensor * kv_tensor) const {
+ if (auto layer = extract_layer_from_name(std::string(kv_tensor->name)); layer.has_value()) {
+ return get_n_heads_kv_for_layer(layer.value());
+ }
+ return m_model_params.n_heads_kv;
+ }
+
int get_past_kv_len() const { return m_compute_params.past_kv_len; }
int get_input_len() const { return m_compute_params.input_len; }
@@ -355,6 +397,10 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
return op->op == GGML_OP_SET_ROWS && op->src[1] == tensor;
}
+ bool is_swa_mask(const ggml_tensor * tensor) const {
+ return m_model_params.swa_mask != nullptr && tensor == m_model_params.swa_mask;
+ }
+
inline static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE &&
op->src[1]->op == GGML_OP_NONE;
@@ -373,8 +419,22 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
if (is_inp_emb(tensor, op)) {
return "embd";
}
- if (is_stateful() && is_inp_mask(tensor, op)) {
- return std::string(tensor->name).find("swa") == std::string::npos ? "self_kq_mask" : "self_kq_mask_swa";
+ if (is_inp_mask(tensor, op)) {
+ // Give the two attention masks distinct OV parameter names.
+ //
+ // An interleaved-SWA model builds one full-attention mask and one sliding-window mask,
+ // but build_attn_inp_kq_mask() names them identically, so keying a parameter off
+ // tensor->name alone makes the second mask OVERWRITE the first in m_model_inputs: both
+ // attention types then read a single parameter, and the windowed layers silently run
+ // against an unbanded mask. Disambiguate using the SWA layer set computed in
+ // compute_llm_params(), which classifies by mask tensor identity rather than by name.
+ //
+ // When no SWA layer was found there is only one mask in play, so the plain name is
+ // correct and no _swa parameter is created.
+ if (m_model_params.swa_layers.empty()) {
+ return "self_kq_mask";
+ }
+ return is_swa_mask(tensor) ? "self_kq_mask_swa" : "self_kq_mask";
}
return tensor->name;
}
diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp
index 36c749244f83..e6f808543dfb 100644
--- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp
+++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp
@@ -31,7 +31,10 @@ void ggml_openvino_device_config::init() {
// String values (use ggml_openvino_getenv_str)
"GGML_OPENVINO_DEVICE",
"GGML_OPENVINO_CACHE_DIR",
+ "GGML_OPENVINO_SPILL_DIR",
"GGML_OPENVINO_DEBUG_NODE",
+ "GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR",
+ "GGML_OPENVINO_NPU_COMPILE_CONFIG",
// Integer values (use ggml_openvino_getenv_int)
"GGML_OPENVINO_PREFILL_CHUNK_SIZE",
// Boolean toggles (treated as int flags via ggml_openvino_getenv_int)
@@ -41,6 +44,9 @@ void ggml_openvino_device_config::init() {
"GGML_OPENVINO_DUMP_IR",
"GGML_OPENVINO_DEBUG_INPUT",
"GGML_OPENVINO_DEBUG_OUTPUT",
+ // Force the static (NPU-shape) compute path on any device, e.g. GGML_OPENVINO_DEVICE=CPU,
+ // to test the static-shape translation without NPUW/real NPU hardware in the loop.
+ "GGML_OPENVINO_FORCE_STATIC",
"GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS",
"GGML_OPENVINO_ENABLE_CACHE",
"GGML_OPENVINO_DISABLE_CACHE",
@@ -50,7 +56,11 @@ void ggml_openvino_device_config::init() {
"GGML_OPENVINO_MEMORY_OPTIMIZE",
"GGML_OPENVINO_RELEASE_WEIGHTS",
"GGML_OPENVINO_REDUCE_COMPILE_MEM",
- "GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR",
+ "GGML_OPENVINO_LOG_UNSUPPORTED_OPS",
+ "GGML_OPENVINO_LOG_SWA_LAYERS",
+ "GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS",
+ "GGML_OPENVINO_REQUANT_KQUANT",
+ "GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT",
};
for (const char * const & env_var : env_var_names) {
@@ -85,11 +95,20 @@ void ggml_openvino_device_config::init() {
compile_config["NPUW_CACHE_DIR"] = cache_dir;
compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE));
}
+ const char * compilation_mode_params =
+ ggml_openvino_getenv_str("GGML_OPENVINO_NPU_COMPILE_CONFIG");
+ if (compilation_mode_params && strlen(compilation_mode_params) > 0) {
+ compile_config["NPU_COMPILATION_MODE_PARAMS"] = compilation_mode_params;
+ }
} else if (cache_dir && strlen(cache_dir) > 0) {
compile_config.insert(ov::cache_dir(cache_dir));
compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE));
}
+ if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING") >= 2) {
+ compile_config.insert(ov::enable_profiling(true));
+ }
+
// Initialize remote context with queue sharing for GPU
if (device_name == "GPU") {
// Create OpenCL context and queue
@@ -114,7 +133,14 @@ void ggml_openvino_device_config::init() {
return;
}
- cl_queue = clCreateCommandQueueWithProperties(cl_ctx, cl_device, nullptr, &err);
+ const cl_queue_properties profiling_properties[] = {
+ CL_QUEUE_PROPERTIES,
+ CL_QUEUE_PROFILING_ENABLE,
+ 0,
+ };
+ const cl_queue_properties * queue_properties =
+ ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING") >= 2 ? profiling_properties : nullptr;
+ cl_queue = clCreateCommandQueueWithProperties(cl_ctx, cl_device, queue_properties, &err);
if (err != CL_SUCCESS) {
GGML_LOG_ERROR("Failed to create OpenCL command queue: %d\n", err);
clReleaseContext(cl_ctx);
@@ -253,9 +279,66 @@ std::optional ggml_openvino_get_requant_type(const ggml_tensor *
if (ggml_openvino_is_npu()) {
return ExtraQuantType::Q4_0_128;
}
+ // By default Q6_K/Q5_K are requantized to Q8_0_C, which *inflates* 6- and 5-bit weights to 8
+ // while the rest of the model stays at 4 bits, and Q4_K keeps its native group-32 layout
+ // (an f16 scale plus an f16 zero point per 32 weights = 0.125 B/weight of metadata).
+ // Decode of a large model is bandwidth-bound, so both cost throughput.
+ //
+ // GGML_OPENVINO_REQUANT_KQUANT selects a 4-bit target instead. Names are
+ // q4_[_all]: says whether a per-group zero point is kept,
+ // is the group size, and the _all suffix sends Q4_K down the same path (without it only
+ // Q6_K/Q5_K are touched):
+ // q4_sym128 Q6_K/Q5_K -> Q4_0_128 (u4, group 128, symmetric)
+ // q4_sym128_all and Q4_K too -- drops Q4_K's per-32 zero point, which costs some accuracy
+ // q4_asym64_all Q6_K/Q5_K and Q4_K -> Q4_1_64 (u4, group 64, asymmetric) -- most of the
+ // metadata saving while keeping a real zero point
+ // native no requantization at all (keep Q6_K/Q5_K as they are)
+ //
+ // The asymmetric target is only offered in its _all form: leaving Q4_K at its native group 32
+ // while Q6_K/Q5_K move to group 64 gives the Q/K/V projections different group counts, and the
+ // GPU plugin's FullyConnectedHorizontalFusion concatenates their scale constants, which then
+ // fails shape inference. Requantizing all three keeps the group size uniform.
+ const char * rq = ggml_openvino_getenv_str("GGML_OPENVINO_REQUANT_KQUANT");
+ auto is_opt = [rq](const char * name) {
+ return rq && strcmp(rq, name) == 0;
+ };
+ const bool sym128 = is_opt("q4_sym128");
+ const bool sym128_all = is_opt("q4_sym128_all");
+ const bool asym64_all = is_opt("q4_asym64_all");
+
+ if (tensor->type == GGML_TYPE_Q4_K) {
+ if (sym128_all) {
+ return ExtraQuantType::Q4_0_128;
+ }
+ if (asym64_all) {
+ return ExtraQuantType::Q4_1_64;
+ }
+ }
+ // MoE expert weights (3D, ne[2] = n_expert) stored as Q5_1/Q8_0 are the expert-side
+ // equivalent of Q6_K/Q5_K: kept at 8 bits by default while the rest of the model is at 4
+ // (gemma-4 26B-A4B keeps its down projection there). Send them to 4 bits under the same
+ // option, at group 64 rather than 128: the down expert has k=704, which 64 divides
+ // (704/64 = 11) and 128 does not.
+ if (tensor->ne[2] > 1 && (tensor->type == GGML_TYPE_Q5_1 || tensor->type == GGML_TYPE_Q8_0)) {
+ if (sym128 || sym128_all) {
+ return ExtraQuantType::Q4_0_64;
+ }
+ if (asym64_all) {
+ return ExtraQuantType::Q4_1_64;
+ }
+ }
switch (tensor->type) {
case GGML_TYPE_Q6_K:
case GGML_TYPE_Q5_K:
+ if (sym128 || sym128_all) {
+ return ExtraQuantType::Q4_0_128;
+ }
+ if (asym64_all) {
+ return ExtraQuantType::Q4_1_64;
+ }
+ if (is_opt("native")) {
+ return std::nullopt;
+ }
return ExtraQuantType::Q8_0_C;
default:
return std::nullopt;
@@ -321,6 +404,16 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
layout.weights_per_block = 128;
layout.is_symmetric = true;
break;
+ case ExtraQuantType::Q4_1_64:
+ layout.is_u4 = true;
+ layout.weights_per_block = 64;
+ layout.is_symmetric = false;
+ break;
+ case ExtraQuantType::Q4_0_64:
+ layout.is_u4 = true;
+ layout.weights_per_block = 64;
+ layout.is_symmetric = true;
+ break;
case ExtraQuantType::Q4_0_C:
layout.is_u4 = true;
layout.weights_per_block = tensor->ne[0];
diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.h b/ggml/src/ggml-openvino/ggml-openvino-extra.h
index 0916b416258f..9d827d969452 100644
--- a/ggml/src/ggml-openvino/ggml-openvino-extra.h
+++ b/ggml/src/ggml-openvino/ggml-openvino-extra.h
@@ -15,7 +15,10 @@
#include
// ExtraQuantType enum - defines requantization target formats
-enum class ExtraQuantType { F16, Q4_0_C, Q8_1_C, Q4_0_128, Q8_0_C, Q8_0_32 };
+// Q4_1_64: u4, group 64, *true* asymmetric (per-group scale and zero point). Note that
+// Q4_0_128/Q4_0_C are symmetric despite taking the unsigned branch of quantize_q4_0 -- that branch
+// pins zp to 8 with d = max/-8, which is algebraically symmetric.
+enum class ExtraQuantType { F16, Q4_0_C, Q8_1_C, Q4_0_128, Q4_0_64, Q8_0_C, Q8_0_32, Q4_1_64 };
ov::Core & ov_singleton_core();
diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp
index e299e16c778a..0b2028b4dda4 100644
--- a/ggml/src/ggml-openvino/ggml-openvino.cpp
+++ b/ggml/src/ggml-openvino/ggml-openvino.cpp
@@ -10,7 +10,10 @@
#include "ggml.h"
#include
+#include
+#include
#include
+#include
#include
#include
#include
@@ -25,6 +28,11 @@
#include
#include
+#ifndef _WIN32
+# include
+# include
+#endif
+
#if defined(_WIN32)
# define WIN32_LEAN_AND_MEAN
# ifndef NOMINMAX
@@ -64,6 +72,11 @@ struct ggml_backend_openvino_buffer_context {
size_t size;
bool is_remote;
+ // Set when the buffer is a file-backed spill mapping (GGML_OPENVINO_SPILL_DIR); it must be
+ // munmap'd rather than freed.
+ void * spill_mapping = nullptr;
+ size_t spill_size = 0;
+
// Wrapping of the buffer
std::shared_ptr ov_buffer;
@@ -98,10 +111,56 @@ struct ggml_backend_openvino_buffer_context {
data = usm_tensor.get();
ov_buffer = std::make_shared(std::move(usm_tensor));
} else {
- data = ggml_aligned_malloc(size);
- GGML_ASSERT(data);
- memset(data, 0, size);
- ov_buffer = std::make_shared(ov::element::u8, ov::Shape{size}, data);
+#ifndef _WIN32
+ if (const char * spill_dir = ggml_openvino_getenv_str("GGML_OPENVINO_SPILL_DIR")) {
+ // Disk-backed weight buffer: back the repacked weights with a temp file via MAP_SHARED
+ // instead of anonymous memory. Anonymous pages can only be evicted to swap, so the
+ // repacked buffer stays pinned alongside the mmap'd source and both are resident at once
+ // -- that double residency is the load-time peak. File-backed pages are reclaimable: the
+ // kernel can write them back and drop them under pressure, then re-read on demand, so RSS
+ // becomes a working set rather than the whole buffer. The file is unlinked immediately,
+ // so it disappears when the process exits.
+ //
+ // The directory must be real storage. Pointing this at a tmpfs mount (/tmp on many
+ // systems) backs the "spill" with RAM and makes matters worse.
+ char path[PATH_MAX];
+ snprintf(path, sizeof(path), "%s/ggml-ov-weights-%d-XXXXXX", spill_dir, (int) getpid());
+ int fd = mkstemp(path);
+ if (fd < 0) {
+ GGML_LOG_ERROR("%s: mkstemp(%s) failed: %s\n", __func__, path, strerror(errno));
+ return;
+ }
+ unlink(path); // anonymous-but-file-backed: freed on process exit
+ if (ftruncate(fd, (off_t) size) != 0) {
+ GGML_LOG_ERROR("%s: ftruncate(%zu) failed: %s\n", __func__, size, strerror(errno));
+ close(fd);
+ return;
+ }
+ void * m = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+ close(fd); // the mapping keeps the file alive
+ if (m == MAP_FAILED) {
+ GGML_LOG_ERROR("%s: mmap(%zu) failed: %s\n", __func__, size, strerror(errno));
+ return;
+ }
+ data = m;
+ spill_mapping = m;
+ spill_size = size;
+ GGML_LOG_INFO("%s: weight buffer spilled to %s (%zu MB, file-backed)\n", __func__, spill_dir,
+ size / 1024 / 1024);
+ ov_buffer = std::make_shared(ov::element::u8, ov::Shape{size}, data);
+ } else
+#endif
+ {
+#ifdef _WIN32
+ if (ggml_openvino_getenv_str("GGML_OPENVINO_SPILL_DIR")) {
+ GGML_LOG_WARN("%s: GGML_OPENVINO_SPILL_DIR is not supported on Windows, ignoring\n", __func__);
+ }
+#endif
+ data = ggml_aligned_malloc(size);
+ GGML_ASSERT(data);
+ memset(data, 0, size);
+ ov_buffer = std::make_shared(ov::element::u8, ov::Shape{size}, data);
+ }
}
if (data == nullptr) {
@@ -124,6 +183,11 @@ struct ggml_backend_openvino_buffer_context {
delete pair.second;
}
tensor_extras.clear();
+#ifndef _WIN32
+ if (spill_mapping != nullptr) {
+ munmap(spill_mapping, spill_size);
+ } else
+#endif
if (!is_remote && data != nullptr) {
ggml_aligned_free(data, size);
}
@@ -908,11 +972,27 @@ static bool has_non_contiguous_view_input(const ggml_tensor * op) {
}
static bool is_supported_flash_attn_pattern(const ggml_tensor * op) {
- // pattern of q,k,v should be q->op==PERMUTE, q->src[0]->op==VIEW, q->src[0]->src[0]->view_src==nullptr
+ // Each Q/K/V input must follow one of:
+ // PERMUTE -> VIEW -> base (view_src==nullptr) (llama KV-cache path)
+ // PERMUTE -> RESHAPE -> base (view_src==nullptr) (whisper Q)
+ // VIEW -> base (view_src==nullptr) (whisper K/V from kv_pad)
for (int i = 0; i < 3; i++) {
const ggml_tensor * src = op->src[i];
- if (src->op != GGML_OP_PERMUTE || src->src[0] == nullptr || src->src[0]->op != GGML_OP_VIEW ||
- src->src[0]->src[0] == nullptr || src->src[0]->src[0]->view_src != nullptr) {
+ if (src->op == GGML_OP_PERMUTE) {
+ if (src->src[0] == nullptr) {
+ return false;
+ }
+ if (src->src[0]->op != GGML_OP_VIEW && src->src[0]->op != GGML_OP_RESHAPE) {
+ return false;
+ }
+ if (src->src[0]->src[0] == nullptr || src->src[0]->src[0]->view_src != nullptr) {
+ return false;
+ }
+ } else if (src->op == GGML_OP_VIEW) {
+ if (src->src[0] == nullptr || src->src[0]->view_src != nullptr) {
+ return false;
+ }
+ } else {
return false;
}
}
@@ -1030,18 +1110,29 @@ static bool is_msa_block_mask_expansion(const ggml_tensor * op) {
return tensor_name_starts_with(src, "msa_block_mask");
}
-static bool is_op_unsupported_case(const ggml_tensor * op) {
+namespace {
+struct ggml_openvino_op_support {
+ bool is_supported = true;
+ std::string reason;
+
+ operator bool() const {
+ return is_supported;
+ }
+};
+} // namespace
+
+static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
if (is_msa_block_mask_expansion(op)) {
- return true;
+ return {false, "MSA block mask expansion is not supported"};
}
switch (op->op) {
case GGML_OP_CONCAT: {
if (op->type == GGML_TYPE_I64) {
- return true;
+ return {false, "CONCAT with I64 type is not supported"};
}
if (ggml_openvino_get_device_name() == "GPU" && op->type == GGML_TYPE_BF16 && has_view_op_input(op)) {
- return true;
+ return {false, "CONCAT with BF16 type and VIEW input is not supported on GPU"};
}
break;
}
@@ -1052,24 +1143,21 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// OpenVINO SET translation currently supports dst layouts that match src0 strides.
if (op->src[0] == nullptr || nb1 != op->src[0]->nb[1] || nb2 != op->src[0]->nb[2] || nb3 != op->src[0]->nb[3]) {
- // std::cout << "Unsupported SET op with dst nb1=" << nb1 << ", nb2=" << nb2 << ", nb3=" << nb3
- // << " that does not match src0 strides nb[1]="
- // << (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[1]) : "null")
- // << ", nb[2]=" << (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[2]) : "null")
- // << ", nb[3]=" << (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[3]) : "null")
- // << std::endl;
- return true;
+ return {false, "SET op with dst nb1=" + std::to_string(nb1) + ", nb2=" + std::to_string(nb2) + ", nb3=" + std::to_string(nb3) +
+ " that does not match src0 strides nb[1]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[1]) : "null") +
+ ", nb[2]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[2]) : "null") +
+ ", nb[3]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[3]) : "null")};
}
break;
}
case GGML_OP_GET_ROWS:
case GGML_OP_SET_ROWS: {
if (op->ne[3] != 1) {
- return true;
+ return {false, "GET_ROWS/SET_ROWS with ne[3] != 1 (ne[3]=" + std::to_string(op->ne[3]) + ") is not supported"};
}
if (op->op == GGML_OP_GET_ROWS && ggml_openvino_get_device_name() == "GPU" &&
op->src[0]->type == GGML_TYPE_BF16) {
- return true;
+ return {false, "GET_ROWS with BF16 src0 is not supported on GPU"};
}
if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K ||
op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_1)) {
@@ -1078,14 +1166,14 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// make_int8_weights/make_int4_weights: dequant is done in f16, not f32, to keep the
// Convert/Subtract/Multiply chain fusable into GatherMatmulCompressed/FullyConnectedCompressed
// for the shared non-test code paths).
- return true;
+ return {false, "GET_ROWS/SET_ROWS with ne[0] == 256 and type " + std::string(ggml_type_name(op->src[0]->type)) +
+ " rejected due to f16-arithmetic dequant rounding errors that intermittently exceed 1e-7 NMSE threshold"};
}
-
break;
}
case GGML_OP_RESHAPE: {
if (strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) {
- return true;
+ return {false, "RESHAPE for ffn_norm_exps is not supported"};
}
break;
}
@@ -1093,11 +1181,13 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
case GGML_OP_MUL:
case GGML_OP_SUB: {
if (op->src[1]->op == GGML_OP_PERMUTE) {
- return true;
+ return {false, "ADD/MUL/SUB with PERMUTE src1 is not supported"};
}
for (int i = 0; i < 4; i++) {
if (op->src[0]->ne[i] != op->src[1]->ne[i] && (op->src[0]->ne[i] != 1 && op->src[1]->ne[i] != 1)) {
- return true;
+ return {false, "ADD/MUL/SUB with incompatible broadcast shapes: src0->ne[" + std::to_string(i) + "]=" +
+ std::to_string(op->src[0]->ne[i]) + ", src1->ne[" + std::to_string(i) + "]=" +
+ std::to_string(op->src[1]->ne[i])};
}
}
break;
@@ -1106,7 +1196,7 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// Keep support aligned with the CPU backend implementation, which only handles f32 inputs/output and i32 ids.
if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32 ||
op->src[2]->type != GGML_TYPE_I32) {
- return true;
+ return {false, "ADD_ID only supports F32 inputs/output and I32 ids"};
}
break;
}
@@ -1116,14 +1206,27 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// until the fused GPU kernel is reliable. (falied case llama-arch-test mpt)
if (ggml_openvino_get_device_name() == "GPU" && op->src[1]->ne[0] == op->ne[0] &&
op->src[1]->ne[1] == 1 && op->src[1]->ne[2] == 1 && op->src[1]->ne[3] == 1) {
- return true;
+ return {false, "DIV per-channel scale broadcast is not supported on GPU"};
+ }
+ break;
+ }
+ case GGML_OP_POOL_2D: {
+ const auto& name = ggml_openvino_get_device_name();
+ if (name == "GPU") {
+ const int32_t * params = op->op_params;
+ const int k0 = params[1];
+ const int k1 = params[2];
+ const int p0 = params[5];
+ const int p1 = params[6];
+ if ((p0 > 0 || p1 > 0) && (k0 < 3 || k1 < 3)) {
+ return {false, "POOL_2D with padding and kernel size < 3 is not supported on " + name};
+ }
}
break;
}
case GGML_OP_SUM_ROWS: {
- // if the input is PERMUTE skip
if (op->src[0]->op == GGML_OP_PERMUTE) {
- return true;
+ return {false, "SUM_ROWS with PERMUTE input is not supported"};
}
break;
}
@@ -1140,54 +1243,51 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// accuracy drift in the OpenVINO path. Restrict by scale=1.0 to avoid
// affecting non-gemma3n models such as Llama-3.2.
if (fabsf(scale - 1.0f) < 1e-6f && is_gemma3n_flash_attn_pattern(op)) {
- return true;
+ return {false, "FLASH_ATTN_EXT gemma3n pattern on GPU is not supported"};
}
if (op->src[4] != nullptr) {
- // GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with sinks\n");
- return true;
+ return {false, "FLASH_ATTN_EXT with sinks is not supported"};
}
if (!is_supported_flash_attn_pattern(op)) {
- return true;
+ return {false, "FLASH_ATTN_EXT unsupported attention pattern"};
}
if (max_bias > 0) {
- // GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with max_bias > 0\n");
- return true;
+ return {false, "FLASH_ATTN_EXT with max_bias > 0 (max_bias=" + std::to_string(max_bias) + ") is not supported"};
}
if (logit_softcap != 0) {
- // GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with logit_softcap != 0\n");
- return true;
+ return {false, "FLASH_ATTN_EXT with logit_softcap != 0 (logit_softcap=" + std::to_string(logit_softcap) + ") is not supported"};
}
break;
}
case GGML_OP_PERMUTE: {
- if (op->type == GGML_TYPE_BF16) {
- // err msg: [GPU] Could not find a suitable kernel for transpose
- // GGML_LOG_WARN("OpenVINO backend does not support PERMUTE with BF16 type\n");
- return true;
+ if (op->type == GGML_TYPE_BF16 && ggml_openvino_get_device_name() == "GPU") {
+ return {false, "PERMUTE with BF16 type is not supported on GPU"};
}
break;
}
case GGML_OP_CPY: {
if (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16) {
- // GGML_LOG_WARN("OpenVINO backend does not support CPY with non-contiguous data or bf16 types\n");
- return true;
+ return {false, "CPY with BF16 src type is not supported"};
}
// CPY to a quantized destination (e.g. f32 -> q4_0) is numerically unstable with OpenVINO backend.
if (ggml_is_quantized(op->type)) {
- return true;
+ return {false, "CPY to quantized destination (e.g. f32 -> q4_0) is numerically unstable"};
}
if (ggml_nelements(op->src[0]) != ggml_nelements(op->src[1])) {
- return true;
+ return {false, "CPY with mismatched element counts is not supported: src0=" + std::to_string(ggml_nelements(op->src[0])) +
+ " != src1=" + std::to_string(ggml_nelements(op->src[1]))};
}
// op test case with non-contiguous src or dst
if ((op->ne[0] == 3 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) ||
(op->ne[0] == 1 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) ||
(op->ne[0] == 2 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2)) {
- return true;
+ return {false, "CPY with non-contiguous shape [" + std::to_string(op->ne[0]) + ", " +
+ std::to_string(op->ne[1]) + ", " + std::to_string(op->ne[2]) + ", " +
+ std::to_string(op->ne[3]) + "] is not supported"};
}
if (!cpy_output_view_is_supported(op)) {
- return true;
+ return {false, "CPY with non-contiguous output view is not supported"};
}
break;
}
@@ -1196,13 +1296,14 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
ggml_is_quantized(op->src[0]->type) && strcmp(op->src[0]->name, "a") == 0 &&
strcmp(op->src[1]->name, "b") == 0 && op->src[0]->ne[1] == 1 && op->src[1]->ne[1] == 64 &&
op->src[0]->ne[0] == 256 && op->src[1]->ne[0] == 256) {
- return true;
+ return {false, "MUL_MAT quantized benchmark test case on GPU is not supported"};
}
if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) {
- return true;
+ return {false, "MUL_MAT with incompatible broadcast on ne[3]: src0->ne[3]=" + std::to_string(op->src[0]->ne[3]) +
+ ", src1->ne[3]=" + std::to_string(op->src[1]->ne[3])};
}
if (op->src[0]->op == GGML_OP_VIEW && op->src[1]->op == GGML_OP_VIEW) {
- return true;
+ return {false, "MUL_MAT with both inputs as VIEW is not supported"};
}
break;
}
@@ -1210,16 +1311,14 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// Single-expert (or empty) MUL_MAT_ID is a degenerate shape that stresses GatherMatmul edge
// cases and never occurs in real MoE; let it fall back to CPU.
if (op->src[0] != nullptr && op->src[0]->ne[2] <= 1) {
- return true;
+ return {false, "MUL_MAT_ID with single-expert or empty ne[2] <= 1 (ne[2]=" +
+ std::to_string(op->src[0]->ne[2]) + ") is not supported"};
}
if (ggml_openvino_get_device_name() == "GPU" && op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_BF16) {
- return true;
+ return {false, "MUL_MAT_ID with BF16 weights on GPU is not supported"};
}
- // GPU MUL_MAT_ID uses a Gather+MatMul fallback because the GPU plugin rejects internal
- // GatherMatmul for these test shapes. Skip cases that would materialize a large selected
- // expert-weight temporary.
- if (ggml_openvino_get_device_name() == "GPU" && mul_mat_id_requires_large_tmp(op)) {
- return true;
+ if (op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_MXFP4 && mul_mat_id_requires_large_tmp(op)) {
+ return {false, "MUL_MAT_ID with MXFP4 weights requires large temporary"};
}
break;
}
@@ -1229,51 +1328,46 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
const int mode = op_params[2];
if (op_params[15] != 0) {
// FIXME: support ggml_rope_set_offset
- return true;
+ return {false, "ggml_rope_set_offset is not supported"};
}
if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX && mode != GGML_ROPE_TYPE_IMROPE) {
- // GGML_LOG_WARN("OpenVINO backend does not support ROPE with mode %d\n", mode);
- return true;
+ return {false, "ROPE with mode " + std::to_string(mode) + " is not supported"};
}
const int64_t head_dim = op->src[0]->ne[0];
const int64_t rope_dims = n_dims == 0 ? head_dim : n_dims;
if (rope_dims <= 0 || rope_dims > head_dim || (rope_dims % 2) != 0) {
- // GGML_LOG_WARN("OpenVINO backend does not support ROPE with n_dims %d and src[0]->ne[0] %ld\n", n_dims,
- // op->src[0]->ne[0]);
- return true;
+ return {false, "ROPE with n_dims=" + std::to_string(n_dims) + ", head_dim=" + std::to_string(head_dim) + " is not supported"};
}
if (op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) {
- // GGML_LOG_WARN("OpenVINO backend does not support ROPE with type %s\n", ggml_type_name(op->type));
- return true;
+ return {false, "ROPE with type " + std::string(ggml_type_name(op->type)) + " is not supported"};
}
if (op->src[0]->op == GGML_OP_VIEW) {
- if (op->src[0]->view_src->ne[1] != op->src[0]->ne[2]) {
- // GGML_LOG_WARN(
- // "OpenVINO backend does not support ROPE with src[0]->view_src->ne[1] %ld != src[0]->ne[2] "
- // "%ld\n",
- // op->src[0]->view_src->ne[1], op->src[0]->ne[2]);
- return true;
+ const struct ggml_tensor * view = op->src[0];
+ const struct ggml_tensor * view_src = view->view_src;
+ if (view_src->ne[1] != view->ne[1] || view_src->ne[2] != view->ne[2] || view_src->ne[3] != view->ne[3]) {
+ return {false, "ROPE with view_src->ne [" + std::to_string(view_src->ne[1]) + ", " +
+ std::to_string(view_src->ne[2]) + ", " + std::to_string(view_src->ne[3]) +
+ "] != view->ne [" + std::to_string(view->ne[1]) + ", " +
+ std::to_string(view->ne[2]) + ", " + std::to_string(view->ne[3]) +
+ "] is not supported"};
}
}
if (mode == GGML_ROPE_TYPE_IMROPE &&
(op->src[2] != 0 || ((const float *) op_params)[6] != 1 || ((const float *) op_params)[7] != 0 ||
((const float *) op_params)[8] != 1)) {
- // GGML_LOG_WARN("OpenVINO backend does not support IMROPE with freq_factors, freq_scale, ext_factor, and attn_factor\n");
- return true;
+ return {false, "IMROPE with freq_factors, freq_scale, ext_factor, and attn_factor is not supported"};
}
break;
}
case GGML_OP_TRANSPOSE: {
- // if the type is bf16, will return true
if (op->type == GGML_TYPE_BF16) {
- // GGML_LOG_WARN("OpenVINO backend does not support CONT with BF16 type\n");
- return true;
+ return {false, "TRANSPOSE with BF16 type is not supported"};
}
break;
}
case GGML_OP_REPEAT: {
if (ggml_openvino_get_device_name() == "GPU" && op->type == GGML_TYPE_BF16) {
- return true;
+ return {false, "REPEAT with BF16 type is not supported on GPU"};
}
break;
}
@@ -1285,15 +1379,15 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// return true;
// }
if (op->src[2]->op == GGML_OP_PERMUTE) {
- return true;
+ return {false, "GATED_DELTA_NET with PERMUTE src2 is not supported"};
}
// kda (per-key-dimension gating) not supported by fused GatedDeltaNet op
if (op->src[3]->ne[0] != 1) {
- return true;
+ return {false, "GATED_DELTA_NET with kda (per-key-dimension gating) is not supported"};
}
// K > 1 (multiple state snapshots) not supported by fused op
if (((const int32_t *) op->op_params)[0] > 1) {
- return true;
+ return {false, "GATED_DELTA_NET with K > 1 (multiple state snapshots) is not supported"};
}
break;
}
@@ -1307,17 +1401,17 @@ static bool is_op_unsupported_case(const ggml_tensor * op) {
// Skip TOPK_MOE fused tests until it is fully supported.
// The argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe.
if (strcmp(op->name, "selected_experts") == 0) {
- return true;
+ return {false, "VIEW for selected_experts (argsort_top_k) is not supported"};
}
break;
}
default:
break;
}
- return false;
+ return {true, ""};
}
-static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
+static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(ggml_backend_dev_t dev, const ggml_tensor * op) {
GGML_ASSERT(dev->reg != nullptr);
static std::unordered_set supported_types{
@@ -1367,48 +1461,41 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
case GGML_OP_UNARY: {
auto supported = supported_unary_ops.find(ggml_get_unary_op(op)) != supported_unary_ops.end();
if (!supported) {
- // GGML_LOG_WARN("OpenVINO backend does not support unary op %s\n", ggml_unary_op_name(ggml_get_unary_op(op)));
- return false;
+ return {false, "unary op " + std::string(ggml_unary_op_name(ggml_get_unary_op(op))) + " has no op translator"};
}
if (ggml_get_unary_op(op) == GGML_UNARY_OP_EXP && op->type == GGML_TYPE_F32) {
- return false;
+ return {false, "UNARY_EXP with F32 type is not supported"};
}
break;
}
case GGML_OP_GLU: {
auto supported = supported_glu_ops.find(ggml_get_glu_op(op)) != supported_glu_ops.end();
if (!supported) {
- // GGML_LOG_WARN("OpenVINO backend does not support GLU op %s\n", ggml_glu_op_name(ggml_get_glu_op(op)));
- return false;
+ return {false, "GLU op " + std::string(ggml_glu_op_name(ggml_get_glu_op(op))) + " has no op translator"};
}
// if (has_view_op_input(op)) {
- // // GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n",
- // // ggml_glu_op_name(ggml_get_glu_op(op)));
- // return false;
+ // return {false, "GLU op " + std::string(ggml_glu_op_name(ggml_get_glu_op(op))) + " with view input is not supported"};
// }
if (op->src[1] == nullptr && op->src[0]->ne[0] % 2 != 0) {
// triggers bug in ov gpu
- return false;
+ return {false, "GLU op with odd src0 ne[0] and null src1 is not supported"};
}
break;
}
default: {
auto supported = supported_ops.find(op->op) != supported_ops.end();
if (!supported) {
- // GGML_LOG_WARN("OpenVINO backend does not support op %s\n", ggml_op_name(op->op));
- return false;
+ return {false, "op " + std::string(ggml_op_name(op->op)) + " has no op translator"};
}
static std::set ops_not_support_view_input{};
if (ops_not_support_view_input.find(op->op) != ops_not_support_view_input.end() && has_view_op_input(op)) {
- // GGML_LOG_WARN("OpenVINO backend does not support op %s with view input\n", ggml_op_name(op->op));
- return false;
+ return {false, "op " + std::string(ggml_op_name(op->op)) + " with VIEW input is not supported"};
}
}
}
if (supported_types.find(op->type) == supported_types.end()) {
- // GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(op->type));
- return false;
+ return {false, "tensor type " + std::string(ggml_type_name(op->type)) + " is not supported"};
}
for (int i = 0; i < GGML_MAX_SRC; i++) {
auto * src = op->src[i];
@@ -1416,21 +1503,32 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
break;
}
if (supported_types.find(src->type) == supported_types.end()) {
- // GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(src->type));
- return false;
+ return {false, "src[" + std::to_string(i) + "] type " + std::string(ggml_type_name(src->type)) + " is not supported"};
}
const bool is_supported_3d_moe_expert =
op->op == GGML_OP_MUL_MAT_ID && i == 0 && (src->type == GGML_TYPE_MXFP4 || src->ne[3] == 1);
if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_moe_expert) {
- // GGML_LOG_WARN("OpenVINO backend does not support 3D quantized tensors\n");
- return false;
+ return {false, "3D quantized tensor for src[" + std::to_string(i) + "] is not supported"};
}
}
- if (is_op_unsupported_case(op)) {
- return false;
+ auto op_support_case = is_op_supported_case(op);
+ if (!op_support_case.is_supported) {
+ return op_support_case;
}
- return true;
+ return {true, ""};
+}
+
+static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
+ auto res = ggml_backend_openvino_device_supports_op_impl(dev, op);
+ if (!res.is_supported) {
+ static const bool log_unsupported = ggml_openvino_getenv_int("GGML_OPENVINO_LOG_UNSUPPORTED_OPS") != 0;
+ if (log_unsupported) {
+ GGML_LOG_WARN("OpenVINO op unsupported: op '%s' (%s), type %s: %s\n",
+ op->name, ggml_op_name(op->op), ggml_type_name(op->type), res.reason.c_str());
+ }
+ }
+ return res.is_supported;
}
static bool ggml_backend_openvino_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp
index 120db01e17cd..93f9e8254aa6 100644
--- a/ggml/src/ggml-openvino/ggml-quants.cpp
+++ b/ggml/src/ggml-openvino/ggml-quants.cpp
@@ -851,7 +851,8 @@ std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor,
const auto * type_traits = ggml_get_type_traits(tensor->type);
const size_t src_row_bytes = ggml_row_size(tensor->type, ne0);
- bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128);
+ bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128 ||
+ requant_type == ExtraQuantType::Q4_0_64 || requant_type == ExtraQuantType::Q4_1_64);
// Streaming dequant (opt-in via GGML_OPENVINO_REDUCE_COMPILE_MEM or
// GGML_OPENVINO_MEMORY_OPTIMIZE): instead of
@@ -879,7 +880,9 @@ std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor,
result->set_friendly_name(tensor->name);
return result;
}
- if (is_u4) {
+ if (requant_type == ExtraQuantType::Q4_1_64) {
+ quantize_q4_1_asym(weights_f32.data(), weights, scales, zp, n_elements, block_size);
+ } else if (is_u4) {
quantize_q4_0(weights_f32.data(), weights, scales, zp, n_elements, block_size);
} else if (requant_type == ExtraQuantType::Q8_1_C) {
quantize_q8_1(weights_f32.data(), weights, scales, zp, n_elements, block_size);
@@ -1178,6 +1181,71 @@ void quantize_q4_0(const float * x,
}
}
+// Asymmetric u4 quantization with a per-group scale and zero point.
+//
+// Unlike quantize_q4_0's unsigned branch, which pins the zero point to 8 and is therefore
+// symmetric, this keeps a real per-group zero point, so a group whose values are not centred on
+// zero does not waste half its range.
+void quantize_q4_1_asym(const float * x,
+ ov::Tensor & weights_arr,
+ ov::Tensor & scales_arr,
+ ov::Tensor & zp_arr,
+ int64_t k,
+ int64_t qk) {
+ assert(k % qk == 0);
+ const int nb = k / qk;
+
+ auto * weights = static_cast(weights_arr.data());
+ auto * scales = scales_arr.data::value_type>();
+ auto * zp = static_cast(zp_arr.data());
+
+ // u4 zero points are packed two per byte, low nibble first, indexed by group -- the same
+ // convention as the unsigned branch of quantize_q4_0.
+ auto store_zp = [zp](int i, uint8_t v) {
+ if (i % 2 == 0) {
+ zp[i / 2] = v & 0x0F;
+ } else {
+ zp[i / 2] |= (uint8_t) ((v & 0x0F) << 4);
+ }
+ };
+
+ for (int i = 0; i < nb; i++) {
+ float vmin = x[i * qk];
+ float vmax = x[i * qk];
+ for (int j = 1; j < qk; j++) {
+ const float v = x[i * qk + j];
+ vmin = std::min(vmin, v);
+ vmax = std::max(vmax, v);
+ }
+ // Include 0 in the range so an all-positive or all-negative group still represents zero
+ // exactly -- these are weights, so an exact zero matters.
+ vmin = std::min(vmin, 0.0f);
+ vmax = std::max(vmax, 0.0f);
+
+ const float d = (vmax - vmin) / 15.0f;
+ if (d == 0.0f) {
+ scales[i] = ov::float16(1.0f);
+ store_zp(i, 0);
+ memset(weights + i * qk / 2, 0, qk / 2);
+ continue;
+ }
+ const float id = 1.0f / d;
+
+ // The zero point is itself a 4-bit integer, so round it and dequantize as (q - zq) * d.
+ const int zq = std::max(0, std::min(15, (int) lroundf(-vmin * id)));
+ scales[i] = ov::float16(d);
+ store_zp(i, (uint8_t) zq);
+
+ for (int j = 0; j < qk / 2; ++j) {
+ const float x0 = x[i * qk + 2 * j] * id;
+ const float x1 = x[i * qk + 2 * j + 1] * id;
+ const uint8_t q0 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x0) + zq));
+ const uint8_t q1 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x1) + zq));
+ weights[i * qk / 2 + j] = (uint8_t) (q0 | (q1 << 4));
+ }
+ }
+}
+
void quantize_q8_0(const float * x,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
diff --git a/ggml/src/ggml-openvino/ggml-quants.h b/ggml/src/ggml-openvino/ggml-quants.h
index e247255a7f77..d5273727e87d 100644
--- a/ggml/src/ggml-openvino/ggml-quants.h
+++ b/ggml/src/ggml-openvino/ggml-quants.h
@@ -122,6 +122,10 @@ inline const char * extra_quant_type_name(ExtraQuantType t) {
return "Q8_0_32";
case ExtraQuantType::Q8_1_C:
return "Q8_1_C";
+ case ExtraQuantType::Q4_0_64:
+ return "Q4_0_64";
+ case ExtraQuantType::Q4_1_64:
+ return "Q4_1_64";
default:
return "unknown";
}
@@ -166,6 +170,12 @@ void quantize_q8_1(const float * x,
int64_t k,
int64_t qk,
int64_t block_offset = 0);
+void quantize_q4_1_asym(const float * x,
+ ov::Tensor & weights_arr,
+ ov::Tensor & scales_arr,
+ ov::Tensor & zp_arr,
+ int64_t k,
+ int64_t qk);
void quantize_q8_0(const float * x,
ov::Tensor & weights_arr,
ov::Tensor & scales_arr,
diff --git a/ggml/src/ggml-openvino/openvino/node_context.h b/ggml/src/ggml-openvino/openvino/node_context.h
index 2e2756037703..80bb595effcc 100644
--- a/ggml/src/ggml-openvino/openvino/node_context.h
+++ b/ggml/src/ggml-openvino/openvino/node_context.h
@@ -33,6 +33,8 @@ class NodeContext : public frontend::NodeContext {
const std::vector & get_input_names() const { return m_input_names; }
+ const std::vector & get_output_names() const { return m_output_names; }
+
size_t get_input_size() const override { return m_decoder->get_input_size(m_node_idx); }
ov::element::Type get_input_type(size_t index) const {
@@ -120,7 +122,10 @@ class NodeContext : public frontend::NodeContext {
auto view_it = m_tensor_map->find(m_input_names[idx]);
if (!base_name.empty() && view_it != m_tensor_map->end()) {
auto base_it = m_tensor_map->find(base_name);
- if (base_it != m_tensor_map->end() &&
+ // A multi-output translator can publish a VIEW directly without materializing
+ // its packed parent (GatedDeltaNet attention/state). In that case the VIEW is the
+ // authoritative value. The node comparison retains the existing resolved-VIEW path.
+ if (base_it == m_tensor_map->end() ||
view_it->second.get_node_shared_ptr() != base_it->second.get_node_shared_ptr()) {
return view_it->second;
}
diff --git a/ggml/src/ggml-openvino/openvino/op/cpy.cpp b/ggml/src/ggml-openvino/openvino/op/cpy.cpp
index 5b387fc50d38..60d3019f2571 100644
--- a/ggml/src/ggml-openvino/openvino/op/cpy.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/cpy.cpp
@@ -3,8 +3,11 @@
#include "../utils.h"
#include
+#include
+#include
#include
-#include
+#include
+#include
#include
#include
#include
@@ -12,9 +15,14 @@
#include
#include
#include
+#include
#include
+#include
#include
#include
+#include
+#include
+#include
namespace ov {
namespace frontend {
@@ -61,14 +69,45 @@ OutputVector translate_cpy(const NodeContext & context) {
return rename_outputs_with_suffix({res}, context.get_name());
}
- // Recurrent state cache writeback into a slot block of the cache. Where the block starts and
- // where the copied data starts in the source are runtime inputs, so the cached model works for
- // any kv head, active sequence count and token count. The result is the full updated cache.
+ // Recurrent state cache writeback into a slot block of the cache. Where the block starts is a
+ // runtime input, so the cached model works for any kv head and active sequence count. The
+ // result is the full updated cache.
// op_case 1: gated-delta-net state, op_case 2: conv state, op_case 3: defrag remainder.
+ if (op_case == 3) {
+ // With -np 1 (and generally whenever there is no defrag remainder) this GET_ROWS gathers
+ // zero rows: nothing to write back, and the cache is unchanged. NPU rejects zero-size
+ // tensors, so short-circuit instead of building a degenerate Slice/Concat chain.
+ bool is_empty = false;
+ if (input_shape.rank().is_static()) {
+ for (const auto & d : input_shape) {
+ if (d.is_static() && d.get_length() == 0) {
+ is_empty = true;
+ break;
+ }
+ }
+ }
+ if (is_empty) {
+ return {context.get_input(1)};
+ }
+ }
+ // op_case 7/8/9 are the single-slot variants; op_case 10 writes native GDN state into a
+ // multi-slot cache without rollback snapshots.
+ const bool single_slot_assign = op_case >= 7 && op_case <= 9;
+ const bool direct_gdn_state = op_case == 7 || op_case == 10;
+ int writeback_case = op_case;
+ if (op_case == 10) {
+ writeback_case = 1;
+ } else if (single_slot_assign) {
+ writeback_case = op_case - 6;
+ }
const std::string slot_begin_name = "rs_slot_begin_" + context.get_name();
- const bool slice_assign =
- context.has_input(slot_begin_name) && !context.is_stateful() && (op_case >= 1 && op_case <= 3);
+ const bool slice_assign = !context.is_stateful() && writeback_case >= 1 && writeback_case <= 3 &&
+ (single_slot_assign || context.has_input(slot_begin_name));
if (slice_assign) {
+ if (single_slot_assign && writeback_case == 3) {
+ return {context.get_input(1)};
+ }
+
const int64_t slot_axis = 2;
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
@@ -78,22 +117,49 @@ OutputVector translate_cpy(const NodeContext & context) {
std::vector{1, 1, -1, output_shape[3].get_length()});
ov::Output src;
- ov::Output begin = context.get_input(slot_begin_name);
+ ov::Output begin;
+ if (!single_slot_assign) {
+ begin = context.get_input(slot_begin_name);
+ }
auto base = context.get_input(1);
- if (op_case == 1) {
- // GDN packs [attn | state snapshots]; the state part runs from src_begin to the end.
- auto src_begin = context.get_input("rs_src_begin_" + context.get_name());
- auto state_part = std::make_shared(context.get_input(0), src_begin, int_max, one, axis);
- src = std::make_shared(state_part, feature, false);
- } else if (op_case == 2) {
- // conv_input is [previous conv state | new tokens]; copy the conv_kernel_size - 1 wide
- // window starting at src_begin, which is the snapshot this writeback corresponds to.
+ if (writeback_case == 1) {
+ if (direct_gdn_state) {
+ // Non-rollback GDN publishes state directly as [active_slots, heads, value_dim,
+ // key_dim]. Flatten each active slot before replacing or updating the cache.
+ src = std::make_shared(context.get_input(0), feature, false);
+ } else {
+ // Multi-slot rollback still consumes GGML's packed [attention | state snapshots]
+ // layout. Slice the state block using the runtime source offset.
+ auto src_begin = context.get_input("rs_src_begin_" + context.get_name());
+ auto state_part =
+ std::make_shared(context.get_input(0), src_begin, int_max, one, axis);
+ src = std::make_shared(state_part, feature, false);
+ }
+ } else if (writeback_case == 2) {
+ // conv_input is [previous conv state | new tokens]; the snapshot is the conv_kernel_size - 1
+ // columns ending at the last *valid* token. Gather (rather than Slice) keeps the output
+ // shape static even though the window start is a runtime value.
auto window_size = (int64_t) input_shape[3].get_length();
- auto src_begin = context.get_input("rs_src_begin_" + context.get_name());
- auto src_end = std::make_shared(
- src_begin, ov::op::v0::Constant::create(ov::element::i64, {1}, {window_size}));
- auto window = std::make_shared(context.get_input(0), src_begin, src_end, one,
- ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
+ ov::Output window;
+ auto col_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
+ const std::string src_begin_name = "rs_src_begin_" + context.get_name();
+ if (context.has_input(src_begin_name)) {
+ auto src_begin = context.get_input(src_begin_name);
+ auto src_end = std::make_shared(
+ src_begin, ov::op::v0::Constant::create(ov::element::i64, {1}, {window_size}));
+ window = std::make_shared(context.get_input(0), src_begin, src_end, one, col_axis);
+ } else if (context.has_input("chunk_valid_len")) {
+ std::vector offsets(window_size);
+ std::iota(offsets.begin(), offsets.end(), 0);
+ auto indices = std::make_shared(
+ ov::op::v0::Constant::create(ov::element::i64, {(size_t) window_size}, offsets),
+ context.get_input("chunk_valid_len"));
+ window = std::make_shared(context.get_input(0), indices, col_axis);
+ } else {
+ auto window_begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {-window_size});
+ window =
+ std::make_shared(context.get_input(0), window_begin, int_max, one, col_axis);
+ }
const auto base_shape = base.get_partial_shape();
FRONT_END_OP_CONVERSION_CHECK(base_shape.rank().is_static() && base_shape.rank().get_length() == 4,
"CPY conv state cache update requires rank-4 base cache");
@@ -122,6 +188,10 @@ OutputVector translate_cpy(const NodeContext & context) {
src = std::make_shared(src, context.get_output_type());
}
+ if (single_slot_assign) {
+ return rename_outputs_with_suffix({src}, context.get_name());
+ }
+
auto src_len = std::make_shared(
std::make_shared(src, ov::element::i64), axis,
ov::op::v0::Constant::create(ov::element::i64, {}, {0}));
@@ -145,6 +215,10 @@ OutputVector translate_cpy(const NodeContext & context) {
src = std::make_shared(src, context.get_output_type());
}
+ if (single_slot_assign) {
+ return rename_outputs_with_suffix({src}, context.get_name());
+ }
+
auto src_len =
std::make_shared(std::make_shared(src, ov::element::i64), axis,
ov::op::v0::Constant::create(ov::element::i64, {}, {0}));
@@ -157,6 +231,63 @@ OutputVector translate_cpy(const NodeContext & context) {
auto input = process_view_input_new(context, 0);
+ if (op_case == 5 || op_case == 6) {
+ auto input_shape = context.get_input_shape(0);
+ auto output_shape = context.get_output_shape();
+ auto dst_ggml_shape = context.get_view_input_ggml_shape(1, 0);
+ auto dst_stride = context.get_view_input_stride(1, 0);
+ size_t offset_bytes = context.get_view_input_offset(1, 0);
+ auto n_state = (int64_t) context.get_input_shape(0)[3].get_length();
+ auto n_state_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_state});
+ auto kv_buf = context.get_input(1); // shape {1,1,1,N}
+
+ Output token_len_per_seq;
+ Output n_write_dyn;
+ if (context.has_input("token_len_per_seq")) {
+ token_len_per_seq = context.get_input("token_len_per_seq");
+ n_write_dyn = std::make_shared(token_len_per_seq, n_state_c);
+ } else {
+ n_write_dyn = ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) dst_ggml_shape[3]});
+ }
+ size_t elem_size = dst_stride[3];
+ FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, "CPY KV cache view update has invalid element size");
+ int64_t start_elem = (int64_t) (offset_bytes / elem_size);
+ // op_case 5: decoder self-attention – write offset advances each step.
+ // op_case 6: encoder self-attn or cross-attn – offset fixed at compile time.
+ const bool is_decoder_self_attn = (op_case == 5);
+ auto ones_c = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{1, 1, 1});
+ auto new_shape = std::make_shared(ov::OutputVector{ones_c, n_write_dyn}, 0);
+
+ auto reshaped = std::make_shared(input, new_shape, false);
+ auto data = std::make_shared(reshaped, context.get_output_type());
+ // Indices [start_elem .. start_elem + n_write) on axis 3 of {1,1,1,N}
+ // For decoder self-attention the write offset advances each step, so compute it
+ // dynamically from the model inputs: start = (attention_size - token_len_per_seq) * n_state.
+ // For encoder self-attn and cross-attn the offset is fixed at graph-compile time.
+ ov::Output start;
+ if (is_decoder_self_attn && context.has_input("attention_size") && context.has_input("token_len_per_seq")) {
+ auto attention_size_in = context.get_input("attention_size");
+ auto token_len_in = context.get_input("token_len_per_seq");
+ auto past_tokens = std::make_shared(attention_size_in, token_len_in);
+ auto new_start = std::make_shared(past_tokens, n_state_c);
+ start = std::make_shared(
+ new_start, ov::op::v0::Constant::create(ov::element::i64, {1}, {start_elem}));
+ } else {
+ start = ov::op::v0::Constant::create(ov::element::i64, {1}, {start_elem});
+ }
+ auto start_squeezed = std::make_shared(start);
+ auto end = std::make_shared(start_squeezed, n_write_dyn);
+ auto end_squeezed = std::make_shared(end);
+ auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+ auto step_squeezed = std::make_shared(step);
+ auto indices =
+ std::make_shared(start_squeezed, end_squeezed, step_squeezed, ov::element::i64);
+ auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
+
+ auto kv_updated = std::make_shared(kv_buf, indices, data, axis);
+ return rename_outputs_with_suffix({kv_updated}, context.get_name());
+ }
+
if (input_shape != output_shape) {
auto new_shape = ov::op::v0::Constant::create(
ov::element::i64, {static_cast(output_shape.rank().get_length())}, output_shape.to_shape());
diff --git a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp
index 582df0130b59..06547f3d2968 100644
--- a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp
@@ -3,8 +3,8 @@
#include "../utils.h"
#include "ggml-openvino/ggml-openvino-extra.h"
+#include
#include
-#include
#include
#include
#include
@@ -15,6 +15,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -24,13 +25,62 @@ namespace ov {
namespace frontend {
namespace ggml {
namespace op {
+static ov::Output reshape_flat_kv(const ov::Output & kv_flat,
+ size_t view_offset_bytes,
+ size_t nb1_bytes,
+ int64_t n_head,
+ int64_t head_size,
+ const ov::Output & attention_size) {
+ int64_t n_state = n_head * head_size;
+ int64_t layer_start_elem = (int64_t) (view_offset_bytes / (nb1_bytes / n_state));
+ // Dynamic slice: [layer_start_elem, layer_start_elem + n_kv * n_state)
+ auto start_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {layer_start_elem});
+ auto n_state_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_state});
+ // end = start + attention_size * n_state (both static + dynamic)
+ auto kv_len_elems = std::make_shared(attention_size, n_state_c);
+ auto end_c = std::make_shared(start_c, kv_len_elems);
+ auto step_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+ auto axis_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
+ auto sliced = std::make_shared(kv_flat, start_c, end_c, step_c, axis_c);
+
+ // KV cache is laid out as {n_kv, n_head, head_size} in memory
+ // Reshape to {1, n_kv, n_head, head_size}, then transpose to {1, n_head, n_kv, head_size}
+ // as required by SDPA.
+ auto one_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+ auto n_head_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_head});
+ auto head_size_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_size});
+ // reshape: {n_kv*n_state} -> {1, n_kv, n_head, head_size}
+ auto new_shape =
+ std::make_shared(ov::OutputVector{one_c, attention_size, n_head_c, head_size_c}, 0);
+ auto reshaped = std::make_shared(sliced, new_shape, false);
+ // transpose: {1, n_kv, n_head, head_size} -> {1, n_head, n_kv, head_size}
+ auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3});
+ auto ret = std::make_shared(reshaped, perm);
+ return ret;
+}
OutputVector translate_flash_attn_ext(const NodeContext & context) {
- num_inputs_check(context, 4, 4);
+ num_inputs_check(context, 3, 4);
+ const bool has_mask = context.get_input_size() == 4;
auto q_f32 = context.get_input(0);
auto k = context.get_input(1);
auto v = context.get_input(2);
- auto mask = context.get_input(3);
+ const int op_case = context.get_op_case();
+
+ if (op_case == 1 || op_case == 2) {
+ int64_t n_state_head = (int64_t) context.get_view_input_ggml_shape(1, 0)[3];
+ int64_t n_head = (int64_t) context.get_view_input_ggml_shape(1, 0)[1];
+ size_t nb1 = context.get_view_input_stride(1, 0)[2];
+ size_t offset = context.get_view_input_offset(1, 0);
+ ov::Output attention_size;
+ if (op_case == 1) {
+ attention_size = context.get_input("attention_size");
+ } else {
+ attention_size = context.get_input("attention_size_static");
+ }
+ k = reshape_flat_kv(k, offset, nb1, n_head, n_state_head, attention_size);
+ v = reshape_flat_kv(v, offset, nb1, n_head, n_state_head, attention_size);
+ }
float * params = reinterpret_cast(context.get_output_op_params());
float scale = params[0];
@@ -43,16 +93,19 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
ov::Output res;
// For stateful
- std::string mask_name = "KQ_mask_sliced";
- if (context.get_input_names()[3].find("swa") != std::string::npos) {
- mask_name = "KQ_mask_swa_sliced";
- }
- if (context.has_input(mask_name)) {
- mask = context.get_input(mask_name);
- }
-
- if (mask.get_element_type() != ov::element::f16) {
- mask = std::make_shared(mask, ov::element::f16);
+ ov::Output mask;
+ if (has_mask) {
+ mask = context.get_input(3);
+ std::string mask_name = "KQ_mask_sliced";
+ if (context.get_input_names()[3].find("swa") != std::string::npos) {
+ mask_name = "KQ_mask_swa_sliced";
+ }
+ if (context.has_input(mask_name)) {
+ mask = context.get_input(mask_name);
+ }
+ if (mask.get_element_type() != ov::element::f16) {
+ mask = std::make_shared(mask, ov::element::f16);
+ }
}
//auto tile_kv = [&](int64_t num_heads, int64_t num_heads_kv, int64_t head_size, ov::Output kv) {
@@ -108,10 +161,14 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
// get [B, 1, 1, S_q, S_k], which NUMPY-broadcasts cleanly against the
// [B, num_heads_kv, factor, S_q, S_k] scores: B==B, then 1→num_heads_kv and
// 1→factor on the head dims.
- auto mask_unsq1 =
- std::make_shared(mask, ov::op::v0::Constant::create(ov::element::i64, {1}, {2}));
- // mask_unsq1: [B, 1, 1, S_q, S_k] (rank 5)
- ov::Output qk_masked = std::make_shared(qk_scaled, mask_unsq1);
+ ov::Output qk_masked;
+ if (has_mask) {
+ auto mask_unsq1 =
+ std::make_shared(mask, ov::op::v0::Constant::create(ov::element::i64, {1}, {2}));
+ qk_masked = std::make_shared(qk_scaled, mask_unsq1);
+ } else {
+ qk_masked = qk_scaled;
+ }
auto softmax = std::make_shared(qk_masked, /*axis=*/-1);
@@ -164,9 +221,16 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
k = tile_kv(num_heads, num_heads_kv, head_size, k);
v = tile_kv(num_heads, num_heads_kv, head_size, v);
- auto sdpa = std::make_shared(q, k, v, mask, scale_node, false);
- res = std::make_shared(sdpa,
- ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
+ constexpr auto causal = false;
+ if (has_mask) {
+ auto sdpa = std::make_shared(q, k, v, mask, scale_node, causal);
+ res = std::make_shared(
+ sdpa, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
+ } else {
+ auto sdpa = std::make_shared(q, k, v, scale_node, causal);
+ res = std::make_shared(
+ sdpa, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
+ }
res = std::make_shared(res, ov::element::f32);
return rename_outputs_with_suffix({res}, context.get_name());
}
diff --git a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
index 66c748283311..2801e92e9704 100644
--- a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp
@@ -7,12 +7,15 @@
#include
#include
#include
+#include
#include
#include
#include
#include
+#include
#include
#include
+#include
#include
#include
#include
@@ -80,6 +83,28 @@ OutputVector translate_gated_delta_net(const NodeContext & context) {
g = std::make_shared(g, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
beta = std::make_shared(beta, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
+ if (context.has_input("chunk_valid_len")) {
+ // The last prefill chunk is padded with fabricated tokens. The recurrence is
+ // S_t = S_{t-1} * exp(g_t) + k_t (x) ((v_t - S_{t-1}^T k_t) * beta_t)
+ // so forcing g = 0 and beta = 0 makes a padded step an exact identity and keeps the final
+ // state equal to the state after the last real token. Attention output at those positions
+ // is garbage but never read.
+ const auto & g_ps = g.get_partial_shape();
+ FRONT_END_OP_CONVERSION_CHECK(g_ps.rank().is_static() && g_ps.rank().get_length() == 3 && g_ps[1].is_static(),
+ "GATED_DELTA_NET pad masking requires a static token dimension");
+ const int64_t n_tokens = g_ps[1].get_length();
+ std::vector positions(n_tokens);
+ std::iota(positions.begin(), positions.end(), 0);
+ auto valid = std::make_shared(
+ ov::op::v0::Constant::create(ov::element::i64, {(size_t) n_tokens}, positions),
+ context.get_input("chunk_valid_len"));
+ auto mask = std::make_shared(
+ std::make_shared(valid, g.get_element_type()),
+ ov::op::v0::Constant::create(ov::element::i64, {2}, std::vector{0, 2}));
+ g = std::make_shared(g, mask);
+ beta = std::make_shared(beta, mask);
+ }
+
// std::cout << "GatedDeltaNet input shapes: q=" << q.get_partial_shape() << ", k=" << k.get_partial_shape()
// << ", v=" << v.get_partial_shape() << ", g=" << g.get_partial_shape()
// << ", beta=" << beta.get_partial_shape() << ", state=" << state.get_partial_shape() << std::endl;
@@ -93,6 +118,13 @@ OutputVector translate_gated_delta_net(const NodeContext & context) {
// Transpose output state back to ggml layout [B, H_v, value_dim, key_dim]
auto state_transposed = std::make_shared(state_4d, state_perm);
+ if (context.get_output_names().size() == 2) {
+ // The canonical graph consumes the packed GGML result only through separate attention
+ // and state VIEWs. Publish the native outputs under those VIEW names to avoid
+ // flatten -> concat -> reshape -> slice -> reshape chains. This also works for B > 1.
+ return rename_outputs_with_suffix({attn_4d, state_transposed}, context.get_name());
+ }
+
auto flat_shape_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
auto attn = std::make_shared(attn_4d, flat_shape_1d, false);
auto new_state = std::make_shared(state_transposed, flat_shape_1d, false);
@@ -285,6 +317,11 @@ static OutputVector translate_gated_delta_net_ref(const NodeContext & context) {
// state: [B*H_v, S_v, S_v] -> [B, H_v, S_v, S_v] -> flatten
auto state_4d_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{B, H_v, S_v, S_v});
auto state_4d = std::make_shared(final_state_out, state_4d_shape, false);
+ if (context.get_output_names().size() == 2) {
+ // Match the fused translator's direct attention/state contract.
+ return rename_outputs_with_suffix({attn_perm, state_4d}, context.get_name());
+ }
+
auto state_1d = std::make_shared(state_4d, flat_shape_1d, false);
// Concat [attn | state] and reshape to final output
diff --git a/ggml/src/ggml-openvino/openvino/op/get_rows.cpp b/ggml/src/ggml-openvino/openvino/op/get_rows.cpp
index 2ac8ec0ba1df..1d0f672384b1 100644
--- a/ggml/src/ggml-openvino/openvino/op/get_rows.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/get_rows.cpp
@@ -27,6 +27,10 @@ OutputVector translate_get_rows(const NodeContext & context) {
auto data = process_view_input_new(context, 0);
auto op_case = context.get_op_case();
+ if (op_case == 3 || op_case == 4) {
+ return {data};
+ }
+
ov::Output indices;
if ((op_case == 1 || op_case == 2) && context.has_input("s_copy_active_slot_len")) {
// Recurrent state reorder (inp->s_copy): slice the active (op_case 1) or extra (op_case 2)
diff --git a/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp b/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp
new file mode 100644
index 000000000000..c6d64aed43aa
--- /dev/null
+++ b/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp
@@ -0,0 +1,64 @@
+#include "../node_context.h"
+#include "../op_table.h"
+#include "../utils.h"
+
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace ov {
+namespace frontend {
+namespace ggml {
+namespace op {
+
+OutputVector translate_glu_geglu_quick(const NodeContext & context) {
+ num_inputs_check(context, 1, 2);
+
+ ov::Output src0;
+ ov::Output src1;
+ if (context.get_input_size() == 2) {
+ src0 = process_view_input_new(context, 0);
+ src1 = process_view_input_new(context, 1);
+ } else {
+ // split along last axis, nc = ne[0] / 2
+ auto combined = process_view_input_new(context, 0);
+ auto combined_shape = combined.get_partial_shape();
+ int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length();
+ int64_t nc = last_dim_val / 2;
+
+ auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
+ auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
+ auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
+ auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});
+ auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc});
+ auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc});
+
+ src0 = std::make_shared(combined, start0, stop0, step, axis);
+ src1 = std::make_shared(combined, start1, stop1, step, axis);
+ }
+
+ int32_t * params = context.get_output_op_params();
+ const int32_t swapped = params[1];
+ if (swapped) {
+ std::swap(src0, src1);
+ }
+
+ // GELU_QUICK(x) = x * sigmoid(1.702 * x)
+ // Create the constant in the same type as src0 to avoid f16/f32 mismatch.
+ auto input_type = src0.get_element_type();
+ auto coef = ov::op::v0::Constant::create(input_type, ov::Shape{}, {1.702f});
+ auto scaled = std::make_shared(src0, coef);
+ auto sigmoid = std::make_shared(scaled);
+ auto gated = std::make_shared(src0, sigmoid);
+ auto res = std::make_shared(gated, src1);
+
+ return rename_outputs_with_suffix({res}, context.get_name());
+}
+
+} // namespace op
+} // namespace ggml
+} // namespace frontend
+} // namespace ov
diff --git a/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp b/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp
new file mode 100644
index 000000000000..07e94c690152
--- /dev/null
+++ b/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp
@@ -0,0 +1,90 @@
+// Copyright (C) 2018-2026 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+// Local mirror of OpenVINO's internal ov::op::internal::MOE and MOECompressed ops.
+//
+// The class bodies are provided by the linked libopenvino.so; only the declarations are
+// needed here so the backend can construct the node directly (same approach as
+// GatherMatmul and GatedDeltaNet). The class layout must stay in sync with
+// openvino/src/core/dev_api/openvino/op/moe.hpp
+// openvino/src/common/transformations/include/ov_ops/moe_compressed.hpp
+//
+// \note MOE op classes are under development and subject to change.
+
+#pragma once
+
+#include
+
+#include "openvino/core/type/element_type.hpp"
+#include "openvino/op/op.hpp"
+
+namespace ov::op::internal {
+
+class OPENVINO_API MOE : public ov::op::Op {
+public:
+ OPENVINO_OP("MOE")
+
+ MOE() = default;
+
+ MOE(const OutputVector & args) : Op(args) {}
+
+ enum class Expert_type { GEMM2_BIAS_SWIGLU_CLAMP, GEMM3_SWIGLU };
+
+ enum class Activation_type { SWIGLU, GEGLU_TANH, GEGLU_ERF };
+
+ struct Config {
+ Expert_type expert_type{ Expert_type::GEMM2_BIAS_SWIGLU_CLAMP };
+ float expert_alpha{ 0.0f };
+ float expert_beta{ 1.0f };
+ size_t gate_idx{ 0 };
+ Activation_type activation_type{ Activation_type::SWIGLU };
+ };
+
+ MOE(const OutputVector & args, const Config & config);
+
+ const Config & get_config() const;
+ void set_config(const Config & config);
+
+ bool visit_attributes(AttributeVisitor & visitor) override;
+ void validate_and_infer_types() override;
+ std::shared_ptr clone_with_new_inputs(const OutputVector & new_args) const override;
+
+private:
+ Config m_config;
+};
+
+class OPENVINO_API MOECompressed : public MOE {
+public:
+ OPENVINO_OP("MOECompressed", "", ov::op::internal::MOE)
+
+ MOECompressed() = default;
+
+ struct Config : public MOE::Config {
+ size_t hidden_size = 0;
+ size_t inter_size = 0;
+ size_t num_expert = 0;
+ size_t num_shared_expert = 0;
+ size_t top_k = 0;
+ // numeric_limits::max() means per_channel compression (single group)
+ size_t group_size = 0;
+ bool has_batch_dim = false;
+ bool has_zp = false;
+ ov::element::Type out_type = ov::element::dynamic;
+ std::optional scale_factor;
+ };
+
+ MOECompressed(const OutputVector & args, const Config & config);
+
+ const Config & get_config() const { return m_config; }
+
+ void set_scale_factor(float scale_factor) { m_config.scale_factor = scale_factor; }
+
+ bool visit_attributes(AttributeVisitor & visitor) override;
+ void validate_and_infer_types() override;
+ std::shared_ptr clone_with_new_inputs(const OutputVector & new_args) const override;
+
+protected:
+ Config m_config;
+};
+
+} // namespace ov::op::internal
diff --git a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp
index f1b28c85d401..0de6161bed85 100644
--- a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp
@@ -56,54 +56,6 @@ ov::Output static_shape_dims_or_shapeof(const ov::Output & i
return get_dimensions(shape, dims);
}
-ov::Output translate_mul_mat_id_gather_matmul_fallback(const NodeContext & context,
- ov::Output expert_weights,
- ov::Output activations,
- ov::Output ids) {
- auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0});
- ov::Output selected_weights = std::make_shared(expert_weights, ids, gather_axis);
-
- const auto output_type = context.get_output_type();
- if (selected_weights.get_element_type() != ov::element::f32) {
- selected_weights = std::make_shared(selected_weights, ov::element::f32);
- }
- if (activations.get_element_type() != ov::element::f32) {
- activations = std::make_shared(activations, ov::element::f32);
- }
-
- auto activations_shape = std::make_shared(activations, ov::element::i64);
- auto ids_shape = std::make_shared(ids, ov::element::i64);
- ov::Output acts_target_dims = std::make_shared(
- ov::OutputVector{
- get_dimensions(activations_shape, {0}),
- get_dimensions(ids_shape, {1}),
- get_dimensions(activations_shape, {2}),
- },
- 0);
- ov::Output acts_broadcasted =
- std::make_shared(activations, acts_target_dims, ov::op::BroadcastType::BIDIRECTIONAL);
-
- auto activations_expanded = std::make_shared(acts_broadcasted, const_i64({2}));
- ov::Output result =
- std::make_shared(activations_expanded, selected_weights, false, true);
-
- auto output_shape = context.get_output_shape();
- FRONT_END_OP_CONVERSION_CHECK(output_shape.rank().is_static() && output_shape.rank().get_length() == 4,
- "Unexpected MUL_MAT_ID output rank");
- FRONT_END_OP_CONVERSION_CHECK(output_shape[3].is_static(), "Expected static row dimension for MUL_MAT_ID output");
-
- auto batch_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
- auto row_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3].get_length()});
- auto result_target_dims = std::make_shared(
- ov::OutputVector{batch_dim, get_dimensions(ids_shape, {0, 1}), row_dim}, 0);
- result = std::make_shared(result, result_target_dims, false);
-
- if (result.get_element_type() != output_type) {
- result = std::make_shared(result, output_type);
- }
- return result;
-}
-
ov::Output translate_mul_mat_id_mxfp4_packed(const NodeContext & context,
ov::Output expert_weights,
ov::Output activations,
@@ -229,7 +181,6 @@ OutputVector translate_mul_mat_id(const NodeContext & context) {
auto expert_weights_rank = expert_weights.get_partial_shape().rank();
FRONT_END_OP_CONVERSION_CHECK(expert_weights_rank.is_static(),
"Expected static rank for MUL_MAT_ID expert weights");
- const bool use_gpu_fallback = ggml_openvino_get_device_name() == "GPU";
if (expert_weights_rank.get_length() == 4) {
auto expert_weights_shape_3d = static_shape_dims_or_shapeof(expert_weights, {1, 2, 3});
expert_weights = std::make_shared(expert_weights, expert_weights_shape_3d, false);
@@ -246,14 +197,9 @@ OutputVector translate_mul_mat_id(const NodeContext & context) {
}
const auto output_type = context.get_output_type();
- if (activations.get_element_type() != ov::element::f32) {
- activations = std::make_shared(activations, ov::element::f32);
- }
-
- if (use_gpu_fallback || !expert_weights.get_partial_shape().is_static() || !activations.get_partial_shape().is_static() ||
- !ids.get_partial_shape().is_static()) {
- return rename_outputs_with_suffix({translate_mul_mat_id_gather_matmul_fallback(context, expert_weights, activations, ids)},
- context.get_name());
+ const auto activations_type = ggml_openvino_get_device_name() == "GPU" ? ov::element::f16 : ov::element::f32;
+ if (activations.get_element_type() != activations_type) {
+ activations = std::make_shared(activations, activations_type);
}
// GatherMatmul's A input is [n_used_or_1, n_tokens, k]; activations_3d is
diff --git a/ggml/src/ggml-openvino/openvino/op/permute.cpp b/ggml/src/ggml-openvino/openvino/op/permute.cpp
index 85550bff396b..df4f038984c5 100644
--- a/ggml/src/ggml-openvino/openvino/op/permute.cpp
+++ b/ggml/src/ggml-openvino/openvino/op/permute.cpp
@@ -45,11 +45,22 @@ OutputVector translate_permute(const NodeContext & context) {
static_cast(perm_values.size() - 1 - input_axis);
}
}
- auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values);
-
if (op_case == 1 || context.is_stateful()) {
+ // The stateful path carries hidden-state tensors in a rank-3 layout (the
+ // leading batch dim is dropped, e.g. Gemma4's per-layer-embedding path). The
+ // perm above is rank-4; when the actual input is rank-3, drop the batch axis
+ // (perm[0], which is always the identity 0 here) and shift the rest down by 1
+ // so the transpose order matches the input rank.
+ std::vector perm_used = perm_values;
+ const auto & src_ps = src.get_partial_shape();
+ if (src_ps.rank().is_static() && src_ps.rank().get_length() == 3 && perm_values.size() == 4 &&
+ perm_values[0] == 0) {
+ perm_used = {perm_values[1] - 1, perm_values[2] - 1, perm_values[3] - 1};
+ }
+ auto perm = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{perm_used.size()}, perm_used);
res = std::make_shared(src, perm);
} else if (op_case == 2) {
+ auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values);
auto output_shape = context.get_output_shape().to_shape();
auto n_heads = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[1]});
auto head_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]});
@@ -68,6 +79,7 @@ OutputVector translate_permute(const NodeContext & context) {
auto reshaped = std::make_shared(src, new_shape, true);
res = std::make_shared(reshaped, perm);
} else {
+ auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values);
auto cache_shape = src.get_partial_shape();
auto output_shape = context.get_output_shape().to_shape();
int64_t head_size = output_shape[3];
diff --git a/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp b/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp
new file mode 100644
index 000000000000..82bff6b898ba
--- /dev/null
+++ b/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp
@@ -0,0 +1,53 @@
+#include "../node_context.h"
+#include "../op_table.h"
+#include "../utils.h"
+
+#include
+#include
+#include
+
+namespace ov {
+namespace frontend {
+namespace ggml {
+namespace op {
+
+OutputVector translate_pool_2d(const NodeContext & context) {
+ num_inputs_check(context, 1, 1);
+ const int32_t * params = context.get_output_op_params();
+
+ const int k0 = params[1];
+ const int k1 = params[2];
+ const int s0 = params[3];
+ const int s1 = params[4];
+ const int p0 = params[5];
+ const int p1 = params[6];
+
+ const int op_case = context.get_op_case();
+ ov::Output input = context.get_input(0);
+ ov::Strides strides{static_cast(s1), static_cast(s0)};
+ ov::Shape pads_begin{static_cast(p1), static_cast(p0)};
+ ov::Shape pads_end{static_cast(p1), static_cast(p0)};
+ ov::Shape kernel{static_cast(k1), static_cast