From 55232edcbdf643d51fdc66926c00664252c1ccac Mon Sep 17 00:00:00 2001
From: Mike Lothian <mike@fireburn.co.uk>
Date: Wed, 21 Oct 2026 12:42:50 +0100
Subject: [PATCH] vulkan : fuse the bias add into the mat-vec for small column
 batches

The fused mat-vec and add path only accepted a single output row, although the
mat-vec shaders already add the bias per column. Accept outputs of up to
mul_mat_vec_max_cols columns that are dispatched as a mat-vec, so the residual
adds of speculative decoding verification steps are fused as well. This removes
78 of 175 separate adds from a five token step on a Radeon AI PRO R9700, about
1% of the step, with identical perplexity.

Assisted-by: Claude Code (Claude Sonnet 5.5)
---
 ggml/src/ggml-vulkan/ggml-vulkan.cpp | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index 8a728fec4..634c9c831 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -13745,9 +13745,15 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr
     auto const &mm_add_ok = [&](const ggml_tensor *mul, const ggml_tensor *add) {
         const ggml_tensor *bias = add->src[0] == mul ? add->src[1] : add->src[0];
 
-        // mat-vec only
+        // mat-vec only: a single row, or a small batch of columns that is dispatched as a mat-vec
         if (ggml_nrows(mul) != 1) {
-            return false;
+            const ggml_tensor * mm_a = mul->src[0];
+            const ggml_tensor * mm_b = mul->src[1];
+            if (mul->op != GGML_OP_MUL_MAT || mul->ne[1] > mul_mat_vec_max_cols || mul->ne[2] != 1 || mul->ne[3] != 1 ||
+                mm_b->ne[2] * mm_b->ne[3] != 1 || !ggml_is_contiguous(mul) ||
+                !(mm_a->type == GGML_TYPE_F32 || mm_a->type == GGML_TYPE_F16 || mm_a->type == GGML_TYPE_BF16 || ggml_is_quantized(mm_a->type))) {
+                return false;
+            }
         }
         // shaders assume the types match
         if (mul->type != bias->type) {
-- 
2.56.0

