fix: CPU vec_dot heap allocation for turbo/TQ types (n > 4096 models)

TheTom · claude · TheTom · commit 21110eb069b4 · 2026-04-02T14:54:24.000-05:00
Stack-allocated float tmp[4096] buffers in CPU vec_dot functions
crashed on models with intermediate_size &gt; 4096 (e.g. TinyLlama 5632,
Qwen 27B 18944). Replaced with heap allocation.

Affects CPU-only inference fallback path. GPU users unaffected.

Reported by @oemc1470 on RX 6600 (gfx1032) where broken HIP forced
CPU fallback.

Tested: Qwen3.5-27B Config I, CPU-only (-ngl 0), intermediate_size=18944.
No crash, no assert.

Co-Authored-By: tturney@psyguard.ai
Co-Authored-By: Claude Opus 4.6 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -3379,15 +3379,16 @@ static void ggml_vec_dot_turbo3_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
     GGML_UNUSED(bs); GGML_UNUSED(bx); GGML_UNUSED(by); GGML_UNUSED(nrc);
 
     // Dequantize turbo3 to f32 temp buffer, then dot
-    float tmp[4096];  // max head_dim
-    GGML_ASSERT(n <= 4096);
+    float * tmp = (float *)malloc(n * sizeof(float));
+    GGML_ASSERT(tmp != NULL);
     ggml_get_type_traits(GGML_TYPE_TURBO3_0)->to_float(vx, tmp, n);
 
     const float * y = (const float *)vy;
     float sum = 0.0f;
     for (int i = 0; i < n; i++) {
         sum += tmp[i] * y[i];
     }
+    free(tmp);
     *s = sum;
 }
 
@@ -3398,15 +3399,16 @@ static void ggml_vec_dot_turbo2_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
     GGML_ASSERT(nrc == 1);
     GGML_UNUSED(bs); GGML_UNUSED(bx); GGML_UNUSED(by); GGML_UNUSED(nrc);
 
-    float tmp[4096];
-    GGML_ASSERT(n <= 4096);
+    float * tmp = (float *)malloc(n * sizeof(float));
+    GGML_ASSERT(tmp != NULL);
     ggml_get_type_traits(GGML_TYPE_TURBO2_0)->to_float(vx, tmp, n);
 
     const float * y = (const float *)vy;
     float sum = 0.0f;
     for (int i = 0; i < n; i++) {
         sum += tmp[i] * y[i];
     }
+    free(tmp);
     *s = sum;
 }
 
@@ -3417,15 +3419,16 @@ static void ggml_vec_dot_turbo4_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
     GGML_ASSERT(nrc == 1);
     GGML_UNUSED(bs); GGML_UNUSED(bx); GGML_UNUSED(by); GGML_UNUSED(nrc);
 
-    float tmp[4096];
-    GGML_ASSERT(n <= 4096);
+    float * tmp = (float *)malloc(n * sizeof(float));
+    GGML_ASSERT(tmp != NULL);
     ggml_get_type_traits(GGML_TYPE_TURBO4_0)->to_float(vx, tmp, n);
 
     const float * y = (const float *)vy;
     float sum = 0.0f;
     for (int i = 0; i < n; i++) {
         sum += tmp[i] * y[i];
     }
+    free(tmp);
     *s = sum;
 }
 
@@ -3437,18 +3440,21 @@ static void ggml_vec_dot_tq3_1s_q8_0(int n, float * GGML_RESTRICT s, size_t bs,
     GGML_ASSERT(nrc == 1);
     GGML_UNUSED(bs); GGML_UNUSED(bx); GGML_UNUSED(by); GGML_UNUSED(nrc);
 
-    float tmp[4096];
-    GGML_ASSERT(n <= 4096);
+    float * tmp = (float *)malloc(n * sizeof(float));
+    GGML_ASSERT(tmp != NULL);
     ggml_get_type_traits(GGML_TYPE_TQ3_1S)->to_float(vx, tmp, n);
 
     // Dequantize q8_0 and dot
-    float tmp2[4096];
+    float * tmp2 = (float *)malloc(n * sizeof(float));
+    GGML_ASSERT(tmp2 != NULL);
     ggml_get_type_traits(GGML_TYPE_Q8_0)->to_float(vy, tmp2, n);
 
     float sum = 0.0f;
     for (int i = 0; i < n; i++) {
         sum += tmp[i] * tmp2[i];
     }
+    free(tmp);
+    free(tmp2);
     *s = sum;
 }
 
@@ -3460,17 +3466,20 @@ static void ggml_vec_dot_tq4_1s_q8_0(int n, float * GGML_RESTRICT s, size_t bs,
     GGML_ASSERT(nrc == 1);
     GGML_UNUSED(bs); GGML_UNUSED(bx); GGML_UNUSED(by); GGML_UNUSED(nrc);
 
-    float tmp[4096];
-    GGML_ASSERT(n <= 4096);
+    float * tmp = (float *)malloc(n * sizeof(float));
+    GGML_ASSERT(tmp != NULL);
     ggml_get_type_traits(GGML_TYPE_TQ4_1S)->to_float(vx, tmp, n);
 
-    float tmp2[4096];
+    float * tmp2 = (float *)malloc(n * sizeof(float));
+    GGML_ASSERT(tmp2 != NULL);
     ggml_get_type_traits(GGML_TYPE_Q8_0)->to_float(vy, tmp2, n);
 
     float sum = 0.0f;
     for (int i = 0; i < n; i++) {
         sum += tmp[i] * tmp2[i];
     }
+    free(tmp);
+    free(tmp2);
     *s = sum;
 }
 
