g_exp_pos = ggml_exp(ctx0, g_cumsum);
        g_exp_neg = ggml_exp(ctx0, ggml_neg(ctx0, g_cumsum));
        ggml_tensor * k_pos_beta = ggml_mul(ctx0, k_beta, g_exp_pos);
        ggml_tensor * k_neg = ggml_mul(ctx0, k, g_exp_neg);
        k_decay = ggml_mul_mat(ctx0, k_pos_beta, k_neg);

    const int64_t CHB = n_chunks * H_k * n_seqs;
    ggml_tensor * gkcs_i = ggml_reshape_4d(ctx0, gk_cumsum, chunk_size, 1, S_k, CHB);  // [chunk_size, 1, S_k, CHB]
    ggml_tensor * gkcs_j = ggml_reshape_4d(ctx0, gkcs_i, 1, chunk_size, S_k, CHB);  // [1, chunk_size, S_k, CHB]

    ggml_tensor * gkcs_j_bc = ggml_repeat_4d(ctx0, gkcs_j, chunk_size, chunk_size, S_k, CHB);  // [1, chunk_size, S_k, CHB] -> [chunk_size, chunk_size, S_k, CHB]
    // decay_mask [chunk_size,chunk_size,S_k,CHB]
    ggml_tensor * decay_mask = ggml_sub(ctx0, gkcs_j_bc, gkcs_i);
    cb(decay_mask, "decay_mask", il);

    decay_mask = ggml_mul(ctx0, decay_mask, diag_mask);
    cb(decay_mask, "decay_masked", il);
    decay_mask = ggml_exp(ctx0, decay_mask);
    decay_mask = ggml_mul(ctx0, decay_mask, diag_mask);

    // decay_mask [S_k,BT_j,BT_i,CHB] *Note* second and third chunk_sizes are switched
    decay_mask = ggml_cont_4d(ctx0, ggml_permute(ctx0, decay_mask, 2, 1, 0, 3), S_k, chunk_size, chunk_size, CHB);

    ggml_tensor * k_i = ggml_cont(ctx0, ggml_reshape_4d(ctx0, k, S_k, chunk_size, 1, CHB));
    ggml_tensor * k_j = ggml_cont(ctx0, ggml_reshape_4d(ctx0, k, S_k, 1, chunk_size, CHB));
    ggml_tensor * q_i = ggml_cont(ctx0, ggml_reshape_4d(ctx0, q, S_k, chunk_size, 1, CHB));

    ggml_tensor * decay_k_i = ggml_mul(ctx0, decay_mask, k_i);
    ggml_tensor * decay_q_i = ggml_mul(ctx0, decay_mask, q_i);

    // decay_k_i [S.BT,BT,CHB] @ k_j [S,1,BT,CHB] = Akk [BT,1,BT,CHB]
    ggml_tensor * Akk = ggml_mul_mat(ctx0, decay_k_i, k_j);
    ggml_tensor * Aqk = ggml_mul_mat(ctx0, decay_q_i, k_j);
    Akk = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_4d(ctx0, Akk, chunk_size, chunk_size, n_chunks, HB)));
    Aqk = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_4d(ctx0, Aqk, chunk_size, chunk_size, n_chunks, HB)));

> /read ggml/src/ggml-quants.c

Loaded text from 'ggml/src/ggml-quants.c'

> Please tell me what quants are available in this library and how each of them works.

Okay, let's break down the quantization methods available in this GGML-based codebase. This file, `ggml-common.c`, is part of the broader **GGML** (GG Machine Learning) library, which is the core engine behind projects like llama.cpp.

The key to understanding quantization here is that **GGML aims to be a universal backend for efficient inference of large language models (LLMs)**. The "quantization" is the process of converting high-precision floating-point weights (like `float32`) into lower-precision integer representations to save memory and accelerate computation, often at the cost of some accuracy.

Here's a breakdown of the quantization schemes implemented in this code, how they work, and their trade-offs:

### 1. **Classical Block Quantization (Q8_0, Q4_0, etc.)**
These are the simplest and oldest quantization methods. The idea is to process data in fixed-size blocks (e.g., `QK8_0 = 32` values) and compute a single scale factor for the entire block.

*   **Q8_0**: Quantizes to 8-bit signed integers.
    *   **How it works**:
        1.  Find the maximum absolute value (`amax`) in the block.
        2.  Compute scale `d = amax / 127` (so that 127 maps to `amax`).
        3.  Quantize each value `x[i]` to `round(x[i] * d)`, clamped to `[-127, 127]`.
        4.  Store the 8-bit quantized values (`qs`) and the scale `d` (in half-precision).
    *   **Structure**: `d` (fp16) + `qs[32]` (uint8_t). Total size: `4 + 32 = 36 bytes` for 128 bits of data (`32 float32`).
    *   **Pros**: Simple, fast, good baseline accuracy.
    *   **Cons**: A single scale per block means all values are compressed equally, which isn't ideal if the data has varying distributions.

*   **Q4_0, Q4_1, Q5_0, Q5_1**: Quantizes to 4-bit integers.
    *   **How it works**:
        *   Similar to Q8_0, but now the scale `d` is used to quantize to 16 levels instead of 256.
        *   `Q4_1` and `Q5_1` also store a minimum value `m` (like `dmin`), making them `min-max quantization`.
    *   **Pros**: Significantly smaller memory footprint (e.g., `Q4_0` uses `2 + 16 = 18 bytes` for the same data).
    *   **Cons**: Lower precision than higher-bit quantization.

### 2. **Block Quantization with Adaptive Scales (Q2_K, Q3_K, Q4_K, Q5_K, Q6_K)**
These are more modern methods that use **multiple scales per block** to capture finer-grained data distributions.

*   **Q2_K, Q3_K, Q4_K, Q5_K, Q6_K**: These are part of the `QX_K` family (e.g., `Q4_K`, `Q5_K`).
*   **How it works**:
    *   The large block (`QK_K = 1024`) is divided into smaller sub-blocks (e.g., `QK_K/16 = 64` values per scale).
    *   **Multiple scales** are computed for these sub-blocks, and **multiple quantized values** are stored per sub-block.
    *   For example, `Q4_K` has a scale per sub-block and a `dmin`, storing 4-bit quantized values.
*   **Pros**: Much better accuracy for the same bit-width compared to `Q8_0` or `Q4_0`, due to adaptive scaling.
*   **Cons**: Slightly more complex, and requires more scale parameters to store.

### 3. **Mixed-Precision Quantization (IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S, IQ1_S, IQ1_M, IQ4_NL, IQ4_XS)**
This is the most advanced family, where the quantization grid is **not uniform** and is defined by pre-computed lookup tables.

*   **IQ2_XXS, IQ2_XS, IQ2_S**: "2-bit" methods using a 2-bit grid.
*   **IQ3_XXS, IQ3_S**: "3-bit" methods using a 3-bit grid.
*   **IQ1_S, IQ1_M**: "1-bit" methods (extremely low precision, using only 3 possible values: `-1`, `0`, `1`).
*   **IQ4_NL, IQ4_XS**: Non-linear quantization, using a lookup table (`kvalues_iq4nl`) to map quantized indices to actual float values. This is key to `IQ4_NL` and `IQ4_XS`.
*   **How it works**:
    1.  **Grid Definition**: A grid of representative float values is pre-computed and stored in lookup tables (`kgrid_...`, `kmap_...`, `kneighbors_...`). These grids are not linearly spaced.
    2.  **Nearest Neighbor Search**: For each sub-block, the algorithm searches for the nearest grid point that minimizes the reconstru    ]
ction error, using the quantization weights if provided.
    3.  **Variable-Length Encoding**: The indices to the grid and sign bits are stored, often in a compact way (e.g., bits packed into bytes).
*   **Pros**: Achieves state-of-the-art accuracy for a given bit-width, especially at very low bit-widths (like 2-bit or 1-bit).
*   **Cons**: Significantly more complex, requires large pre-computed grids and neighbor tables, and the quantization process is much slower (it's not a simple rounding operation).

### 4. **Custom Formats (MXFP4, TQ1_0, TQ2_0)**
These are specialized formats, often inspired by hardware or research papers.

*   **MXFP4**: Based on the "Maximal Floating Point" format used by NVIDIA Hopper GPUs. It uses an 8-bit exponent and a 4-bit mantissa, allowing for a wider dynamic range than pure integer quantization.
*   **TQ1_0, TQ2_0**: "Ternary quantization" (values are `-1`, `0`, `1`), which is the core idea behind the BitNet (b1.58) and TriLMs research. These are extremely memory-efficient.

### 5. **Reference Implementations vs. Optimized Implementations**
The file contains both **`_ref`** (reference) and **optimized** implementations.

*   **`_ref` functions**: These are written in portable C and are meant to be the "ground truth" for quantization. They are slow but easy to understand and debug.
*   **Optimized implementations**: In a real system, the actual quantization routines (`quantize_iq2_xxs`, `quantize_q4_K`, etc.) would use SIMD instructions (AVX2, NEON) for speed. The `_ref` functions are often used as a baseline to ensure correctness.

### How to Use These Quantizations

The key point is that **this file provides the core quantization logic**. In a production system, this would be wrapped by a higher-level API that handles:

1.  **Model Loading**: Reading a float32 model and converting it to a quantized format (e.g., `Q4_K`) using these functions.
2.  **Runtime Inference**: Running the quantized model, which calls functions like `dequantize_row_q4_K` and `ggml_mul_mat_q4_K` to perform matrix multiplication directly on the quantized data (without dequantizing to float32 first).

In summary, the library supports a wide spectrum of quantization methods, from simple fixed-bit quantization to highly adaptive grid-based methods, allowing users to choose a balance between model size, speed, and accuracy.

[ Prompt: 922,4 t/s | Generation: 37,5 t/s ]

// GGML_OP_ACC - block accumulation test
struct test_acc_block: public test_case {
    const ggml_type type;
    const int64_t block_size;
    const int64_t n_blocks;
    const int64_t ne2;
    const int64_t ne3;
    
    std::string vars() override {
        return VARS_TO_STR5(type, block_size, n_blocks, ne2, ne3); 
    }
    
    test_acc_block(ggml_type type = GGML_TYPE_F32,
            int64_t block_size = 16,
            int64_t n_blocks = 4,
            int64_t ne2 = 1,
            int64_t ne3 = 1)
        : type(type), block_size(block_size), n_blocks(n_blocks), ne2(ne2), ne3(ne3) {}

    ggml_tensor * build_graph(ggml_context * ctx) override {
        const int64_t chunk_size = block_size * n_blocks;

        // Base tensor initialized to zero using ggml_clamp
        ggml_tensor * a = ggml_new_tensor_4d(ctx, type, chunk_size, chunk_size, ne2, ne3);
        ggml_set_param(a);
        ggml_set_name(a, "a");
        ggml_tensor * acc = ggml_clamp(ctx, a, 0.0f, 0.0f);
    
        // Source blocks that will be accumulated at different offsets
        // Mimics the lower-triangular block pattern from the original code
        for (int64_t j = 0; j < n_blocks; ++j) {
            for (int64_t i = 0; i <= j; ++i) {
                ggml_tensor * block = ggml_new_tensor_4d(ctx, type,
                    block_size, block_size, ne2, ne3);
                ggml_set_param(block);
    
                char name[64];
                snprintf(name, sizeof(name), "block_%ld_%ld", (long)j, (long)i);
                ggml_set_name(block, name);
    
                // Accumulate block at position [i*block_size, j*block_size]
                // This is the same pattern as the original code:
                //   offset = i_start * nb[0] + j_start * nb[1]
                size_t offset = (i * block_size) * ggml_type_size(type)
                              + (j * block_size) * (chunk_size * ggml_type_size(type));

                acc = ggml_acc(ctx, acc, block,
                    acc->nb[1], acc->nb[2], acc->nb[3],
                    offset);
            }
        }

        ggml_set_name(acc, "out");
        return acc;
    }
};

./build/bin/llama-server -c 16384 --parallel 8 --mmap -m ~/Kimi-Linear-48B-A3B-Instruct-GGUF/Kimi-Linear-48B-A3B-Instruct-jp-imatrix.IQ3_M.gguf -ngl 100