python3 -m sglang.bench_one_batch \
    --model-path meta-llama/Llama-4-Scout-17B-16E \
    --disable-radix-cache \
    --context-length 2048 \
    --attention-backend fa3 \
    --load-format dummy \
    --batch-size 128 \
    --tp-size 8 \
    --input-len 1000 \
    --output-len 1000

Prefill. latency: 2.12386 s, throughput:  60267.49 token/s
Decode 0. Batch size: 128, latency: 0.17131 s, throughput:    747.20 token/s
Decode 1. Batch size: 128, latency: 0.11283 s, throughput:   1134.45 token/s
Decode 2. Batch size: 128, latency: 0.11220 s, throughput:   1140.83 token/s
Decode 3. Batch size: 128, latency: 0.11324 s, throughput:   1130.34 token/s
Decode 4. Batch size: 128, latency: 0.11100 s, throughput:   1153.15 token/s
Decode.  median latency: 0.11281 s, median throughput:   1134.62 token/s
Total. latency:  5.681 s, throughput:  23253.82 token/s
Benchmark ...
Prefill. latency: 1.81066 s, throughput:  70692.62 token/s
Decode 0. Batch size: 128, latency: 0.11238 s, throughput:   1139.00 token/s
Decode 1. Batch size: 128, latency: 0.11217 s, throughput:   1141.11 token/s
Decode 2. Batch size: 128, latency: 0.11157 s, throughput:   1147.31 token/s
Decode 3. Batch size: 128, latency: 0.11164 s, throughput:   1146.54 token/s
Decode 4. Batch size: 128, latency: 0.11063 s, throughput:   1157.03 token/s
Decode.  median latency: 0.15820 s, median throughput:    809.09 token/s
Total. latency: 160.106 s, throughput:   1598.95 token/s

Prefill. latency: 2.03622 s, throughput:  62861.66 token/s
Decode 0. Batch size: 128, latency: 0.08541 s, throughput:   1498.60 token/s
Decode 1. Batch size: 128, latency: 0.01811 s, throughput:   7067.50 token/s
Decode 2. Batch size: 128, latency: 0.01751 s, throughput:   7309.17 token/s
Decode 3. Batch size: 128, latency: 0.01738 s, throughput:   7365.30 token/s
Decode 4. Batch size: 128, latency: 0.01736 s, throughput:   7371.69 token/s
Decode.  median latency: 0.01744 s, median throughput:   7338.32 token/s
Total. latency:  2.646 s, throughput:  49923.53 token/s
Benchmark ...
Prefill. latency: 1.72161 s, throughput:  74349.17 token/s
Decode 0. Batch size: 128, latency: 0.01989 s, throughput:   6435.87 token/s
Decode 1. Batch size: 128, latency: 0.01811 s, throughput:   7067.99 token/s
Decode 2. Batch size: 128, latency: 0.01759 s, throughput:   7276.41 token/s
Decode 3. Batch size: 128, latency: 0.01768 s, throughput:   7241.46 token/s
Decode 4. Batch size: 128, latency: 0.01753 s, throughput:   7303.33 token/s
Decode.  median latency: 0.01828 s, median throughput:   7003.46 token/s
Total. latency: 20.002 s, throughput:  12798.53 token/s