SGLANG_TORCH_PROFILER_DIR=/data/sglang_profilers/QWEN3_tp8_32_3500_25_96G python3 -m sglang.bench_offline_throughput --model-path /data/models/Qwen3-235B-A22B --dataset-path /data/datasets/ShareGPT_V3_unfiltered_cleaned_split.json --dataset-name random --num-prompts 32 --random-input-len 3500 --random-output-len 25 --random-range-ratio 1 --tp-size 8 --trust-remote-code --disable-radix-cache --mem-fraction-static 0.9 --profile --reasoning-parser qwen3 --attention-backend {fa3, flashinfer}

### server
# FA3 (default)
python3 -m sglang.launch_server --model /data/models/Qwen3-235B-A22B --tp 8 --reasoning-parser qwen3 --port 8080

# Flashinfer
python3 -m sglang.launch_server --model /data/models/Qwen3-235B-A22B --tp 8 --reasoning-parser qwen3 --port 8080 --attention-backend flashinfer

# Hybrid (FA3 for prefill and Flashinfer for decode)
python3 -m sglang.launch_server --model /data/models/Qwen3-235B-A22B --tp 8 --reasoning-parser qwen3 --port 8080 --enable-flashinfer-attention-decode

### client
python3 -m sglang.bench_serving --backend sglang \
            --dataset-name random \
            --dataset-path /data/datasets/ShareGPT_V3_unfiltered_cleaned_split.json \
            --random-input-len 3500 \
            --random-output-len 1500 \
            --random-range-ratio 1 \
            --request-rate 32 \
            --max-concurrency 32 \
            --num-prompts 128 \
            --host 0.0.0.0 --port 8080

============ Serving Benchmark Result ============
Backend:                                 sglang    
Traffic request rate:                    32.0      
Max reqeuest concurrency:                32        
Successful requests:                     128       
Benchmark duration (s):                  222.58    
Total input tokens:                      448000    
Total generated tokens:                  192000    
Total generated tokens (retokenized):    191973    
Request throughput (req/s):              0.58      
Input token throughput (tok/s):          2012.80   
Output token throughput (tok/s):         862.63    
Total token throughput (tok/s):          2875.43   
Concurrency:                             31.95     
----------------End-to-End Latency----------------
Mean E2E Latency (ms):                   55551.83  
Median E2E Latency (ms):                 55102.26  
---------------Time to First Token----------------
Mean TTFT (ms):                          4597.17   
Median TTFT (ms):                        4514.20   
P99 TTFT (ms):                           8477.12   
---------------Inter-Token Latency----------------
Mean ITL (ms):                           33.99     
Median ITL (ms):                         31.61     
P95 ITL (ms):                            33.84     
P99 ITL (ms):                            34.42     
Max ITL (ms):                            7467.86   
==================================================

============ Serving Benchmark Result ============
Backend:                                 sglang    
Traffic request rate:                    32.0      
Max reqeuest concurrency:                32        
Successful requests:                     128       
Benchmark duration (s):                  184.63    
Total input tokens:                      448000    
Total generated tokens:                  192000    
Total generated tokens (retokenized):    191985    
Request throughput (req/s):              0.69      
Input token throughput (tok/s):          2426.54   
Output token throughput (tok/s):         1039.95   
Total token throughput (tok/s):          3466.48   
Concurrency:                             31.94     
----------------End-to-End Latency----------------
Mean E2E Latency (ms):                   46065.23  
Median E2E Latency (ms):                 46034.10  
---------------Time to First Token----------------
Mean TTFT (ms):                          4825.35   
Median TTFT (ms):                        4708.10   
P99 TTFT (ms):                           9092.60   
---------------Inter-Token Latency----------------
Mean ITL (ms):                           27.51     
Median ITL (ms):                         24.82     
P95 ITL (ms):                            26.18     
P99 ITL (ms):                            26.53     
Max ITL (ms):                            9292.85   
==================================================

============ Serving Benchmark Result ============
Backend:                                 sglang    
Traffic request rate:                    32.0      
Max reqeuest concurrency:                32        
Successful requests:                     128       
Benchmark duration (s):                  181.07    
Total input tokens:                      448000    
Total generated tokens:                  192000    
Total generated tokens (retokenized):    191987    
Request throughput (req/s):              0.71      
Input token throughput (tok/s):          2474.20   
Output token throughput (tok/s):         1060.37   
Total token throughput (tok/s):          3534.57   
Concurrency:                             31.94     
----------------End-to-End Latency----------------
Mean E2E Latency (ms):                   45176.10  
Median E2E Latency (ms):                 45040.04  
---------------Time to First Token----------------
Mean TTFT (ms):                          4495.42   
Median TTFT (ms):                        4383.04   
P99 TTFT (ms):                           8057.08   
---------------Inter-Token Latency----------------
Mean ITL (ms):                           27.14     
Median ITL (ms):                         24.54     
P95 ITL (ms):                            25.84     
P99 ITL (ms):                            26.23     
Max ITL (ms):                            8783.36   
==================================================