MODEL := "deepseek-ai/DeepSeek-V3.1"

launch_vllm:
    VLLM_ALLREDUCE_USE_SYMM_MEM=0 VLLM_MOE_USE_DEEP_GEMM=0 VLLM_USE_DEEP_GEMM=1 VLLM_TORCH_PROFILER_DIR=$(pwd)/profiles-vllm-dsr1 chg run --gpus 8 -- vllm serve {{MODEL}} -tp 8

bench_decode BATCH_SIZE NUM_PROMPTS:
    vllm bench serve \
        --port {{PORT}} \
        --model {{MODEL}} \
        --dataset-name random \
        --random-input-len 2 \
        --random-output-len 100 \
        --max-concurrency {{BATCH_SIZE}} \
        --num-prompts {{NUM_PROMPTS}} \
        --seed $(date +%M%H%M%S) \
        --percentile-metrics ttft,tpot,itl \
        --ignore-eos

sweep_decode:
    just bench_decode 4 40 && \
    just bench_decode 8 80 && \
    just bench_decode 16 160 && \
    just bench_decode 32 320 && \
    just bench_decode 64 640 && \
    just bench_decode 128 1280

============ Serving Benchmark Result ============
Successful requests:                     160       
Failed requests:                         0         
Maximum request concurrency:             16        
Benchmark duration (s):                  23.38     
Total input tokens:                      160       
Total generated tokens:                  16000     
Request throughput (req/s):              6.84      
Output token throughput (tok/s):         684.26    
Peak output token throughput (tok/s):    704.00    
Peak concurrent requests:                32.00     
Total Token throughput (tok/s):          691.10    
---------------Time to First Token----------------
Mean TTFT (ms):                          44.03     
Median TTFT (ms):                        43.80     
P99 TTFT (ms):                           64.44     
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms):                          23.15     
Median TPOT (ms):                        23.13     
P99 TPOT (ms):                           23.41     
---------------Inter-token Latency----------------
Mean ITL (ms):                           23.15     
Median ITL (ms):                         23.16     
P99 ITL (ms):                            25.07     
==================