python3 -m sglang.launch_server --model /shared/public/elr-models/meta-llama/Meta-Llama-3.1-8B-Instruct/07eb05b21d191a58c577b4a45982fe0c049d0693  --speculative-algorithm EAGLE3 --speculative-draft-model-path /shared/public/elr-models/jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B/e5ed08d66f528a95ce89f5d4fd136a28f6def714 --speculative-num-steps 3         --speculative-eagle-topk 1 --speculative-num-draft-tokens 4  --trust-remote-code --dtype float16 --enable-torch-compile --attention-backend triton --cuda-graph-max-bs 2 --enable-deterministic-inference

python benchmark/gsm8k/bench_sglang.py --data-path /shared/public/data/gsm8k/test.jsonl
100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [00:18<00:00, 10.70it/s]
Accuracy: 0.770
Invalid: 0.000
Latency: 18.762 s
Output throughput: 909.155 token/s

python3 -m sglang.test.test_deterministic --test-mode prefix --n-trials 50
Prompt 0 with prefix length 1: total samples: 286, Unique samples: 1
Prompt 1 with prefix length 511: total samples: 333, Unique samples: 1
Prompt 2 with prefix length 2048: total samples: 311, Unique samples: 1
Prompt 3 with prefix length 4097: total samples: 345, Unique samples: 1