Skip to content

Commit 5da1648

Browse files
committed
feat: integrate HomeSec Benchmark as option 3 (LLM only)
1 parent c75b50a commit 5da1648

1 file changed

Lines changed: 35 additions & 5 deletions

File tree

run_benchmark.sh

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,17 @@ echo ""
1111
echo "Select Action:"
1212
echo "1) Test 1: Automated Context & Memory Profile (TPS & RAM matrix)"
1313
echo "2) Test 2: Prompt Cache & Sliding Window Regression Test"
14-
echo "3) Model Maintain List and Delete"
15-
echo "4) Quit"
16-
read -p "Option (1-4): " suite_opt
14+
echo "3) Test 3: HomeSec Benchmark (LLM Only)"
15+
echo "4) Model Maintain List and Delete"
16+
echo "5) Quit"
17+
read -p "Option (1-5): " suite_opt
1718

18-
if [ "$suite_opt" == "4" ] || [ -z "$suite_opt" ]; then
19+
if [ "$suite_opt" == "5" ] || [ -z "$suite_opt" ]; then
1920
echo "Exiting."
2021
exit 0
2122
fi
2223

23-
if [ "$suite_opt" == "3" ]; then
24+
if [ "$suite_opt" == "4" ]; then
2425
echo ""
2526
echo "=> Downloaded Models Maintenance"
2627
CACHE_DIR="$HOME/.cache/huggingface/hub"
@@ -167,6 +168,35 @@ if [ "$suite_opt" == "2" ]; then
167168
exit 0
168169
fi
169170

171+
if [ "$suite_opt" == "3" ]; then
172+
echo ""
173+
echo "=> Starting HomeSec Benchmark (LLM Only) on $FULL_MODEL"
174+
175+
echo "Starting Server in background..."
176+
killall SwiftLM 2>/dev/null
177+
mkdir -p tmp
178+
$BIN --model "$FULL_MODEL" --port 5431 --turbo-kv --stream-experts --ctx-size 8192 > ./tmp/homesec_server.log 2>&1 &
179+
SERVER_PID=$!
180+
181+
echo "Waiting for server to be ready on port 5431 (this may take a minute if downloading)..."
182+
for i in {1..300}; do
183+
if curl -s http://127.0.0.1:5431/health > /dev/null; then break; fi
184+
sleep 1
185+
done
186+
187+
echo ""
188+
echo "Server is up! Executing DeepCamera HomeSec Benchmark..."
189+
190+
# Run the benchmark against the LLM gateway. Not specifying --vlm disables VLM tests.
191+
node ../DeepCamera/skills/analysis/home-security-benchmark/scripts/run-benchmark.cjs --gateway http://127.0.0.1:5431/v1
192+
193+
echo ""
194+
echo "Cleaning up..."
195+
killall SwiftLM
196+
wait $SERVER_PID 2>/dev/null
197+
exit 0
198+
fi
199+
170200
# Fallback to Test 1 for anything else
171201
echo ""
172202
read -p "Enter context lengths to test [default: 512,40000,100000]: " CONTEXTS

0 commit comments

Comments
 (0)