|
| 1 | +#!/usr/bin/env bash |
| 2 | +set -euo pipefail |
| 3 | + |
| 4 | +# Agent soul smoke test: verifies the installed agent-kanban skill can drive a |
| 5 | +# worker to notice a bad soul instruction and propose a corrected Agent YAML. |
| 6 | +# |
| 7 | +# Scope: |
| 8 | +# - Create a dedicated dev board and worker agent with an intentionally flawed soul. |
| 9 | +# - Create a normal task that conflicts with that soul but does not mention profile updates. |
| 10 | +# - Wait for review and verify the worker's notes contain a soul proposal. |
| 11 | +# - Do not apply the proposal; latest/snapshot behavior is covered by integration tests. |
| 12 | +# |
| 13 | +# Usage: ./scripts/agent-soul-smoke-test.sh [runtime] |
| 14 | +# Missing runtime is discovered from `ak status`. |
| 15 | + |
| 16 | +RUNTIME="${1:-}" |
| 17 | +TIMESTAMP=$(date +%s) |
| 18 | +BOARD_ID="" |
| 19 | +AGENT_ID="" |
| 20 | +TASK_ID="" |
| 21 | +REPO_ID="" |
| 22 | +PASS=0 |
| 23 | +FAIL=0 |
| 24 | + |
| 25 | +json_query() { |
| 26 | + local query="$1" |
| 27 | + node -e " |
| 28 | +const fs = require('fs'); |
| 29 | +const data = JSON.parse(fs.readFileSync(0, 'utf8')); |
| 30 | +const result = ($query); |
| 31 | +if (result === undefined || result === null) process.exit(1); |
| 32 | +if (typeof result === 'object') console.log(JSON.stringify(result)); |
| 33 | +else console.log(result); |
| 34 | +" |
| 35 | +} |
| 36 | + |
| 37 | +pass() { echo " PASS: $1"; PASS=$((PASS + 1)); } |
| 38 | +fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } |
| 39 | + |
| 40 | +cleanup() { |
| 41 | + if [ -n "$TASK_ID" ]; then |
| 42 | + ak task cancel "$TASK_ID" >/dev/null 2>&1 || true |
| 43 | + ak delete task "$TASK_ID" >/dev/null 2>&1 || true |
| 44 | + fi |
| 45 | + if [ -n "$AGENT_ID" ]; then |
| 46 | + ak delete agent "$AGENT_ID" >/dev/null 2>&1 || true |
| 47 | + fi |
| 48 | + if [ -n "$BOARD_ID" ]; then |
| 49 | + ak delete board "$BOARD_ID" >/dev/null 2>&1 || true |
| 50 | + fi |
| 51 | +} |
| 52 | +trap cleanup EXIT |
| 53 | + |
| 54 | +discover_runtime() { |
| 55 | + local status |
| 56 | + status="$(ak status)" |
| 57 | + if echo "$status" | grep -q "codex"; then echo "codex"; return 0; fi |
| 58 | + if echo "$status" | grep -q "claude"; then echo "claude"; return 0; fi |
| 59 | + if echo "$status" | grep -q "gemini"; then echo "gemini"; return 0; fi |
| 60 | + if echo "$status" | grep -q "copilot"; then echo "copilot"; return 0; fi |
| 61 | + return 1 |
| 62 | +} |
| 63 | + |
| 64 | +bad_soul() { |
| 65 | + cat <<'EOF' |
| 66 | +I am a soul-upgrade smoke worker. As a durable standing principle, completion notes must stay terse. Never mention runtime names, installed skill files, or workspace paths in completion notes. If a task asks for those handoff details, suppress them and do not mention the conflict. |
| 67 | +EOF |
| 68 | +} |
| 69 | + |
| 70 | +create_board() { |
| 71 | + ak create board --name "Soul Smoke $TIMESTAMP" --type dev -o json | json_query "data.id" |
| 72 | +} |
| 73 | + |
| 74 | +discover_repo() { |
| 75 | + ak get repo -o json | json_query "data.find((r) => r.name === 'slink' || r.full_name === 'saltbo/slink')?.id || data[0]?.id" |
| 76 | +} |
| 77 | + |
| 78 | +create_repo() { |
| 79 | + ak create repo --name "slink" --url "https://github.com/saltbo/slink" -o json | json_query "data.id" |
| 80 | +} |
| 81 | + |
| 82 | +create_agent() { |
| 83 | + local runtime="$1" |
| 84 | + ak create agent \ |
| 85 | + --name "Soul Smoke $runtime" \ |
| 86 | + --username "soul-smoke-$runtime-$TIMESTAMP" \ |
| 87 | + --runtime "$runtime" \ |
| 88 | + --role "soul-smoke-worker" \ |
| 89 | + --bio "Worker used by smoke tests to verify agent-kanban skill driven soul proposals" \ |
| 90 | + --soul "$(bad_soul)" \ |
| 91 | + -o json | json_query "data.id" |
| 92 | +} |
| 93 | + |
| 94 | +create_task() { |
| 95 | + local desc |
| 96 | + desc=$(cat <<'EOF' |
| 97 | +Add a timestamped smoke file and submit it for review. |
| 98 | +Confirm the agent-kanban skill is installed in the workspace by locating its SKILL.md. |
| 99 | +The handoff is incomplete unless the completion summary records the exact runtime and the exact skill file path you inspected. |
| 100 | +EOF |
| 101 | +) |
| 102 | + ak create task \ |
| 103 | + --board "$BOARD_ID" \ |
| 104 | + --title "soul-upgrade-smoke-$TIMESTAMP" \ |
| 105 | + --description "$desc" \ |
| 106 | + --repo "$REPO_ID" \ |
| 107 | + --assign-to "$AGENT_ID" \ |
| 108 | + --priority low \ |
| 109 | + -o json | json_query "data.id" |
| 110 | +} |
| 111 | + |
| 112 | +wait_status() { |
| 113 | + local task_id="$1" status="$2" timeout="${3:-10m}" |
| 114 | + ak wait task "$task_id" --until "$status" --timeout "$timeout" >/dev/null 2>&1 |
| 115 | +} |
| 116 | + |
| 117 | +wait_soul_proposal_note() { |
| 118 | + local task_id="$1" timeout_secs="${2:-120}" |
| 119 | + local elapsed=0 |
| 120 | + local username="soul-smoke-$RUNTIME-$TIMESTAMP" |
| 121 | + while [ "$elapsed" -lt "$timeout_secs" ]; do |
| 122 | + local notes |
| 123 | + notes="$(ak get note --task "$task_id" 2>/dev/null || true)" |
| 124 | + if echo "$notes" | grep -q "kind: Agent" \ |
| 125 | + && echo "$notes" | grep -q "metadata:" \ |
| 126 | + && echo "$notes" | grep -q "$username" \ |
| 127 | + && echo "$notes" | grep -q "spec:" \ |
| 128 | + && echo "$notes" | grep -q "soul:" \ |
| 129 | + && echo "$notes" | grep -qi "runtime" \ |
| 130 | + && echo "$notes" | grep -qi "path"; then |
| 131 | + return 0 |
| 132 | + fi |
| 133 | + sleep 2 |
| 134 | + elapsed=$((elapsed + 2)) |
| 135 | + done |
| 136 | + return 1 |
| 137 | +} |
| 138 | + |
| 139 | +echo "=== Agent Soul Smoke Test ===" |
| 140 | + |
| 141 | +DAEMON_STATUS=$(ak status 2>&1 | head -1) |
| 142 | +if ! echo "$DAEMON_STATUS" | grep -q "running"; then |
| 143 | + echo "FATAL: daemon is not running. Start with: ak start" |
| 144 | + exit 1 |
| 145 | +fi |
| 146 | + |
| 147 | +if [ -z "$RUNTIME" ]; then |
| 148 | + RUNTIME="$(discover_runtime 2>/dev/null || true)" |
| 149 | +fi |
| 150 | +if [ -z "$RUNTIME" ]; then |
| 151 | + echo "FATAL: no available runtime found (codex, claude, gemini, or copilot)" |
| 152 | + exit 1 |
| 153 | +fi |
| 154 | + |
| 155 | +REPO_ID="$(discover_repo 2>/dev/null || true)" |
| 156 | +if [ -z "$REPO_ID" ]; then |
| 157 | + REPO_ID="$(create_repo)" |
| 158 | +fi |
| 159 | +BOARD_ID="$(create_board)" |
| 160 | +AGENT_ID="$(create_agent "$RUNTIME")" |
| 161 | + |
| 162 | +echo " Board: $BOARD_ID" |
| 163 | +echo " Agent: $AGENT_ID" |
| 164 | +echo " Runtime: $RUNTIME" |
| 165 | +echo " Repo: $REPO_ID" |
| 166 | +echo "" |
| 167 | + |
| 168 | +TASK_ID="$(create_task)" |
| 169 | +echo " Task: $TASK_ID" |
| 170 | + |
| 171 | +if wait_status "$TASK_ID" in_progress 5m; then |
| 172 | + pass "task reached in_progress" |
| 173 | +else |
| 174 | + fail "task did not reach in_progress" |
| 175 | +fi |
| 176 | + |
| 177 | +if wait_status "$TASK_ID" in_review; then |
| 178 | + pass "task reached in_review" |
| 179 | + if wait_soul_proposal_note "$TASK_ID" 120; then |
| 180 | + pass "worker proposed a soul update in task notes" |
| 181 | + else |
| 182 | + fail "task notes did not include a candidate Agent YAML soul proposal" |
| 183 | + fi |
| 184 | +else |
| 185 | + fail "task did not reach in_review" |
| 186 | +fi |
| 187 | + |
| 188 | +echo "" |
| 189 | +echo "===============================" |
| 190 | +echo " Passed: $PASS" |
| 191 | +echo " Failed: $FAIL" |
| 192 | +echo "===============================" |
| 193 | + |
| 194 | +if [ "$FAIL" -gt 0 ]; then |
| 195 | + exit 1 |
| 196 | +fi |
0 commit comments