command -v nmcli &>/dev/null || return 0

local changed=0

# Walk all ethernet connection files, not just the active one

for _NM_FILE in /etc/NetworkManager/system-connections/*.nmconnection; do

[ -f "$_NM_FILE" ] || continue

if grep -q 'permissions=user:root:;' "$_NM_FILE" 2>/dev/null; then

sudo sed -i 's/permissions=user:root:;//' "$_NM_FILE" 2>/dev/null

changed=1

fi

if grep -q 'autoconnect=false' "$_NM_FILE" 2>/dev/null; then

sudo sed -i 's/autoconnect=false/autoconnect=true/' "$_NM_FILE" 2>/dev/null

changed=1

fi

done

[ "$changed" = "1" ] && nmcli connection reload 2>/dev/null || true

command -v nmcli &>/dev/null || return 0

local NM_DOCKER_CONF="/etc/NetworkManager/conf.d/docker-unmanaged.conf"

local changed=0

if [ ! -f "$NM_DOCKER_CONF" ]; then

sudo tee "$NM_DOCKER_CONF" > /dev/null << 'NMDEOF'

changed=1

fi

if [ "$changed" = "1" ]; then

sudo nmcli general reload 2>/dev/null || true

fi

# Ensure Docker uses nftables backend to avoid iptables/nftables conflicts

# with firewalld on RHEL 9 / AlmaLinux 9 OVAs.

local DAEMON_JSON="/etc/docker/daemon.json"

if [ -f "$DAEMON_JSON" ] && command -v python3 &>/dev/null; then

if ! python3 -c "import json,sys; d=json.load(open('$DAEMON_JSON')); sys.exit(0 if d.get('firewall-backend')=='nftables' and d.get('bip')=='100.64.0.1/24' else 1)" 2>/dev/null; then

python3 - "$DAEMON_JSON" << 'PYEOF'

sudo systemctl reload-or-restart docker 2>/dev/null || true

fi

fi

if [ -f "$POSTGRES_OVERRIDE_FILE" ]; then

# Extract version from calltelemetry/postgres:XX image line

local version=$(grep -o 'calltelemetry/postgres:[0-9]\+' "$POSTGRES_OVERRIDE_FILE" | grep -o '[0-9]\+$')

if [ -n "$version" ]; then

echo "$version"

return

fi

fi

echo "$POSTGRES_DEFAULT_VERSION"

local version="$1"

if ! echo "$POSTGRES_SUPPORTED_VERSIONS" | grep -qw "$version"; then

echo "Invalid PostgreSQL version: $version"

echo "Supported versions: $POSTGRES_SUPPORTED_VERSIONS"

return 1

fi

local override_url="${POSTGRES_OVERRIDE_URL}/postgres-${version}.yaml"

echo "Downloading PostgreSQL $version override..."

if wget -q "$override_url" -O "$POSTGRES_OVERRIDE_FILE"; then

echo "PostgreSQL version set to $version"

echo "Override file: $POSTGRES_OVERRIDE_FILE"

return 0

else

echo "Failed to download override file from: $override_url"

return 1

fi

# Check override file first

if [ -f "$POSTGRES_OVERRIDE_FILE" ]; then

local override_image=$(grep -o 'image: *"[^"]*"' "$POSTGRES_OVERRIDE_FILE" | head -1 | sed 's/image: *"\([^"]*\)"/\1/')

if [ -n "$override_image" ]; then

echo "$override_image"

return

fi

fi

# Fall back to main compose file

if [ -f "$ORIGINAL_FILE" ]; then

grep -A1 "^ db:" "$ORIGINAL_FILE" | grep "image:" | sed 's/.*image: *"\?\([^"]*\)"\?.*/\1/' | head -1

else

echo "unknown"

fi

local image="$1"

local helper=""

local script_dir

script_dir="$(cd "$(dirname "$CURRENT_SCRIPT_PATH")" && pwd)"

for candidate in \

"${INSTALL_DIR}/postgres-bitnami-convert.sh" \

"${script_dir}/postgres-bitnami-convert.sh" \

"${PWD}/postgres-bitnami-convert.sh"; do

if [ -f "$candidate" ]; then

helper="$candidate"

break

fi

done

if [ -z "$helper" ]; then

echo "[FAIL] PostgreSQL compatibility repair helper not found." >&2

echo " Expected postgres-bitnami-convert.sh in ${INSTALL_DIR} or next to cli.sh." >&2

return 1

fi

chmod +x "$helper" 2>/dev/null || true

if [ -n "$image" ] && [ "$image" != "unknown" ]; then

bash "$helper" --image "$image" --data-dir "${INSTALL_DIR}/postgres-data/data"

else

bash "$helper" --data-dir "${INSTALL_DIR}/postgres-data/data"

fi

local compose_file="${1:-$ORIGINAL_FILE}"

local pg_conf="${INSTALL_DIR}/postgres-data/data/postgresql.conf"

local changed=false

if [ ! -f "$compose_file" ]; then

return 0

fi

# Bad release bundles stripped TimescaleDB from the compose command. The

# command line overrides postgresql.conf, so restore preload before restart

# to prevent migrations from failing on existing TimescaleDB databases.

if grep -q "shared_preload_libraries='pg_stat_statements'" "$compose_file" 2>/dev/null; then

sed -i "s/shared_preload_libraries='pg_stat_statements'/shared_preload_libraries='timescaledb,pg_stat_statements'/" "$compose_file"

changed=true

elif grep -q "shared_preload_libraries='timescaledb'" "$compose_file" 2>/dev/null; then

sed -i "s/shared_preload_libraries='timescaledb'/shared_preload_libraries='timescaledb,pg_stat_statements'/" "$compose_file"

changed=true

fi

if grep -q "shared_preload_libraries='timescaledb,pg_stat_statements'" "$compose_file" 2>/dev/null; then

if ! grep -q "timescaledb.max_background_workers" "$compose_file" 2>/dev/null; then

sed -i "/shared_preload_libraries='timescaledb,pg_stat_statements'/a\\ -c timescaledb.max_background_workers=\${TS_MAX_BACKGROUND_WORKERS:-8}\\

-c timescaledb.bgw_scheduler_restart_time=\${TS_BGW_SCHEDULER_RESTART_TIME:-60000}" "$compose_file"

changed=true

fi

fi

if [ -f "$pg_conf" ] && grep -q '^shared_preload_libraries' "$pg_conf" 2>/dev/null && \

! grep -q "^shared_preload_libraries.*timescaledb" "$pg_conf" 2>/dev/null; then

sudo sed -i -E "s/^shared_preload_libraries[[:space:]]*=.*/shared_preload_libraries = 'timescaledb,pg_stat_statements'/" "$pg_conf"

changed=true

fi

if [ "$changed" = true ]; then

echo "[OK] TimescaleDB preload settings verified"

fi

if ! grep -q "shared_preload_libraries='timescaledb,pg_stat_statements'" "$ORIGINAL_FILE" 2>/dev/null; then

return 0

fi

local wait_time=0

local preload=""

while [ "$wait_time" -lt 120 ]; do

preload=$($DOCKER_COMPOSE_CMD exec -T db psql -U calltelemetry -d calltelemetry_prod -tAc "SHOW shared_preload_libraries;" 2>/dev/null || true)

if echo "$preload" | grep -q "timescaledb"; then

echo "[OK] PostgreSQL runtime preload includes TimescaleDB"

return 0

fi

sleep 5

wait_time=$((wait_time + 5))

done

echo "[FAIL] PostgreSQL started without TimescaleDB in shared_preload_libraries."

echo " Current value: ${preload:-unavailable}"

echo " Refusing to continue because migrations can fail against existing TimescaleDB databases."

return 1

local key="$1"

if [ -f "$ENV_FILE" ]; then

grep "^${key}=" "$ENV_FILE" 2>/dev/null | head -1 | cut -d= -f2-

fi

local key="$1" value="$2"

if [ ! -f "$ENV_FILE" ]; then

echo "${key}=${value}" > "$ENV_FILE"

return

fi

if grep -q "^${key}=" "$ENV_FILE" 2>/dev/null; then

sed -i "s|^${key}=.*|${key}=${value}|" "$ENV_FILE"

else

echo "${key}=${value}" >> "$ENV_FILE"

fi

local key="$1" value="$2"

if [ -z "$(env_get "$key")" ]; then

env_set "$key" "$value"

fi

echo "Ensuring PostgreSQL defaults..."

env_set_default "PG_PROFILE" "small"

env_set_default "PG_MAX_CONNECTIONS" "100"

env_set_default "DB_POOL_SIZE" "15"

env_set_default "DB_CALL_CONTROL_POOL_SIZE" "15"

env_set_default "DB_BACKGROUND_POOL_SIZE" "15"

env_set_default "DB_DISCOVERY_POOL_SIZE" "15"

env_set_default "DB_OBAN_POOL_SIZE" "15"

echo "[OK] PostgreSQL defaults applied (existing values preserved)"

local profile="$1"

case "$profile" in

small)

env_set "PG_PROFILE" "small"

env_set "PG_MAX_CONNECTIONS" "100"

env_set "DB_POOL_SIZE" "15"

env_set "DB_CALL_CONTROL_POOL_SIZE" "15"

env_set "DB_BACKGROUND_POOL_SIZE" "15"

env_set "DB_DISCOVERY_POOL_SIZE" "15"

env_set "DB_OBAN_POOL_SIZE" "15"

;;

medium)

env_set "PG_PROFILE" "medium"

env_set "PG_MAX_CONNECTIONS" "200"

env_set "DB_POOL_SIZE" "20"

env_set "DB_CALL_CONTROL_POOL_SIZE" "20"

env_set "DB_BACKGROUND_POOL_SIZE" "20"

env_set "DB_DISCOVERY_POOL_SIZE" "20"

env_set "DB_OBAN_POOL_SIZE" "20"

;;

large)

env_set "PG_PROFILE" "large"

env_set "PG_MAX_CONNECTIONS" "300"

env_set "DB_POOL_SIZE" "25"

env_set "DB_CALL_CONTROL_POOL_SIZE" "25"

env_set "DB_BACKGROUND_POOL_SIZE" "25"

env_set "DB_DISCOVERY_POOL_SIZE" "25"

env_set "DB_OBAN_POOL_SIZE" "25"

;;

*)

echo "Usage: cli.sh postgres profile <small|medium|large|show>"

return 1

;;

esac

echo "PostgreSQL profile set to: $profile"

echo " max_connections: $(env_get PG_MAX_CONNECTIONS)"

echo " db_pool (main): $(env_get DB_POOL_SIZE)"

echo " db_pool (callctl): $(env_get DB_CALL_CONTROL_POOL_SIZE)"

echo " db_pool (background): $(env_get DB_BACKGROUND_POOL_SIZE)"

echo " db_pool (discovery): $(env_get DB_DISCOVERY_POOL_SIZE)"

echo " db_pool (oban): $(env_get DB_OBAN_POOL_SIZE)"

echo ""

echo "Restarting services to apply new profile..."

$DOCKER_COMPOSE_CMD down db 2>/dev/null

$DOCKER_COMPOSE_CMD up -d 2>/dev/null

echo "[OK] Services restarted with $profile profile"

if [ -f "$JTAPI_STATE_FILE" ]; then

echo "Migrating JTAPI state from .jtapi-enabled to .env COMPOSE_PROFILES..."

local profiles

profiles=$(env_get "COMPOSE_PROFILES")

if ! echo "$profiles" | grep -q "jtapi"; then

if [ -n "$profiles" ]; then

env_set "COMPOSE_PROFILES" "${profiles},jtapi"

else

env_set "COMPOSE_PROFILES" "jtapi"

fi

fi

# Set JTAPI env vars if not already present

[ -z "$(env_get JTAPI_MODE)" ] && env_set "JTAPI_MODE" "direct"

[ -z "$(env_get JTAPI_SIDECAR_ENDPOINT)" ] && env_set "JTAPI_SIDECAR_ENDPOINT" "jtapi-sidecar:50051"

[ -z "$(env_get JTAPI_SIDECAR_URL)" ] && env_set "JTAPI_SIDECAR_URL" "http://jtapi-sidecar:8080"

[ -z "$(env_get S3_ENABLED)" ] && env_set "S3_ENABLED" "true"

[ -z "$(env_get CT_MEDIA_ENDPOINT)" ] && env_set "CT_MEDIA_ENDPOINT" "ct-media:50053"

rm -f "$JTAPI_STATE_FILE"

echo "Migration complete. .jtapi-enabled removed."

fi

local profiles

profiles=$(env_get "COMPOSE_PROFILES")

echo "$profiles" | grep -q "jtapi"

local current

current=$(sysctl -n net.ipv4.ip_forward 2>/dev/null || echo "0")

if [ "$current" != "1" ]; then

echo "[cli.sh] Enabling IPv4 forwarding (was disabled — Docker requires this)"

sysctl -w net.ipv4.ip_forward=1 >/dev/null 2>&1 || true

fi

# Always ensure persistent config exists

if ! grep -q "net.ipv4.ip_forward = 1" /etc/sysctl.d/99-docker-ipforward.conf 2>/dev/null; then

echo "net.ipv4.ip_forward = 1" > /etc/sysctl.d/99-docker-ipforward.conf 2>/dev/null || true

fi

local SERVICE_FILE="/etc/systemd/system/docker-compose-app.service"

[ -f "$SERVICE_FILE" ] || return 0

local compose_files

compose_files=$(get_compose_files)

local cmd_path

if [ "$DOCKER_COMPOSE_CMD" = "docker compose" ]; then

cmd_path="/usr/bin/docker compose"

else

cmd_path="/usr/bin/docker-compose"

fi

local new_start="ExecStart=$cmd_path $compose_files up -d"

local new_stop="ExecStop=$cmd_path $compose_files down"

if ! grep -qF "$new_start" "$SERVICE_FILE" 2>/dev/null; then

sudo cp "$SERVICE_FILE" "${SERVICE_FILE}.backup" 2>/dev/null

# Strip any leftover -f docker-compose-jtapi.yml from systemd

sudo sed -i "s|^ExecStart=.*|$new_start|" "$SERVICE_FILE"

sudo sed -i "s|^ExecStop=.*|$new_stop|" "$SERVICE_FILE"

sudo systemctl daemon-reload

echo "Updated systemd service compose files"

fi

local compose_file="${1:-$ORIGINAL_FILE}"

[ -f "$compose_file" ] || return 0

if ! grep -q 'grafana/grafana' "$compose_file" 2>/dev/null; then

return 0

fi

local tmp_file

tmp_file="$(mktemp)"

awk '

' "$compose_file" > "$tmp_file"

if cmp -s "$compose_file" "$tmp_file"; then

rm -f "$tmp_file"

return 0

fi

mv "$tmp_file" "$compose_file"

echo "[OK] Removed direct Grafana host port binding; use /grafana via Caddy."

local subcmd="${1:-}"

shift 2>/dev/null || true

# Auto-migrate legacy state file on any jtapi command

migrate_jtapi_state

case "$subcmd" in

enable)

# Add jtapi to COMPOSE_PROFILES

local profiles

profiles=$(env_get "COMPOSE_PROFILES")

if ! echo "$profiles" | grep -q "jtapi"; then

if [ -n "$profiles" ]; then

env_set "COMPOSE_PROFILES" "${profiles},jtapi"

else

env_set "COMPOSE_PROFILES" "jtapi"

fi

fi

# Set JTAPI env vars

env_set "JTAPI_MODE" "direct"

env_set "JTAPI_SIDECAR_ENDPOINT" "jtapi-sidecar:50051"

env_set "JTAPI_SIDECAR_URL" "http://jtapi-sidecar:8080"

env_set "S3_ENABLED" "true"

env_set "CT_MEDIA_ENDPOINT" "ct-media:50053"

# Clean up legacy state file if present

rm -f "$JTAPI_STATE_FILE"

fix_systemd_compose_files

echo "[OK] JTAPI enabled — restarting services..."

echo ""

if ! restart_service "jtapi enable"; then

echo "[FAIL] Service restart failed. JTAPI config was saved but services may not be running."

echo " Retry with: systemctl restart docker-compose-app.service"

return 1

fi

# Force recreate web to pick up new env vars

sleep 3

$DOCKER_COMPOSE_CMD $(get_compose_files) up -d --force-recreate web 2>/dev/null

echo "Services restarted."

echo ""

echo "Next steps:"

echo " 1. Wait for sidecar to start (~90s)"

echo " 2. Upload JTAPI JAR via UI (Settings > JTAPI)"

echo " 3. Sidecar auto-restarts when JAR is received"

echo " 4. Add CUCM server via UI (Settings > JTAPI > Servers)"

echo " 5. Sidecar auto-connects when credentials appear in NATS KV"

;;

disable)

# Remove jtapi from COMPOSE_PROFILES

local profiles

profiles=$(env_get "COMPOSE_PROFILES")

local new_profiles

new_profiles=$(echo "$profiles" | sed 's/,*jtapi,*//' | sed 's/^,//' | sed 's/,$//')

env_set "COMPOSE_PROFILES" "$new_profiles"

# Clear JTAPI env vars

env_remove "JTAPI_MODE"

env_remove "JTAPI_SIDECAR_ENDPOINT"

env_remove "JTAPI_SIDECAR_URL"

env_remove "S3_ENABLED"

env_remove "CT_MEDIA_ENDPOINT"

# Clean up legacy state file if present

rm -f "$JTAPI_STATE_FILE"

fix_systemd_compose_files

echo "[OK] JTAPI disabled — restarting services..."

if ! restart_service "jtapi disable"; then

echo "[FAIL] Service restart failed. JTAPI config was saved but services may not be running."

echo " Retry with: systemctl restart docker-compose-app.service"

return 1

fi

echo "JTAPI services removed."

;;

status)

if is_jtapi_enabled; then

echo "JTAPI: enabled"

$DOCKER_COMPOSE_CMD $(get_compose_files) ps jtapi-sidecar ct-media seaweedfs 2>/dev/null || true

else

echo "JTAPI: disabled"

fi

;;

troubleshoot)

echo "=== JTAPI Troubleshooting ==="

echo ""

# --- 1. Feature State ---

echo "--- Feature State ---"

if is_jtapi_enabled; then

echo "✓ JTAPI enabled (COMPOSE_PROFILES includes jtapi)"

else

echo "[FAIL] JTAPI disabled (COMPOSE_PROFILES does not include jtapi)"

fi

echo " COMPOSE_PROFILES=$(env_get COMPOSE_PROFILES)"

echo " Compose files: $(get_compose_files)"

echo ""

# --- 2. Container Health ---

echo "--- Container Health ---"

export DEFAULT_IPV4="${DEFAULT_IPV4:-}"

for svc in jtapi-jar-init jtapi-sidecar ct-media seaweedfs; do

local cid

cid=$($DOCKER_COMPOSE_CMD $(get_compose_files) ps -q "$svc" 2>/dev/null)

if [ -z "$cid" ]; then

# Init container may not show in ps after completion — check all containers

cid=$(docker ps -a --filter "name=${svc}" --format '{{.ID}}' 2>/dev/null | head -1)

fi

if [ -z "$cid" ]; then

if [ "$svc" = "jtapi-jar-init" ]; then

echo "[WARN] $svc: not found (may have been cleaned up after successful run)"

else

echo "[FAIL] $svc: not found (not deployed or not in compose files)"

fi

else

local cstate

cstate=$(docker inspect --format='{{.State.Status}}' "$cid" 2>/dev/null || echo "unknown")

local exit_code

exit_code=$(docker inspect --format='{{.State.ExitCode}}' "$cid" 2>/dev/null || echo "N/A")

if [ "$svc" = "jtapi-jar-init" ]; then

# Init container is expected to exit with code 0

if [ "$cstate" = "exited" ] && [ "$exit_code" = "0" ]; then

echo "✓ $svc: completed successfully (exit 0)"

elif [ "$cstate" = "exited" ]; then

echo "[FAIL] $svc: failed (exit $exit_code)"

else

echo "[WARN] $svc: $cstate"

fi

elif [ "$cstate" = "running" ]; then

echo "✓ $svc: running"

else

echo "[FAIL] $svc: $cstate (exit $exit_code)"

fi

fi

done

# Show restart counts

echo ""

echo " Container restart counts:"

for svc in jtapi-sidecar ct-media seaweedfs; do

local restart_count

restart_count=$(docker inspect --format='{{.RestartCount}}' "$($DOCKER_COMPOSE_CMD $(get_compose_files) ps -q "$svc" 2>/dev/null)" 2>/dev/null || echo "N/A")

echo " $svc: $restart_count restarts"

done

echo ""

# --- 3. NATS Connectivity ---

echo "--- NATS Connectivity ---"

# Check NATS is accepting connections via health endpoint

local nats_health

nats_health=$($DOCKER_COMPOSE_CMD $(get_compose_files) exec -T nats wget -q -O- http://127.0.0.1:8222/healthz 2>&1)

if echo "$nats_health" | grep -qi "ok\|status"; then

echo "✓ NATS server is healthy"

else

# Fallback: check container health status (pgrep not available in nats:2.11 image)

local nats_status

nats_status=$($DOCKER_COMPOSE_CMD $(get_compose_files) ps nats --format '{{.Status}}' 2>/dev/null || echo "unknown")

echo " NATS: $nats_status"

fi

echo ""

# Use nats-box for KV/ObjectStore checks (nats CLI not in server image)

local CT_NETWORK

CT_NETWORK=$($DOCKER_COMPOSE_CMD $(get_compose_files) ps --format '{{.Networks}}' nats 2>/dev/null | head -1 | cut -d',' -f1)

if [ -z "$CT_NETWORK" ]; then

CT_NETWORK="calltelemetry_ct"

fi

echo " NATS KV buckets:"

docker run --rm --network "$CT_NETWORK" natsio/nats-box:0.14.5 nats -s nats://nats:4222 kv ls 2>&1 | sed 's/^/ /' || echo " [FAIL] Could not list KV buckets"

echo ""

echo " NATS ObjectStore (jtapi-jars-1):"

local objstore_result

objstore_result=$(docker run --rm --network "$CT_NETWORK" natsio/nats-box:0.14.5 nats -s nats://nats:4222 object ls jtapi-jars-1 2>&1)

if echo "$objstore_result" | grep -q "jtapi.jar"; then

echo " ✓ jtapi.jar found in NATS ObjectStore"

elif echo "$objstore_result" | grep -qi "not found\|no such\|error"; then

echo " [WARN] jtapi-jars-1 bucket: $objstore_result"

else

echo " $objstore_result" | sed 's/^/ /'

fi

echo ""

echo "--- NATS ObjectStore Buckets ---"

docker run --rm --network "$CT_NETWORK" natsio/nats-box:0.14.5 \

sh -c 'nats -s nats://nats:4222 object ls 2>/dev/null' 2>/dev/null | sed 's/^/ /' || echo " Failed to list ObjectStore buckets"

echo ""

echo "--- jtapi-jars-1 bucket contents ---"

docker run --rm --network "$CT_NETWORK" natsio/nats-box:0.14.5 \

sh -c 'nats -s nats://nats:4222 object ls jtapi-jars-1 2>/dev/null' 2>/dev/null | sed 's/^/ /' || echo " Bucket jtapi-jars-1 not found or empty"

echo ""

# --- 4. JAR Status ---

echo "--- JAR Status ---"

local jar_vol_check

jar_vol_check=$(docker run --rm -v calltelemetry_jtapi-jars:/jars alpine ls -la /jars/jtapi.jar 2>&1)

if echo "$jar_vol_check" | grep -q "jtapi.jar"; then

echo "✓ JAR found in Docker volume"

echo " $jar_vol_check"

else

echo "[FAIL] JAR not found in Docker volume"

echo " $jar_vol_check"

fi

echo ""

echo " NATS ObjectStore JAR info:"

docker run --rm --network "$CT_NETWORK" natsio/nats-box:0.14.5 nats -s nats://nats:4222 object info jtapi-jars-1 jtapi.jar 2>&1 | sed 's/^/ /' || echo " [FAIL] Could not query NATS ObjectStore"

echo ""

# --- 5. Sidecar Health ---

echo "--- Sidecar Health ---"

local sidecar_health

sidecar_health=$($DOCKER_COMPOSE_CMD $(get_compose_files) exec -T jtapi-sidecar wget -q -O- http://127.0.0.1:8080/actuator/health 2>&1 || echo "Sidecar not responding")

if echo "$sidecar_health" | grep -qi '"status":"UP"\|"status":"up"'; then

echo "✓ Sidecar health: $sidecar_health"

else

echo "[WARN] Sidecar health: $sidecar_health"

fi

echo ""

echo " Last 20 lines of sidecar logs:"

$DOCKER_COMPOSE_CMD $(get_compose_files) logs --tail=20 jtapi-sidecar 2>&1 | sed 's/^/ /'

echo ""

# --- 6. SeaweedFS Health ---

echo "--- SeaweedFS Health ---"

local seaweedfs_health

seaweedfs_health=$($DOCKER_COMPOSE_CMD $(get_compose_files) exec -T seaweedfs curl -sf http://127.0.0.1:9333/cluster/healthz 2>&1)

if [ $? -eq 0 ]; then

echo "✓ SeaweedFS is healthy"

else

echo "[FAIL] SeaweedFS health check failed: $seaweedfs_health"

fi

echo ""

echo " SeaweedFS buckets:"

$DOCKER_COMPOSE_CMD $(get_compose_files) exec -T seaweedfs curl -sf http://localhost:9333/cluster/healthz 2>&1 | sed 's/^/ /' || echo " [WARN] Could not check SeaweedFS cluster status"

echo ""

# --- 7. ct-media Health ---

echo "--- ct-media Health ---"

local media_state

media_state=$($DOCKER_COMPOSE_CMD $(get_compose_files) ps --format '{{.State}}' ct-media 2>/dev/null || echo "not found")

if [ "$media_state" = "running" ]; then

echo "✓ ct-media is running"

else

echo "[FAIL] ct-media state: ${media_state:-not found}"

fi

echo ""

echo " Last 10 lines of ct-media logs:"

$DOCKER_COMPOSE_CMD $(get_compose_files) logs --tail=10 ct-media 2>&1 | sed 's/^/ /'

echo ""

# --- 8. Web Service JTAPI Config ---

echo "--- Web Service JTAPI Config ---"

echo " Environment variables:"

$DOCKER_COMPOSE_CMD $(get_compose_files) exec -T web env 2>/dev/null | grep -E 'JTAPI|S3_|CT_MEDIA|NATS_URL' | sort | sed 's/^/ /' || echo " [FAIL] Could not read web container env"

echo ""

# --- 9. CallManager CTI Credentials Check ---

echo "--- CallManager CTI Credentials ---"

local release_bin

release_bin=$(get_release_binary)

$DOCKER_COMPOSE_CMD $(get_compose_files) exec -T web "$release_bin" rpc '

' 2>&1 | sed 's/^/ /' || echo " [FAIL] Could not query JTAPI server configuration (web container may not be running)"

echo ""

# --- 10. JTAPI Runtime Diagnostics (from web logs) ---

echo ""

echo "=== JTAPI Runtime Diagnostics (from web logs) ==="

echo ""

# Check for NatsSupervisor startup messages

echo "--- NATS Supervisor Startup ---"

local nats_sup_logs

nats_sup_logs=$($DOCKER_COMPOSE_CMD $(get_compose_files) logs web --tail=200 2>/dev/null | grep -i "NatsSupervisor\|jtapi_gnat\|JtapiGreeting" | tail -5)

if [ -n "$nats_sup_logs" ]; then

echo "$nats_sup_logs"

else

echo " No JTAPI NATS supervisor messages found in recent logs"

fi

# Check for ObjectStore / JarManager errors

echo ""

echo "--- JAR Manager / ObjectStore Errors ---"

local jar_logs

jar_logs=$($DOCKER_COMPOSE_CMD $(get_compose_files) logs web --tail=500 2>/dev/null | grep -i "JarManager\|ObjectStore\|object_store\|jtapi.*jar\|bucket" | tail -10)

if [ -n "$jar_logs" ]; then

echo "$jar_logs"

else

echo " No JAR/ObjectStore errors found in recent logs"

fi

# Check for gRPC client errors

echo ""

echo "--- gRPC Client Status ---"

local grpc_logs

grpc_logs=$($DOCKER_COMPOSE_CMD $(get_compose_files) logs web --tail=200 2>/dev/null | grep -i "DirectGrpcClient\|grpc.*connect\|GRPC.*error\|sidecar.*endpoint" | tail -5)

if [ -n "$grpc_logs" ]; then

echo "$grpc_logs"

else

echo " No gRPC client messages found in recent logs"

fi

# --- 11. JTAPI Health API Response ---

echo ""

echo "=== JTAPI Health API Response ==="

echo ""

# Detect Docker Compose network name (reuse CT_NETWORK if already set)

if [ -z "${CT_NETWORK:-}" ]; then

CT_NETWORK=$(docker network ls --format '{{.Name}}' 2>/dev/null | grep '_ct$' | head -1)

if [ -z "$CT_NETWORK" ]; then

CT_NETWORK="${COMPOSE_PROJECT_NAME:-$(basename "$(pwd)")}_ct"

fi