diff --git a/bin/agent-health.sh b/bin/agent-health.sh new file mode 100755 index 0000000..00c877a --- /dev/null +++ b/bin/agent-health.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# Operator health monitor for Muse agents on bl. +# Checks each agent via API every 5 minutes. If unresponsive: +# 1. Restart the browser +# 2. Re-check +# 3. Log failure if still down +# +# Run via systemd timer or cron: */5 * * * * /home/super/Projects/NetVM/bin/agent-health.sh +# +# Agents are defined in ACCOUNTS.md. This script reads the active ones. + +NETVM_BIN="$(cd "$(dirname "$0")" && pwd)" +LOG="/tmp/agent-health.log" + +check_agent() { + local agent=$1 + local node=$2 + local cdp_port=$3 + + # Try API messages command (lightweight check) + if timeout 30 "$NETVM_BIN/netvm-exec.sh" "$node" -- python3 "$NETVM_BIN/muse-chat-api.py" --account "$agent" messages 1 > /dev/null 2>&1; then + echo "$(date -Iseconds) $agent: OK" >> "$LOG" + return 0 + else + echo "$(date -Iseconds) $agent: FAIL (api timeout)" >> "$LOG" + return 1 + fi +} + +restart_browser() { + local agent=$1 + local cdp_port=$2 + + echo "$(date -Iseconds) $agent: restarting browser..." >> "$LOG" + # Kill existing + pkill -f "$agent.*$cdp_port" 2>/dev/null + sleep 3 + # Restart via netvm-chrome.sh + cd "$NETVM_BIN" + nohup ./netvm-chrome.sh --headless --cdp-port "$cdp_port" "$agent" https://muse.ai > "/tmp/bl-$agent.log" 2>&1 & + sleep 15 + echo "$(date -Iseconds) $agent: browser restarted" >> "$LOG" +} + +# Main +echo "=== Health check $(date -Iseconds) ===" >> "$LOG" + +# muse (node muse, CDP 9410) +if ! check_agent "muse" "muse" "9410"; then + restart_browser "muse" "9410" + sleep 5 + if ! check_agent "muse" "muse" "9410"; then + echo "$(date -Iseconds) muse: CRITICAL - still down after restart" >> "$LOG" + # TODO: Alert operator (e.g., via board post or email) + else + echo "$(date -Iseconds) muse: RECOVERED after restart" >> "$LOG" + fi +fi + +# pip (node pip, CDP 9420) +if ! check_agent "pip" "pip" "9420"; then + restart_browser "pip" "9420" + sleep 5 + if ! check_agent "pip" "pip" "9420"; then + echo "$(date -Iseconds) pip: CRITICAL - still down after restart" >> "$LOG" + else + echo "$(date -Iseconds) pip: RECOVERED after restart" >> "$LOG" + fi +fi + +# Trim log (keep last 1000 lines) +tail -1000 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG"