agent-health.sh: operator health monitor

2026-10-03 18:03:17 +00:00
parent 4be49314d3
commit baae422b5f
1 changed files with 72 additions and 0 deletions
@@ -0,0 +1,72 @@
+#!/bin/bash
+# Operator health monitor for Muse agents on bl.
+# Checks each agent via API every 5 minutes. If unresponsive:
+# 1. Restart the browser
+# 2. Re-check
+# 3. Log failure if still down
+#
+# Run via systemd timer or cron: */5 * * * * /home/super/Projects/NetVM/bin/agent-health.sh
+#
+# Agents are defined in ACCOUNTS.md. This script reads the active ones.
+
+NETVM_BIN="$(cd "$(dirname "$0")" && pwd)"
+LOG="/tmp/agent-health.log"
+
+check_agent() {
+    local agent=$1
+    local node=$2
+    local cdp_port=$3
+    
+    # Try API messages command (lightweight check)
+    if timeout 30 "$NETVM_BIN/netvm-exec.sh" "$node" -- python3 "$NETVM_BIN/muse-chat-api.py" --account "$agent" messages 1 > /dev/null 2>&1; then
+        echo "$(date -Iseconds) $agent: OK" >> "$LOG"
+        return 0
+    else
+        echo "$(date -Iseconds) $agent: FAIL (api timeout)" >> "$LOG"
+        return 1
+    fi
+}
+
+restart_browser() {
+    local agent=$1
+    local cdp_port=$2
+    
+    echo "$(date -Iseconds) $agent: restarting browser..." >> "$LOG"
+    # Kill existing
+    pkill -f "$agent.*$cdp_port" 2>/dev/null
+    sleep 3
+    # Restart via netvm-chrome.sh
+    cd "$NETVM_BIN"
+    nohup ./netvm-chrome.sh --headless --cdp-port "$cdp_port" "$agent" https://muse.ai > "/tmp/bl-$agent.log" 2>&1 &
+    sleep 15
+    echo "$(date -Iseconds) $agent: browser restarted" >> "$LOG"
+}
+
+# Main
+echo "=== Health check $(date -Iseconds) ===" >> "$LOG"
+
+# muse (node muse, CDP 9410)
+if ! check_agent "muse" "muse" "9410"; then
+    restart_browser "muse" "9410"
+    sleep 5
+    if ! check_agent "muse" "muse" "9410"; then
+        echo "$(date -Iseconds) muse: CRITICAL - still down after restart" >> "$LOG"
+        # TODO: Alert operator (e.g., via board post or email)
+    else
+        echo "$(date -Iseconds) muse: RECOVERED after restart" >> "$LOG"
+    fi
+fi
+
+# pip (node pip, CDP 9420)
+if ! check_agent "pip" "pip" "9420"; then
+    restart_browser "pip" "9420"
+    sleep 5
+    if ! check_agent "pip" "pip" "9420"; then
+        echo "$(date -Iseconds) pip: CRITICAL - still down after restart" >> "$LOG"
+    else
+        echo "$(date -Iseconds) pip: RECOVERED after restart" >> "$LOG"
+    fi
+fi
+
+# Trim log (keep last 1000 lines)
+tail -1000 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG"