operator-main
5ab157b3b0
agent-health.sh used bare node names for 'ip netns exec' but netns are
named warp-<node> since the NetVM layout; the 6189793 CDP-liveness check
always failed ('No such file or directory'), logging false CRITICALs and
kill -9'ing healthy browsers every 5 min. Use warp-$node.
fleet-alert-check.sh: new 5-min critical-condition detector (per-node CDP
liveness via warp-<node> netns). Consecutive-failure state machine:
page after 2 consecutive failures, re-page every 30 min while critical,
quiet-hours-aware (first alert always pages). Emits ALERT/RECOVERY
records to ~/.local/share/fleet-alert/outbox.jsonl for the container
fleet-alert-relay hook; best-effort box-ctl notify to healthy agents.
Session: sidechat/critical-alerting-pipeline
111 lines
4.1 KiB
Bash
Executable File
111 lines
4.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# GOLDEN PATH: container -> VM (34.139.37.135) -> bl (100.123.153.75) -> netns -> browser -> agent
|
|
# This script is the operator's heartbeat. If it stops, agents go dark.
|
|
# When debugging: trace each hop. Don't assume -- verify.
|
|
|
|
# Operator health monitor for Muse agents on bl.
|
|
# Checks each agent via API every 5 minutes. If unresponsive:
|
|
# 1. Restart the browser
|
|
# 2. Re-check
|
|
# 3. Log failure if still down
|
|
#
|
|
# Run via systemd timer or cron: */5 * * * * /home/super/Projects/NetVM/bin/agent-health.sh
|
|
#
|
|
# Agents are defined in ACCOUNTS.md. This script reads the active ones.
|
|
|
|
NETVM_BIN="$(cd "$(dirname "$0")" && pwd)"
|
|
LOG="/tmp/agent-health.log"
|
|
|
|
check_cdp_port() {
|
|
# Verify CDP port is actually listening in the netns.
|
|
# A browser can be running but not bound to CDP (zombie state).
|
|
local node=$1
|
|
local cdp_port=$2
|
|
if sudo ip netns exec "warp-$node" ss -tln 2>/dev/null | grep -q ":$cdp_port "; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
check_agent() {
|
|
local agent=$1
|
|
local node=$2
|
|
local cdp_port=$3
|
|
|
|
# First: verify CDP port is listening (catches zombie browsers)
|
|
if ! check_cdp_port "$node" "$cdp_port"; then
|
|
echo "$(date -Iseconds) $agent: FAIL (cdp port $cdp_port not listening)" >> "$LOG"
|
|
return 1
|
|
fi
|
|
|
|
# Then: try API messages command (lightweight check)
|
|
if timeout 30 "$NETVM_BIN/netvm-exec.sh" "$node" -- python3 "$NETVM_BIN/muse-chat-api.py" --account "$agent" messages 1 > /dev/null 2>&1; then
|
|
echo "$(date -Iseconds) $agent: OK" >> "$LOG"
|
|
return 0
|
|
else
|
|
echo "$(date -Iseconds) $agent: FAIL (api timeout)" >> "$LOG"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
restart_browser() {
|
|
local agent=$1
|
|
local cdp_port=$2
|
|
|
|
echo "$(date -Iseconds) $agent: restarting browser..." >> "$LOG"
|
|
# Kill existing by exact PIDs (pkill patterns are unreliable)
|
|
# Find chromium processes for this profile
|
|
for pid in $(pgrep -f "chromium.*profiles/$agent" 2>/dev/null); do
|
|
kill -9 "$pid" 2>/dev/null
|
|
done
|
|
sleep 3
|
|
# Verify port is free before restart
|
|
if sudo ip netns exec "warp-$agent" ss -tln 2>/dev/null | grep -q ":$cdp_port "; then
|
|
echo "$(date -Iseconds) $agent: WARNING - port $cdp_port still bound after kill" >> "$LOG"
|
|
fi
|
|
# Restart via netvm-chrome.sh in its own systemd scope.
|
|
# This oneshot service runs with KillMode=control-group, so anything
|
|
# spawned directly under it (nohup AND setsid both stay in the cgroup)
|
|
# is SIGKILLed when the service exits — observed 2026-10-03: every
|
|
# restart "recovered" then died at service teardown, looping forever.
|
|
# A transient scope escapes the service cgroup and survives.
|
|
# NOTE: systemd-run --scope WAITS for the scope's processes (even with
|
|
# --no-block, verified 2026-10-03), so background it — the scope itself
|
|
# is an independent unit and outlives the wrapper.
|
|
cd "$NETVM_BIN"
|
|
systemd-run --user --scope --unit="netvm-chrome-$agent-$(date +%s)" \
|
|
./netvm-chrome.sh --headless --cdp-port "$cdp_port" "$agent" https://muse.ai \
|
|
> "/tmp/bl-$agent.log" 2>&1 &
|
|
sleep 15
|
|
echo "$(date -Iseconds) $agent: browser restarted" >> "$LOG"
|
|
}
|
|
|
|
# Main
|
|
echo "=== Health check $(date -Iseconds) ===" >> "$LOG"
|
|
|
|
check_one() {
|
|
local agent=$1
|
|
local cdp_port=$2
|
|
# node == agent == profile (unified naming)
|
|
if ! check_agent "$agent" "$agent" "$cdp_port"; then
|
|
restart_browser "$agent" "$cdp_port"
|
|
sleep 5
|
|
if ! check_agent "$agent" "$agent" "$cdp_port"; then
|
|
echo "$(date -Iseconds) $agent: CRITICAL - still down after restart" >> "$LOG"
|
|
# TODO: Alert operator (e.g., via board post or email)
|
|
else
|
|
echo "$(date -Iseconds) $agent: RECOVERED after restart" >> "$LOG"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Every active node in the NODES.md registry gets checked — new nodes
|
|
# propagate automatically, no per-node blocks to add.
|
|
"$NETVM_BIN/netvm-registry.py" 2>/dev/null | while IFS=: read -r node port; do
|
|
[ -n "$node" ] && [ -n "$port" ] && check_one "$node" "$port"
|
|
done
|
|
|
|
# Trim log (keep last 1000 lines)
|
|
tail -1000 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG"
|