Watchdog upgrades: stage-specific logging + new CDP relay watchdog

chromebox-watchdog.sh: HEALTH_FAIL_REASON pinpoints which health stage failed (no process / CDP unreachable / no Chat page); chromium stdout redirected to per-profile chromebox-<profile>.log; 10MB log rotation (one generation); Chat title match relaxed to .*Chat. bin/cdp-relay-watchdog.sh (new): keeps per-node CDP relays alive. Two-stage check: (1) host veth IP assigned (fail-loud, no auto-fix — veth recreation touches WireGuard/iptables), (2) relay connectivity via curl to veth IP:port (never trust pidfiles — observed stale 2026-10-04). Restarts dead/misrouted relays in-netns. Runs via systemd timer every 5min. Pattern mirrors chromebox-watchdog.sh. Session: sidechat/chromebox-ops
2026-10-04 12:40:56 +00:00
parent 3dd9d9ffaf
commit 343920e0c3
2 changed files with 149 additions and 6 deletions
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+# cdp-relay-watchdog.sh — keep per-node CDP relays alive and correctly routed.
+# Two-stage health check:
+#   1. Host veth IP must be assigned (veth_healthy). Without it the relay is
+#      unreachable from the host no matter how many times we restart it —
+#      observed 2026-10-04 (muse/pip veths existed but had no IPs). FAIL_LOUD
+#      in the log; do NOT auto-fix (veth recreation touches WireGuard/iptables).
+#   2. Relay connectivity (relay_healthy): curl to veth IP:port, not pidfile
+#      (which goes stale and lies — observed 2026-10-04).
+# If a relay is down or misrouted: kill it and restart via the exact
+# netvm-node-up.sh relay invocation inside the node's netns.
+# Runs every 5 min via systemd timer cdp-relay-watchdog.timer.
+# Pattern mirrors chromebox-watchdog.sh (stage-specific logging, rotation).
+set -euo pipefail
+LOCK="/tmp/cdp-relay-watchdog.lock"
+exec 9>"$LOCK"
+if ! flock -n 9; then
+  echo "[$(date -u +%FT%TZ)] another relay watchdog run in progress, skipping" >&2
+  exit 0
+fi
+
+NETVM_BIN="/home/super/Projects/NetVM/bin"
+LOG="/home/super/Projects/NetVM/cdp-relay-watchdog.log"
+
+rotate_log() {
+  local f="$1"
+  [ -f "$f" ] || return 0
+  local sz
+  sz=$(stat -c%s "$f" 2>/dev/null || echo 0)
+  if [ "$sz" -gt 10485760 ]; then
+    mv -f "$f" "$f.1"
+    echo "[$(date -u +%FT%TZ)] log rotated" > "$f"
+  fi
+}
+rotate_log "$LOG"
+log() { echo "[$(date -u +%FT%TZ)] $*" | tee -a "$LOG"; }
+
+# node -> "veth_ip:port" via netvm-names.sh (hash-derived, don't hardcode)
+relay_target() {
+  local node="$1"
+  # shellcheck disable=SC1091
+  . "$NETVM_BIN/netvm-names.sh"
+  netvm_names "$node" || return 1
+  # CDP_PORT_OVERRIDE pins registry ports; fall back to hash-derived
+  local port="${CDP_PORT_OVERRIDE:-$CDP_PORT}"
+  case "$node" in
+    muse) port=9410 ;; pip) port=9420 ;; 646) port=9430 ;; opm) port=9440 ;;
+  esac
+  echo "$PEER_IP:$port"
+}
+node_port() { echo "${1##*:}"; }
+
+# node -> "VETH GW" via netvm-names.sh
+node_veth() {
+  local node="$1"
+  # shellcheck disable=SC1091
+  . "$NETVM_BIN/netvm-names.sh"
+  netvm_names "$node" || return 1
+  echo "$VETH $GW"
+}
+
+# Is the host-side veth IP assigned? The relay listens on the netns-side peer
+# IP; the host reaches it via the veth interface's GW address. If the GW IP is
+# missing, the relay is unreachable from the host — restarting the relay is
+# pointless and masks the real problem.
+veth_healthy() {
+  local node="$1" veth gw
+  read -r veth gw <<< "$(node_veth "$node")" || return 1
+  ip addr show dev "$veth" 2>/dev/null | grep -q "inet ${gw}/" || return 1
+  return 0
+}
+
+relay_healthy() {
+  local node="$1" target
+  target="$(relay_target "$node")" || return 1
+  curl -s -m 8 "http://$target/json/version" 2>/dev/null | grep -q '"Browser"' || return 1
+  return 0
+}
+
+restart_relay() {
+  local node="$1" target port veth_ip netns
+  target="$(relay_target "$node")"
+  veth_ip="${target%%:*}"
+  port="${target##*:}"
+  netns="warp-$node"
+  # Kill any existing relay for this node's port (correct or not)
+  pkill -f "netvm-cdp-relay.py .* $port 127.0.0.1 $port" 2>/dev/null || true
+  sleep 2
+  # Launch inside the netns, listening on the veth IP (host-reachable)
+  sudo -n ip netns exec "$netns" setsid nohup python3 \
+    "$NETVM_BIN/netvm-cdp-relay.py" "$veth_ip" "$port" 127.0.0.1 "$port" \
+    >>"$LOG" 2>&1 < /dev/null &
+  sleep 5
+  if relay_healthy "$node"; then
+    log "[$node] relay restarted OK on $target"
+    return 0
+  else
+    log "[$node] relay restart FAILED on $target — needs operator attention"
+    return 1
+  fi
+}
+
+FAILED=0
+for node in muse pip 646 opm; do
+  # Stage 1: host veth IP. Fail loud, skip relay restart (pointless).
+  if ! veth_healthy "$node"; then
+    read -r veth gw <<< "$(node_veth "$node")"
+    log "[$node] FAIL_LOUD: host veth $veth missing IP $gw — relay unreachable, needs netvm-node-up.sh $node (manual)"
+    FAILED=1
+    continue
+  fi
+  # Stage 2: relay connectivity.
+  if relay_healthy "$node"; then
+    continue
+  fi
+  target="$(relay_target "$node")"
+  log "[$node] relay unhealthy on $target, restarting"
+  restart_relay "$node" || FAILED=1
+done
+
+exit $FAILED