feat: setup-fed watchdog supervision for all registry nodes

Close the def/dev supervision gap at the source: every node brought
up gets watched, and every supervisor enumerates the registry.

- bin/ensure-node-supervision.sh (new, idempotent): appends the
  NODES.md row (netvm-names port, honors CDP_PORT_OVERRIDE so it
  never fights provision's picker) and installs/enables
  chromebox-watchdog-<node>.timer. --all heals drift (registry +
  /etc/netvm identities). Template verified byte-identical to the
  installed def unit.
- netvm-node-up.sh: calls ensure (non-fatal) at the end. Provision
  and the onboarding pipeline reach it transitively.
- relay-health-check.sh, cdp-latency-check.sh: registry-driven
  watched_nodes() + LIB_ONLY guards (were hardcoded 4 nodes).
- tests/test_node_supervision.py (6): row add/idempotent/override,
  timer render, node-up wiring, both watched_nodes().
- CHROMEBOX-RUNBOOK.md: setup-fed supervision section.

Pairs with the registry-driven relay/chromebox watchdogs: new rows
are picked up on the next run with no per-node code edits.
This commit is contained in:
Muse Sidechat
2026-10-06 19:28:29 +00:00
parent c9143a558b
commit 66c8900a58
6 changed files with 249 additions and 3 deletions
+19 -1
View File
@@ -17,7 +17,25 @@ source "$BIN_DIR/netvm-names.sh"
TIMEOUT_S=10
for node in muse pip 646 opm; do
# Registry-driven node list (was hardcoded 4 nodes; def/dev had no
# cdp-latency coverage — 2026-10-06).
watched_nodes() {
"$BIN_DIR/netvm-registry.py" 2>/dev/null | cut -d: -f1
}
# Allow sourcing for tests without running checks.
if [ "${CDP_LATENCY_CHECK_LIB_ONLY:-}" = "1" ]; then
return 0 2>/dev/null || exit 0
fi
NODES="$(watched_nodes)"
if [ -z "$NODES" ]; then
echo "node registry empty/unreadable" >&2
exit 1
fi
# shellcheck disable=SC2086 (intended word splitting: one node per word)
for node in $NODES; do
netvm_names "$node"
url="http://${PEER_IP}:${CDP_PORT}/json/version"
probe=$(curl -s -m "$TIMEOUT_S" -o /dev/null -w "%{time_total} %{http_code}" "$url" 2>/dev/null)
+95
View File
@@ -0,0 +1,95 @@
#!/usr/bin/env bash
# ensure-node-supervision.sh <node> | --all — feed a node to the watchdogs.
#
# Setup (netvm-node-up.sh, hence netvm-provision-node.sh and the onboarding
# pipeline) calls this so every node gets supervision without manual wiring:
# 1. NODES.md registry row (idempotent) — feeds the registry-driven
# supervisors: cdp-relay-watchdog, agent-health.sh, relay-health-check,
# cdp-latency-check. Port from netvm-names pinning (honors
# CDP_PORT_OVERRIDE, so provision's picked port wins when present).
# 2. chromebox-watchdog-<node>.timer unit + enable --now — the one
# supervisor that needs a per-node systemd unit (the @.service
# template already exists). Needs root for the real unit dir.
#
# Env overrides (tests): NODES_MD, UNIT_DIR. systemctl is skipped when
# UNIT_DIR is not the real system dir.
#
# Runs at the end of netvm-node-up.sh (as root); safe to re-run anytime:
# sudo bin/ensure-node-supervision.sh --all
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
NODES_MD="${NODES_MD:-$SCRIPT_DIR/../NODES.md}"
UNIT_DIR="${UNIT_DIR:-/etc/systemd/system}"
# shellcheck disable=SC1091
. "$SCRIPT_DIR/netvm-names.sh"
usage() { echo "usage: ensure-node-supervision.sh <node> | --all" >&2; exit 1; }
ensure_registry_row() {
local node="$1"
if grep -qE "^\|[[:space:]]*$node[[:space:]]*\|" "$NODES_MD" 2>/dev/null; then
echo "registry: $node already in NODES.md"
return 0
fi
netvm_names "$node" || { echo "registry: unknown node $node" >&2; return 1; }
printf '| %s | %s | unknown | %s | active | %s (auto-registered) |\n' \
"$node" "$NETNS" "$CDP_PORT" "$node" >> "$NODES_MD"
echo "registry: added $node (port $CDP_PORT)"
}
ensure_timer() {
local node="$1" unit
unit="$UNIT_DIR/chromebox-watchdog-$node.timer"
if [ -f "$unit" ]; then
echo "timer: chromebox-watchdog-$node.timer already installed"
else
if [ "$UNIT_DIR" = "/etc/systemd/system" ] && [ "$(id -u)" -ne 0 ]; then
echo "timer: need root to install chromebox-watchdog-$node.timer (run with sudo)" >&2
return 1
fi
cat > "$unit" <<EOF
[Unit]
Description=Run chromebox watchdog for $node every 2 minutes
[Timer]
RandomizedDelaySec=30s
OnBootSec=2min
OnUnitActiveSec=2min
Unit=chromebox-watchdog@$node.service
[Install]
WantedBy=timers.target
EOF
echo "timer: installed chromebox-watchdog-$node.timer"
fi
if [ "$UNIT_DIR" = "/etc/systemd/system" ]; then
systemctl daemon-reload
systemctl enable --now "chromebox-watchdog-$node.timer" >/dev/null 2>&1
echo "timer: enabled chromebox-watchdog-$node.timer"
fi
}
ensure_node() {
local node="$1"
ensure_registry_row "$node"
ensure_timer "$node"
}
case "${1:-}" in
--all)
nodes="$(python3 "$SCRIPT_DIR/netvm-registry.py" 2>/dev/null | cut -d: -f1)"
for conf in /etc/netvm/*.conf; do
[ -f "$conf" ] || continue
nodes="$nodes $(basename "$conf" .conf)"
done
seen=""
# shellcheck disable=SC2086 (intended word splitting)
for node in $nodes; do
case " $seen " in *" $node "*) continue;; esac
seen="$seen $node"
ensure_node "$node" || echo "supervision: $node failed (continuing)" >&2
done
;;
""|-h|--help) usage;;
*) ensure_node "$1";;
esac
+5
View File
@@ -104,3 +104,8 @@ else
EGRESS=$(nsexec curl -sk --max-time 15 'https://1.1.1.1/cdn-cgi/trace' 2>/dev/null | grep -oP '^ip=\K.*' || true)
fi
echo "node=$NODE netns=$NETNS ifaces=$WG/$VETH egress=${EGRESS:-unknown}"
# Feed the watchdogs: registry row + chromebox timer (non-fatal — the
# node is up regardless, and supervision heals on the next run).
"$SCRIPT_DIR/ensure-node-supervision.sh" "$NODE" \
|| echo "supervision ensure failed for $NODE (non-fatal)" >&2
+19 -2
View File
@@ -1,5 +1,5 @@
#!/bin/bash
# relay-health-check.sh — check all four CDP relay endpoints on bl.
# relay-health-check.sh — check all registry CDP relay endpoints on bl.
# Self-contained: no nested SSH quoting. Sources pinned ports from netvm-names.sh.
#
# Output: "name:code" per relay on stdout.
@@ -12,8 +12,25 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "$SCRIPT_DIR/netvm-names.sh"
# Registry-driven node list (was hardcoded 4 nodes; def/dev had no
# relay-health coverage — 2026-10-06).
watched_nodes() {
"$SCRIPT_DIR/netvm-registry.py" 2>/dev/null | cut -d: -f1
}
# Allow sourcing for tests without running checks.
if [ "${RELAY_HEALTH_CHECK_LIB_ONLY:-}" = "1" ]; then
return 0 2>/dev/null || exit 0
fi
NODES="$(watched_nodes)"
if [ -z "$NODES" ]; then
echo "node registry empty/unreadable" >&2
exit 1
fi
FAILED=0
for node in muse pip 646 opm; do
# shellcheck disable=SC2086 (intended word splitting: one node per word)
for node in $NODES; do
netvm_names "$node"
url="http://${PEER_IP}:${CDP_PORT}/json/version"
code=$(curl -s -m 8 -o /dev/null -w "%{http_code}" "$url" 2>/dev/null || echo "000")
+10
View File
@@ -203,6 +203,16 @@ a 30-min half-open probe or any successful check. Manual reset:
Then fix the actual API-layer failure (account session/auth/chat-state),
not the browser.
### Setup-fed supervision (new nodes automatically watched)
**Wiring:** `netvm-node-up.sh` ends with `ensure-node-supervision.sh <node>`
(idempotent): appends the NODES.md registry row (port from netvm-names
pinning, honors CDP_PORT_OVERRIDE) and installs/enables
`chromebox-watchdog-<node>.timer`. Provision/onboarding reach it
transitively via node-up. Registry-driven supervisors (relay watchdog,
agent-health, relay-health/cdp-latency checks) pick up new rows on
their next run — no per-node code edits. Heal drift anytime:
`sudo bin/ensure-node-supervision.sh --all`.
### Relay on wrong IP
**Symptoms:** relay process exists but on the wrong veth IP (e.g., muse's relay
on pip's `10.201.87.2` instead of muse's `10.201.35.2`). Port responds on the
+101
View File
@@ -0,0 +1,101 @@
"""Tests for setup-fed watchdog supervision.
Covers bin/ensure-node-supervision.sh (registry row + timer unit,
idempotent), its hook in netvm-node-up.sh, and the registry-driven
node lists in relay-health-check.sh / cdp-latency-check.sh.
Shell is driven for real (LIB_ONLY sourcing where the script runs on
import); UNIT_DIR/NODES_MD overrides keep everything in scratch dirs.
"""
import subprocess
import tempfile
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
BIN = REPO_ROOT / "bin"
def _bash(prog, extra_env=None):
env = {"PATH": "/usr/bin:/bin"}
env.update(extra_env or {})
return subprocess.run(["bash", "-c", prog], capture_output=True,
text=True, env=env, timeout=30)
class EnsureSupervision(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.unit_dir = str(Path(self.tmp.name) / "units")
Path(self.unit_dir).mkdir()
self.nodes_md = str(Path(self.tmp.name) / "NODES.md")
Path(self.nodes_md).write_text(
"# NetVM Nodes (bl)\n\n"
"| node | netns | egress_ip | cdp_port | status | agent |\n"
"|------|-------|-----------|----------|--------|-------|\n"
"| muse | warp-muse | 1.2.3.4 | 9410 | active | muse |\n")
self.env = {"NODES_MD": self.nodes_md, "UNIT_DIR": self.unit_dir}
def tearDown(self):
self.tmp.cleanup()
def ensure(self, *args):
return _bash("'%s' %s" % (
BIN / "ensure-node-supervision.sh",
" ".join("'%s'" % a for a in args)), self.env)
def test_adds_registry_row_and_timer(self):
r = self.ensure("pip")
self.assertEqual(r.returncode, 0, r.stderr)
body = Path(self.nodes_md).read_text()
self.assertRegex(body, r"\| pip \| warp-pip \| \S+ \| 9420 \| active \|")
unit = Path(self.unit_dir) / "chromebox-watchdog-pip.timer"
self.assertTrue(unit.exists())
text = unit.read_text()
self.assertIn("Unit=chromebox-watchdog@pip.service", text)
self.assertIn("OnUnitActiveSec=2min", text)
def test_idempotent(self):
self.assertEqual(self.ensure("pip").returncode, 0)
before = Path(self.nodes_md).read_text()
r = self.ensure("pip")
self.assertEqual(r.returncode, 0, r.stderr)
self.assertEqual(Path(self.nodes_md).read_text(), before)
self.assertIn("already", r.stdout)
def test_honors_port_override(self):
env = dict(self.env, CDP_PORT_OVERRIDE="9470")
r = _bash("'%s' newnode" % (BIN / "ensure-node-supervision.sh"), env)
self.assertEqual(r.returncode, 0, r.stderr)
self.assertRegex(Path(self.nodes_md).read_text(),
r"\| newnode \| warp-newnode \| \S+ \| 9470 \| active \|")
def test_node_up_hooks_ensure(self):
text = (BIN / "netvm-node-up.sh").read_text()
self.assertIn("ensure-node-supervision.sh", text)
class CheckScriptCoverage(unittest.TestCase):
def watched(self, script, guard):
prog = "source '%s'\nwatched_nodes\n" % (BIN / script)
return _bash(prog, {guard: "1"})
def test_relay_health_covers_registry(self):
r = self.watched("relay-health-check.sh",
"RELAY_HEALTH_CHECK_LIB_ONLY")
self.assertEqual(r.returncode, 0, r.stderr)
nodes = set(r.stdout.split())
for n in ("muse", "pip", "646", "opm", "def", "dev"):
self.assertIn(n, nodes)
def test_cdp_latency_covers_registry(self):
r = self.watched("cdp-latency-check.sh",
"CDP_LATENCY_CHECK_LIB_ONLY")
self.assertEqual(r.returncode, 0, r.stderr)
nodes = set(r.stdout.split())
for n in ("muse", "pip", "646", "opm", "def", "dev"):
self.assertIn(n, nodes)
if __name__ == "__main__":
unittest.main()