From 66c8900a58ba60c2cfd7d79bd8f586f376d5901c Mon Sep 17 00:00:00 2001 From: Muse Sidechat Date: Tue, 6 Oct 2026 19:28:29 +0000 Subject: [PATCH] feat: setup-fed watchdog supervision for all registry nodes Close the def/dev supervision gap at the source: every node brought up gets watched, and every supervisor enumerates the registry. - bin/ensure-node-supervision.sh (new, idempotent): appends the NODES.md row (netvm-names port, honors CDP_PORT_OVERRIDE so it never fights provision's picker) and installs/enables chromebox-watchdog-.timer. --all heals drift (registry + /etc/netvm identities). Template verified byte-identical to the installed def unit. - netvm-node-up.sh: calls ensure (non-fatal) at the end. Provision and the onboarding pipeline reach it transitively. - relay-health-check.sh, cdp-latency-check.sh: registry-driven watched_nodes() + LIB_ONLY guards (were hardcoded 4 nodes). - tests/test_node_supervision.py (6): row add/idempotent/override, timer render, node-up wiring, both watched_nodes(). - CHROMEBOX-RUNBOOK.md: setup-fed supervision section. Pairs with the registry-driven relay/chromebox watchdogs: new rows are picked up on the next run with no per-node code edits. --- bin/cdp-latency-check.sh | 20 ++++++- bin/ensure-node-supervision.sh | 95 +++++++++++++++++++++++++++++++ bin/netvm-node-up.sh | 5 ++ bin/relay-health-check.sh | 21 ++++++- docs/CHROMEBOX-RUNBOOK.md | 10 ++++ tests/test_node_supervision.py | 101 +++++++++++++++++++++++++++++++++ 6 files changed, 249 insertions(+), 3 deletions(-) create mode 100755 bin/ensure-node-supervision.sh create mode 100644 tests/test_node_supervision.py diff --git a/bin/cdp-latency-check.sh b/bin/cdp-latency-check.sh index e528c7a..14c97f8 100755 --- a/bin/cdp-latency-check.sh +++ b/bin/cdp-latency-check.sh @@ -17,7 +17,25 @@ source "$BIN_DIR/netvm-names.sh" TIMEOUT_S=10 -for node in muse pip 646 opm; do +# Registry-driven node list (was hardcoded 4 nodes; def/dev had no +# cdp-latency coverage — 2026-10-06). +watched_nodes() { + "$BIN_DIR/netvm-registry.py" 2>/dev/null | cut -d: -f1 +} + +# Allow sourcing for tests without running checks. +if [ "${CDP_LATENCY_CHECK_LIB_ONLY:-}" = "1" ]; then + return 0 2>/dev/null || exit 0 +fi + +NODES="$(watched_nodes)" +if [ -z "$NODES" ]; then + echo "node registry empty/unreadable" >&2 + exit 1 +fi + +# shellcheck disable=SC2086 (intended word splitting: one node per word) +for node in $NODES; do netvm_names "$node" url="http://${PEER_IP}:${CDP_PORT}/json/version" probe=$(curl -s -m "$TIMEOUT_S" -o /dev/null -w "%{time_total} %{http_code}" "$url" 2>/dev/null) diff --git a/bin/ensure-node-supervision.sh b/bin/ensure-node-supervision.sh new file mode 100755 index 0000000..74cd31b --- /dev/null +++ b/bin/ensure-node-supervision.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# ensure-node-supervision.sh | --all — feed a node to the watchdogs. +# +# Setup (netvm-node-up.sh, hence netvm-provision-node.sh and the onboarding +# pipeline) calls this so every node gets supervision without manual wiring: +# 1. NODES.md registry row (idempotent) — feeds the registry-driven +# supervisors: cdp-relay-watchdog, agent-health.sh, relay-health-check, +# cdp-latency-check. Port from netvm-names pinning (honors +# CDP_PORT_OVERRIDE, so provision's picked port wins when present). +# 2. chromebox-watchdog-.timer unit + enable --now — the one +# supervisor that needs a per-node systemd unit (the @.service +# template already exists). Needs root for the real unit dir. +# +# Env overrides (tests): NODES_MD, UNIT_DIR. systemctl is skipped when +# UNIT_DIR is not the real system dir. +# +# Runs at the end of netvm-node-up.sh (as root); safe to re-run anytime: +# sudo bin/ensure-node-supervision.sh --all +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +NODES_MD="${NODES_MD:-$SCRIPT_DIR/../NODES.md}" +UNIT_DIR="${UNIT_DIR:-/etc/systemd/system}" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/netvm-names.sh" + +usage() { echo "usage: ensure-node-supervision.sh | --all" >&2; exit 1; } + +ensure_registry_row() { + local node="$1" + if grep -qE "^\|[[:space:]]*$node[[:space:]]*\|" "$NODES_MD" 2>/dev/null; then + echo "registry: $node already in NODES.md" + return 0 + fi + netvm_names "$node" || { echo "registry: unknown node $node" >&2; return 1; } + printf '| %s | %s | unknown | %s | active | %s (auto-registered) |\n' \ + "$node" "$NETNS" "$CDP_PORT" "$node" >> "$NODES_MD" + echo "registry: added $node (port $CDP_PORT)" +} + +ensure_timer() { + local node="$1" unit + unit="$UNIT_DIR/chromebox-watchdog-$node.timer" + if [ -f "$unit" ]; then + echo "timer: chromebox-watchdog-$node.timer already installed" + else + if [ "$UNIT_DIR" = "/etc/systemd/system" ] && [ "$(id -u)" -ne 0 ]; then + echo "timer: need root to install chromebox-watchdog-$node.timer (run with sudo)" >&2 + return 1 + fi + cat > "$unit" </dev/null 2>&1 + echo "timer: enabled chromebox-watchdog-$node.timer" + fi +} + +ensure_node() { + local node="$1" + ensure_registry_row "$node" + ensure_timer "$node" +} + +case "${1:-}" in + --all) + nodes="$(python3 "$SCRIPT_DIR/netvm-registry.py" 2>/dev/null | cut -d: -f1)" + for conf in /etc/netvm/*.conf; do + [ -f "$conf" ] || continue + nodes="$nodes $(basename "$conf" .conf)" + done + seen="" + # shellcheck disable=SC2086 (intended word splitting) + for node in $nodes; do + case " $seen " in *" $node "*) continue;; esac + seen="$seen $node" + ensure_node "$node" || echo "supervision: $node failed (continuing)" >&2 + done + ;; + ""|-h|--help) usage;; + *) ensure_node "$1";; +esac diff --git a/bin/netvm-node-up.sh b/bin/netvm-node-up.sh index 4cb666b..ecf3f8b 100755 --- a/bin/netvm-node-up.sh +++ b/bin/netvm-node-up.sh @@ -104,3 +104,8 @@ else EGRESS=$(nsexec curl -sk --max-time 15 'https://1.1.1.1/cdn-cgi/trace' 2>/dev/null | grep -oP '^ip=\K.*' || true) fi echo "node=$NODE netns=$NETNS ifaces=$WG/$VETH egress=${EGRESS:-unknown}" + +# Feed the watchdogs: registry row + chromebox timer (non-fatal — the +# node is up regardless, and supervision heals on the next run). +"$SCRIPT_DIR/ensure-node-supervision.sh" "$NODE" \ + || echo "supervision ensure failed for $NODE (non-fatal)" >&2 diff --git a/bin/relay-health-check.sh b/bin/relay-health-check.sh index a9d6414..6e9f9b9 100755 --- a/bin/relay-health-check.sh +++ b/bin/relay-health-check.sh @@ -1,5 +1,5 @@ #!/bin/bash -# relay-health-check.sh — check all four CDP relay endpoints on bl. +# relay-health-check.sh — check all registry CDP relay endpoints on bl. # Self-contained: no nested SSH quoting. Sources pinned ports from netvm-names.sh. # # Output: "name:code" per relay on stdout. @@ -12,8 +12,25 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=/dev/null source "$SCRIPT_DIR/netvm-names.sh" +# Registry-driven node list (was hardcoded 4 nodes; def/dev had no +# relay-health coverage — 2026-10-06). +watched_nodes() { + "$SCRIPT_DIR/netvm-registry.py" 2>/dev/null | cut -d: -f1 +} + +# Allow sourcing for tests without running checks. +if [ "${RELAY_HEALTH_CHECK_LIB_ONLY:-}" = "1" ]; then + return 0 2>/dev/null || exit 0 +fi + +NODES="$(watched_nodes)" +if [ -z "$NODES" ]; then + echo "node registry empty/unreadable" >&2 + exit 1 +fi FAILED=0 -for node in muse pip 646 opm; do +# shellcheck disable=SC2086 (intended word splitting: one node per word) +for node in $NODES; do netvm_names "$node" url="http://${PEER_IP}:${CDP_PORT}/json/version" code=$(curl -s -m 8 -o /dev/null -w "%{http_code}" "$url" 2>/dev/null || echo "000") diff --git a/docs/CHROMEBOX-RUNBOOK.md b/docs/CHROMEBOX-RUNBOOK.md index aea3650..2cd1bcc 100644 --- a/docs/CHROMEBOX-RUNBOOK.md +++ b/docs/CHROMEBOX-RUNBOOK.md @@ -203,6 +203,16 @@ a 30-min half-open probe or any successful check. Manual reset: Then fix the actual API-layer failure (account session/auth/chat-state), not the browser. +### Setup-fed supervision (new nodes automatically watched) +**Wiring:** `netvm-node-up.sh` ends with `ensure-node-supervision.sh ` +(idempotent): appends the NODES.md registry row (port from netvm-names +pinning, honors CDP_PORT_OVERRIDE) and installs/enables +`chromebox-watchdog-.timer`. Provision/onboarding reach it +transitively via node-up. Registry-driven supervisors (relay watchdog, +agent-health, relay-health/cdp-latency checks) pick up new rows on +their next run — no per-node code edits. Heal drift anytime: +`sudo bin/ensure-node-supervision.sh --all`. + ### Relay on wrong IP **Symptoms:** relay process exists but on the wrong veth IP (e.g., muse's relay on pip's `10.201.87.2` instead of muse's `10.201.35.2`). Port responds on the diff --git a/tests/test_node_supervision.py b/tests/test_node_supervision.py new file mode 100644 index 0000000..8f50600 --- /dev/null +++ b/tests/test_node_supervision.py @@ -0,0 +1,101 @@ +"""Tests for setup-fed watchdog supervision. + +Covers bin/ensure-node-supervision.sh (registry row + timer unit, +idempotent), its hook in netvm-node-up.sh, and the registry-driven +node lists in relay-health-check.sh / cdp-latency-check.sh. + +Shell is driven for real (LIB_ONLY sourcing where the script runs on +import); UNIT_DIR/NODES_MD overrides keep everything in scratch dirs. +""" +import subprocess +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +BIN = REPO_ROOT / "bin" + + +def _bash(prog, extra_env=None): + env = {"PATH": "/usr/bin:/bin"} + env.update(extra_env or {}) + return subprocess.run(["bash", "-c", prog], capture_output=True, + text=True, env=env, timeout=30) + + +class EnsureSupervision(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.unit_dir = str(Path(self.tmp.name) / "units") + Path(self.unit_dir).mkdir() + self.nodes_md = str(Path(self.tmp.name) / "NODES.md") + Path(self.nodes_md).write_text( + "# NetVM Nodes (bl)\n\n" + "| node | netns | egress_ip | cdp_port | status | agent |\n" + "|------|-------|-----------|----------|--------|-------|\n" + "| muse | warp-muse | 1.2.3.4 | 9410 | active | muse |\n") + self.env = {"NODES_MD": self.nodes_md, "UNIT_DIR": self.unit_dir} + + def tearDown(self): + self.tmp.cleanup() + + def ensure(self, *args): + return _bash("'%s' %s" % ( + BIN / "ensure-node-supervision.sh", + " ".join("'%s'" % a for a in args)), self.env) + + def test_adds_registry_row_and_timer(self): + r = self.ensure("pip") + self.assertEqual(r.returncode, 0, r.stderr) + body = Path(self.nodes_md).read_text() + self.assertRegex(body, r"\| pip \| warp-pip \| \S+ \| 9420 \| active \|") + unit = Path(self.unit_dir) / "chromebox-watchdog-pip.timer" + self.assertTrue(unit.exists()) + text = unit.read_text() + self.assertIn("Unit=chromebox-watchdog@pip.service", text) + self.assertIn("OnUnitActiveSec=2min", text) + + def test_idempotent(self): + self.assertEqual(self.ensure("pip").returncode, 0) + before = Path(self.nodes_md).read_text() + r = self.ensure("pip") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(Path(self.nodes_md).read_text(), before) + self.assertIn("already", r.stdout) + + def test_honors_port_override(self): + env = dict(self.env, CDP_PORT_OVERRIDE="9470") + r = _bash("'%s' newnode" % (BIN / "ensure-node-supervision.sh"), env) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertRegex(Path(self.nodes_md).read_text(), + r"\| newnode \| warp-newnode \| \S+ \| 9470 \| active \|") + + def test_node_up_hooks_ensure(self): + text = (BIN / "netvm-node-up.sh").read_text() + self.assertIn("ensure-node-supervision.sh", text) + + +class CheckScriptCoverage(unittest.TestCase): + def watched(self, script, guard): + prog = "source '%s'\nwatched_nodes\n" % (BIN / script) + return _bash(prog, {guard: "1"}) + + def test_relay_health_covers_registry(self): + r = self.watched("relay-health-check.sh", + "RELAY_HEALTH_CHECK_LIB_ONLY") + self.assertEqual(r.returncode, 0, r.stderr) + nodes = set(r.stdout.split()) + for n in ("muse", "pip", "646", "opm", "def", "dev"): + self.assertIn(n, nodes) + + def test_cdp_latency_covers_registry(self): + r = self.watched("cdp-latency-check.sh", + "CDP_LATENCY_CHECK_LIB_ONLY") + self.assertEqual(r.returncode, 0, r.stderr) + nodes = set(r.stdout.split()) + for n in ("muse", "pip", "646", "opm", "def", "dev"): + self.assertIn(n, nodes) + + +if __name__ == "__main__": + unittest.main()