feat: setup-fed watchdog supervision for all registry nodes
Close the def/dev supervision gap at the source: every node brought up gets watched, and every supervisor enumerates the registry. - bin/ensure-node-supervision.sh (new, idempotent): appends the NODES.md row (netvm-names port, honors CDP_PORT_OVERRIDE so it never fights provision's picker) and installs/enables chromebox-watchdog-<node>.timer. --all heals drift (registry + /etc/netvm identities). Template verified byte-identical to the installed def unit. - netvm-node-up.sh: calls ensure (non-fatal) at the end. Provision and the onboarding pipeline reach it transitively. - relay-health-check.sh, cdp-latency-check.sh: registry-driven watched_nodes() + LIB_ONLY guards (were hardcoded 4 nodes). - tests/test_node_supervision.py (6): row add/idempotent/override, timer render, node-up wiring, both watched_nodes(). - CHROMEBOX-RUNBOOK.md: setup-fed supervision section. Pairs with the registry-driven relay/chromebox watchdogs: new rows are picked up on the next run with no per-node code edits.
This commit is contained in:
Muse Sidechat
@@ -17,7 +17,25 @@ source "$BIN_DIR/netvm-names.sh"
|
||||
|
||||
TIMEOUT_S=10
|
||||
|
||||
for node in muse pip 646 opm; do
|
||||
# Registry-driven node list (was hardcoded 4 nodes; def/dev had no
|
||||
# cdp-latency coverage — 2026-10-06).
|
||||
watched_nodes() {
|
||||
"$BIN_DIR/netvm-registry.py" 2>/dev/null | cut -d: -f1
|
||||
}
|
||||
|
||||
# Allow sourcing for tests without running checks.
|
||||
if [ "${CDP_LATENCY_CHECK_LIB_ONLY:-}" = "1" ]; then
|
||||
return 0 2>/dev/null || exit 0
|
||||
fi
|
||||
|
||||
NODES="$(watched_nodes)"
|
||||
if [ -z "$NODES" ]; then
|
||||
echo "node registry empty/unreadable" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2086 (intended word splitting: one node per word)
|
||||
for node in $NODES; do
|
||||
netvm_names "$node"
|
||||
url="http://${PEER_IP}:${CDP_PORT}/json/version"
|
||||
probe=$(curl -s -m "$TIMEOUT_S" -o /dev/null -w "%{time_total} %{http_code}" "$url" 2>/dev/null)
|
||||
|
||||
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
# ensure-node-supervision.sh <node> | --all — feed a node to the watchdogs.
|
||||
#
|
||||
# Setup (netvm-node-up.sh, hence netvm-provision-node.sh and the onboarding
|
||||
# pipeline) calls this so every node gets supervision without manual wiring:
|
||||
# 1. NODES.md registry row (idempotent) — feeds the registry-driven
|
||||
# supervisors: cdp-relay-watchdog, agent-health.sh, relay-health-check,
|
||||
# cdp-latency-check. Port from netvm-names pinning (honors
|
||||
# CDP_PORT_OVERRIDE, so provision's picked port wins when present).
|
||||
# 2. chromebox-watchdog-<node>.timer unit + enable --now — the one
|
||||
# supervisor that needs a per-node systemd unit (the @.service
|
||||
# template already exists). Needs root for the real unit dir.
|
||||
#
|
||||
# Env overrides (tests): NODES_MD, UNIT_DIR. systemctl is skipped when
|
||||
# UNIT_DIR is not the real system dir.
|
||||
#
|
||||
# Runs at the end of netvm-node-up.sh (as root); safe to re-run anytime:
|
||||
# sudo bin/ensure-node-supervision.sh --all
|
||||
set -euo pipefail
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
NODES_MD="${NODES_MD:-$SCRIPT_DIR/../NODES.md}"
|
||||
UNIT_DIR="${UNIT_DIR:-/etc/systemd/system}"
|
||||
# shellcheck disable=SC1091
|
||||
. "$SCRIPT_DIR/netvm-names.sh"
|
||||
|
||||
usage() { echo "usage: ensure-node-supervision.sh <node> | --all" >&2; exit 1; }
|
||||
|
||||
ensure_registry_row() {
|
||||
local node="$1"
|
||||
if grep -qE "^\|[[:space:]]*$node[[:space:]]*\|" "$NODES_MD" 2>/dev/null; then
|
||||
echo "registry: $node already in NODES.md"
|
||||
return 0
|
||||
fi
|
||||
netvm_names "$node" || { echo "registry: unknown node $node" >&2; return 1; }
|
||||
printf '| %s | %s | unknown | %s | active | %s (auto-registered) |\n' \
|
||||
"$node" "$NETNS" "$CDP_PORT" "$node" >> "$NODES_MD"
|
||||
echo "registry: added $node (port $CDP_PORT)"
|
||||
}
|
||||
|
||||
ensure_timer() {
|
||||
local node="$1" unit
|
||||
unit="$UNIT_DIR/chromebox-watchdog-$node.timer"
|
||||
if [ -f "$unit" ]; then
|
||||
echo "timer: chromebox-watchdog-$node.timer already installed"
|
||||
else
|
||||
if [ "$UNIT_DIR" = "/etc/systemd/system" ] && [ "$(id -u)" -ne 0 ]; then
|
||||
echo "timer: need root to install chromebox-watchdog-$node.timer (run with sudo)" >&2
|
||||
return 1
|
||||
fi
|
||||
cat > "$unit" <<EOF
|
||||
[Unit]
|
||||
Description=Run chromebox watchdog for $node every 2 minutes
|
||||
|
||||
[Timer]
|
||||
RandomizedDelaySec=30s
|
||||
OnBootSec=2min
|
||||
OnUnitActiveSec=2min
|
||||
Unit=chromebox-watchdog@$node.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
EOF
|
||||
echo "timer: installed chromebox-watchdog-$node.timer"
|
||||
fi
|
||||
if [ "$UNIT_DIR" = "/etc/systemd/system" ]; then
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now "chromebox-watchdog-$node.timer" >/dev/null 2>&1
|
||||
echo "timer: enabled chromebox-watchdog-$node.timer"
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_node() {
|
||||
local node="$1"
|
||||
ensure_registry_row "$node"
|
||||
ensure_timer "$node"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
--all)
|
||||
nodes="$(python3 "$SCRIPT_DIR/netvm-registry.py" 2>/dev/null | cut -d: -f1)"
|
||||
for conf in /etc/netvm/*.conf; do
|
||||
[ -f "$conf" ] || continue
|
||||
nodes="$nodes $(basename "$conf" .conf)"
|
||||
done
|
||||
seen=""
|
||||
# shellcheck disable=SC2086 (intended word splitting)
|
||||
for node in $nodes; do
|
||||
case " $seen " in *" $node "*) continue;; esac
|
||||
seen="$seen $node"
|
||||
ensure_node "$node" || echo "supervision: $node failed (continuing)" >&2
|
||||
done
|
||||
;;
|
||||
""|-h|--help) usage;;
|
||||
*) ensure_node "$1";;
|
||||
esac
|
||||
@@ -104,3 +104,8 @@ else
|
||||
EGRESS=$(nsexec curl -sk --max-time 15 'https://1.1.1.1/cdn-cgi/trace' 2>/dev/null | grep -oP '^ip=\K.*' || true)
|
||||
fi
|
||||
echo "node=$NODE netns=$NETNS ifaces=$WG/$VETH egress=${EGRESS:-unknown}"
|
||||
|
||||
# Feed the watchdogs: registry row + chromebox timer (non-fatal — the
|
||||
# node is up regardless, and supervision heals on the next run).
|
||||
"$SCRIPT_DIR/ensure-node-supervision.sh" "$NODE" \
|
||||
|| echo "supervision ensure failed for $NODE (non-fatal)" >&2
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/bin/bash
|
||||
# relay-health-check.sh — check all four CDP relay endpoints on bl.
|
||||
# relay-health-check.sh — check all registry CDP relay endpoints on bl.
|
||||
# Self-contained: no nested SSH quoting. Sources pinned ports from netvm-names.sh.
|
||||
#
|
||||
# Output: "name:code" per relay on stdout.
|
||||
@@ -12,8 +12,25 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=/dev/null
|
||||
source "$SCRIPT_DIR/netvm-names.sh"
|
||||
|
||||
# Registry-driven node list (was hardcoded 4 nodes; def/dev had no
|
||||
# relay-health coverage — 2026-10-06).
|
||||
watched_nodes() {
|
||||
"$SCRIPT_DIR/netvm-registry.py" 2>/dev/null | cut -d: -f1
|
||||
}
|
||||
|
||||
# Allow sourcing for tests without running checks.
|
||||
if [ "${RELAY_HEALTH_CHECK_LIB_ONLY:-}" = "1" ]; then
|
||||
return 0 2>/dev/null || exit 0
|
||||
fi
|
||||
|
||||
NODES="$(watched_nodes)"
|
||||
if [ -z "$NODES" ]; then
|
||||
echo "node registry empty/unreadable" >&2
|
||||
exit 1
|
||||
fi
|
||||
FAILED=0
|
||||
for node in muse pip 646 opm; do
|
||||
# shellcheck disable=SC2086 (intended word splitting: one node per word)
|
||||
for node in $NODES; do
|
||||
netvm_names "$node"
|
||||
url="http://${PEER_IP}:${CDP_PORT}/json/version"
|
||||
code=$(curl -s -m 8 -o /dev/null -w "%{http_code}" "$url" 2>/dev/null || echo "000")
|
||||
|
||||
Reference in New Issue
Block a user