box/bin/ensure-node-supervision.sh

#!/usr/bin/env bash
# ensure-node-supervision.sh <node> | --all — feed a node to the watchdogs.
#
# Setup (netvm-node-up.sh, hence netvm-provision-node.sh and the onboarding
# pipeline) calls this so every node gets supervision without manual wiring:
#   1. NODES.md registry row (idempotent) — feeds the registry-driven
#      supervisors: cdp-relay-watchdog, agent-health.sh, relay-health-check,
#      cdp-latency-check. Port from netvm-names pinning (honors
#      CDP_PORT_OVERRIDE, so provision's picked port wins when present).
#      Example/verify/probe names retire on sight (never active, no timer).
#   2. chromebox-watchdog-<node>.timer unit + enable --now — the one
#      supervisor that needs a per-node systemd unit (the @.service
#      template already exists). Needs root for the real unit dir.
#
# Env overrides (tests): NODES_MD, UNIT_DIR. systemctl is skipped when
# UNIT_DIR is not the real system dir.
#
# Runs at the end of netvm-node-up.sh (as root); safe to re-run anytime:
#   sudo bin/ensure-node-supervision.sh --all
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
NODES_MD="${NODES_MD:-$SCRIPT_DIR/../NODES.md}"
UNIT_DIR="${UNIT_DIR:-/etc/systemd/system}"
# shellcheck disable=SC1091
. "$SCRIPT_DIR/netvm-names.sh"

usage() { echo "usage: ensure-node-supervision.sh <node> | --all" >&2; exit 1; }

# Example/verify/probe nodes (onboarding drills, id-verify examples) must
# never join active supervision: they carry no warp identity, wedge the
# pinned registry contract, and spin chrome restarts forever. Match is
# deliberately narrow (examp anywhere, test-/verify- prefixes) so real
# node names containing those substrings elsewhere stay active.
is_example_node() {
  case "$1" in
    *examp*|test*|verify-*|*-verify-*) return 0;;
    *) return 1;;
  esac
}

row_is_retired() {
  local node="$1"
  grep -qE "^\|[[:space:]]*$node[[:space:]]*\|[^|]*\|[^|]*\|[^|]*\|[[:space:]]*retired[[:space:]]*\|" \
    "$NODES_MD" 2>/dev/null
}

ensure_registry_row() {
  local node="$1" status="active" note="auto-registered"
  if grep -qE "^\|[[:space:]]*$node[[:space:]]*\|" "$NODES_MD" 2>/dev/null; then
    echo "registry: $node already in NODES.md"
    return 0
  fi
  netvm_names "$node" || { echo "registry: unknown node $node" >&2; return 1; }
  if is_example_node "$node"; then
    status="retired"
    note="auto-registered example — retired"
  fi
  printf '| %s | %s | unknown | %s | %s | %s (%s) |\n' \
    "$node" "$NETNS" "$CDP_PORT" "$status" "$node" "$note" >> "$NODES_MD"
  echo "registry: added $node (port $CDP_PORT, $status)"
}

ensure_timer() {
  local node="$1" unit
  unit="$UNIT_DIR/chromebox-watchdog-$node.timer"
  if [ -f "$unit" ]; then
    echo "timer: chromebox-watchdog-$node.timer already installed"
  else
    if [ "$UNIT_DIR" = "/etc/systemd/system" ] && [ "$(id -u)" -ne 0 ]; then
      echo "timer: need root to install chromebox-watchdog-$node.timer (run with sudo)" >&2
      return 1
    fi
    cat > "$unit" <<EOF
[Unit]
Description=Run chromebox watchdog for $node every 2 minutes

[Timer]
RandomizedDelaySec=30s
OnBootSec=2min
OnUnitActiveSec=2min
Unit=chromebox-watchdog@$node.service

[Install]
WantedBy=timers.target
EOF
    echo "timer: installed chromebox-watchdog-$node.timer"
  fi
  if [ "$UNIT_DIR" = "/etc/systemd/system" ]; then
    systemctl daemon-reload
    systemctl enable --now "chromebox-watchdog-$node.timer" >/dev/null 2>&1
    echo "timer: enabled chromebox-watchdog-$node.timer"
  fi
}

ensure_node() {
  local node="$1"
  ensure_registry_row "$node"
  if row_is_retired "$node"; then
    echo "timer: $node retired, skipping supervision"
    return 0
  fi
  ensure_timer "$node"
}

case "${1:-}" in
  --all)
    nodes="$(python3 "$SCRIPT_DIR/netvm-registry.py" 2>/dev/null | cut -d: -f1)"
    for conf in /etc/netvm/*.conf; do
      [ -f "$conf" ] || continue
      nodes="$nodes $(basename "$conf" .conf)"
    done
    seen=""
    # shellcheck disable=SC2086 (intended word splitting)
    for node in $nodes; do
      case " $seen " in *" $node "*) continue;; esac
      seen="$seen $node"
      ensure_node "$node" || echo "supervision: $node failed (continuing)" >&2
    done
    ;;
  ""|-h|--help) usage;;
  *) ensure_node "$1";;
esac