chore(fleet): sync operator memory, hatch menu dialogs, and watchdog alerts

This commit is contained in:
operator
2026-10-07 00:25:51 +00:00
parent 0065d11e97
commit 34b0ef9fe2
38 changed files with 2205 additions and 444 deletions
+33 -12
View File
@@ -13,10 +13,14 @@
# Pattern mirrors chromebox-watchdog.sh (stage-specific logging, rotation).
set -euo pipefail
LOCK="/tmp/cdp-relay-watchdog.lock"
exec 9>"$LOCK"
if ! flock -n 9; then
echo "[$(date -u +%FT%TZ)] another relay watchdog run in progress, skipping" >&2
exit 0
# Tests source this file with CDP_RELAY_WATCHDOG_LIB_ONLY=1: they call
# helpers without running checks, so no lock is needed.
if [ "${CDP_RELAY_WATCHDOG_LIB_ONLY:-}" != "1" ]; then
exec 9>"$LOCK"
if ! flock -n 9; then
echo "[$(date -u +%FT%TZ)] another relay watchdog run in progress, skipping" >&2
exit 0
fi
fi
NETVM_BIN="/home/super/Projects/NetVM/bin"
@@ -37,16 +41,16 @@ log() { echo "[$(date -u +%FT%TZ)] $*" | tee -a "$LOG"; }
# node -> "veth_ip:port" via netvm-names.sh (hash-derived, don't hardcode)
relay_target() {
local node="$1"
local node="$1" reg_port=""
# shellcheck disable=SC1091
. "$NETVM_BIN/netvm-names.sh"
netvm_names "$node" || return 1
# CDP_PORT_OVERRIDE pins registry ports; fall back to hash-derived
local port="${CDP_PORT_OVERRIDE:-$CDP_PORT}"
case "$node" in
muse) port=9410 ;; pip) port=9420 ;; 646) port=9430 ;; opm) port=9440 ;;
esac
echo "$PEER_IP:$port"
# The registry is the source of truth for ports (new nodes propagate
# automatically); netvm-names pinning is the fallback.
if reg_port=$("$NETVM_BIN/netvm-registry.py" "$node" 2>/dev/null); then
[ -n "$reg_port" ] && CDP_PORT="$reg_port"
fi
echo "$PEER_IP:$CDP_PORT"
}
node_port() { echo "${1##*:}"; }
@@ -103,8 +107,25 @@ restart_relay() {
fi
}
# Registry-driven node list: every active node gets relay supervision
# (the old hardcoded 4-node list left def/dev unsupervised — 2026-10-06).
watched_nodes() {
"$NETVM_BIN/netvm-registry.py" 2>/dev/null | cut -d: -f1
}
# Allow sourcing for tests without running checks.
if [ "${CDP_RELAY_WATCHDOG_LIB_ONLY:-}" = "1" ]; then
return 0 2>/dev/null || exit 0
fi
FAILED=0
for node in muse pip 646 opm; do
NODES="$(watched_nodes)"
if [ -z "$NODES" ]; then
log "FAIL_LOUD: node registry empty/unreadable, skipping run"
exit 1
fi
# shellcheck disable=SC2086 (intended word splitting: one node per word)
for node in $NODES; do
# Stage 1: host veth IP. Fail loud, skip relay restart (pointless).
if ! veth_healthy "$node"; then
read -r veth gw <<< "$(node_veth "$node")"