chore(fleet): sync operator memory, hatch menu dialogs, and watchdog alerts
This commit is contained in:
+33
-12
@@ -13,10 +13,14 @@
|
||||
# Pattern mirrors chromebox-watchdog.sh (stage-specific logging, rotation).
|
||||
set -euo pipefail
|
||||
LOCK="/tmp/cdp-relay-watchdog.lock"
|
||||
exec 9>"$LOCK"
|
||||
if ! flock -n 9; then
|
||||
echo "[$(date -u +%FT%TZ)] another relay watchdog run in progress, skipping" >&2
|
||||
exit 0
|
||||
# Tests source this file with CDP_RELAY_WATCHDOG_LIB_ONLY=1: they call
|
||||
# helpers without running checks, so no lock is needed.
|
||||
if [ "${CDP_RELAY_WATCHDOG_LIB_ONLY:-}" != "1" ]; then
|
||||
exec 9>"$LOCK"
|
||||
if ! flock -n 9; then
|
||||
echo "[$(date -u +%FT%TZ)] another relay watchdog run in progress, skipping" >&2
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
NETVM_BIN="/home/super/Projects/NetVM/bin"
|
||||
@@ -37,16 +41,16 @@ log() { echo "[$(date -u +%FT%TZ)] $*" | tee -a "$LOG"; }
|
||||
|
||||
# node -> "veth_ip:port" via netvm-names.sh (hash-derived, don't hardcode)
|
||||
relay_target() {
|
||||
local node="$1"
|
||||
local node="$1" reg_port=""
|
||||
# shellcheck disable=SC1091
|
||||
. "$NETVM_BIN/netvm-names.sh"
|
||||
netvm_names "$node" || return 1
|
||||
# CDP_PORT_OVERRIDE pins registry ports; fall back to hash-derived
|
||||
local port="${CDP_PORT_OVERRIDE:-$CDP_PORT}"
|
||||
case "$node" in
|
||||
muse) port=9410 ;; pip) port=9420 ;; 646) port=9430 ;; opm) port=9440 ;;
|
||||
esac
|
||||
echo "$PEER_IP:$port"
|
||||
# The registry is the source of truth for ports (new nodes propagate
|
||||
# automatically); netvm-names pinning is the fallback.
|
||||
if reg_port=$("$NETVM_BIN/netvm-registry.py" "$node" 2>/dev/null); then
|
||||
[ -n "$reg_port" ] && CDP_PORT="$reg_port"
|
||||
fi
|
||||
echo "$PEER_IP:$CDP_PORT"
|
||||
}
|
||||
node_port() { echo "${1##*:}"; }
|
||||
|
||||
@@ -103,8 +107,25 @@ restart_relay() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Registry-driven node list: every active node gets relay supervision
|
||||
# (the old hardcoded 4-node list left def/dev unsupervised — 2026-10-06).
|
||||
watched_nodes() {
|
||||
"$NETVM_BIN/netvm-registry.py" 2>/dev/null | cut -d: -f1
|
||||
}
|
||||
|
||||
# Allow sourcing for tests without running checks.
|
||||
if [ "${CDP_RELAY_WATCHDOG_LIB_ONLY:-}" = "1" ]; then
|
||||
return 0 2>/dev/null || exit 0
|
||||
fi
|
||||
|
||||
FAILED=0
|
||||
for node in muse pip 646 opm; do
|
||||
NODES="$(watched_nodes)"
|
||||
if [ -z "$NODES" ]; then
|
||||
log "FAIL_LOUD: node registry empty/unreadable, skipping run"
|
||||
exit 1
|
||||
fi
|
||||
# shellcheck disable=SC2086 (intended word splitting: one node per word)
|
||||
for node in $NODES; do
|
||||
# Stage 1: host veth IP. Fail loud, skip relay restart (pointless).
|
||||
if ! veth_healthy "$node"; then
|
||||
read -r veth gw <<< "$(node_veth "$node")"
|
||||
|
||||
Reference in New Issue
Block a user