operator
98 lines
3.6 KiB
Bash
Executable File
98 lines
3.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# accounts-health.sh — account session vitality reporter for the front-door network.
|
|
#
|
|
# Reads ACCOUNTS.md, probes each account's browser via CDP (inside its NetVM
|
|
# netns) for session liveness, and emits a JSON report. Optionally signs and
|
|
# POSTs it to the board health ingest, following the health-report.sh
|
|
# convention: payload is <machine>\n<ts>\n<facts-json>, namespace "health".
|
|
#
|
|
# Usage: accounts-health.sh [--no-post]
|
|
#
|
|
# Cron (on bl, every 15 min):
|
|
# */15 * * * * ~/Projects/NetVM/bin/accounts-health.sh >/dev/null 2>&1
|
|
#
|
|
# Health key setup (once, on bl):
|
|
# ssh-keygen -t ed25519 -N "" -f ~/.ssh/muse-health
|
|
# # operator registers the pubkey on the VM:
|
|
# echo "bl $(cat ~/.ssh/muse-health.pub)" \
|
|
# | ssh super@34.139.37.135 "sudo tee -a /srv/board/health_signers"
|
|
set -u
|
|
|
|
NETVM_DIR="${NETVM_DIR:-$HOME/Projects/NetVM}"
|
|
ACCOUNTS="$NETVM_DIR/ACCOUNTS.md"
|
|
CHECKER="$NETVM_DIR/bin/accounts-health.py"
|
|
MACHINE="${MUSE_MACHINE:-bl}"
|
|
KEY="${HEALTH_KEY:-$HOME/.ssh/muse-health}"
|
|
ENDPOINT="${HEALTH_ENDPOINT:-https://board.muse-dev.online/api/health/report}"
|
|
POST=1
|
|
[ "${1:-}" = "--no-post" ] && POST=0
|
|
|
|
[ -f "$ACCOUNTS" ] || { echo "accounts-health: $ACCOUNTS missing" >&2; exit 1; }
|
|
[ -f "$CHECKER" ] || { echo "accounts-health: $CHECKER missing" >&2; exit 1; }
|
|
|
|
TS="$(date +%s)"
|
|
TMP="$(mktemp -d)"
|
|
trap 'rm -rf "$TMP"' EXIT
|
|
|
|
# Parse ACCOUNTS.md pipe table, probe each account inside its netns
|
|
NETVM_DIR="$NETVM_DIR" python3 - > "$TMP/facts.json" <<'PYEOF'
|
|
import json, os, subprocess, time
|
|
netvm = os.environ["NETVM_DIR"]
|
|
rows = []
|
|
for line in open(os.path.join(netvm, "ACCOUNTS.md")):
|
|
line = line.strip()
|
|
if not line.startswith("|"):
|
|
continue
|
|
cells = [c.strip() for c in line.strip("|").split("|")]
|
|
if len(cells) < 13 or cells[0] in ("agent", "-------", ""):
|
|
continue
|
|
if cells[10] not in ("", "-"):
|
|
rows.append({"agent": cells[0], "node": cells[1],
|
|
"status": cells[8], "cdp_port": cells[10]})
|
|
checker = os.path.join(netvm, "bin", "accounts-health.py")
|
|
out = {}
|
|
for a in rows:
|
|
try:
|
|
r = subprocess.run(
|
|
["sudo", "-n", "ip", "netns", "exec", f"warp-{a['node']}",
|
|
"python3", checker, a["cdp_port"]],
|
|
capture_output=True, text=True, timeout=60)
|
|
res = json.loads(r.stdout.strip().splitlines()[-1])
|
|
res["registry_status"] = a["status"]
|
|
out[a["agent"]] = res
|
|
except Exception as e:
|
|
out[a["agent"]] = {"browser_up": False, "session_alive": None,
|
|
"detail": f"probe failed: {e}",
|
|
"registry_status": a["status"]}
|
|
print(json.dumps({"accounts": out, "checked_at": int(time.time())}, indent=2))
|
|
PYEOF
|
|
|
|
if [ "$POST" -eq 0 ]; then
|
|
cat "$TMP/facts.json"
|
|
exit 0
|
|
fi
|
|
|
|
if [ ! -f "$KEY" ]; then
|
|
echo "accounts-health: $KEY missing — printing JSON, not posting (see header for key setup)" >&2
|
|
cat "$TMP/facts.json"
|
|
exit 0
|
|
fi
|
|
|
|
printf '%s\n%s\n' "$MACHINE" "$TS" > "$TMP/payload"
|
|
FACTS_JSON="$(cat "$TMP/facts.json")"
|
|
printf '%s' "$FACTS_JSON" >> "$TMP/payload"
|
|
ssh-keygen -Y sign -f "$KEY" -n health "$TMP/payload" >/dev/null 2>&1
|
|
SIG="$(cat "$TMP/payload.sig")"
|
|
python3 - "$MACHINE" "$TS" "$FACTS_JSON" "$SIG" <<'PYEOF' > "$TMP/body.json"
|
|
import json, sys
|
|
machine, ts, facts_json, sig = sys.argv[1], int(sys.argv[2]), sys.argv[3], sys.argv[4]
|
|
body = {"machine": machine, "ts": ts,
|
|
"facts": json.loads(facts_json), "facts_json": facts_json,
|
|
"signature": sig}
|
|
print(json.dumps(body))
|
|
PYEOF
|
|
|
|
curl -s -X POST "$ENDPOINT" -H 'Content-Type: application/json' \
|
|
--data @"$TMP/body.json" | head -c 300
|
|
echo
|