101 lines
3.8 KiB
Python
101 lines
3.8 KiB
Python
|
"""Watchdog coverage: every registry node is supervised.
|
||
|
|
|
||
|
|
Regression test for the dev/def outage (2026-10-06): both watchdogs
|
||
|
|
hardcoded the original four nodes (muse/pip/646/opm), so dev/def had no
|
||
|
|
browser supervision, no Warp-tunnel auto-recovery, and no relay
|
||
|
|
supervision. A dead tunnel paged CRITICAL partition alerts forever with
|
||
|
|
nothing acting on it.
|
||
|
|
|
||
|
|
Both scripts now resolve nodes/ports from the fleet registry
|
||
|
|
(bin/netvm-registry.py). These tests drive the real shell code sourced
|
||
|
|
with a LIB_ONLY guard (same pattern as test_agent_health.py).
|
||
|
|
"""
|
||
|
|
import importlib.util
|
||
|
|
import subprocess
|
||
|
|
import unittest
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||
|
|
CHROMEBOX_WD = REPO_ROOT / "bin" / "chromebox-watchdog.sh"
|
||
|
|
RELAY_WD = REPO_ROOT / "bin" / "cdp-relay-watchdog.sh"
|
||
|
|
|
||
|
|
# The fleet's pinned contract (also pinned in bin/netvm-names.sh).
|
||
|
|
EXPECTED_PORTS = {
|
||
|
|
"muse": 9410, "pip": 9420, "646": 9430,
|
||
|
|
"opm": 9440, "def": 9450, "dev": 9455,
|
||
|
|
}
|
||
|
|
EXPECTED_PEERS = {
|
||
|
|
"muse": "10.201.35.2", "pip": "10.201.87.2", "646": "10.201.202.2",
|
||
|
|
"opm": "10.201.157.2", "def": "10.201.66.2", "dev": "10.201.36.2",
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def _load(mod_name, rel_path):
|
||
|
|
spec = importlib.util.spec_from_file_location(mod_name, REPO_ROOT / rel_path)
|
||
|
|
mod = importlib.util.module_from_spec(spec)
|
||
|
|
spec.loader.exec_module(mod)
|
||
|
|
return mod
|
||
|
|
|
||
|
|
|
||
|
|
def _chromebox(profile, snippet):
|
||
|
|
prog = "set -- '%s'\nsource '%s'\n%s\n" % (profile, CHROMEBOX_WD, snippet)
|
||
|
|
env = {"PATH": "/usr/bin:/bin", "CHROMEBOX_WATCHDOG_LIB_ONLY": "1"}
|
||
|
|
return subprocess.run(["bash", "-c", prog], capture_output=True,
|
||
|
|
text=True, env=env, timeout=30)
|
||
|
|
|
||
|
|
|
||
|
|
def _relay(snippet):
|
||
|
|
prog = "source '%s'\n%s\n" % (RELAY_WD, snippet)
|
||
|
|
env = {"PATH": "/usr/bin:/bin", "CDP_RELAY_WATCHDOG_LIB_ONLY": "1"}
|
||
|
|
return subprocess.run(["bash", "-c", prog], capture_output=True,
|
||
|
|
text=True, env=env, timeout=30)
|
||
|
|
|
||
|
|
|
||
|
|
class RegistryContract(unittest.TestCase):
|
||
|
|
def test_registry_matches_pinned_ports(self):
|
||
|
|
reg = _load("netvm_registry_cov", "bin/netvm-registry.py")
|
||
|
|
self.assertEqual(reg.active_nodes(), EXPECTED_PORTS)
|
||
|
|
for node, peer in EXPECTED_PEERS.items():
|
||
|
|
self.assertEqual(reg.peer_ip_for(node), peer)
|
||
|
|
|
||
|
|
|
||
|
|
class ChromeboxWatchdogPorts(unittest.TestCase):
|
||
|
|
def test_all_registry_nodes_resolve_to_pinned_ports(self):
|
||
|
|
for node, port in sorted(EXPECTED_PORTS.items()):
|
||
|
|
with self.subTest(node=node):
|
||
|
|
r = _chromebox(node, "echo \"PORT=$CDP_PORT\"")
|
||
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
||
|
|
line = next((ln for ln in r.stdout.splitlines()
|
||
|
|
if ln.startswith("PORT=")), None)
|
||
|
|
self.assertIsNotNone(
|
||
|
|
line, "no PORT line: stdout=%r stderr=%r"
|
||
|
|
% (r.stdout, r.stderr))
|
||
|
|
self.assertEqual(int(line.split("=", 1)[1]), port)
|
||
|
|
|
||
|
|
def test_unknown_profile_rejected(self):
|
||
|
|
r = _chromebox("ghost", "echo UNREACHABLE")
|
||
|
|
self.assertNotEqual(r.returncode, 0)
|
||
|
|
self.assertIn("unknown profile", r.stderr)
|
||
|
|
self.assertNotIn("UNREACHABLE", r.stdout)
|
||
|
|
|
||
|
|
|
||
|
|
class RelayWatchdogCoverage(unittest.TestCase):
|
||
|
|
def test_watched_nodes_covers_registry(self):
|
||
|
|
r = _relay("watched_nodes")
|
||
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
||
|
|
nodes = set(r.stdout.split())
|
||
|
|
for node in EXPECTED_PORTS:
|
||
|
|
self.assertIn(node, nodes)
|
||
|
|
|
||
|
|
def test_relay_targets_use_pinned_ports(self):
|
||
|
|
for node, port in sorted(EXPECTED_PORTS.items()):
|
||
|
|
with self.subTest(node=node):
|
||
|
|
r = _relay("relay_target %s" % node)
|
||
|
|
self.assertEqual(r.returncode, 0, r.stderr)
|
||
|
|
self.assertEqual(r.stdout.strip(),
|
||
|
|
"%s:%d" % (EXPECTED_PEERS[node], port))
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
unittest.main()
|