"""Watchdog coverage: every registry node is supervised. Regression test for the dev/def outage (2026-10-06): both watchdogs hardcoded the original four nodes (muse/pip/646/opm), so dev/def had no browser supervision, no Warp-tunnel auto-recovery, and no relay supervision. A dead tunnel paged CRITICAL partition alerts forever with nothing acting on it. Both scripts now resolve nodes/ports from the fleet registry (bin/netvm-registry.py). These tests drive the real shell code sourced with a LIB_ONLY guard (same pattern as test_agent_health.py). """ import importlib.util import subprocess import unittest from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent CHROMEBOX_WD = REPO_ROOT / "bin" / "chromebox-watchdog.sh" RELAY_WD = REPO_ROOT / "bin" / "cdp-relay-watchdog.sh" # The fleet's pinned contract (also pinned in bin/netvm-names.sh). EXPECTED_PORTS = { "muse": 9410, "pip": 9420, "646": 9430, "opm": 9440, "def": 9450, "dev": 9455, } EXPECTED_PEERS = { "muse": "10.201.35.2", "pip": "10.201.87.2", "646": "10.201.202.2", "opm": "10.201.157.2", "def": "10.201.66.2", "dev": "10.201.36.2", } def _load(mod_name, rel_path): spec = importlib.util.spec_from_file_location(mod_name, REPO_ROOT / rel_path) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod def _chromebox(profile, snippet): prog = "set -- '%s'\nsource '%s'\n%s\n" % (profile, CHROMEBOX_WD, snippet) env = {"PATH": "/usr/bin:/bin", "CHROMEBOX_WATCHDOG_LIB_ONLY": "1"} return subprocess.run(["bash", "-c", prog], capture_output=True, text=True, env=env, timeout=30) def _relay(snippet): prog = "source '%s'\n%s\n" % (RELAY_WD, snippet) env = {"PATH": "/usr/bin:/bin", "CDP_RELAY_WATCHDOG_LIB_ONLY": "1"} return subprocess.run(["bash", "-c", prog], capture_output=True, text=True, env=env, timeout=30) class RegistryContract(unittest.TestCase): def test_registry_matches_pinned_ports(self): reg = _load("netvm_registry_cov", "bin/netvm-registry.py") self.assertEqual(reg.active_nodes(), EXPECTED_PORTS) for node, peer in EXPECTED_PEERS.items(): self.assertEqual(reg.peer_ip_for(node), peer) class ChromeboxWatchdogPorts(unittest.TestCase): def test_all_registry_nodes_resolve_to_pinned_ports(self): for node, port in sorted(EXPECTED_PORTS.items()): with self.subTest(node=node): r = _chromebox(node, "echo \"PORT=$CDP_PORT\"") self.assertEqual(r.returncode, 0, r.stderr) line = next((ln for ln in r.stdout.splitlines() if ln.startswith("PORT=")), None) self.assertIsNotNone( line, "no PORT line: stdout=%r stderr=%r" % (r.stdout, r.stderr)) self.assertEqual(int(line.split("=", 1)[1]), port) def test_unknown_profile_rejected(self): r = _chromebox("ghost", "echo UNREACHABLE") self.assertNotEqual(r.returncode, 0) self.assertIn("unknown profile", r.stderr) self.assertNotIn("UNREACHABLE", r.stdout) class RelayWatchdogCoverage(unittest.TestCase): def test_watched_nodes_covers_registry(self): r = _relay("watched_nodes") self.assertEqual(r.returncode, 0, r.stderr) nodes = set(r.stdout.split()) for node in EXPECTED_PORTS: self.assertIn(node, nodes) def test_relay_targets_use_pinned_ports(self): for node, port in sorted(EXPECTED_PORTS.items()): with self.subTest(node=node): r = _relay("relay_target %s" % node) self.assertEqual(r.returncode, 0, r.stderr) self.assertEqual(r.stdout.strip(), "%s:%d" % (EXPECTED_PEERS[node], port)) if __name__ == "__main__": unittest.main()