feat(supervision): add choice watcher daemon, HTTPS spec docs, and test suites

- bin/muse_choice_watcher.py + systemd/muse-choices-reconcile.*: automatic choice answering and timer reconciliation - bin/digest.py: fleet log and health summarization - docs/BOX-*-HTTPS.md: comprehensive HTTPS execution contracts and API documentation - docs/MUSE-CHOICES-POLICY.md & docs/SUPERVISION-SPEC.md: autonomous execution specs - tests/test_*.py: unit test suites for HTTPS API, choice watcher, fleet heal, and swarm pruning
2026-10-07 00:25:46 +00:00
parent 04339bad14
commit 0065d11e97
29 changed files with 6419 additions and 7 deletions
@@ -0,0 +1,100 @@
+"""Watchdog coverage: every registry node is supervised.
+
+Regression test for the dev/def outage (2026-10-06): both watchdogs
+hardcoded the original four nodes (muse/pip/646/opm), so dev/def had no
+browser supervision, no Warp-tunnel auto-recovery, and no relay
+supervision. A dead tunnel paged CRITICAL partition alerts forever with
+nothing acting on it.
+
+Both scripts now resolve nodes/ports from the fleet registry
+(bin/netvm-registry.py). These tests drive the real shell code sourced
+with a LIB_ONLY guard (same pattern as test_agent_health.py).
+"""
+import importlib.util
+import subprocess
+import unittest
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+CHROMEBOX_WD = REPO_ROOT / "bin" / "chromebox-watchdog.sh"
+RELAY_WD = REPO_ROOT / "bin" / "cdp-relay-watchdog.sh"
+
+# The fleet's pinned contract (also pinned in bin/netvm-names.sh).
+EXPECTED_PORTS = {
+    "muse": 9410, "pip": 9420, "646": 9430,
+    "opm": 9440, "def": 9450, "dev": 9455,
+}
+EXPECTED_PEERS = {
+    "muse": "10.201.35.2", "pip": "10.201.87.2", "646": "10.201.202.2",
+    "opm": "10.201.157.2", "def": "10.201.66.2", "dev": "10.201.36.2",
+}
+
+
+def _load(mod_name, rel_path):
+    spec = importlib.util.spec_from_file_location(mod_name, REPO_ROOT / rel_path)
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+
+
+def _chromebox(profile, snippet):
+    prog = "set -- '%s'\nsource '%s'\n%s\n" % (profile, CHROMEBOX_WD, snippet)
+    env = {"PATH": "/usr/bin:/bin", "CHROMEBOX_WATCHDOG_LIB_ONLY": "1"}
+    return subprocess.run(["bash", "-c", prog], capture_output=True,
+                          text=True, env=env, timeout=30)
+
+
+def _relay(snippet):
+    prog = "source '%s'\n%s\n" % (RELAY_WD, snippet)
+    env = {"PATH": "/usr/bin:/bin", "CDP_RELAY_WATCHDOG_LIB_ONLY": "1"}
+    return subprocess.run(["bash", "-c", prog], capture_output=True,
+                          text=True, env=env, timeout=30)
+
+
+class RegistryContract(unittest.TestCase):
+    def test_registry_matches_pinned_ports(self):
+        reg = _load("netvm_registry_cov", "bin/netvm-registry.py")
+        self.assertEqual(reg.active_nodes(), EXPECTED_PORTS)
+        for node, peer in EXPECTED_PEERS.items():
+            self.assertEqual(reg.peer_ip_for(node), peer)
+
+
+class ChromeboxWatchdogPorts(unittest.TestCase):
+    def test_all_registry_nodes_resolve_to_pinned_ports(self):
+        for node, port in sorted(EXPECTED_PORTS.items()):
+            with self.subTest(node=node):
+                r = _chromebox(node, "echo \"PORT=$CDP_PORT\"")
+                self.assertEqual(r.returncode, 0, r.stderr)
+                line = next((ln for ln in r.stdout.splitlines()
+                             if ln.startswith("PORT=")), None)
+                self.assertIsNotNone(
+                    line, "no PORT line: stdout=%r stderr=%r"
+                    % (r.stdout, r.stderr))
+                self.assertEqual(int(line.split("=", 1)[1]), port)
+
+    def test_unknown_profile_rejected(self):
+        r = _chromebox("ghost", "echo UNREACHABLE")
+        self.assertNotEqual(r.returncode, 0)
+        self.assertIn("unknown profile", r.stderr)
+        self.assertNotIn("UNREACHABLE", r.stdout)
+
+
+class RelayWatchdogCoverage(unittest.TestCase):
+    def test_watched_nodes_covers_registry(self):
+        r = _relay("watched_nodes")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        nodes = set(r.stdout.split())
+        for node in EXPECTED_PORTS:
+            self.assertIn(node, nodes)
+
+    def test_relay_targets_use_pinned_ports(self):
+        for node, port in sorted(EXPECTED_PORTS.items()):
+            with self.subTest(node=node):
+                r = _relay("relay_target %s" % node)
+                self.assertEqual(r.returncode, 0, r.stderr)
+                self.assertEqual(r.stdout.strip(),
+                                 "%s:%d" % (EXPECTED_PEERS[node], port))
+
+
+if __name__ == "__main__":
+    unittest.main()