feat(supervision): add choice watcher daemon, HTTPS spec docs, and test suites
- bin/muse_choice_watcher.py + systemd/muse-choices-reconcile.*: automatic choice answering and timer reconciliation - bin/digest.py: fleet log and health summarization - docs/BOX-*-HTTPS.md: comprehensive HTTPS execution contracts and API documentation - docs/MUSE-CHOICES-POLICY.md & docs/SUPERVISION-SPEC.md: autonomous execution specs - tests/test_*.py: unit test suites for HTTPS API, choice watcher, fleet heal, and swarm pruning
This commit is contained in:
operator
@@ -0,0 +1,309 @@
|
||||
"""Tests for `box fleet heal` and `box watchdog` in super-cli.py.
|
||||
|
||||
Heal drives lock cleanup, watchdog-timer install, tunnel restart, and
|
||||
egress/relay verification; watchdog status/run expose the systemd
|
||||
watchdog layer. Shell-outs are faked; no sudo/systemctl/netns touch
|
||||
the host.
|
||||
"""
|
||||
import argparse
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import unittest
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
SPEC = importlib.util.spec_from_file_location(
|
||||
"super_cli_heal", REPO_ROOT / "bin" / "super-cli.py")
|
||||
super_cli = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(super_cli)
|
||||
|
||||
|
||||
def _ns(**over):
|
||||
kw = dict(node=None, target=None, json=True)
|
||||
kw.update(over)
|
||||
return argparse.Namespace(**kw)
|
||||
|
||||
|
||||
class FakeSh:
|
||||
"""Programmable stand-in for super_cli._sh; records calls."""
|
||||
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
self.handlers = []
|
||||
|
||||
def on(self, *needles, rc=0, out=""):
|
||||
self.handlers.append((needles, rc, out))
|
||||
return self
|
||||
|
||||
def __call__(self, cmd, timeout=120, input_text=None):
|
||||
self.calls.append((list(cmd), timeout, input_text))
|
||||
blob = " ".join(cmd)
|
||||
for needles, rc, out in self.handlers:
|
||||
if all(n in blob for n in needles):
|
||||
return rc, out() if callable(out) else out
|
||||
raise AssertionError("unexpected command: %r" % (cmd,))
|
||||
|
||||
|
||||
class HealBase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
import tempfile
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.root = Path(self.tmp.name)
|
||||
self.systemd = self.root / "systemd"
|
||||
self.systemd.mkdir()
|
||||
self.locks = self.root / "locks"
|
||||
self.locks.mkdir()
|
||||
self.patchers = [
|
||||
mock.patch.object(super_cli, "SYSTEMD_SYSTEM_DIR", self.systemd),
|
||||
mock.patch.object(super_cli, "WATCHDOG_LOCK_TMPL",
|
||||
str(self.locks / "chromebox-watchdog-{node}.lock")),
|
||||
]
|
||||
for p in self.patchers:
|
||||
p.start()
|
||||
self.addCleanup(self._unpatch)
|
||||
self.sh = FakeSh()
|
||||
self.sh_mock = mock.patch.object(super_cli, "_sh", self.sh)
|
||||
self.sh_mock.start()
|
||||
self.addCleanup(self.sh_mock.stop)
|
||||
|
||||
def _unpatch(self):
|
||||
for p in self.patchers:
|
||||
p.stop()
|
||||
|
||||
def run_heal(self, node="dev", as_json=True):
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
with self.assertRaises(SystemExit) as cm:
|
||||
super_cli.cmd_fleet_heal(_ns(node=node, json=as_json))
|
||||
return cm.exception.code, buf.getvalue()
|
||||
|
||||
|
||||
class FleetHealTests(HealBase):
|
||||
def _healthy_sh(self):
|
||||
(self.sh
|
||||
.on("systemctl", "enable", "--now", rc=0, out="")
|
||||
.on("netvm-node-up.sh", "dev", rc=0,
|
||||
out="tunnel already up (egress=1.2.3.4), skipping handshake wait\n"
|
||||
"node=dev netns=warp-dev egress=1.2.3.4")
|
||||
.on("curl", rc=0, out="ip=1.2.3.4\nfoo=bar"))
|
||||
|
||||
def test_heal_recovered(self):
|
||||
(self.systemd / "chromebox-watchdog-dev.timer").write_text("x")
|
||||
self._healthy_sh()
|
||||
with mock.patch.object(super_cli, "probe_cdp_status",
|
||||
return_value={"ok": True, "latency_ms": 12}):
|
||||
code, out = self.run_heal()
|
||||
self.assertEqual(code, 0)
|
||||
data = json.loads(out)
|
||||
self.assertEqual(data["verdict"], "RECOVERED")
|
||||
self.assertTrue(data["ok"])
|
||||
self.assertEqual([s["step"] for s in data["steps"]],
|
||||
["lock", "timer", "tunnel", "egress", "relay"])
|
||||
self.assertTrue(all(s["ok"] for s in data["steps"]))
|
||||
|
||||
def test_heal_down_when_egress_fails(self):
|
||||
(self.systemd / "chromebox-watchdog-dev.timer").write_text("x")
|
||||
(self.sh
|
||||
.on("systemctl", "enable", "--now", rc=0, out="")
|
||||
.on("netvm-node-up.sh", "dev", rc=0,
|
||||
out="no handshake yet (endpoint=162.159.192.1)\n"
|
||||
"node=dev netns=warp-dev egress=unknown")
|
||||
.on("curl", rc=7, out="curl: (7) couldn't connect"))
|
||||
with mock.patch.object(super_cli, "probe_cdp_status",
|
||||
return_value={"ok": False, "error": "refused",
|
||||
"latency_ms": None}):
|
||||
code, out = self.run_heal()
|
||||
self.assertEqual(code, 1)
|
||||
data = json.loads(out)
|
||||
self.assertEqual(data["verdict"], "DOWN")
|
||||
self.assertFalse(data["ok"])
|
||||
by_step = {s["step"]: s for s in data["steps"]}
|
||||
self.assertFalse(by_step["tunnel"]["ok"])
|
||||
self.assertFalse(by_step["egress"]["ok"])
|
||||
|
||||
def test_heal_down_prints_identity_guidance(self):
|
||||
(self.systemd / "chromebox-watchdog-dev.timer").write_text("x")
|
||||
(self.sh
|
||||
.on("systemctl", rc=0, out="")
|
||||
.on("netvm-node-up.sh", rc=0, out="node=dev egress=unknown")
|
||||
.on("curl", rc=7, out="fail"))
|
||||
with mock.patch.object(super_cli, "probe_cdp_status",
|
||||
return_value={"ok": False, "error": "x",
|
||||
"latency_ms": None}):
|
||||
code, out = self.run_heal(as_json=False)
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("netvm-new-identity.sh dev", out)
|
||||
|
||||
def test_heal_installs_missing_timer(self):
|
||||
seen = {}
|
||||
|
||||
def fake(cmd, timeout=120, input_text=None):
|
||||
blob = " ".join(cmd)
|
||||
self.sh.calls.append((list(cmd), timeout, input_text))
|
||||
if "tee" in blob:
|
||||
seen["tee_target"] = cmd[-1]
|
||||
seen["tee_input"] = input_text
|
||||
return 0, ""
|
||||
if "daemon-reload" in blob:
|
||||
seen["reload"] = True
|
||||
return 0, ""
|
||||
if "enable" in blob:
|
||||
return 0, ""
|
||||
if "netvm-node-up.sh" in blob:
|
||||
return 0, "node=dev egress=1.2.3.4"
|
||||
if "curl" in blob:
|
||||
return 0, "ip=1.2.3.4"
|
||||
raise AssertionError("unexpected: %r" % (cmd,))
|
||||
|
||||
with mock.patch.object(super_cli, "_sh", fake):
|
||||
with mock.patch.object(
|
||||
super_cli, "probe_cdp_status",
|
||||
return_value={"ok": True, "latency_ms": 3}):
|
||||
code, _ = self.run_heal()
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(seen["tee_target"],
|
||||
str(self.systemd / "chromebox-watchdog-dev.timer"))
|
||||
self.assertIn("Unit=chromebox-watchdog@dev.service", seen["tee_input"])
|
||||
self.assertTrue(seen["reload"])
|
||||
|
||||
def test_heal_clears_unwritable_lock(self):
|
||||
(self.systemd / "chromebox-watchdog-dev.timer").write_text("x")
|
||||
lock = self.locks / "chromebox-watchdog-dev.lock"
|
||||
lock.write_text("")
|
||||
lock.chmod(0o444)
|
||||
self._healthy_sh()
|
||||
self.sh.on("rm", str(lock), rc=0, out="")
|
||||
with mock.patch.object(super_cli, "probe_cdp_status",
|
||||
return_value={"ok": True, "latency_ms": 3}):
|
||||
code, out = self.run_heal()
|
||||
self.assertEqual(code, 0)
|
||||
data = json.loads(out)
|
||||
by_step = {s["step"]: s for s in data["steps"]}
|
||||
self.assertTrue(by_step["lock"]["ok"])
|
||||
self.assertIn("removed stale lock", by_step["lock"]["detail"])
|
||||
self.assertTrue(any("rm" in " ".join(c[0]) for c in self.sh.calls))
|
||||
|
||||
def test_heal_rejects_unknown_node(self):
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
with self.assertRaises(SystemExit) as cm:
|
||||
super_cli.cmd_fleet_heal(_ns(node="ghost"))
|
||||
self.assertEqual(cm.exception.code, 1)
|
||||
|
||||
|
||||
class ShHelperTests(unittest.TestCase):
|
||||
def test_timeout_and_oserror(self):
|
||||
import subprocess as real_subprocess
|
||||
with mock.patch.object(super_cli.subprocess, "run",
|
||||
side_effect=real_subprocess.TimeoutExpired("x", 1)):
|
||||
self.assertEqual(super_cli._sh(["x"], timeout=1)[0], 124)
|
||||
with mock.patch.object(super_cli.subprocess, "run",
|
||||
side_effect=OSError("nope")):
|
||||
self.assertEqual(super_cli._sh(["x"])[0], 127)
|
||||
|
||||
|
||||
class WatchdogTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.sh = FakeSh()
|
||||
self.sh_mock = mock.patch.object(super_cli, "_sh", self.sh)
|
||||
self.sh_mock.start()
|
||||
self.addCleanup(self.sh_mock.stop)
|
||||
|
||||
def _unit_states(self, missing=()):
|
||||
def fake(cmd, timeout=120, input_text=None):
|
||||
self.sh.calls.append((list(cmd), timeout, input_text))
|
||||
unit = cmd[-1]
|
||||
if "is-enabled" in cmd or "is-active" in cmd:
|
||||
return (1, "") if unit in missing else (0, "")
|
||||
if cmd[:3] == ["sudo", "systemctl", "start"]:
|
||||
return 0, ""
|
||||
raise AssertionError("unexpected: %r" % (cmd,))
|
||||
return fake
|
||||
|
||||
def test_status_json(self):
|
||||
stub = mock.Mock()
|
||||
stub.collect.return_value = {
|
||||
n: {"browser": "healthy", "browser_detail": "silent",
|
||||
"cdp": "healthy", "cdp_detail": "silent"}
|
||||
for n in super_cli.VALID_NODES}
|
||||
with mock.patch.object(super_cli, "_sh", self._unit_states(
|
||||
missing=("chromebox-watchdog-def.timer",))):
|
||||
with mock.patch.dict(sys.modules, {"host_evidence": stub}):
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
super_cli.cmd_watchdog_status(_ns(json=True))
|
||||
data = json.loads(buf.getvalue())
|
||||
self.assertTrue(data["ok"])
|
||||
self.assertEqual(len(data["nodes"]), 6)
|
||||
by_node = {n["node"]: n for n in data["nodes"]}
|
||||
self.assertFalse(by_node["def"]["enabled"])
|
||||
self.assertFalse(by_node["def"]["active"])
|
||||
self.assertTrue(by_node["dev"]["active"])
|
||||
self.assertEqual(by_node["dev"]["browser"], "healthy")
|
||||
self.assertIn("timer", data["relay"])
|
||||
|
||||
def test_status_survives_missing_evidence(self):
|
||||
with mock.patch.object(super_cli, "_sh", self._unit_states()):
|
||||
with mock.patch.dict(sys.modules, {"host_evidence": None}):
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
super_cli.cmd_watchdog_status(_ns(json=True))
|
||||
data = json.loads(buf.getvalue())
|
||||
self.assertTrue(data["ok"])
|
||||
self.assertEqual(data["nodes"][0]["browser"], "unknown")
|
||||
|
||||
def test_run_node_and_relay(self):
|
||||
for target, unit in (("dev", "chromebox-watchdog@dev.service"),
|
||||
("relay", "cdp-relay-watchdog.service")):
|
||||
with self.subTest(target=target):
|
||||
with mock.patch.object(super_cli, "_sh",
|
||||
self._unit_states()) as _:
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
with self.assertRaises(SystemExit) as cm:
|
||||
super_cli.cmd_watchdog_run(
|
||||
_ns(target=target, json=True))
|
||||
self.assertEqual(cm.exception.code, 0)
|
||||
data = json.loads(buf.getvalue())
|
||||
self.assertTrue(data["ok"])
|
||||
self.assertEqual(data["unit"], unit)
|
||||
|
||||
def test_run_rejects_bad_target(self):
|
||||
with self.assertRaises(SystemExit) as cm:
|
||||
super_cli.cmd_watchdog_run(_ns(target="ghost"))
|
||||
self.assertEqual(cm.exception.code, 1)
|
||||
|
||||
def test_run_reports_start_failure(self):
|
||||
def fake(cmd, timeout=120, input_text=None):
|
||||
return 1, "Failed to start"
|
||||
|
||||
with mock.patch.object(super_cli, "_sh", fake):
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
with self.assertRaises(SystemExit) as cm:
|
||||
super_cli.cmd_watchdog_run(_ns(target="dev", json=True))
|
||||
self.assertEqual(cm.exception.code, 1)
|
||||
self.assertFalse(json.loads(buf.getvalue())["ok"])
|
||||
|
||||
|
||||
class ParserTests(unittest.TestCase):
|
||||
def test_fleet_heal_parses(self):
|
||||
args = super_cli.build_parser().parse_args(["fleet", "heal", "dev"])
|
||||
self.assertEqual((args.domain, args.action, args.node),
|
||||
("fleet", "heal", "dev"))
|
||||
|
||||
def test_watchdog_parses(self):
|
||||
args = super_cli.build_parser().parse_args(["watchdog", "run", "relay"])
|
||||
self.assertEqual((args.domain, args.action, args.target),
|
||||
("watchdog", "run", "relay"))
|
||||
args = super_cli.build_parser().parse_args(["watchdog"])
|
||||
self.assertEqual((args.domain, args.action), ("watchdog", "status"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user