fix: truthful fleet status in blind shells + agent-health circuit breaker
box fleet status / approvals check misreported every node as STOPPED / CDP-unreachable from sandboxed shells (own PID+net namespaces: pgrep blind, no route to 10.201.x.x, no sudo). Fleet was healthy throughout. - bin/host_evidence.py (new): host watchdog evidence fallback. Recent timer runs (journal -o json, exact UNIT match) with no newer failure line in cdp-relay-watchdog.log / chromebox-watchdog.log (both silent-when-healthy) prove a node is up. def/dev have no watchdog coverage: browser verdict via chromebox-<node>.log freshness (alive-only), CDP verdict unknown. - super-cli.py: effective status/source/evidence per node. Host evidence decides ONLY the fully-blind pattern (both local probes negative); live local signals always win. New UNKNOWN badge, [*] footnote; approvals UNREACHABLE splits into BLIND / OFFLINE(host agrees) / unreachable-evidence-inconclusive, with honest footer. proc_alive/cdp_ok keep local-probe meaning; status/source/evidence are new JSON fields. - approvals.py: host_cdp_ok flag on the unreachable path. - agent-health.sh: restart circuit breaker. 3 consecutive futile restarts (restart leaves agent still failing) opens the circuit: no more kills for 1800s, ALERT to log+journal, half-open probe after cooldown, reset on any success. Stops the def murder loop (57 restarts / 155 API FAILs for an account-layer failure). - tests/test_fleet_status.py (25), tests/test_agent_health.py (6). - CHROMEBOX-RUNBOOK.md: blind-shell status + futile-restart sections. Tests: 98/98 focused green (agent_health + fleet_status + completion + tool_calls). Live-verified: 4 ACTIVE [*] + 2 UNKNOWN.
This commit is contained in:
Muse Sidechat
@@ -0,0 +1,87 @@
|
||||
"""Tests for the agent-health.sh restart circuit breaker.
|
||||
|
||||
Drives the real shell functions (sourced with AGENT_HEALTH_LIB_ONLY=1)
|
||||
against a scratch STATE_DIR/LOG: futile-restart counting, circuit open,
|
||||
half-open probe after cooldown, and reset on success.
|
||||
"""
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT = REPO_ROOT / "bin" / "agent-health.sh"
|
||||
|
||||
|
||||
def _run(state_dir, log, snippet):
|
||||
prog = (
|
||||
"source '%s'\n"
|
||||
"STATE_DIR='%s'; LOG='%s'\n"
|
||||
"%s\n" % (SCRIPT, state_dir, log, snippet)
|
||||
)
|
||||
env = {"AGENT_HEALTH_LIB_ONLY": "1", "PATH": "/usr/bin:/bin"}
|
||||
return subprocess.run(["bash", "-c", prog], capture_output=True,
|
||||
text=True, env=env, timeout=30)
|
||||
|
||||
|
||||
class CircuitBreaker(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.state = str(Path(self.tmp.name) / "state")
|
||||
Path(self.state).mkdir()
|
||||
self.log = str(Path(self.tmp.name) / "log")
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def bash(self, snippet):
|
||||
return _run(self.state, self.log, snippet)
|
||||
|
||||
def test_allows_when_closed(self):
|
||||
r = self.bash("circuit_allows def")
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
|
||||
def test_allows_below_threshold(self):
|
||||
r = self.bash("circuit_note_restart def fail\n"
|
||||
"circuit_note_restart def fail\n"
|
||||
"circuit_allows def")
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertEqual((Path(self.state) / "futile-def").read_text().strip(), "2")
|
||||
|
||||
def test_opens_after_threshold_and_alerts(self):
|
||||
r = self.bash("circuit_note_restart def fail\n"
|
||||
"circuit_note_restart def fail\n"
|
||||
"circuit_note_restart def fail")
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertTrue((Path(self.state) / "circuit-def").exists())
|
||||
self.assertIn("ALERT", r.stdout)
|
||||
self.assertIn("ALERT", Path(self.log).read_text())
|
||||
r2 = self.bash("circuit_allows def")
|
||||
self.assertNotEqual(r2.returncode, 0)
|
||||
self.assertIn("CIRCUIT OPEN", Path(self.log).read_text())
|
||||
|
||||
def test_half_open_after_cooldown(self):
|
||||
old = "echo $(( $(date +%%s) - 1900 )) > '%s/circuit-def'" % self.state
|
||||
r = self.bash(old + "\ncircuit_allows def")
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertIn("half-open", Path(self.log).read_text())
|
||||
|
||||
def test_ok_resets(self):
|
||||
r = self.bash("circuit_note_restart def fail\n"
|
||||
"circuit_note_restart def fail\n"
|
||||
"circuit_note_restart def fail\n"
|
||||
"circuit_note_restart def ok")
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertFalse((Path(self.state) / "futile-def").exists())
|
||||
self.assertFalse((Path(self.state) / "circuit-def").exists())
|
||||
|
||||
def test_per_agent_isolation(self):
|
||||
r = self.bash("circuit_note_restart def fail\n"
|
||||
"circuit_note_restart def fail\n"
|
||||
"circuit_note_restart def fail\n"
|
||||
"circuit_allows pip")
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,280 @@
|
||||
"""Tests for host-side fleet evidence (bin/host_evidence.py).
|
||||
|
||||
Covers log-line classification, verdict assembly (silent-when-healthy
|
||||
runs vs. fresh failures vs. stale/missing runs), and the effective
|
||||
status mapping used by `box fleet status` in PID/net-blind shells.
|
||||
"""
|
||||
import importlib.util
|
||||
import unittest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def _load(mod_name, rel_path):
|
||||
spec = importlib.util.spec_from_file_location(mod_name, REPO_ROOT / rel_path)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
he = _load("host_evidence_test", "bin/host_evidence.py")
|
||||
|
||||
NOW = datetime(2026, 10, 6, 8, 40, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _ts(dt):
|
||||
return dt.strftime("[%Y-%m-%dT%H:%M:%SZ]")
|
||||
|
||||
|
||||
def _run(minutes_ago):
|
||||
return NOW - timedelta(minutes=minutes_ago)
|
||||
|
||||
|
||||
class ParseLogTs(unittest.TestCase):
|
||||
def test_valid(self):
|
||||
self.assertEqual(
|
||||
he.parse_log_ts("[2026-10-06T05:55:25Z] [opm] relay restart FAILED"),
|
||||
datetime(2026, 10, 6, 5, 55, 25, tzinfo=timezone.utc))
|
||||
|
||||
def test_garbage(self):
|
||||
self.assertIsNone(he.parse_log_ts("cdp relay already running"))
|
||||
self.assertIsNone(he.parse_log_ts(""))
|
||||
self.assertIsNone(he.parse_log_ts("[2026-13-99T99:99:99Z] [muse] x"))
|
||||
|
||||
|
||||
class ClassifyChromebox(unittest.TestCase):
|
||||
def test_down(self):
|
||||
self.assertEqual(
|
||||
he.classify_chromebox_line("[muse] relaunch FAILED (x) — needs operator attention"),
|
||||
"down")
|
||||
|
||||
def test_healthy(self):
|
||||
self.assertEqual(
|
||||
he.classify_chromebox_line("[muse] relaunch OK (attempt 2)"), "healthy")
|
||||
self.assertEqual(
|
||||
he.classify_chromebox_line(
|
||||
"[muse] tunnel restart recovered warp egress, chrome relaunch not needed"),
|
||||
"healthy")
|
||||
|
||||
def test_degraded(self):
|
||||
for line in (
|
||||
"[muse] relaunch attempt 3 not healthy yet (CDP up but no page)",
|
||||
"[muse] tunnel restart did not recover egress (x), proceeding with chrome relaunch",
|
||||
"[muse] unhealthy (x), relaunching chromebox",
|
||||
"[muse] warp partition detected (x), restarting tunnel via netvm-node-up.sh",
|
||||
"[muse] browser launched recently (pid 1), skipping relaunch (probably still starting)",
|
||||
):
|
||||
self.assertEqual(he.classify_chromebox_line(line), "degraded", line)
|
||||
|
||||
def test_no_verdict(self):
|
||||
for line in (
|
||||
"[2026-10-06T08:00:00Z] [muse] log rotated",
|
||||
"cdp relay already running",
|
||||
"tunnel already up (egress=1.2.3.4), skipping handshake wait",
|
||||
"node=muse netns=warp-muse egress=1.2.3.4",
|
||||
"Traceback (most recent call last):",
|
||||
):
|
||||
self.assertIsNone(he.classify_chromebox_line(line), line)
|
||||
|
||||
|
||||
class ClassifyRelay(unittest.TestCase):
|
||||
def test_down(self):
|
||||
self.assertEqual(
|
||||
he.classify_relay_line("[opm] relay restart FAILED on 10.201.157.2:9440 — needs operator"),
|
||||
"down")
|
||||
self.assertEqual(
|
||||
he.classify_relay_line("[pip] FAIL_LOUD: host veth ve-x missing IP 10.201.87.1"),
|
||||
"down")
|
||||
# Trailing "restarting" with no OK/FAILED after it: crashed mid-restart.
|
||||
self.assertEqual(
|
||||
he.classify_relay_line("[muse] relay unhealthy on 10.201.35.2:9410, restarting"),
|
||||
"down")
|
||||
|
||||
def test_healthy(self):
|
||||
self.assertEqual(
|
||||
he.classify_relay_line("[646] relay restarted OK on 10.201.202.2:9430"),
|
||||
"healthy")
|
||||
|
||||
def test_no_verdict(self):
|
||||
self.assertIsNone(he.classify_relay_line(" File \"/x/netvm-cdp-relay.py\", line 37, in main"))
|
||||
self.assertIsNone(he.classify_relay_line(""))
|
||||
|
||||
|
||||
class LastVerdict(unittest.TestCase):
|
||||
def test_newest_wins_per_node(self):
|
||||
lines = [
|
||||
"%s [muse] relaunch OK (attempt 1)" % _ts(_run(30)),
|
||||
"%s [pip] relaunch FAILED (x)" % _ts(_run(20)),
|
||||
"%s [muse] unhealthy (x), relaunching chromebox" % _ts(_run(5)),
|
||||
"%s [muse] log rotated" % _ts(_run(1)),
|
||||
]
|
||||
v, ts, _ = he.last_verdict(lines, "muse", he.classify_chromebox_line)
|
||||
self.assertEqual(v, "degraded")
|
||||
self.assertEqual(ts, _run(5))
|
||||
v, _, _ = he.last_verdict(lines, "pip", he.classify_chromebox_line)
|
||||
self.assertEqual(v, "down")
|
||||
|
||||
def test_none_when_no_verdict(self):
|
||||
self.assertIsNone(he.last_verdict(
|
||||
["[2026-10-06T08:00:00Z] [muse] log rotated", "noise"],
|
||||
"muse", he.classify_chromebox_line))
|
||||
self.assertIsNone(he.last_verdict([], "muse", he.classify_chromebox_line))
|
||||
|
||||
|
||||
class BrowserVerdict(unittest.TestCase):
|
||||
def test_silent_run_is_healthy(self):
|
||||
v, d = he.browser_verdict("muse", [], _run(2), now=NOW)
|
||||
self.assertEqual(v, "healthy")
|
||||
v, _ = he.browser_verdict(
|
||||
"muse", ["%s [muse] relaunch FAILED (x)" % _ts(_run(60))],
|
||||
_run(2), now=NOW)
|
||||
self.assertEqual(v, "healthy") # old failure, silent since
|
||||
|
||||
def test_fresh_failure_wins(self):
|
||||
v, _ = he.browser_verdict(
|
||||
"muse", ["%s [muse] relaunch FAILED (x)" % _ts(_run(1))],
|
||||
_run(2), now=NOW)
|
||||
self.assertEqual(v, "down")
|
||||
v, _ = he.browser_verdict(
|
||||
"muse", ["%s [muse] unhealthy (x), relaunching chromebox" % _ts(_run(1))],
|
||||
_run(2), now=NOW)
|
||||
self.assertEqual(v, "degraded")
|
||||
|
||||
def test_fresh_recovery_is_healthy(self):
|
||||
v, _ = he.browser_verdict(
|
||||
"muse", ["%s [muse] tunnel restart recovered warp egress, chrome relaunch not needed"
|
||||
% _ts(_run(1))],
|
||||
_run(2), now=NOW)
|
||||
self.assertEqual(v, "healthy")
|
||||
|
||||
def test_stale_or_missing_run_is_unknown(self):
|
||||
v, _ = he.browser_verdict("muse", [], _run(30), now=NOW)
|
||||
self.assertEqual(v, "unknown")
|
||||
v, _ = he.browser_verdict("muse", [], None, now=NOW)
|
||||
self.assertEqual(v, "unknown")
|
||||
|
||||
def test_def_uses_watchdog_like_core_nodes(self):
|
||||
v, _ = he.browser_verdict("def", [], _run(2), now=NOW)
|
||||
self.assertEqual(v, "healthy") # silent run
|
||||
v, _ = he.browser_verdict("def", [], None, now=NOW)
|
||||
self.assertEqual(v, "unknown") # no timer run (yet)
|
||||
v, _ = he.browser_verdict(
|
||||
"def", ["%s [def] relaunch FAILED (x)" % _ts(_run(1))],
|
||||
_run(2), now=NOW)
|
||||
self.assertEqual(v, "down")
|
||||
|
||||
def test_uncovered_node_uses_chrome_log_freshness(self):
|
||||
v, _ = he.browser_verdict("ghost", [], None,
|
||||
chrome_log_mtime=_run(5), now=NOW)
|
||||
self.assertEqual(v, "healthy")
|
||||
v, d = he.browser_verdict("ghost", [], None,
|
||||
chrome_log_mtime=_run(60), now=NOW)
|
||||
self.assertEqual(v, "unknown")
|
||||
v, _ = he.browser_verdict("ghost", [], None,
|
||||
chrome_log_mtime=None, now=NOW)
|
||||
self.assertEqual(v, "unknown")
|
||||
|
||||
|
||||
class CdpVerdict(unittest.TestCase):
|
||||
def test_silent_run_is_healthy(self):
|
||||
v, _ = he.cdp_verdict("opm", [], _run(4), now=NOW)
|
||||
self.assertEqual(v, "healthy")
|
||||
|
||||
def test_fresh_failure_is_down(self):
|
||||
v, _ = he.cdp_verdict(
|
||||
"opm", ["%s [opm] relay restart FAILED on 10.201.157.2:9440" % _ts(_run(1))],
|
||||
_run(4), now=NOW)
|
||||
self.assertEqual(v, "down")
|
||||
|
||||
def test_stale_or_missing_run_is_unknown(self):
|
||||
v, _ = he.cdp_verdict("opm", [], _run(60), now=NOW)
|
||||
self.assertEqual(v, "unknown")
|
||||
v, _ = he.cdp_verdict("opm", [], None, now=NOW)
|
||||
self.assertEqual(v, "unknown")
|
||||
|
||||
def test_dev_def_monitored_like_core_nodes(self):
|
||||
for node in ("def", "dev"):
|
||||
v, _ = he.cdp_verdict(node, [], _run(4), now=NOW)
|
||||
self.assertEqual(v, "healthy", node) # silent run
|
||||
v, _ = he.cdp_verdict(
|
||||
node,
|
||||
["%s [%s] relay restart FAILED on 10.0.0.1:1" % (_ts(_run(1)), node)],
|
||||
_run(4), now=NOW)
|
||||
self.assertEqual(v, "down", node)
|
||||
|
||||
def test_unmonitored_node_is_unknown(self):
|
||||
v, d = he.cdp_verdict("ghost", [], _run(1), now=NOW)
|
||||
self.assertEqual(v, "unknown")
|
||||
self.assertIn("no relay-monitor coverage", d)
|
||||
|
||||
|
||||
class EffectiveStatus(unittest.TestCase):
|
||||
def test_live_local_signals_win(self):
|
||||
self.assertEqual(he.effective_status(True, True, "down", "down"),
|
||||
("ACTIVE", "local"))
|
||||
self.assertEqual(he.effective_status(True, False, "healthy", "healthy"),
|
||||
("CDP_DOWN", "local"))
|
||||
|
||||
def test_cdp_without_proc_is_active(self):
|
||||
self.assertEqual(he.effective_status(False, True, "unknown", "unknown"),
|
||||
("ACTIVE", "local"))
|
||||
|
||||
def test_blind_with_evidence(self):
|
||||
self.assertEqual(
|
||||
he.effective_status(False, False, "healthy", "healthy"),
|
||||
("ACTIVE", "host-evidence"))
|
||||
self.assertEqual(
|
||||
he.effective_status(False, False, "degraded", "healthy"),
|
||||
("ACTIVE", "host-evidence"))
|
||||
self.assertEqual(
|
||||
he.effective_status(False, False, "down", "healthy"),
|
||||
("STOPPED", "host-evidence"))
|
||||
self.assertEqual(
|
||||
he.effective_status(False, False, "healthy", "down"),
|
||||
("CDP_DOWN", "host-evidence"))
|
||||
|
||||
def test_blind_without_evidence_is_unknown(self):
|
||||
self.assertEqual(
|
||||
he.effective_status(False, False, "unknown", "unknown"),
|
||||
("UNKNOWN", "host-evidence"))
|
||||
self.assertEqual(
|
||||
he.effective_status(False, False, "healthy", "unknown"),
|
||||
("UNKNOWN", "host-evidence"))
|
||||
self.assertEqual(
|
||||
he.effective_status(False, False, "unknown", "healthy"),
|
||||
("UNKNOWN", "host-evidence"))
|
||||
|
||||
|
||||
class CollectSeams(unittest.TestCase):
|
||||
def test_assembles_per_node_verdicts(self):
|
||||
starts = {"cdp-relay-watchdog.service": _run(4),
|
||||
"chromebox-watchdog@muse.service": _run(1),
|
||||
"chromebox-watchdog@def.service": _run(1)}
|
||||
relay_lines = ["%s [muse] relay restarted OK on 10.201.35.2:9410" % _ts(_run(3))]
|
||||
chrome_lines = ["%s [muse] relaunch OK (attempt 1)" % _ts(_run(1))]
|
||||
out = he.collect(["muse", "def"],
|
||||
_journal_starts=starts,
|
||||
_relay_lines=relay_lines,
|
||||
_chromebox_lines=chrome_lines,
|
||||
_chrome_mtimes={"def": _run(5)},
|
||||
_now=NOW)
|
||||
self.assertEqual(out["muse"]["browser"], "healthy")
|
||||
self.assertEqual(out["muse"]["cdp"], "healthy")
|
||||
# def is watchdog-covered: silent runs read healthy (chrome-log
|
||||
# freshness is no longer consulted for covered nodes).
|
||||
self.assertEqual(out["def"]["browser"], "healthy")
|
||||
self.assertEqual(out["def"]["cdp"], "healthy")
|
||||
|
||||
|
||||
class CoverageTuples(unittest.TestCase):
|
||||
def test_watchdog_tuples_cover_registry(self):
|
||||
reg = _load("netvm_registry_cov", "bin/netvm-registry.py")
|
||||
for node in reg.active_nodes():
|
||||
self.assertIn(node, he.RELAY_NODES)
|
||||
self.assertIn(node, he.CHROMEBOX_NODES)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user