Muse Sidechat
c9143a558b
box fleet status / approvals check misreported every node as STOPPED / CDP-unreachable from sandboxed shells (own PID+net namespaces: pgrep blind, no route to 10.201.x.x, no sudo). Fleet was healthy throughout. - bin/host_evidence.py (new): host watchdog evidence fallback. Recent timer runs (journal -o json, exact UNIT match) with no newer failure line in cdp-relay-watchdog.log / chromebox-watchdog.log (both silent-when-healthy) prove a node is up. def/dev have no watchdog coverage: browser verdict via chromebox-<node>.log freshness (alive-only), CDP verdict unknown. - super-cli.py: effective status/source/evidence per node. Host evidence decides ONLY the fully-blind pattern (both local probes negative); live local signals always win. New UNKNOWN badge, [*] footnote; approvals UNREACHABLE splits into BLIND / OFFLINE(host agrees) / unreachable-evidence-inconclusive, with honest footer. proc_alive/cdp_ok keep local-probe meaning; status/source/evidence are new JSON fields. - approvals.py: host_cdp_ok flag on the unreachable path. - agent-health.sh: restart circuit breaker. 3 consecutive futile restarts (restart leaves agent still failing) opens the circuit: no more kills for 1800s, ALERT to log+journal, half-open probe after cooldown, reset on any success. Stops the def murder loop (57 restarts / 155 API FAILs for an account-layer failure). - tests/test_fleet_status.py (25), tests/test_agent_health.py (6). - CHROMEBOX-RUNBOOK.md: blind-shell status + futile-restart sections. Tests: 98/98 focused green (agent_health + fleet_status + completion + tool_calls). Live-verified: 4 ACTIVE [*] + 2 UNKNOWN.
281 lines
11 KiB
Python
281 lines
11 KiB
Python
"""Tests for host-side fleet evidence (bin/host_evidence.py).
|
|
|
|
Covers log-line classification, verdict assembly (silent-when-healthy
|
|
runs vs. fresh failures vs. stale/missing runs), and the effective
|
|
status mapping used by `box fleet status` in PID/net-blind shells.
|
|
"""
|
|
import importlib.util
|
|
import unittest
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
|
|
def _load(mod_name, rel_path):
|
|
spec = importlib.util.spec_from_file_location(mod_name, REPO_ROOT / rel_path)
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
return mod
|
|
|
|
|
|
he = _load("host_evidence_test", "bin/host_evidence.py")
|
|
|
|
NOW = datetime(2026, 10, 6, 8, 40, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
def _ts(dt):
|
|
return dt.strftime("[%Y-%m-%dT%H:%M:%SZ]")
|
|
|
|
|
|
def _run(minutes_ago):
|
|
return NOW - timedelta(minutes=minutes_ago)
|
|
|
|
|
|
class ParseLogTs(unittest.TestCase):
|
|
def test_valid(self):
|
|
self.assertEqual(
|
|
he.parse_log_ts("[2026-10-06T05:55:25Z] [opm] relay restart FAILED"),
|
|
datetime(2026, 10, 6, 5, 55, 25, tzinfo=timezone.utc))
|
|
|
|
def test_garbage(self):
|
|
self.assertIsNone(he.parse_log_ts("cdp relay already running"))
|
|
self.assertIsNone(he.parse_log_ts(""))
|
|
self.assertIsNone(he.parse_log_ts("[2026-13-99T99:99:99Z] [muse] x"))
|
|
|
|
|
|
class ClassifyChromebox(unittest.TestCase):
|
|
def test_down(self):
|
|
self.assertEqual(
|
|
he.classify_chromebox_line("[muse] relaunch FAILED (x) — needs operator attention"),
|
|
"down")
|
|
|
|
def test_healthy(self):
|
|
self.assertEqual(
|
|
he.classify_chromebox_line("[muse] relaunch OK (attempt 2)"), "healthy")
|
|
self.assertEqual(
|
|
he.classify_chromebox_line(
|
|
"[muse] tunnel restart recovered warp egress, chrome relaunch not needed"),
|
|
"healthy")
|
|
|
|
def test_degraded(self):
|
|
for line in (
|
|
"[muse] relaunch attempt 3 not healthy yet (CDP up but no page)",
|
|
"[muse] tunnel restart did not recover egress (x), proceeding with chrome relaunch",
|
|
"[muse] unhealthy (x), relaunching chromebox",
|
|
"[muse] warp partition detected (x), restarting tunnel via netvm-node-up.sh",
|
|
"[muse] browser launched recently (pid 1), skipping relaunch (probably still starting)",
|
|
):
|
|
self.assertEqual(he.classify_chromebox_line(line), "degraded", line)
|
|
|
|
def test_no_verdict(self):
|
|
for line in (
|
|
"[2026-10-06T08:00:00Z] [muse] log rotated",
|
|
"cdp relay already running",
|
|
"tunnel already up (egress=1.2.3.4), skipping handshake wait",
|
|
"node=muse netns=warp-muse egress=1.2.3.4",
|
|
"Traceback (most recent call last):",
|
|
):
|
|
self.assertIsNone(he.classify_chromebox_line(line), line)
|
|
|
|
|
|
class ClassifyRelay(unittest.TestCase):
|
|
def test_down(self):
|
|
self.assertEqual(
|
|
he.classify_relay_line("[opm] relay restart FAILED on 10.201.157.2:9440 — needs operator"),
|
|
"down")
|
|
self.assertEqual(
|
|
he.classify_relay_line("[pip] FAIL_LOUD: host veth ve-x missing IP 10.201.87.1"),
|
|
"down")
|
|
# Trailing "restarting" with no OK/FAILED after it: crashed mid-restart.
|
|
self.assertEqual(
|
|
he.classify_relay_line("[muse] relay unhealthy on 10.201.35.2:9410, restarting"),
|
|
"down")
|
|
|
|
def test_healthy(self):
|
|
self.assertEqual(
|
|
he.classify_relay_line("[646] relay restarted OK on 10.201.202.2:9430"),
|
|
"healthy")
|
|
|
|
def test_no_verdict(self):
|
|
self.assertIsNone(he.classify_relay_line(" File \"/x/netvm-cdp-relay.py\", line 37, in main"))
|
|
self.assertIsNone(he.classify_relay_line(""))
|
|
|
|
|
|
class LastVerdict(unittest.TestCase):
|
|
def test_newest_wins_per_node(self):
|
|
lines = [
|
|
"%s [muse] relaunch OK (attempt 1)" % _ts(_run(30)),
|
|
"%s [pip] relaunch FAILED (x)" % _ts(_run(20)),
|
|
"%s [muse] unhealthy (x), relaunching chromebox" % _ts(_run(5)),
|
|
"%s [muse] log rotated" % _ts(_run(1)),
|
|
]
|
|
v, ts, _ = he.last_verdict(lines, "muse", he.classify_chromebox_line)
|
|
self.assertEqual(v, "degraded")
|
|
self.assertEqual(ts, _run(5))
|
|
v, _, _ = he.last_verdict(lines, "pip", he.classify_chromebox_line)
|
|
self.assertEqual(v, "down")
|
|
|
|
def test_none_when_no_verdict(self):
|
|
self.assertIsNone(he.last_verdict(
|
|
["[2026-10-06T08:00:00Z] [muse] log rotated", "noise"],
|
|
"muse", he.classify_chromebox_line))
|
|
self.assertIsNone(he.last_verdict([], "muse", he.classify_chromebox_line))
|
|
|
|
|
|
class BrowserVerdict(unittest.TestCase):
|
|
def test_silent_run_is_healthy(self):
|
|
v, d = he.browser_verdict("muse", [], _run(2), now=NOW)
|
|
self.assertEqual(v, "healthy")
|
|
v, _ = he.browser_verdict(
|
|
"muse", ["%s [muse] relaunch FAILED (x)" % _ts(_run(60))],
|
|
_run(2), now=NOW)
|
|
self.assertEqual(v, "healthy") # old failure, silent since
|
|
|
|
def test_fresh_failure_wins(self):
|
|
v, _ = he.browser_verdict(
|
|
"muse", ["%s [muse] relaunch FAILED (x)" % _ts(_run(1))],
|
|
_run(2), now=NOW)
|
|
self.assertEqual(v, "down")
|
|
v, _ = he.browser_verdict(
|
|
"muse", ["%s [muse] unhealthy (x), relaunching chromebox" % _ts(_run(1))],
|
|
_run(2), now=NOW)
|
|
self.assertEqual(v, "degraded")
|
|
|
|
def test_fresh_recovery_is_healthy(self):
|
|
v, _ = he.browser_verdict(
|
|
"muse", ["%s [muse] tunnel restart recovered warp egress, chrome relaunch not needed"
|
|
% _ts(_run(1))],
|
|
_run(2), now=NOW)
|
|
self.assertEqual(v, "healthy")
|
|
|
|
def test_stale_or_missing_run_is_unknown(self):
|
|
v, _ = he.browser_verdict("muse", [], _run(30), now=NOW)
|
|
self.assertEqual(v, "unknown")
|
|
v, _ = he.browser_verdict("muse", [], None, now=NOW)
|
|
self.assertEqual(v, "unknown")
|
|
|
|
def test_def_uses_watchdog_like_core_nodes(self):
|
|
v, _ = he.browser_verdict("def", [], _run(2), now=NOW)
|
|
self.assertEqual(v, "healthy") # silent run
|
|
v, _ = he.browser_verdict("def", [], None, now=NOW)
|
|
self.assertEqual(v, "unknown") # no timer run (yet)
|
|
v, _ = he.browser_verdict(
|
|
"def", ["%s [def] relaunch FAILED (x)" % _ts(_run(1))],
|
|
_run(2), now=NOW)
|
|
self.assertEqual(v, "down")
|
|
|
|
def test_uncovered_node_uses_chrome_log_freshness(self):
|
|
v, _ = he.browser_verdict("ghost", [], None,
|
|
chrome_log_mtime=_run(5), now=NOW)
|
|
self.assertEqual(v, "healthy")
|
|
v, d = he.browser_verdict("ghost", [], None,
|
|
chrome_log_mtime=_run(60), now=NOW)
|
|
self.assertEqual(v, "unknown")
|
|
v, _ = he.browser_verdict("ghost", [], None,
|
|
chrome_log_mtime=None, now=NOW)
|
|
self.assertEqual(v, "unknown")
|
|
|
|
|
|
class CdpVerdict(unittest.TestCase):
|
|
def test_silent_run_is_healthy(self):
|
|
v, _ = he.cdp_verdict("opm", [], _run(4), now=NOW)
|
|
self.assertEqual(v, "healthy")
|
|
|
|
def test_fresh_failure_is_down(self):
|
|
v, _ = he.cdp_verdict(
|
|
"opm", ["%s [opm] relay restart FAILED on 10.201.157.2:9440" % _ts(_run(1))],
|
|
_run(4), now=NOW)
|
|
self.assertEqual(v, "down")
|
|
|
|
def test_stale_or_missing_run_is_unknown(self):
|
|
v, _ = he.cdp_verdict("opm", [], _run(60), now=NOW)
|
|
self.assertEqual(v, "unknown")
|
|
v, _ = he.cdp_verdict("opm", [], None, now=NOW)
|
|
self.assertEqual(v, "unknown")
|
|
|
|
def test_dev_def_monitored_like_core_nodes(self):
|
|
for node in ("def", "dev"):
|
|
v, _ = he.cdp_verdict(node, [], _run(4), now=NOW)
|
|
self.assertEqual(v, "healthy", node) # silent run
|
|
v, _ = he.cdp_verdict(
|
|
node,
|
|
["%s [%s] relay restart FAILED on 10.0.0.1:1" % (_ts(_run(1)), node)],
|
|
_run(4), now=NOW)
|
|
self.assertEqual(v, "down", node)
|
|
|
|
def test_unmonitored_node_is_unknown(self):
|
|
v, d = he.cdp_verdict("ghost", [], _run(1), now=NOW)
|
|
self.assertEqual(v, "unknown")
|
|
self.assertIn("no relay-monitor coverage", d)
|
|
|
|
|
|
class EffectiveStatus(unittest.TestCase):
|
|
def test_live_local_signals_win(self):
|
|
self.assertEqual(he.effective_status(True, True, "down", "down"),
|
|
("ACTIVE", "local"))
|
|
self.assertEqual(he.effective_status(True, False, "healthy", "healthy"),
|
|
("CDP_DOWN", "local"))
|
|
|
|
def test_cdp_without_proc_is_active(self):
|
|
self.assertEqual(he.effective_status(False, True, "unknown", "unknown"),
|
|
("ACTIVE", "local"))
|
|
|
|
def test_blind_with_evidence(self):
|
|
self.assertEqual(
|
|
he.effective_status(False, False, "healthy", "healthy"),
|
|
("ACTIVE", "host-evidence"))
|
|
self.assertEqual(
|
|
he.effective_status(False, False, "degraded", "healthy"),
|
|
("ACTIVE", "host-evidence"))
|
|
self.assertEqual(
|
|
he.effective_status(False, False, "down", "healthy"),
|
|
("STOPPED", "host-evidence"))
|
|
self.assertEqual(
|
|
he.effective_status(False, False, "healthy", "down"),
|
|
("CDP_DOWN", "host-evidence"))
|
|
|
|
def test_blind_without_evidence_is_unknown(self):
|
|
self.assertEqual(
|
|
he.effective_status(False, False, "unknown", "unknown"),
|
|
("UNKNOWN", "host-evidence"))
|
|
self.assertEqual(
|
|
he.effective_status(False, False, "healthy", "unknown"),
|
|
("UNKNOWN", "host-evidence"))
|
|
self.assertEqual(
|
|
he.effective_status(False, False, "unknown", "healthy"),
|
|
("UNKNOWN", "host-evidence"))
|
|
|
|
|
|
class CollectSeams(unittest.TestCase):
|
|
def test_assembles_per_node_verdicts(self):
|
|
starts = {"cdp-relay-watchdog.service": _run(4),
|
|
"chromebox-watchdog@muse.service": _run(1),
|
|
"chromebox-watchdog@def.service": _run(1)}
|
|
relay_lines = ["%s [muse] relay restarted OK on 10.201.35.2:9410" % _ts(_run(3))]
|
|
chrome_lines = ["%s [muse] relaunch OK (attempt 1)" % _ts(_run(1))]
|
|
out = he.collect(["muse", "def"],
|
|
_journal_starts=starts,
|
|
_relay_lines=relay_lines,
|
|
_chromebox_lines=chrome_lines,
|
|
_chrome_mtimes={"def": _run(5)},
|
|
_now=NOW)
|
|
self.assertEqual(out["muse"]["browser"], "healthy")
|
|
self.assertEqual(out["muse"]["cdp"], "healthy")
|
|
# def is watchdog-covered: silent runs read healthy (chrome-log
|
|
# freshness is no longer consulted for covered nodes).
|
|
self.assertEqual(out["def"]["browser"], "healthy")
|
|
self.assertEqual(out["def"]["cdp"], "healthy")
|
|
|
|
|
|
class CoverageTuples(unittest.TestCase):
|
|
def test_watchdog_tuples_cover_registry(self):
|
|
reg = _load("netvm_registry_cov", "bin/netvm-registry.py")
|
|
for node in reg.active_nodes():
|
|
self.assertIn(node, he.RELAY_NODES)
|
|
self.assertIn(node, he.CHROMEBOX_NODES)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|