fix: truthful fleet status in blind shells + agent-health circuit breaker
box fleet status / approvals check misreported every node as STOPPED / CDP-unreachable from sandboxed shells (own PID+net namespaces: pgrep blind, no route to 10.201.x.x, no sudo). Fleet was healthy throughout. - bin/host_evidence.py (new): host watchdog evidence fallback. Recent timer runs (journal -o json, exact UNIT match) with no newer failure line in cdp-relay-watchdog.log / chromebox-watchdog.log (both silent-when-healthy) prove a node is up. def/dev have no watchdog coverage: browser verdict via chromebox-<node>.log freshness (alive-only), CDP verdict unknown. - super-cli.py: effective status/source/evidence per node. Host evidence decides ONLY the fully-blind pattern (both local probes negative); live local signals always win. New UNKNOWN badge, [*] footnote; approvals UNREACHABLE splits into BLIND / OFFLINE(host agrees) / unreachable-evidence-inconclusive, with honest footer. proc_alive/cdp_ok keep local-probe meaning; status/source/evidence are new JSON fields. - approvals.py: host_cdp_ok flag on the unreachable path. - agent-health.sh: restart circuit breaker. 3 consecutive futile restarts (restart leaves agent still failing) opens the circuit: no more kills for 1800s, ALERT to log+journal, half-open probe after cooldown, reset on any success. Stops the def murder loop (57 restarts / 155 API FAILs for an account-layer failure). - tests/test_fleet_status.py (25), tests/test_agent_health.py (6). - CHROMEBOX-RUNBOOK.md: blind-shell status + futile-restart sections. Tests: 98/98 focused green (agent_health + fleet_status + completion + tool_calls). Live-verified: 4 ACTIVE [*] + 2 UNKNOWN.
This commit is contained in:
Muse Sidechat
+80
-6
@@ -252,6 +252,53 @@ def get_queue_depth(node: str) -> int:
|
||||
# ---------------------------------------------------------------------------
|
||||
# Domain: FLEET
|
||||
# ---------------------------------------------------------------------------
|
||||
def _host_evidence_for(results):
|
||||
"""Host watchdog evidence, or None when unneeded/unavailable.
|
||||
|
||||
Queried only when at least one node is fully blind (both local
|
||||
probes negative). Never raises: evidence must not break status.
|
||||
"""
|
||||
if not any(not r["proc_alive"] and not r["cdp_ok"] for r in results):
|
||||
return None
|
||||
try:
|
||||
import host_evidence
|
||||
return host_evidence.collect([r["node"] for r in results])
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_statuses(results) -> None:
|
||||
"""Attach effective status/source/evidence to each collected node.
|
||||
|
||||
proc_alive/cdp_ok keep their local-probe meaning; status is the
|
||||
display verdict. Host evidence only decides the fully-blind pattern
|
||||
(both local probes negative); live local signals always win.
|
||||
"""
|
||||
try:
|
||||
import host_evidence
|
||||
have_mod = True
|
||||
except ImportError:
|
||||
have_mod = False
|
||||
ev_map = _host_evidence_for(results) if have_mod else None
|
||||
for r in results:
|
||||
ev = (ev_map or {}).get(r["node"]) if ev_map else None
|
||||
if not r["proc_alive"] and r["cdp_ok"]:
|
||||
# CDP answers: the browser is definitionally alive even when
|
||||
# pgrep cannot see it (PID-blind shell, renamed profile path).
|
||||
r["status"], r["source"] = "ACTIVE", "local"
|
||||
elif r["proc_alive"] and not r["cdp_ok"]:
|
||||
r["status"], r["source"] = "CDP_DOWN", "local"
|
||||
elif r["proc_alive"] and r["cdp_ok"]:
|
||||
r["status"], r["source"] = "ACTIVE", "local"
|
||||
elif ev is None:
|
||||
r["status"], r["source"] = "STOPPED", "local"
|
||||
else:
|
||||
r["status"], r["source"] = host_evidence.effective_status(
|
||||
False, False, ev.get("browser", "unknown"),
|
||||
ev.get("cdp", "unknown"))
|
||||
r["evidence"] = ev
|
||||
|
||||
|
||||
def collect_fleet_data() -> list:
|
||||
results = []
|
||||
for node in VALID_NODES:
|
||||
@@ -287,6 +334,7 @@ def collect_fleet_data() -> list:
|
||||
"approval_pending": approval_pending,
|
||||
"approval_detail": approval_detail,
|
||||
})
|
||||
_resolve_statuses(results)
|
||||
return results
|
||||
|
||||
def cmd_fleet_status(args):
|
||||
@@ -301,14 +349,16 @@ def cmd_fleet_status(args):
|
||||
rows = []
|
||||
has_any_approval = False
|
||||
for item in data:
|
||||
# Status calculation
|
||||
# Status calculation (effective status; see _resolve_statuses)
|
||||
if item.get("approval_pending"):
|
||||
status = badge_warn("APPROVAL_REQ")
|
||||
has_any_approval = True
|
||||
elif item["proc_alive"] and item["cdp_ok"]:
|
||||
elif item.get("status") == "ACTIVE":
|
||||
status = badge_ok("ACTIVE")
|
||||
elif item["proc_alive"] and not item["cdp_ok"]:
|
||||
elif item.get("status") == "CDP_DOWN":
|
||||
status = badge_warn("CDP_DOWN")
|
||||
elif item.get("status") == "UNKNOWN":
|
||||
status = badge_dim("UNKNOWN")
|
||||
else:
|
||||
status = badge_err("STOPPED")
|
||||
|
||||
@@ -338,6 +388,8 @@ def cmd_fleet_status(args):
|
||||
])
|
||||
|
||||
print_table(headers, rows)
|
||||
if any(r.get("source") == "host-evidence" for r in data):
|
||||
print(c_dim(" [*] status via host watchdog evidence (local probes blind in this shell)"))
|
||||
if has_any_approval:
|
||||
print("\n" + c_yellow(" ⚠ Agent(s) held up on browser approval. Run 'box approvals' to inspect/allow."))
|
||||
print("\n" + c_dim(" Commands: super fleet watch | super fleet restart <node> | box approvals [check|allow|auto]") + "\n")
|
||||
@@ -446,9 +498,16 @@ def cmd_approvals(args):
|
||||
btns = c_dim("answer in task")
|
||||
input_wait_count += 1
|
||||
elif st == "UNREACHABLE":
|
||||
badge = badge_dim("OFFLINE")
|
||||
if it.get("host_cdp_ok") is True:
|
||||
badge = badge_dim("BLIND")
|
||||
purp = c_dim("CDP ok on host; blind here")
|
||||
elif it.get("host_cdp_ok") is False:
|
||||
badge = badge_err("OFFLINE")
|
||||
purp = c_dim("CDP down (host agrees)")
|
||||
else:
|
||||
badge = badge_dim("OFFLINE")
|
||||
purp = c_dim("CDP unreachable")
|
||||
target = "-"
|
||||
purp = c_dim("CDP unreachable")
|
||||
trust = "-"
|
||||
btns = "-"
|
||||
elif st == "ERROR":
|
||||
@@ -478,7 +537,22 @@ def cmd_approvals(args):
|
||||
print(c_cyan(" All pending approvals are for trusted infrastructure. Run 'box approvals auto' to resolve."))
|
||||
print()
|
||||
elif input_wait_count == 0:
|
||||
print("\n" + c_green(" ✔ All agent approval queues clear. No agents blocked.") + "\n")
|
||||
unreach = [it for it in fleet if it.get("status") == "UNREACHABLE"]
|
||||
blind_ok = [it["node"] for it in unreach
|
||||
if it.get("host_cdp_ok") is True]
|
||||
blind_unknown = [it["node"] for it in unreach
|
||||
if it.get("host_cdp_ok") is None]
|
||||
if blind_ok:
|
||||
print("\n" + c_dim(" ? %d node(s) blind from this shell "
|
||||
"(CDP ok on host); queues unverified: %s."
|
||||
% (len(blind_ok), ", ".join(blind_ok))) + "\n")
|
||||
if blind_unknown:
|
||||
print("\n" + c_dim(" ? %d node(s) unreachable, host evidence "
|
||||
"inconclusive: %s."
|
||||
% (len(blind_unknown),
|
||||
", ".join(blind_unknown))) + "\n")
|
||||
if not blind_ok and not blind_unknown:
|
||||
print("\n" + c_green(" ✔ All agent approval queues clear. No agents blocked.") + "\n")
|
||||
|
||||
# Verbose or inspect breakdown
|
||||
is_verbose = getattr(args, "verbose", False) or action == "inspect"
|
||||
|
||||
Reference in New Issue
Block a user