feat(work): add pre-flight health gate for Hatch, Restore, and Git Config

This commit is contained in:
operator
2026-10-09 23:02:33 +00:00
parent 6d4909e1be
commit a6e5f565f5
3 changed files with 221 additions and 1 deletions
+200 -1
View File
@@ -219,7 +219,167 @@ def check_tunnel_ports():
s.close()
return ports_status
def cmd_status(args):
def badge_status(status: str) -> str:
if status == "PASS":
return c_green("PASS")
elif status == "WARN":
return c_yellow("WARN")
else:
return c_red("FAIL")
def check_agent_preflight(agent_name: str) -> dict:
"""Ensures hatch, restore, and git config health before assigning work to cloud muse agents."""
worker = next((w for w in WORKERS if w["name"] == agent_name), None)
if not worker and agent_name != "super":
return {
"agent": agent_name,
"port": 0,
"hatch": {"status": "FAIL", "details": f"Unknown agent '{agent_name}'"},
"restore": {"status": "FAIL", "details": "Not listed in fleet topology"},
"git": {"status": "FAIL", "details": "No partition entry"},
"overall": "FAIL",
"ready": False,
"reasons": [f"Agent '{agent_name}' is not in fleet topology"]
}
port = worker["port"] if worker else 2224
tunnel_ports = check_tunnel_ports()
port_status = tunnel_ports.get(port, "DARK")
reasons = []
# 1. HATCH HEALTH (tunnel listener + responsive chat)
hatch_status = "PASS"
hatch_details = []
if port_status == "UP":
hatch_details.append(f"Port {port} listener UP")
else:
hatch_status = "FAIL"
hatch_details.append(f"Port {port} reverse tunnel DARK")
reasons.append(f"Hatch tunnel is DOWN on port {port}. Container is offline or unreachable.")
last_chats = get_last_agent_chats()
chat_ev = last_chats.get(agent_name)
if chat_ev:
ts_str = chat_ev.get("ts", "")[:19].replace("T", " ")
hatch_details.append(f"Chat active ({ts_str})")
else:
hatch_details.append("No recent chat entries")
# 2. RESTORE HEALTH (NODES.md, supervisor persistence)
restore_status = "PASS"
restore_details = []
nodes_file = REPO_ROOT / "NODES.md"
node_in_registry = False
if nodes_file.exists():
try:
with open(nodes_file) as f:
content = f.read()
if f"| {agent_name} |" in content or f"warp-{agent_name}" in content:
node_in_registry = True
except Exception:
pass
if node_in_registry or agent_name in ("muse-main", "super"):
restore_details.append("Registered in NODES.md")
else:
restore_status = "WARN"
restore_details.append("Not found in NODES.md")
if port_status == "UP":
restore_details.append("Watchdog/Supervisor persistent")
else:
restore_status = "FAIL"
restore_details.append("Container rebuild / tunnel recovery pending")
reasons.append("Container requires recovery/restore (run recover-after-rebuild or inspect watchdog).")
# 3. GIT CONFIG HEALTH (partition token, collaborator access, branches)
git_status = "PASS"
git_details = []
token = ""
if PARTITION_TABLE_PATH.exists():
try:
with open(PARTITION_TABLE_PATH) as f:
pt = json.load(f)
contributor = pt.get("contributors", {}).get(agent_name)
if contributor:
token = contributor.get("token", "")
git_details.append("Token in partition-table")
else:
git_status = "FAIL"
git_details.append("Missing from partition-table")
reasons.append(f"Agent '{agent_name}' has no credentials in fleet/partition-table.json")
except Exception as e:
git_status = "WARN"
git_details.append(f"Partition table error: {e}")
collab_check = gitea_api_request(f"/repos/super/box/collaborators/{agent_name}")
if isinstance(collab_check, dict) and collab_check.get("error") and collab_check.get("error") not in (200, 204):
git_status = "FAIL"
git_details.append("Not a repository collaborator")
reasons.append(f"Gitea user '{agent_name}' lacks write/collaborator access")
else:
git_details.append("Gitea collaborator OK")
branches = gitea_api_request("/repos/super/box/branches")
agent_branch = False
if isinstance(branches, list):
for b in branches:
bname = b.get("name", "")
if bname.startswith(f"dev/{agent_name}/") or bname.startswith(f"builder/{agent_name}/"):
agent_branch = True
break
if agent_branch:
git_details.append("Branch verified in Gitea")
else:
git_details.append("No active branch")
overall = "PASS"
if hatch_status == "FAIL" or restore_status == "FAIL" or git_status == "FAIL":
overall = "FAIL"
elif hatch_status == "WARN" or restore_status == "WARN" or git_status == "WARN":
overall = "WARN"
return {
"agent": agent_name,
"port": port,
"hatch": {"status": hatch_status, "details": ", ".join(hatch_details)},
"restore": {"status": restore_status, "details": ", ".join(restore_details)},
"git": {"status": git_status, "details": ", ".join(git_details)},
"overall": overall,
"ready": (overall != "FAIL"),
"reasons": reasons
}
def cmd_check(args):
target_agent = getattr(args, "agent", None)
targets = [target_agent] if target_agent else [w["name"] for w in WORKERS if w["role"] != "host"]
print(c_bold("\n=== BOX WORK: PRE-FLIGHT HEALTH VERIFICATION ===\n"))
header = f"{'AGENT':<12} {'HATCH':<12} {'RESTORE':<12} {'GIT CONFIG':<12} {'STATUS'}"
print(c_dim(header))
print(c_dim("-" * len(header)))
for ag in targets:
res = check_agent_preflight(ag)
h_badge = badge_status(res["hatch"]["status"])
r_badge = badge_status(res["restore"]["status"])
g_badge = badge_status(res["git"]["status"])
overall_badge = c_green("🟢 READY") if res["ready"] else c_red("🔴 BLOCKED")
print(f"{c_bold(ag):<21} {h_badge:<21} {r_badge:<21} {g_badge:<21} {overall_badge}")
print()
blocked = [ag for ag in targets if not check_agent_preflight(ag)["ready"]]
if blocked:
print(c_bold("--- PRE-FLIGHT DIAGNOSTIC DETAILS ---"))
for ag in blocked:
res = check_agent_preflight(ag)
print(f" {c_bold(ag)}:")
print(f" • Hatch: {res['hatch']['details']}")
print(f" • Restore: {res['restore']['details']}")
print(f" • Git: {res['git']['details']}")
for r in res["reasons"]:
print(f" {c_yellow('!')} {r}")
print()
api_base, _ = get_gitea_config()
print(c_bold(f"\n=== BOX WORK: FLEET & BUILD PIPELINE ({api_base}) ===\n"))
@@ -390,6 +550,23 @@ def cmd_start(args):
agent = args.agent
body = args.goal or f"Work task for {agent}: {title}"
# 0. Pre-flight health gate: Hatch, Restore, Git Config
preflight = check_agent_preflight(agent)
if not preflight["ready"] and not getattr(args, "force", False):
print(c_red(f"\n[BLOCKED] Agent '{agent}' failed pre-flight health verification:"))
print(f" • Hatch: {badge_status(preflight['hatch']['status'])} - {preflight['hatch']['details']}")
print(f" • Restore: {badge_status(preflight['restore']['status'])} - {preflight['restore']['details']}")
print(f" • Git: {badge_status(preflight['git']['status'])} - {preflight['git']['details']}")
print(c_yellow("\nBlocking reasons:"))
for r in preflight["reasons"]:
print(f" - {r}")
print(c_dim(f"\nTo inspect full health: box work check {agent}\nTo bypass pre-flight: box work start '{title}' --to {agent} --force\n"))
sys.exit(1)
elif not preflight["ready"] and getattr(args, "force", False):
print(c_yellow(f"[WARNING] Overriding failed pre-flight checks on {agent} (--force specified).\n"))
else:
print(c_green(f"✓ Pre-flight checks passed (Hatch: OK, Restore: OK, Git Config: OK) for {agent}"))
print(c_bold(f"Initiating work ticket for agent {agent}..."))
# 1. Ensure label exists in Gitea
@@ -443,6 +620,20 @@ def cmd_start(args):
def cmd_assign(args):
issue_num = args.issue
agent = args.agent
# 0. Pre-flight health gate: Hatch, Restore, Git Config
preflight = check_agent_preflight(agent)
if not preflight["ready"] and not getattr(args, "force", False):
print(c_red(f"\n[BLOCKED] Agent '{agent}' failed pre-flight health verification:"))
print(f" • Hatch: {badge_status(preflight['hatch']['status'])} - {preflight['hatch']['details']}")
print(f" • Restore: {badge_status(preflight['restore']['status'])} - {preflight['restore']['details']}")
print(f" • Git: {badge_status(preflight['git']['status'])} - {preflight['git']['details']}")
print(c_yellow("\nBlocking reasons:"))
for r in preflight["reasons"]:
print(f" - {r}")
print(c_dim(f"\nTo bypass pre-flight: box work assign {issue_num} --to {agent} --force\n"))
sys.exit(1)
print(c_bold(f"Assigning Ticket #{issue_num} to {agent}..."))
payload = {
@@ -501,14 +692,20 @@ def main():
sub.add_parser("status", help="Show full operational work dashboard")
# box work check [agent]
p_check = sub.add_parser("check", help="Run pre-flight health verification (Hatch, Restore, Git Config)")
p_check.add_argument("agent", nargs="?", help="Optional specific agent name to check")
p_start = sub.add_parser("start", help="Instantly start and assign new build ticket to an agent")
p_start.add_argument("title", help="Ticket title / summary")
p_start.add_argument("--to", dest="agent", required=True, help="Agent username (opm, 646, dev, pip, def, muse)")
p_start.add_argument("--goal", help="Optional detailed goal description")
p_start.add_argument("--force", action="store_true", help="Bypass pre-flight health gate")
p_assign = sub.add_parser("assign", help="Assign existing ticket to an agent")
p_assign.add_argument("issue", type=int, help="Issue number (e.g. 215)")
p_assign.add_argument("--to", dest="agent", required=True, help="Agent username")
p_assign.add_argument("--force", action="store_true", help="Bypass pre-flight health gate")
p_merge = sub.add_parser("merge", help="Merge an open PR into master")
p_merge.add_argument("pr", type=int, help="Pull request number (e.g. 214)")
@@ -522,6 +719,8 @@ def main():
if not action or action == "status":
cmd_status(args)
elif action == "check":
cmd_check(args)
elif action == "start":
cmd_start(args)
elif action == "assign":
+6
View File
@@ -1483,6 +1483,8 @@ def cmd_work(args):
box_work.cmd_assign(args)
elif action == "merge":
box_work.cmd_merge(args)
elif action == "check":
box_work.cmd_check(args)
elif action == "chats":
box_work.cmd_chats(args)
else:
@@ -6620,13 +6622,17 @@ def build_parser():
p_work = subparsers.add_parser("work", parents=[common], help="Fleet workspace, task orchestration, worker scope, and active signals")
work_sub = p_work.add_subparsers(dest="work_action")
p_w_status = work_sub.add_parser("status", parents=[common], help="Show full operational work dashboard (default)")
p_w_check = work_sub.add_parser("check", parents=[common], help="Run pre-flight health checks (Hatch, Restore, Git Config)")
p_w_check.add_argument("agent", nargs="?", help="Optional specific agent name to check")
p_w_start = work_sub.add_parser("start", parents=[common], help="Instantly start and assign new build ticket to an agent")
p_w_start.add_argument("title", help="Ticket title / summary")
p_w_start.add_argument("--to", dest="agent", required=True, help="Agent username (opm, 646, dev, pip, def, muse)")
p_w_start.add_argument("--goal", help="Optional detailed goal description")
p_w_start.add_argument("--force", action="store_true", help="Bypass pre-flight health gate")
p_w_assign = work_sub.add_parser("assign", parents=[common], help="Assign existing ticket to an agent")
p_w_assign.add_argument("issue", type=int, help="Issue number (e.g. 215)")
p_w_assign.add_argument("--to", dest="agent", required=True, help="Agent username")
p_w_assign.add_argument("--force", action="store_true", help="Bypass pre-flight health gate")
p_w_merge = work_sub.add_parser("merge", parents=[common], help="Merge an open PR into master")
p_w_merge.add_argument("pr", type=int, help="Pull request number (e.g. 214)")
p_w_chats = work_sub.add_parser("chats", parents=[common], help="View recent live chat activity")
+15
View File
@@ -43,5 +43,20 @@ class TestBoxWork(unittest.TestCase):
res = box_work.get_recent_done_tasks(limit=5)
self.assertIsInstance(res, list)
def test_check_agent_preflight_structure(self):
res = box_work.check_agent_preflight("opm")
self.assertIn("hatch", res)
self.assertIn("restore", res)
self.assertIn("git", res)
self.assertIn("status", res["hatch"])
self.assertIn("status", res["restore"])
self.assertIn("status", res["git"])
self.assertIn("ready", res)
def test_check_agent_preflight_unknown_fails(self):
res = box_work.check_agent_preflight("nonexistent_agent_xyz")
self.assertFalse(res["ready"])
self.assertEqual(res["overall"], "FAIL")
if __name__ == "__main__":
unittest.main()