feat(work): add pre-flight health gate for Hatch, Restore, and Git Config
This commit is contained in:
+200
-1
@@ -219,7 +219,167 @@ def check_tunnel_ports():
|
|||||||
s.close()
|
s.close()
|
||||||
return ports_status
|
return ports_status
|
||||||
|
|
||||||
def cmd_status(args):
|
def badge_status(status: str) -> str:
|
||||||
|
if status == "PASS":
|
||||||
|
return c_green("PASS")
|
||||||
|
elif status == "WARN":
|
||||||
|
return c_yellow("WARN")
|
||||||
|
else:
|
||||||
|
return c_red("FAIL")
|
||||||
|
|
||||||
|
def check_agent_preflight(agent_name: str) -> dict:
|
||||||
|
"""Ensures hatch, restore, and git config health before assigning work to cloud muse agents."""
|
||||||
|
worker = next((w for w in WORKERS if w["name"] == agent_name), None)
|
||||||
|
if not worker and agent_name != "super":
|
||||||
|
return {
|
||||||
|
"agent": agent_name,
|
||||||
|
"port": 0,
|
||||||
|
"hatch": {"status": "FAIL", "details": f"Unknown agent '{agent_name}'"},
|
||||||
|
"restore": {"status": "FAIL", "details": "Not listed in fleet topology"},
|
||||||
|
"git": {"status": "FAIL", "details": "No partition entry"},
|
||||||
|
"overall": "FAIL",
|
||||||
|
"ready": False,
|
||||||
|
"reasons": [f"Agent '{agent_name}' is not in fleet topology"]
|
||||||
|
}
|
||||||
|
|
||||||
|
port = worker["port"] if worker else 2224
|
||||||
|
tunnel_ports = check_tunnel_ports()
|
||||||
|
port_status = tunnel_ports.get(port, "DARK")
|
||||||
|
reasons = []
|
||||||
|
|
||||||
|
# 1. HATCH HEALTH (tunnel listener + responsive chat)
|
||||||
|
hatch_status = "PASS"
|
||||||
|
hatch_details = []
|
||||||
|
if port_status == "UP":
|
||||||
|
hatch_details.append(f"Port {port} listener UP")
|
||||||
|
else:
|
||||||
|
hatch_status = "FAIL"
|
||||||
|
hatch_details.append(f"Port {port} reverse tunnel DARK")
|
||||||
|
reasons.append(f"Hatch tunnel is DOWN on port {port}. Container is offline or unreachable.")
|
||||||
|
|
||||||
|
last_chats = get_last_agent_chats()
|
||||||
|
chat_ev = last_chats.get(agent_name)
|
||||||
|
if chat_ev:
|
||||||
|
ts_str = chat_ev.get("ts", "")[:19].replace("T", " ")
|
||||||
|
hatch_details.append(f"Chat active ({ts_str})")
|
||||||
|
else:
|
||||||
|
hatch_details.append("No recent chat entries")
|
||||||
|
|
||||||
|
# 2. RESTORE HEALTH (NODES.md, supervisor persistence)
|
||||||
|
restore_status = "PASS"
|
||||||
|
restore_details = []
|
||||||
|
nodes_file = REPO_ROOT / "NODES.md"
|
||||||
|
node_in_registry = False
|
||||||
|
if nodes_file.exists():
|
||||||
|
try:
|
||||||
|
with open(nodes_file) as f:
|
||||||
|
content = f.read()
|
||||||
|
if f"| {agent_name} |" in content or f"warp-{agent_name}" in content:
|
||||||
|
node_in_registry = True
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if node_in_registry or agent_name in ("muse-main", "super"):
|
||||||
|
restore_details.append("Registered in NODES.md")
|
||||||
|
else:
|
||||||
|
restore_status = "WARN"
|
||||||
|
restore_details.append("Not found in NODES.md")
|
||||||
|
|
||||||
|
if port_status == "UP":
|
||||||
|
restore_details.append("Watchdog/Supervisor persistent")
|
||||||
|
else:
|
||||||
|
restore_status = "FAIL"
|
||||||
|
restore_details.append("Container rebuild / tunnel recovery pending")
|
||||||
|
reasons.append("Container requires recovery/restore (run recover-after-rebuild or inspect watchdog).")
|
||||||
|
|
||||||
|
# 3. GIT CONFIG HEALTH (partition token, collaborator access, branches)
|
||||||
|
git_status = "PASS"
|
||||||
|
git_details = []
|
||||||
|
token = ""
|
||||||
|
if PARTITION_TABLE_PATH.exists():
|
||||||
|
try:
|
||||||
|
with open(PARTITION_TABLE_PATH) as f:
|
||||||
|
pt = json.load(f)
|
||||||
|
contributor = pt.get("contributors", {}).get(agent_name)
|
||||||
|
if contributor:
|
||||||
|
token = contributor.get("token", "")
|
||||||
|
git_details.append("Token in partition-table")
|
||||||
|
else:
|
||||||
|
git_status = "FAIL"
|
||||||
|
git_details.append("Missing from partition-table")
|
||||||
|
reasons.append(f"Agent '{agent_name}' has no credentials in fleet/partition-table.json")
|
||||||
|
except Exception as e:
|
||||||
|
git_status = "WARN"
|
||||||
|
git_details.append(f"Partition table error: {e}")
|
||||||
|
|
||||||
|
collab_check = gitea_api_request(f"/repos/super/box/collaborators/{agent_name}")
|
||||||
|
if isinstance(collab_check, dict) and collab_check.get("error") and collab_check.get("error") not in (200, 204):
|
||||||
|
git_status = "FAIL"
|
||||||
|
git_details.append("Not a repository collaborator")
|
||||||
|
reasons.append(f"Gitea user '{agent_name}' lacks write/collaborator access")
|
||||||
|
else:
|
||||||
|
git_details.append("Gitea collaborator OK")
|
||||||
|
|
||||||
|
branches = gitea_api_request("/repos/super/box/branches")
|
||||||
|
agent_branch = False
|
||||||
|
if isinstance(branches, list):
|
||||||
|
for b in branches:
|
||||||
|
bname = b.get("name", "")
|
||||||
|
if bname.startswith(f"dev/{agent_name}/") or bname.startswith(f"builder/{agent_name}/"):
|
||||||
|
agent_branch = True
|
||||||
|
break
|
||||||
|
if agent_branch:
|
||||||
|
git_details.append("Branch verified in Gitea")
|
||||||
|
else:
|
||||||
|
git_details.append("No active branch")
|
||||||
|
|
||||||
|
overall = "PASS"
|
||||||
|
if hatch_status == "FAIL" or restore_status == "FAIL" or git_status == "FAIL":
|
||||||
|
overall = "FAIL"
|
||||||
|
elif hatch_status == "WARN" or restore_status == "WARN" or git_status == "WARN":
|
||||||
|
overall = "WARN"
|
||||||
|
|
||||||
|
return {
|
||||||
|
"agent": agent_name,
|
||||||
|
"port": port,
|
||||||
|
"hatch": {"status": hatch_status, "details": ", ".join(hatch_details)},
|
||||||
|
"restore": {"status": restore_status, "details": ", ".join(restore_details)},
|
||||||
|
"git": {"status": git_status, "details": ", ".join(git_details)},
|
||||||
|
"overall": overall,
|
||||||
|
"ready": (overall != "FAIL"),
|
||||||
|
"reasons": reasons
|
||||||
|
}
|
||||||
|
|
||||||
|
def cmd_check(args):
|
||||||
|
target_agent = getattr(args, "agent", None)
|
||||||
|
targets = [target_agent] if target_agent else [w["name"] for w in WORKERS if w["role"] != "host"]
|
||||||
|
|
||||||
|
print(c_bold("\n=== BOX WORK: PRE-FLIGHT HEALTH VERIFICATION ===\n"))
|
||||||
|
header = f"{'AGENT':<12} {'HATCH':<12} {'RESTORE':<12} {'GIT CONFIG':<12} {'STATUS'}"
|
||||||
|
print(c_dim(header))
|
||||||
|
print(c_dim("-" * len(header)))
|
||||||
|
|
||||||
|
for ag in targets:
|
||||||
|
res = check_agent_preflight(ag)
|
||||||
|
h_badge = badge_status(res["hatch"]["status"])
|
||||||
|
r_badge = badge_status(res["restore"]["status"])
|
||||||
|
g_badge = badge_status(res["git"]["status"])
|
||||||
|
overall_badge = c_green("🟢 READY") if res["ready"] else c_red("🔴 BLOCKED")
|
||||||
|
print(f"{c_bold(ag):<21} {h_badge:<21} {r_badge:<21} {g_badge:<21} {overall_badge}")
|
||||||
|
|
||||||
|
print()
|
||||||
|
blocked = [ag for ag in targets if not check_agent_preflight(ag)["ready"]]
|
||||||
|
if blocked:
|
||||||
|
print(c_bold("--- PRE-FLIGHT DIAGNOSTIC DETAILS ---"))
|
||||||
|
for ag in blocked:
|
||||||
|
res = check_agent_preflight(ag)
|
||||||
|
print(f" {c_bold(ag)}:")
|
||||||
|
print(f" • Hatch: {res['hatch']['details']}")
|
||||||
|
print(f" • Restore: {res['restore']['details']}")
|
||||||
|
print(f" • Git: {res['git']['details']}")
|
||||||
|
for r in res["reasons"]:
|
||||||
|
print(f" {c_yellow('!')} {r}")
|
||||||
|
print()
|
||||||
api_base, _ = get_gitea_config()
|
api_base, _ = get_gitea_config()
|
||||||
print(c_bold(f"\n=== BOX WORK: FLEET & BUILD PIPELINE ({api_base}) ===\n"))
|
print(c_bold(f"\n=== BOX WORK: FLEET & BUILD PIPELINE ({api_base}) ===\n"))
|
||||||
|
|
||||||
@@ -390,6 +550,23 @@ def cmd_start(args):
|
|||||||
agent = args.agent
|
agent = args.agent
|
||||||
body = args.goal or f"Work task for {agent}: {title}"
|
body = args.goal or f"Work task for {agent}: {title}"
|
||||||
|
|
||||||
|
# 0. Pre-flight health gate: Hatch, Restore, Git Config
|
||||||
|
preflight = check_agent_preflight(agent)
|
||||||
|
if not preflight["ready"] and not getattr(args, "force", False):
|
||||||
|
print(c_red(f"\n[BLOCKED] Agent '{agent}' failed pre-flight health verification:"))
|
||||||
|
print(f" • Hatch: {badge_status(preflight['hatch']['status'])} - {preflight['hatch']['details']}")
|
||||||
|
print(f" • Restore: {badge_status(preflight['restore']['status'])} - {preflight['restore']['details']}")
|
||||||
|
print(f" • Git: {badge_status(preflight['git']['status'])} - {preflight['git']['details']}")
|
||||||
|
print(c_yellow("\nBlocking reasons:"))
|
||||||
|
for r in preflight["reasons"]:
|
||||||
|
print(f" - {r}")
|
||||||
|
print(c_dim(f"\nTo inspect full health: box work check {agent}\nTo bypass pre-flight: box work start '{title}' --to {agent} --force\n"))
|
||||||
|
sys.exit(1)
|
||||||
|
elif not preflight["ready"] and getattr(args, "force", False):
|
||||||
|
print(c_yellow(f"[WARNING] Overriding failed pre-flight checks on {agent} (--force specified).\n"))
|
||||||
|
else:
|
||||||
|
print(c_green(f"✓ Pre-flight checks passed (Hatch: OK, Restore: OK, Git Config: OK) for {agent}"))
|
||||||
|
|
||||||
print(c_bold(f"Initiating work ticket for agent {agent}..."))
|
print(c_bold(f"Initiating work ticket for agent {agent}..."))
|
||||||
|
|
||||||
# 1. Ensure label exists in Gitea
|
# 1. Ensure label exists in Gitea
|
||||||
@@ -443,6 +620,20 @@ def cmd_start(args):
|
|||||||
def cmd_assign(args):
|
def cmd_assign(args):
|
||||||
issue_num = args.issue
|
issue_num = args.issue
|
||||||
agent = args.agent
|
agent = args.agent
|
||||||
|
|
||||||
|
# 0. Pre-flight health gate: Hatch, Restore, Git Config
|
||||||
|
preflight = check_agent_preflight(agent)
|
||||||
|
if not preflight["ready"] and not getattr(args, "force", False):
|
||||||
|
print(c_red(f"\n[BLOCKED] Agent '{agent}' failed pre-flight health verification:"))
|
||||||
|
print(f" • Hatch: {badge_status(preflight['hatch']['status'])} - {preflight['hatch']['details']}")
|
||||||
|
print(f" • Restore: {badge_status(preflight['restore']['status'])} - {preflight['restore']['details']}")
|
||||||
|
print(f" • Git: {badge_status(preflight['git']['status'])} - {preflight['git']['details']}")
|
||||||
|
print(c_yellow("\nBlocking reasons:"))
|
||||||
|
for r in preflight["reasons"]:
|
||||||
|
print(f" - {r}")
|
||||||
|
print(c_dim(f"\nTo bypass pre-flight: box work assign {issue_num} --to {agent} --force\n"))
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
print(c_bold(f"Assigning Ticket #{issue_num} to {agent}..."))
|
print(c_bold(f"Assigning Ticket #{issue_num} to {agent}..."))
|
||||||
|
|
||||||
payload = {
|
payload = {
|
||||||
@@ -501,14 +692,20 @@ def main():
|
|||||||
|
|
||||||
sub.add_parser("status", help="Show full operational work dashboard")
|
sub.add_parser("status", help="Show full operational work dashboard")
|
||||||
|
|
||||||
|
# box work check [agent]
|
||||||
|
p_check = sub.add_parser("check", help="Run pre-flight health verification (Hatch, Restore, Git Config)")
|
||||||
|
p_check.add_argument("agent", nargs="?", help="Optional specific agent name to check")
|
||||||
|
|
||||||
p_start = sub.add_parser("start", help="Instantly start and assign new build ticket to an agent")
|
p_start = sub.add_parser("start", help="Instantly start and assign new build ticket to an agent")
|
||||||
p_start.add_argument("title", help="Ticket title / summary")
|
p_start.add_argument("title", help="Ticket title / summary")
|
||||||
p_start.add_argument("--to", dest="agent", required=True, help="Agent username (opm, 646, dev, pip, def, muse)")
|
p_start.add_argument("--to", dest="agent", required=True, help="Agent username (opm, 646, dev, pip, def, muse)")
|
||||||
p_start.add_argument("--goal", help="Optional detailed goal description")
|
p_start.add_argument("--goal", help="Optional detailed goal description")
|
||||||
|
p_start.add_argument("--force", action="store_true", help="Bypass pre-flight health gate")
|
||||||
|
|
||||||
p_assign = sub.add_parser("assign", help="Assign existing ticket to an agent")
|
p_assign = sub.add_parser("assign", help="Assign existing ticket to an agent")
|
||||||
p_assign.add_argument("issue", type=int, help="Issue number (e.g. 215)")
|
p_assign.add_argument("issue", type=int, help="Issue number (e.g. 215)")
|
||||||
p_assign.add_argument("--to", dest="agent", required=True, help="Agent username")
|
p_assign.add_argument("--to", dest="agent", required=True, help="Agent username")
|
||||||
|
p_assign.add_argument("--force", action="store_true", help="Bypass pre-flight health gate")
|
||||||
|
|
||||||
p_merge = sub.add_parser("merge", help="Merge an open PR into master")
|
p_merge = sub.add_parser("merge", help="Merge an open PR into master")
|
||||||
p_merge.add_argument("pr", type=int, help="Pull request number (e.g. 214)")
|
p_merge.add_argument("pr", type=int, help="Pull request number (e.g. 214)")
|
||||||
@@ -522,6 +719,8 @@ def main():
|
|||||||
|
|
||||||
if not action or action == "status":
|
if not action or action == "status":
|
||||||
cmd_status(args)
|
cmd_status(args)
|
||||||
|
elif action == "check":
|
||||||
|
cmd_check(args)
|
||||||
elif action == "start":
|
elif action == "start":
|
||||||
cmd_start(args)
|
cmd_start(args)
|
||||||
elif action == "assign":
|
elif action == "assign":
|
||||||
|
|||||||
@@ -1483,6 +1483,8 @@ def cmd_work(args):
|
|||||||
box_work.cmd_assign(args)
|
box_work.cmd_assign(args)
|
||||||
elif action == "merge":
|
elif action == "merge":
|
||||||
box_work.cmd_merge(args)
|
box_work.cmd_merge(args)
|
||||||
|
elif action == "check":
|
||||||
|
box_work.cmd_check(args)
|
||||||
elif action == "chats":
|
elif action == "chats":
|
||||||
box_work.cmd_chats(args)
|
box_work.cmd_chats(args)
|
||||||
else:
|
else:
|
||||||
@@ -6620,13 +6622,17 @@ def build_parser():
|
|||||||
p_work = subparsers.add_parser("work", parents=[common], help="Fleet workspace, task orchestration, worker scope, and active signals")
|
p_work = subparsers.add_parser("work", parents=[common], help="Fleet workspace, task orchestration, worker scope, and active signals")
|
||||||
work_sub = p_work.add_subparsers(dest="work_action")
|
work_sub = p_work.add_subparsers(dest="work_action")
|
||||||
p_w_status = work_sub.add_parser("status", parents=[common], help="Show full operational work dashboard (default)")
|
p_w_status = work_sub.add_parser("status", parents=[common], help="Show full operational work dashboard (default)")
|
||||||
|
p_w_check = work_sub.add_parser("check", parents=[common], help="Run pre-flight health checks (Hatch, Restore, Git Config)")
|
||||||
|
p_w_check.add_argument("agent", nargs="?", help="Optional specific agent name to check")
|
||||||
p_w_start = work_sub.add_parser("start", parents=[common], help="Instantly start and assign new build ticket to an agent")
|
p_w_start = work_sub.add_parser("start", parents=[common], help="Instantly start and assign new build ticket to an agent")
|
||||||
p_w_start.add_argument("title", help="Ticket title / summary")
|
p_w_start.add_argument("title", help="Ticket title / summary")
|
||||||
p_w_start.add_argument("--to", dest="agent", required=True, help="Agent username (opm, 646, dev, pip, def, muse)")
|
p_w_start.add_argument("--to", dest="agent", required=True, help="Agent username (opm, 646, dev, pip, def, muse)")
|
||||||
p_w_start.add_argument("--goal", help="Optional detailed goal description")
|
p_w_start.add_argument("--goal", help="Optional detailed goal description")
|
||||||
|
p_w_start.add_argument("--force", action="store_true", help="Bypass pre-flight health gate")
|
||||||
p_w_assign = work_sub.add_parser("assign", parents=[common], help="Assign existing ticket to an agent")
|
p_w_assign = work_sub.add_parser("assign", parents=[common], help="Assign existing ticket to an agent")
|
||||||
p_w_assign.add_argument("issue", type=int, help="Issue number (e.g. 215)")
|
p_w_assign.add_argument("issue", type=int, help="Issue number (e.g. 215)")
|
||||||
p_w_assign.add_argument("--to", dest="agent", required=True, help="Agent username")
|
p_w_assign.add_argument("--to", dest="agent", required=True, help="Agent username")
|
||||||
|
p_w_assign.add_argument("--force", action="store_true", help="Bypass pre-flight health gate")
|
||||||
p_w_merge = work_sub.add_parser("merge", parents=[common], help="Merge an open PR into master")
|
p_w_merge = work_sub.add_parser("merge", parents=[common], help="Merge an open PR into master")
|
||||||
p_w_merge.add_argument("pr", type=int, help="Pull request number (e.g. 214)")
|
p_w_merge.add_argument("pr", type=int, help="Pull request number (e.g. 214)")
|
||||||
p_w_chats = work_sub.add_parser("chats", parents=[common], help="View recent live chat activity")
|
p_w_chats = work_sub.add_parser("chats", parents=[common], help="View recent live chat activity")
|
||||||
|
|||||||
@@ -43,5 +43,20 @@ class TestBoxWork(unittest.TestCase):
|
|||||||
res = box_work.get_recent_done_tasks(limit=5)
|
res = box_work.get_recent_done_tasks(limit=5)
|
||||||
self.assertIsInstance(res, list)
|
self.assertIsInstance(res, list)
|
||||||
|
|
||||||
|
def test_check_agent_preflight_structure(self):
|
||||||
|
res = box_work.check_agent_preflight("opm")
|
||||||
|
self.assertIn("hatch", res)
|
||||||
|
self.assertIn("restore", res)
|
||||||
|
self.assertIn("git", res)
|
||||||
|
self.assertIn("status", res["hatch"])
|
||||||
|
self.assertIn("status", res["restore"])
|
||||||
|
self.assertIn("status", res["git"])
|
||||||
|
self.assertIn("ready", res)
|
||||||
|
|
||||||
|
def test_check_agent_preflight_unknown_fails(self):
|
||||||
|
res = box_work.check_agent_preflight("nonexistent_agent_xyz")
|
||||||
|
self.assertFalse(res["ready"])
|
||||||
|
self.assertEqual(res["overall"], "FAIL")
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user