feat(work): add pre-flight health gate for Hatch, Restore, and Git Config
This commit is contained in:
+200
-1
@@ -219,7 +219,167 @@ def check_tunnel_ports():
|
||||
s.close()
|
||||
return ports_status
|
||||
|
||||
def cmd_status(args):
|
||||
def badge_status(status: str) -> str:
|
||||
if status == "PASS":
|
||||
return c_green("PASS")
|
||||
elif status == "WARN":
|
||||
return c_yellow("WARN")
|
||||
else:
|
||||
return c_red("FAIL")
|
||||
|
||||
def check_agent_preflight(agent_name: str) -> dict:
|
||||
"""Ensures hatch, restore, and git config health before assigning work to cloud muse agents."""
|
||||
worker = next((w for w in WORKERS if w["name"] == agent_name), None)
|
||||
if not worker and agent_name != "super":
|
||||
return {
|
||||
"agent": agent_name,
|
||||
"port": 0,
|
||||
"hatch": {"status": "FAIL", "details": f"Unknown agent '{agent_name}'"},
|
||||
"restore": {"status": "FAIL", "details": "Not listed in fleet topology"},
|
||||
"git": {"status": "FAIL", "details": "No partition entry"},
|
||||
"overall": "FAIL",
|
||||
"ready": False,
|
||||
"reasons": [f"Agent '{agent_name}' is not in fleet topology"]
|
||||
}
|
||||
|
||||
port = worker["port"] if worker else 2224
|
||||
tunnel_ports = check_tunnel_ports()
|
||||
port_status = tunnel_ports.get(port, "DARK")
|
||||
reasons = []
|
||||
|
||||
# 1. HATCH HEALTH (tunnel listener + responsive chat)
|
||||
hatch_status = "PASS"
|
||||
hatch_details = []
|
||||
if port_status == "UP":
|
||||
hatch_details.append(f"Port {port} listener UP")
|
||||
else:
|
||||
hatch_status = "FAIL"
|
||||
hatch_details.append(f"Port {port} reverse tunnel DARK")
|
||||
reasons.append(f"Hatch tunnel is DOWN on port {port}. Container is offline or unreachable.")
|
||||
|
||||
last_chats = get_last_agent_chats()
|
||||
chat_ev = last_chats.get(agent_name)
|
||||
if chat_ev:
|
||||
ts_str = chat_ev.get("ts", "")[:19].replace("T", " ")
|
||||
hatch_details.append(f"Chat active ({ts_str})")
|
||||
else:
|
||||
hatch_details.append("No recent chat entries")
|
||||
|
||||
# 2. RESTORE HEALTH (NODES.md, supervisor persistence)
|
||||
restore_status = "PASS"
|
||||
restore_details = []
|
||||
nodes_file = REPO_ROOT / "NODES.md"
|
||||
node_in_registry = False
|
||||
if nodes_file.exists():
|
||||
try:
|
||||
with open(nodes_file) as f:
|
||||
content = f.read()
|
||||
if f"| {agent_name} |" in content or f"warp-{agent_name}" in content:
|
||||
node_in_registry = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if node_in_registry or agent_name in ("muse-main", "super"):
|
||||
restore_details.append("Registered in NODES.md")
|
||||
else:
|
||||
restore_status = "WARN"
|
||||
restore_details.append("Not found in NODES.md")
|
||||
|
||||
if port_status == "UP":
|
||||
restore_details.append("Watchdog/Supervisor persistent")
|
||||
else:
|
||||
restore_status = "FAIL"
|
||||
restore_details.append("Container rebuild / tunnel recovery pending")
|
||||
reasons.append("Container requires recovery/restore (run recover-after-rebuild or inspect watchdog).")
|
||||
|
||||
# 3. GIT CONFIG HEALTH (partition token, collaborator access, branches)
|
||||
git_status = "PASS"
|
||||
git_details = []
|
||||
token = ""
|
||||
if PARTITION_TABLE_PATH.exists():
|
||||
try:
|
||||
with open(PARTITION_TABLE_PATH) as f:
|
||||
pt = json.load(f)
|
||||
contributor = pt.get("contributors", {}).get(agent_name)
|
||||
if contributor:
|
||||
token = contributor.get("token", "")
|
||||
git_details.append("Token in partition-table")
|
||||
else:
|
||||
git_status = "FAIL"
|
||||
git_details.append("Missing from partition-table")
|
||||
reasons.append(f"Agent '{agent_name}' has no credentials in fleet/partition-table.json")
|
||||
except Exception as e:
|
||||
git_status = "WARN"
|
||||
git_details.append(f"Partition table error: {e}")
|
||||
|
||||
collab_check = gitea_api_request(f"/repos/super/box/collaborators/{agent_name}")
|
||||
if isinstance(collab_check, dict) and collab_check.get("error") and collab_check.get("error") not in (200, 204):
|
||||
git_status = "FAIL"
|
||||
git_details.append("Not a repository collaborator")
|
||||
reasons.append(f"Gitea user '{agent_name}' lacks write/collaborator access")
|
||||
else:
|
||||
git_details.append("Gitea collaborator OK")
|
||||
|
||||
branches = gitea_api_request("/repos/super/box/branches")
|
||||
agent_branch = False
|
||||
if isinstance(branches, list):
|
||||
for b in branches:
|
||||
bname = b.get("name", "")
|
||||
if bname.startswith(f"dev/{agent_name}/") or bname.startswith(f"builder/{agent_name}/"):
|
||||
agent_branch = True
|
||||
break
|
||||
if agent_branch:
|
||||
git_details.append("Branch verified in Gitea")
|
||||
else:
|
||||
git_details.append("No active branch")
|
||||
|
||||
overall = "PASS"
|
||||
if hatch_status == "FAIL" or restore_status == "FAIL" or git_status == "FAIL":
|
||||
overall = "FAIL"
|
||||
elif hatch_status == "WARN" or restore_status == "WARN" or git_status == "WARN":
|
||||
overall = "WARN"
|
||||
|
||||
return {
|
||||
"agent": agent_name,
|
||||
"port": port,
|
||||
"hatch": {"status": hatch_status, "details": ", ".join(hatch_details)},
|
||||
"restore": {"status": restore_status, "details": ", ".join(restore_details)},
|
||||
"git": {"status": git_status, "details": ", ".join(git_details)},
|
||||
"overall": overall,
|
||||
"ready": (overall != "FAIL"),
|
||||
"reasons": reasons
|
||||
}
|
||||
|
||||
def cmd_check(args):
|
||||
target_agent = getattr(args, "agent", None)
|
||||
targets = [target_agent] if target_agent else [w["name"] for w in WORKERS if w["role"] != "host"]
|
||||
|
||||
print(c_bold("\n=== BOX WORK: PRE-FLIGHT HEALTH VERIFICATION ===\n"))
|
||||
header = f"{'AGENT':<12} {'HATCH':<12} {'RESTORE':<12} {'GIT CONFIG':<12} {'STATUS'}"
|
||||
print(c_dim(header))
|
||||
print(c_dim("-" * len(header)))
|
||||
|
||||
for ag in targets:
|
||||
res = check_agent_preflight(ag)
|
||||
h_badge = badge_status(res["hatch"]["status"])
|
||||
r_badge = badge_status(res["restore"]["status"])
|
||||
g_badge = badge_status(res["git"]["status"])
|
||||
overall_badge = c_green("🟢 READY") if res["ready"] else c_red("🔴 BLOCKED")
|
||||
print(f"{c_bold(ag):<21} {h_badge:<21} {r_badge:<21} {g_badge:<21} {overall_badge}")
|
||||
|
||||
print()
|
||||
blocked = [ag for ag in targets if not check_agent_preflight(ag)["ready"]]
|
||||
if blocked:
|
||||
print(c_bold("--- PRE-FLIGHT DIAGNOSTIC DETAILS ---"))
|
||||
for ag in blocked:
|
||||
res = check_agent_preflight(ag)
|
||||
print(f" {c_bold(ag)}:")
|
||||
print(f" • Hatch: {res['hatch']['details']}")
|
||||
print(f" • Restore: {res['restore']['details']}")
|
||||
print(f" • Git: {res['git']['details']}")
|
||||
for r in res["reasons"]:
|
||||
print(f" {c_yellow('!')} {r}")
|
||||
print()
|
||||
api_base, _ = get_gitea_config()
|
||||
print(c_bold(f"\n=== BOX WORK: FLEET & BUILD PIPELINE ({api_base}) ===\n"))
|
||||
|
||||
@@ -390,6 +550,23 @@ def cmd_start(args):
|
||||
agent = args.agent
|
||||
body = args.goal or f"Work task for {agent}: {title}"
|
||||
|
||||
# 0. Pre-flight health gate: Hatch, Restore, Git Config
|
||||
preflight = check_agent_preflight(agent)
|
||||
if not preflight["ready"] and not getattr(args, "force", False):
|
||||
print(c_red(f"\n[BLOCKED] Agent '{agent}' failed pre-flight health verification:"))
|
||||
print(f" • Hatch: {badge_status(preflight['hatch']['status'])} - {preflight['hatch']['details']}")
|
||||
print(f" • Restore: {badge_status(preflight['restore']['status'])} - {preflight['restore']['details']}")
|
||||
print(f" • Git: {badge_status(preflight['git']['status'])} - {preflight['git']['details']}")
|
||||
print(c_yellow("\nBlocking reasons:"))
|
||||
for r in preflight["reasons"]:
|
||||
print(f" - {r}")
|
||||
print(c_dim(f"\nTo inspect full health: box work check {agent}\nTo bypass pre-flight: box work start '{title}' --to {agent} --force\n"))
|
||||
sys.exit(1)
|
||||
elif not preflight["ready"] and getattr(args, "force", False):
|
||||
print(c_yellow(f"[WARNING] Overriding failed pre-flight checks on {agent} (--force specified).\n"))
|
||||
else:
|
||||
print(c_green(f"✓ Pre-flight checks passed (Hatch: OK, Restore: OK, Git Config: OK) for {agent}"))
|
||||
|
||||
print(c_bold(f"Initiating work ticket for agent {agent}..."))
|
||||
|
||||
# 1. Ensure label exists in Gitea
|
||||
@@ -443,6 +620,20 @@ def cmd_start(args):
|
||||
def cmd_assign(args):
|
||||
issue_num = args.issue
|
||||
agent = args.agent
|
||||
|
||||
# 0. Pre-flight health gate: Hatch, Restore, Git Config
|
||||
preflight = check_agent_preflight(agent)
|
||||
if not preflight["ready"] and not getattr(args, "force", False):
|
||||
print(c_red(f"\n[BLOCKED] Agent '{agent}' failed pre-flight health verification:"))
|
||||
print(f" • Hatch: {badge_status(preflight['hatch']['status'])} - {preflight['hatch']['details']}")
|
||||
print(f" • Restore: {badge_status(preflight['restore']['status'])} - {preflight['restore']['details']}")
|
||||
print(f" • Git: {badge_status(preflight['git']['status'])} - {preflight['git']['details']}")
|
||||
print(c_yellow("\nBlocking reasons:"))
|
||||
for r in preflight["reasons"]:
|
||||
print(f" - {r}")
|
||||
print(c_dim(f"\nTo bypass pre-flight: box work assign {issue_num} --to {agent} --force\n"))
|
||||
sys.exit(1)
|
||||
|
||||
print(c_bold(f"Assigning Ticket #{issue_num} to {agent}..."))
|
||||
|
||||
payload = {
|
||||
@@ -501,14 +692,20 @@ def main():
|
||||
|
||||
sub.add_parser("status", help="Show full operational work dashboard")
|
||||
|
||||
# box work check [agent]
|
||||
p_check = sub.add_parser("check", help="Run pre-flight health verification (Hatch, Restore, Git Config)")
|
||||
p_check.add_argument("agent", nargs="?", help="Optional specific agent name to check")
|
||||
|
||||
p_start = sub.add_parser("start", help="Instantly start and assign new build ticket to an agent")
|
||||
p_start.add_argument("title", help="Ticket title / summary")
|
||||
p_start.add_argument("--to", dest="agent", required=True, help="Agent username (opm, 646, dev, pip, def, muse)")
|
||||
p_start.add_argument("--goal", help="Optional detailed goal description")
|
||||
p_start.add_argument("--force", action="store_true", help="Bypass pre-flight health gate")
|
||||
|
||||
p_assign = sub.add_parser("assign", help="Assign existing ticket to an agent")
|
||||
p_assign.add_argument("issue", type=int, help="Issue number (e.g. 215)")
|
||||
p_assign.add_argument("--to", dest="agent", required=True, help="Agent username")
|
||||
p_assign.add_argument("--force", action="store_true", help="Bypass pre-flight health gate")
|
||||
|
||||
p_merge = sub.add_parser("merge", help="Merge an open PR into master")
|
||||
p_merge.add_argument("pr", type=int, help="Pull request number (e.g. 214)")
|
||||
@@ -522,6 +719,8 @@ def main():
|
||||
|
||||
if not action or action == "status":
|
||||
cmd_status(args)
|
||||
elif action == "check":
|
||||
cmd_check(args)
|
||||
elif action == "start":
|
||||
cmd_start(args)
|
||||
elif action == "assign":
|
||||
|
||||
@@ -1483,6 +1483,8 @@ def cmd_work(args):
|
||||
box_work.cmd_assign(args)
|
||||
elif action == "merge":
|
||||
box_work.cmd_merge(args)
|
||||
elif action == "check":
|
||||
box_work.cmd_check(args)
|
||||
elif action == "chats":
|
||||
box_work.cmd_chats(args)
|
||||
else:
|
||||
@@ -6620,13 +6622,17 @@ def build_parser():
|
||||
p_work = subparsers.add_parser("work", parents=[common], help="Fleet workspace, task orchestration, worker scope, and active signals")
|
||||
work_sub = p_work.add_subparsers(dest="work_action")
|
||||
p_w_status = work_sub.add_parser("status", parents=[common], help="Show full operational work dashboard (default)")
|
||||
p_w_check = work_sub.add_parser("check", parents=[common], help="Run pre-flight health checks (Hatch, Restore, Git Config)")
|
||||
p_w_check.add_argument("agent", nargs="?", help="Optional specific agent name to check")
|
||||
p_w_start = work_sub.add_parser("start", parents=[common], help="Instantly start and assign new build ticket to an agent")
|
||||
p_w_start.add_argument("title", help="Ticket title / summary")
|
||||
p_w_start.add_argument("--to", dest="agent", required=True, help="Agent username (opm, 646, dev, pip, def, muse)")
|
||||
p_w_start.add_argument("--goal", help="Optional detailed goal description")
|
||||
p_w_start.add_argument("--force", action="store_true", help="Bypass pre-flight health gate")
|
||||
p_w_assign = work_sub.add_parser("assign", parents=[common], help="Assign existing ticket to an agent")
|
||||
p_w_assign.add_argument("issue", type=int, help="Issue number (e.g. 215)")
|
||||
p_w_assign.add_argument("--to", dest="agent", required=True, help="Agent username")
|
||||
p_w_assign.add_argument("--force", action="store_true", help="Bypass pre-flight health gate")
|
||||
p_w_merge = work_sub.add_parser("merge", parents=[common], help="Merge an open PR into master")
|
||||
p_w_merge.add_argument("pr", type=int, help="Pull request number (e.g. 214)")
|
||||
p_w_chats = work_sub.add_parser("chats", parents=[common], help="View recent live chat activity")
|
||||
|
||||
Reference in New Issue
Block a user