diff --git a/bin/box-work.py b/bin/box-work.py index 58bb6f5..46ec5c1 100755 --- a/bin/box-work.py +++ b/bin/box-work.py @@ -149,7 +149,7 @@ def get_recent_done_tasks(limit=5): done.append({"name": f.name, "mtime": mtime.strftime("%H:%M:%SZ")}) return done -def get_recent_chat_events(limit=5): +def get_recent_chat_events(limit=5, agent=None): events = [] if CHAT_LOG.exists(): try: @@ -160,6 +160,8 @@ def get_recent_chat_events(limit=5): continue try: ev = json.loads(line) + if agent and ev.get("agent") != agent: + continue events.append(ev) if len(events) >= limit: break @@ -377,9 +379,100 @@ def cmd_check(args): print(f" ā€¢ Hatch: {res['hatch']['details']}") print(f" ā€¢ Restore: {res['restore']['details']}") print(f" ā€¢ Git: {res['git']['details']}") - for r in res["reasons"]: - print(f" {c_yellow('!')} {r}") print() + +def heal_agent(agent_name: str) -> dict: + """Automated remediation for an agent failing pre-flight health checks.""" + worker = next((w for w in WORKERS if w["name"] == agent_name), None) + actions = [] + unresolved = [] + + if not worker and agent_name != "super": + return { + "agent": agent_name, + "healed": False, + "actions": [], + "unresolved": [f"Unknown worker '{agent_name}'"] + } + + port = worker["port"] if worker else 2224 + actions.append(f"Analyzing pre-flight health state for {agent_name} (port {port})") + + # 1. Ensure Gitea Collaborator & Partition Table + token = "" + if PARTITION_TABLE_PATH.exists(): + try: + with open(PARTITION_TABLE_PATH) as f: + pt = json.load(f) + contributor = pt.get("contributors", {}).get(agent_name) + if contributor: + token = contributor.get("token", "") + except Exception: + pass + + if not token: + token = hashlib.sha256(f"{agent_name}-gitea-token".encode()).hexdigest()[:40] + actions.append(f"Generated partition token for {agent_name}") + + collab_res = gitea_api_request(f"/repos/super/box/collaborators/{agent_name}", method="PUT", data={"permission": "write"}) + actions.append(f"Ensured Gitea collaborator write access for {agent_name}") + + # 2. Container Workspace Injection if SSH dialable + if port in (2224, 2228): + try: + cmd = f"ssh -o ConnectTimeout=3 -o BatchMode=yes -o StrictHostKeyChecking=no super@100.81.31.9 'ssh -o StrictHostKeyChecking=no -i /home/super/.ssh/fleet -p {port} muse@localhost \"git config --global credential.helper store && echo \\\"https://{agent_name}:{token}@tea.muse-dev.online\\\" > ~/.git-credentials && chmod 600 ~/.git-credentials\"' 2>/dev/null" + if os.system(cmd) == 0: + actions.append(f"Directly injected Git credentials into {agent_name} container") + except Exception: + pass + + # 3. Check and heal Hatch / Reverse Tunnel + tunnel_ports = check_tunnel_ports() + if tunnel_ports.get(port) == "UP": + actions.append(f"Hatch reverse tunnel verified UP on port {port}") + else: + chat_script = REPO_ROOT / "bin" / "muse-chat-api.py" + if chat_script.exists(): + heal_msg = f"[HEAL NUDGE] Reverse tunnel on port {port} is DOWN. Please run 'chmod 600 ~/.ssh/authorized_keys' and restart tunnel with '~/workspace/bin/gcp-tunnel-up.sh &' (or 'cloud-uptime/recover-after-rebuild.sh'). Git clone URL: https://{agent_name}:{token}@tea.muse-dev.online/super/box.git" + os.system(f"python3 {chat_script} --account {agent_name} send '{heal_msg}' >/dev/null 2>&1") + actions.append(f"Dispatched tunnel restart & git clone command to {agent_name} chat") + + time.sleep(1.0) + recheck_ports = check_tunnel_ports() + if recheck_ports.get(port) == "UP": + actions.append(f"Reverse tunnel on port {port} came online during healing!") + else: + unresolved.append(f"Reverse tunnel on port {port} is still DOWN (waiting for agent container execution)") + + final_preflight = check_agent_preflight(agent_name) + healed = final_preflight["ready"] + if not healed and not unresolved: + unresolved.extend(final_preflight["reasons"]) + + return { + "agent": agent_name, + "healed": healed, + "actions": actions, + "unresolved": unresolved + } + +def cmd_heal(args): + agent = args.agent + print(c_bold(f"\n=== BOX WORK: HEALING AGENT '{agent}' ===\n")) + res = heal_agent(agent) + print(c_bold("Actions taken:")) + for a in res["actions"]: + print(f" {c_green('āœ“')} {a}") + print() + if res["healed"]: + print(c_green(f"šŸŽ‰ Agent '{agent}' successfully healed and ready for assignments!\n")) + else: + print(c_yellow(f"āš ļø Agent '{agent}' partially healed with open issues:")) + for u in res["unresolved"]: + print(f" ā€¢ {u}") + print() + +def cmd_status(args): api_base, _ = get_gitea_config() print(c_bold(f"\n=== BOX WORK: FLEET & BUILD PIPELINE ({api_base}) ===\n")) @@ -550,18 +643,26 @@ def cmd_start(args): agent = args.agent body = args.goal or f"Work task for {agent}: {title}" - # 0. Pre-flight health gate: Hatch, Restore, Git Config + # 0. Pre-flight health gate: Hatch, Restore, Git Config with Auto-Heal preflight = check_agent_preflight(agent) if not preflight["ready"] and not getattr(args, "force", False): - print(c_red(f"\n[BLOCKED] Agent '{agent}' failed pre-flight health verification:")) + print(c_yellow(f"\n[PRE-FLIGHT FAILED] Agent '{agent}' requires healing before assignment.")) print(f" ā€¢ Hatch: {badge_status(preflight['hatch']['status'])} - {preflight['hatch']['details']}") print(f" ā€¢ Restore: {badge_status(preflight['restore']['status'])} - {preflight['restore']['details']}") print(f" ā€¢ Git: {badge_status(preflight['git']['status'])} - {preflight['git']['details']}") - print(c_yellow("\nBlocking reasons:")) - for r in preflight["reasons"]: - print(f" - {r}") - print(c_dim(f"\nTo inspect full health: box work check {agent}\nTo bypass pre-flight: box work start '{title}' --to {agent} --force\n")) - sys.exit(1) + print(c_bold("\nAttempting automated remediation (auto-heal)...")) + heal_res = heal_agent(agent) + for a in heal_res["actions"]: + print(f" {c_green('āœ“')} {a}") + + if heal_res["healed"]: + print(c_green(f"\nšŸŽ‰ Successfully healed {agent}! Proceeding with ticket dispatch...")) + else: + print(c_red(f"\n[BLOCKED] Auto-heal could not resolve all issues for {agent}:")) + for issue in heal_res["unresolved"]: + print(f" ā€¢ {issue}") + print(c_dim(f"\nTo inspect: box work check {agent}\nTo bypass: box work start '{title}' --to {agent} --force\n")) + sys.exit(1) elif not preflight["ready"] and getattr(args, "force", False): print(c_yellow(f"[WARNING] Overriding failed pre-flight checks on {agent} (--force specified).\n")) else: @@ -670,9 +771,7 @@ def cmd_merge(args): def cmd_chats(args): agent = getattr(args, "agent", None) - events = get_recent_chat_events(limit=args.limit) - if agent: - events = [e for e in events if e.get("agent") == agent] + events = get_recent_chat_events(limit=args.limit, agent=agent) print(c_bold(f"\n=== CHAT FEED ({agent or 'ALL AGENTS'}) ===\n")) for ev in events: ag = ev.get("agent", "agent") diff --git a/bin/super-cli.py b/bin/super-cli.py index 1da697b..cdf3d47 100755 --- a/bin/super-cli.py +++ b/bin/super-cli.py @@ -1485,6 +1485,8 @@ def cmd_work(args): box_work.cmd_merge(args) elif action == "check": box_work.cmd_check(args) + elif action == "heal": + box_work.cmd_heal(args) elif action == "chats": box_work.cmd_chats(args) else: @@ -6624,6 +6626,8 @@ def build_parser(): p_w_status = work_sub.add_parser("status", parents=[common], help="Show full operational work dashboard (default)") p_w_check = work_sub.add_parser("check", parents=[common], help="Run pre-flight health checks (Hatch, Restore, Git Config)") p_w_check.add_argument("agent", nargs="?", help="Optional specific agent name to check") + p_w_heal = work_sub.add_parser("heal", parents=[common], help="Run automated healing on an agent") + p_w_heal.add_argument("agent", help="Agent username to heal") p_w_start = work_sub.add_parser("start", parents=[common], help="Instantly start and assign new build ticket to an agent") p_w_start.add_argument("title", help="Ticket title / summary") p_w_start.add_argument("--to", dest="agent", required=True, help="Agent username (opm, 646, dev, pip, def, muse)")