From 7fc15eb127295dd83be939427ba0243fb4df26a4 Mon Sep 17 00:00:00 2001 From: operator Date: Mon, 5 Oct 2026 17:15:50 +0000 Subject: [PATCH] feat(operators): amendment workflow and automated drive watchdog daemon --- bin/agent-drive-watchdog.py | 200 +++++++++++++++++++++++++++++++++ bin/agent_md.py | 133 ++++++++++++++++++++++ bin/box-ctl.py | 84 ++++++++++++++ bin/super-cli.py | 140 +++++++++++++++++++++++ docs/OPERATOR-DRIVE-RUNBOOK.md | 29 ++++- 5 files changed, 584 insertions(+), 2 deletions(-) create mode 100755 bin/agent-drive-watchdog.py diff --git a/bin/agent-drive-watchdog.py b/bin/agent-drive-watchdog.py new file mode 100755 index 0000000..b5e35b7 --- /dev/null +++ b/bin/agent-drive-watchdog.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +""" +agent-drive-watchdog.py — Automated Drive Watchdog & Healing Daemon for Muse Agents. + +Monitors operational DRIVE scores across the agent fleet (muse, pip, 646, opm, def, dev) +via Hatch WebSocket RPC. If any agent's DRIVE score drops below 100 or critical drive +files (HEARTBEAT.md, PROACTIVE_PREFERENCES.md, SOUL.md) are degraded or missing: + 1. Detects degraded state and missing checklist/preferences. + 2. Selectively auto-heals core drive files using canonical shared templates. + 3. Preserves MEMORY.md and agent-generated workspace files. + 4. Records state & healing history to /tmp/agent-drive-watchdog.json. + 5. Emits structured telemetry to stdout/journal. + +Can be run: + - Once: python3 bin/agent-drive-watchdog.py --once + - Continuous loop: python3 bin/agent-drive-watchdog.py --interval 600 + - Via systemd timer: agent-drive-watchdog.timer (every 10m) +""" + +import argparse +import datetime +import json +import logging +import os +import sys +import time +from pathlib import Path + +# Add NetVM bin to path +BASE_DIR = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(BASE_DIR / "bin")) + +from agent_md import audit_agents, write_md, read_md, SHARED_OPERATORS, VALID_ACCOUNTS + +STATE_FILE = Path("/tmp/agent-drive-watchdog.json") +LOG_FILE = Path("/tmp/agent-drive-watchdog.log") + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.StreamHandler(sys.stdout), + logging.FileHandler(LOG_FILE, mode="a", encoding="utf-8") + ] +) + + +def load_state() -> dict: + if STATE_FILE.exists(): + try: + return json.loads(STATE_FILE.read_text(encoding="utf-8")) + except Exception as e: + logging.warning(f"Failed to read existing state file: {e}") + return { + "last_run": None, + "history": [], + "agent_status": {}, + } + + +def save_state(state: dict): + try: + # Keep last 50 history entries + if len(state.get("history", [])) > 50: + state["history"] = state["history"][-50:] + STATE_FILE.write_text(json.dumps(state, indent=2), encoding="utf-8") + except Exception as e: + logging.error(f"Failed to save state file: {e}") + + +def heal_agent_drive(account: str, audit_data: dict) -> dict: + """Selectively auto-heal core drive files for an agent.""" + healed = [] + issues = audit_data.get("issues", []) + + # Check what specifically needs healing + need_soul = any("SOUL.md" in iss for iss in issues) or audit_data.get("files", {}).get("SOUL.md", {}).get("size", 0) < 1000 + need_pro = any("PROACTIVE_PREFERENCES.md" in iss for iss in issues) or audit_data.get("files", {}).get("PROACTIVE_PREFERENCES.md", {}).get("size", 0) < 800 + need_hb = any("HEARTBEAT.md" in iss for iss in issues) or audit_data.get("files", {}).get("HEARTBEAT.md", {}).get("size", 0) < 300 + need_context = any("TOOLS.md or USER.md" in iss for iss in issues) + + logging.info(f"[{account.upper()}] Auto-healing drive... need_soul={need_soul}, need_pro={need_pro}, need_hb={need_hb}, need_context={need_context}") + + if need_soul: + soul_text = (SHARED_OPERATORS / "SOUL.md").read_text(encoding="utf-8") + res = write_md(account, "SOUL.md", soul_text, overwrite=True) + healed.append({"file": "SOUL.md", "bytes": res.get("bytes_written")}) + + if need_pro: + pro_text = (SHARED_OPERATORS / "PROACTIVE_PREFERENCES.md").read_text(encoding="utf-8") + res = write_md(account, "PROACTIVE_PREFERENCES.md", pro_text, overwrite=True) + healed.append({"file": "PROACTIVE_PREFERENCES.md", "bytes": res.get("bytes_written")}) + + if need_hb: + hb_text = (SHARED_OPERATORS / "HEARTBEAT.md").read_text(encoding="utf-8") + res = write_md(account, "HEARTBEAT.md", hb_text, overwrite=True) + healed.append({"file": "HEARTBEAT.md", "bytes": res.get("bytes_written")}) + + if need_context: + tools_text = (SHARED_OPERATORS / "TOOLS.md").read_text(encoding="utf-8") + res_t = write_md(account, "TOOLS.md", tools_text, overwrite=True) + healed.append({"file": "TOOLS.md", "bytes": res_t.get("bytes_written")}) + + user_text = (SHARED_OPERATORS / "USER.md").read_text(encoding="utf-8") + res_u = write_md(account, "USER.md", user_text, overwrite=True) + healed.append({"file": "USER.md", "bytes": res_u.get("bytes_written")}) + + return {"ok": True, "account": account, "healed": healed} + + +def run_cycle(auto_heal: bool = True) -> dict: + """Run an audit and healing cycle across all fleet accounts.""" + now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat() + logging.info("Starting fleet drive watchdog audit cycle...") + + state = load_state() + state["last_run"] = now_iso + + try: + audit_results = audit_agents() + except Exception as e: + logging.error(f"Audit failed during cycle: {e}") + return {"ok": False, "error": str(e)} + + cycle_report = { + "timestamp": now_iso, + "total_agents": len(audit_results), + "high_drive": 0, + "degraded": 0, + "healed_agents": [], + } + + for account, a_data in audit_results.items(): + score = a_data.get("drive_score", 0) + issues = a_data.get("issues", []) + status = "HIGH_DRIVE" if score == 100 else "DEGRADED" + + if status == "HIGH_DRIVE": + cycle_report["high_drive"] += 1 + logging.info(f"Agent {account.upper():6}: DRIVE score 100/100 (HIGH DRIVE)") + else: + cycle_report["degraded"] += 1 + logging.warning(f"Agent {account.upper():6}: DRIVE score {score}/100 ({status}) - Issues: {', '.join(issues)}") + + if auto_heal: + try: + heal_res = heal_agent_drive(account, a_data) + healed_files = [h["file"] for h in heal_res.get("healed", [])] + logging.info(f"Agent {account.upper():6}: Successfully healed files: {', '.join(healed_files)}") + cycle_report["healed_agents"].append({ + "account": account, + "prior_score": score, + "healed_files": healed_files, + }) + except Exception as e: + logging.error(f"Agent {account.upper():6}: Healing failed: {e}") + + state["agent_status"][account] = { + "score": score, + "status": status, + "issues": issues, + "last_checked": now_iso, + } + + state["history"].append(cycle_report) + save_state(state) + + logging.info(f"Watchdog cycle complete. High drive: {cycle_report['high_drive']}/{cycle_report['total_agents']}. Degraded: {cycle_report['degraded']}. Healed: {len(cycle_report['healed_agents'])}.") + return cycle_report + + +def main(): + parser = argparse.ArgumentParser(description="NetVM Automated Agent Drive Watchdog & Healing Daemon") + parser.add_argument("--once", action="store_true", help="Run a single audit/healing pass and exit") + parser.add_argument("--no-heal", action="store_true", help="Audit only; do not auto-heal degraded agents") + parser.add_argument("--interval", type=int, default=600, help="Loop interval in seconds (default: 600s / 10m)") + parser.add_argument("--status", action="store_true", help="Print recent watchdog status and exit") + + args = parser.parse_args() + + if args.status: + state = load_state() + print(json.dumps(state, indent=2)) + return + + if args.once: + run_cycle(auto_heal=not args.no_heal) + return + + logging.info(f"Starting NetVM Agent Drive Watchdog daemon (interval: {args.interval}s)...") + while True: + try: + run_cycle(auto_heal=not args.no_heal) + except Exception as e: + logging.error(f"Unexpected error in watchdog loop: {e}", exc_info=True) + time.sleep(args.interval) + + +if __name__ == "__main__": + main() diff --git a/bin/agent_md.py b/bin/agent_md.py index 2eeda68..b8d247e 100755 --- a/bin/agent_md.py +++ b/bin/agent_md.py @@ -239,6 +239,92 @@ def diff_md(account: str, filename: str) -> dict: } +def amend_md(filename: str, content: str, author: str = "operator", reason: str = "") -> dict: + """Amend a centralized shared operator template in shared/operators/ with safety validation and git commit.""" + import subprocess + + local_path = SHARED_OPERATORS / filename + if not local_path.exists(): + raise FileNotFoundError(f"Shared operator file {filename} does not exist in {SHARED_OPERATORS}") + + # Drive safety validation + if filename == "HEARTBEAT.md": + # Ensure checklist is not gutted + non_comment_lines = [l for l in content.splitlines() if l.strip() and not l.strip().startswith("#")] + checklist_items = [l for l in non_comment_lines if l.strip().startswith("- [")] + if not checklist_items: + raise ValueError("Safety rejection: amendment removes all active checklist items from HEARTBEAT.md") + + elif filename == "PROACTIVE_PREFERENCES.md": + if len(content.strip()) < 400: + raise ValueError("Safety rejection: amendment would reduce PROACTIVE_PREFERENCES.md to unconfigured state") + + elif filename == "SOUL.md": + if "Be a guest in someone's life" in content and "AUTONOMOUS OPERATIONAL DRIVE" not in content: + raise ValueError("Safety rejection: amendment reverts SOUL.md to passive stock template") + + old_content = local_path.read_text(encoding="utf-8") + local_path.write_text(content, encoding="utf-8") + + # Git auto-commit if in git repo + git_committed = False + git_hash = None + try: + commit_msg = f"amend(operators): update {filename} via {author}" + if reason: + commit_msg += f" - {reason}" + subprocess.run(["git", "add", str(local_path)], cwd=str(NETVM_ROOT), check=True, capture_output=True) + cr = subprocess.run(["git", "commit", "-m", commit_msg], cwd=str(NETVM_ROOT), capture_output=True, text=True) + if cr.returncode == 0: + git_committed = True + hr = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=str(NETVM_ROOT), capture_output=True, text=True) + git_hash = hr.stdout.strip() + except Exception: + pass + + return { + "ok": True, + "action": "amend", + "filename": filename, + "author": author, + "reason": reason, + "bytes_written": len(content.encode("utf-8")), + "git_committed": git_committed, + "commit": git_hash, + } + + +def append_md(filename: str, text: str, author: str = "operator", section: str = None) -> dict: + """Safely append an amendment or lesson to a centralized shared template.""" + local_path = SHARED_OPERATORS / filename + if not local_path.exists(): + raise FileNotFoundError(f"Shared operator file {filename} does not exist in {SHARED_OPERATORS}") + + current = local_path.read_text(encoding="utf-8") + header = f"\n\n\n" + if section: + header += f"### {section}\n" + new_content = current.rstrip() + header + text.strip() + "\n" + + return amend_md(filename, new_content, author=author, reason=f"append {section or 'note'}") + + +def pull_md(account: str, filename: str) -> dict: + """Pull the canonical centralized template from shared/operators/ into an agent's container.""" + local_path = SHARED_OPERATORS / filename + if not local_path.exists(): + raise FileNotFoundError(f"Shared operator file {filename} does not exist in {SHARED_OPERATORS}") + content = local_path.read_text(encoding="utf-8") + res = write_md(account, filename, content, overwrite=True) + return { + "ok": True, + "account": account, + "filename": filename, + "bytes_written": res.get("bytes_written"), + "message": f"Successfully pulled canonical {filename} into {account} container", + } + + def inject_drive(account: str, force: bool = False) -> dict: """Inject high-drive operational instructions into the agent's container.""" updates = [] @@ -344,6 +430,23 @@ def main(): p_diff.add_argument("account", help="Agent account") p_diff.add_argument("filename", help="Filename (e.g. SOUL.md)") + p_amend = sub.add_parser("amend", help="Amend a shared operator file with validation and git commit") + p_amend.add_argument("filename", help="Filename (e.g. AGENTS.md, TOOLS.md)") + p_amend.add_argument("--content", help="New content") + p_amend.add_argument("--file", help="File with new content") + p_amend.add_argument("--author", default="operator", help="Author of amendment") + p_amend.add_argument("--reason", default="", help="Reason for amendment") + + p_append = sub.add_parser("append", help="Safely append a note or lesson to a shared operator file") + p_append.add_argument("filename", help="Filename (e.g. AGENTS.md)") + p_append.add_argument("text", help="Text to append") + p_append.add_argument("--author", default="operator", help="Author of amendment") + p_append.add_argument("--section", default=None, help="Optional section header") + + p_pull = sub.add_parser("pull", help="Pull canonical shared file into an agent's container") + p_pull.add_argument("account", help="Agent account") + p_pull.add_argument("filename", help="Filename (e.g. SOUL.md)") + p_drive = sub.add_parser("inject-drive", help="Inject high-drive operator files into agent") p_drive.add_argument("account", help="Agent account") p_drive.add_argument("--force", action="store_true", help="Force overwrite") @@ -406,6 +509,36 @@ def main(): print(f"{args.account}:{args.filename} matches local shared/operators/{args.filename} exactly.") else: print(res["diff"]) + + elif args.cmd == "amend": + content = args.content + if args.file: + content = Path(args.file).read_text(encoding="utf-8") + if content is None: + print("Error: provide --content or --file", file=sys.stderr) + sys.exit(2) + try: + res = amend_md(args.filename, content, author=args.author, reason=args.reason) + print(json.dumps(res, indent=2)) + except Exception as e: + print(json.dumps({"ok": False, "error": str(e)}), indent=2) + sys.exit(1) + + elif args.cmd == "append": + try: + res = append_md(args.filename, args.text, author=args.author, section=args.section) + print(json.dumps(res, indent=2)) + except Exception as e: + print(json.dumps({"ok": False, "error": str(e)}), indent=2) + sys.exit(1) + + elif args.cmd == "pull": + try: + res = pull_md(args.account, args.filename) + print(json.dumps(res, indent=2)) + except Exception as e: + print(json.dumps({"ok": False, "error": str(e)}), indent=2) + sys.exit(1) elif args.cmd == "inject-drive": res = inject_drive(args.account, force=args.force) diff --git a/bin/box-ctl.py b/bin/box-ctl.py index 2ab4927..e4eb2ab 100755 --- a/bin/box-ctl.py +++ b/bin/box-ctl.py @@ -1542,6 +1542,39 @@ def act_md_diff(account, filename): out(res.pop("ok", True), **res) +def act_md_amend(filename, content, author="operator", reason=""): + """Amend centralized shared template with safety checks and git commit.""" + import agent_md + audit("md-amend", f"{author}:{filename}") + try: + res = agent_md.amend_md(filename, content, author=author, reason=reason) + out(res.pop("ok", True), **res) + except Exception as e: + fail("AMEND_FAILED", str(e)) + + +def act_md_append(filename, text, author="operator", section=None): + """Safely append an amendment or lesson to a centralized shared template.""" + import agent_md + audit("md-append", f"{author}:{filename}") + try: + res = agent_md.append_md(filename, text, author=author, section=section) + out(res.pop("ok", True), **res) + except Exception as e: + fail("APPEND_FAILED", str(e)) + + +def act_md_pull(account, filename): + """Pull canonical shared template into an agent container.""" + import agent_md + audit("md-pull", f"{account}:{filename}") + try: + res = agent_md.pull_md(account, filename) + out(res.pop("ok", True), **res) + except Exception as e: + fail("PULL_FAILED", str(e)) + + def act_md_inject_drive(account, force=False): """Inject high-drive operational templates into an agent container via Hatch.""" import agent_md @@ -3127,6 +3160,57 @@ def main(argv): act_md_inject_drive(args[0], force=("--force" in args[1:])) elif sub == "sync-all": act_md_sync_all(force=("--force" in args)) + elif sub == "amend": + if len(args) < 2: + fail("BAD_ARGS", "usage: md amend [--content text | --file path] [--author name] [--reason why]") + filename = args[0] + content = None + author = "operator" + reason = "" + idx = 1 + while idx < len(args): + if args[idx] == "--file" and idx + 1 < len(args): + content = Path(args[idx + 1]).read_text(encoding="utf-8") + idx += 2 + elif args[idx] == "--content" and idx + 1 < len(args): + content = args[idx + 1] + idx += 2 + elif args[idx] == "--author" and idx + 1 < len(args): + author = args[idx + 1] + idx += 2 + elif args[idx] == "--reason" and idx + 1 < len(args): + reason = args[idx + 1] + idx += 2 + elif content is None and not args[idx].startswith("--"): + content = args[idx] + idx += 1 + else: + idx += 1 + if content is None: + fail("BAD_ARGS", "usage: md amend [--author name] [--reason why]") + act_md_amend(filename, content, author=author, reason=reason) + elif sub == "append": + if len(args) < 2: + fail("BAD_ARGS", "usage: md append [--author name] [--section header]") + filename = args[0] + text = args[1] + author = "operator" + section = None + idx = 2 + while idx < len(args): + if args[idx] == "--author" and idx + 1 < len(args): + author = args[idx + 1] + idx += 2 + elif args[idx] == "--section" and idx + 1 < len(args): + section = args[idx + 1] + idx += 2 + else: + idx += 1 + act_md_append(filename, text, author=author, section=section) + elif sub == "pull": + if len(args) < 2: + fail("BAD_ARGS", "usage: md pull ") + act_md_pull(args[0], args[1]) else: fail("BAD_NAME", f"unknown md subcommand: {sub}") elif action == "loop-remediate": diff --git a/bin/super-cli.py b/bin/super-cli.py index 1d850f1..00ce094 100755 --- a/bin/super-cli.py +++ b/bin/super-cli.py @@ -3350,6 +3350,117 @@ def cmd_md_sync_all(args): sys.stdout.write(res.stdout) +def cmd_md_amend(args): + cmd = ["python3", str(BIN_DIR / "box-ctl.py"), "md", "amend", args.filename] + if args.file: + cmd.extend(["--file", args.file]) + elif args.content: + cmd.extend(["--content", args.content]) + if args.author: + cmd.extend(["--author", args.author]) + if args.reason: + cmd.extend(["--reason", args.reason]) + + res = subprocess.run(cmd, capture_output=True, text=True) + if args.json: + sys.stdout.write(res.stdout) + return + try: + d = json.loads(res.stdout) + if d.get("ok"): + commit_info = f" (git commit: {d.get('commit')})" if d.get("commit") else "" + print(c_green(f"\n✓ Successfully amended shared/operators/{args.filename}{commit_info}")) + print(f" Author: {c_cyan(d.get('author'))}") + print(f" Bytes: {d.get('bytes_written')}\n") + print(c_dim(" Broadcast to fleet with: box md sync-all\n")) + else: + print(c_red(f"✗ Failed: {d.get('error')}")) + except Exception: + sys.stdout.write(res.stdout) + + +def cmd_md_append(args): + cmd = ["python3", str(BIN_DIR / "box-ctl.py"), "md", "append", args.filename, args.text] + if args.author: + cmd.extend(["--author", args.author]) + if args.section: + cmd.extend(["--section", args.section]) + + res = subprocess.run(cmd, capture_output=True, text=True) + if args.json: + sys.stdout.write(res.stdout) + return + try: + d = json.loads(res.stdout) + if d.get("ok"): + commit_info = f" (git commit: {d.get('commit')})" if d.get("commit") else "" + print(c_green(f"\n✓ Successfully appended note to shared/operators/{args.filename}{commit_info}")) + print(f" Author: {c_cyan(d.get('author'))}\n") + print(c_dim(" Broadcast to fleet with: box md sync-all\n")) + else: + print(c_red(f"✗ Failed: {d.get('error')}")) + except Exception: + sys.stdout.write(res.stdout) + + +def cmd_md_pull(args): + cmd = ["python3", str(BIN_DIR / "box-ctl.py"), "md", "pull", args.account, args.filename] + res = subprocess.run(cmd, capture_output=True, text=True) + if args.json: + sys.stdout.write(res.stdout) + return + try: + d = json.loads(res.stdout) + if d.get("ok"): + print(c_green(f"✓ Pulled canonical shared/operators/{args.filename} into {args.account} container ({d.get('bytes_written')} B)")) + else: + print(c_red(f"✗ Failed: {d.get('error')}")) + except Exception: + sys.stdout.write(res.stdout) + + +def cmd_md_watchdog(args): + watchdog_py = BIN_DIR / "agent-drive-watchdog.py" + if args.sub_action == "status": + cmd = ["python3", str(watchdog_py), "--status"] + res = subprocess.run(cmd, capture_output=True, text=True) + if args.json: + sys.stdout.write(res.stdout) + return + try: + d = json.loads(res.stdout) + print("\n" + c_bold("=== NETVM FLEET AGENT DRIVE WATCHDOG STATUS ===") + "\n") + print(f" Last Run: {c_cyan(d.get('last_run', 'never'))}") + st = d.get("agent_status", {}) + headers = ["AGENT", "DRIVE SCORE", "STATUS", "LAST CHECKED", "ISSUES"] + rows = [] + for acct, info in st.items(): + score = info.get("score", 0) + score_str = c_green(f"{score}/100") if score == 100 else c_red(f"{score}/100") + stat = c_green("HIGH DRIVE") if info.get("status") == "HIGH_DRIVE" else c_red(info.get("status", "DEGRADED")) + dt = (info.get("last_checked") or "-")[:19] + iss = ", ".join(info.get("issues", [])) or c_dim("none") + rows.append([c_bold(acct.upper()), score_str, stat, dt, iss]) + print_table(headers, rows) + + # Show timer status + tr = subprocess.run(["systemctl", "--user", "is-active", "agent-drive-watchdog.timer"], capture_output=True, text=True) + t_stat = tr.stdout.strip() + t_badge = c_green("ACTIVE (every 10m)") if t_stat == "active" else c_yellow(t_stat) + print(f"\n Systemd Timer: {t_badge}\n") + except Exception: + sys.stdout.write(res.stdout) + elif args.sub_action == "run": + cmd = ["python3", str(watchdog_py), "--once"] + if getattr(args, "no_heal", False): + cmd.append("--no-heal") + print("\n" + c_bold("=== RUNNING FLEET AGENT DRIVE WATCHDOG CYCLE ===") + "\n") + subprocess.run(cmd) + print() + else: + print("Usage: box md watchdog {status|run}") + + def cmd_swarm_list(args): cmd = ["python3", str(BIN_DIR / "box-ctl.py"), "swarm-list"] res = subprocess.run(cmd, capture_output=True, text=True) @@ -3709,6 +3820,27 @@ def build_parser(): p_md_sync = md_sub.add_parser("sync-all", parents=[common], help="Inject high-drive operator files across all active fleet agents") p_md_sync.add_argument("--force", action="store_true", help="Force overwrite") + p_md_amend = md_sub.add_parser("amend", parents=[common], help="Amend centralized shared operator file with validation and git commit") + p_md_amend.add_argument("filename", help="Filename (e.g. AGENTS.md, TOOLS.md)") + p_md_amend.add_argument("--content", help="New content") + p_md_amend.add_argument("--file", help="File with new content") + p_md_amend.add_argument("--author", default="operator", help="Author of amendment") + p_md_amend.add_argument("--reason", default="", help="Reason for amendment") + + p_md_append = md_sub.add_parser("append", parents=[common], help="Safely append a lesson or note to a shared operator file") + p_md_append.add_argument("filename", help="Filename (e.g. AGENTS.md)") + p_md_append.add_argument("text", help="Text to append") + p_md_append.add_argument("--author", default="operator", help="Author of amendment") + p_md_append.add_argument("--section", default=None, help="Optional section header") + + p_md_pull = md_sub.add_parser("pull", parents=[common], help="Pull canonical shared template into an agent container") + p_md_pull.add_argument("account", help="Agent account") + p_md_pull.add_argument("filename", help="Filename (e.g. SOUL.md)") + + p_md_wd = md_sub.add_parser("watchdog", parents=[common], help="Fleet agent drive watchdog daemon & status") + p_md_wd.add_argument("sub_action", choices=["status", "run"], nargs="?", default="status", help="Watchdog action: status (default) or run (execute cycle)") + p_md_wd.add_argument("--no-heal", action="store_true", help="Audit only; do not auto-heal degraded agents") + # Domain: HARVEST p_harvest = subparsers.add_parser("harvest", parents=[common], help="Readback & response harvesting engine") harvest_sub = p_harvest.add_subparsers(dest="action") @@ -4069,6 +4201,14 @@ def main(): cmd_md_inject_drive(args) elif act == "sync-all": cmd_md_sync_all(args) + elif act == "amend": + cmd_md_amend(args) + elif act == "append": + cmd_md_append(args) + elif act == "pull": + cmd_md_pull(args) + elif act == "watchdog": + cmd_md_watchdog(args) else: p_md.print_help() elif args.domain == "harvest": diff --git a/docs/OPERATOR-DRIVE-RUNBOOK.md b/docs/OPERATOR-DRIVE-RUNBOOK.md index 4ac4a1d..5d6ea34 100644 --- a/docs/OPERATOR-DRIVE-RUNBOOK.md +++ b/docs/OPERATOR-DRIVE-RUNBOOK.md @@ -84,6 +84,30 @@ box md write 646 .ssh/authorized_keys --file ~/.ssh/id_ed25519.pub box md write 646 HEARTBEAT.md --content "- [ ] Check local reverse SSH tunnel every 5 minutes" ``` +### F. Proposing & Appending Amendments to Centralized Share +Agents and operators can safely propose updates and amendments to `shared/operators/`: +```bash +# Safely append a lesson learned or operational discovery +box md append AGENTS.md "Dev CDP relay verified on port 9455." --author "dev" --section "Relay Verification" + +# Amend a full shared template (with safety checks and automatic git commit) +box md amend TOOLS.md --file /path/to/updated_tools.md --author "646" --reason "Updated ttyd watchdog command" + +# Pull the latest canonical shared file into an agent's container +box md pull pip AGENTS.md +``` + +### G. Automated Drive Watchdog & Healing Daemon +A background service continuously verifies DRIVE scores and auto-heals degraded or missing checklists: +```bash +# Check watchdog status and systemd timer +box md watchdog status + +# Trigger an immediate audit and healing cycle +box md watchdog run +``` +The watchdog runs via systemd timer (`agent-drive-watchdog.timer`) every 10 minutes on the host, logging state to `/tmp/agent-drive-watchdog.json`. + --- ## 3. Container SSH Access & Reverse Tunnel Architecture @@ -123,5 +147,6 @@ cat shared/operators/SOUL.md | ssh -o StrictHostKeyChecking=no -J super@34.139.3 ## 4. Key Takeaways & Best Practices 1. **Never leave `HEARTBEAT.md` empty**: If an agent has an empty checklist, its background runner will remain completely dormant. -2. **Relative Paths in Hatch RPC**: Hatch WebSocket RPC rejects absolute paths (`/SOUL.md` fails; `SOUL.md` succeeds). -3. **Dual Access Redundancy**: If SSH reverse tunnels drop, Hatch WebSocket RPC is independent of SSH and can be used immediately to inspect logs, repair `authorized_keys`, or restart watchdog scripts. +2. **Safety Gates on Amendments**: `box md amend` automatically validates that amendments do not remove checklists or revert `SOUL.md` to passive templates. +3. **Relative Paths in Hatch RPC**: Hatch WebSocket RPC rejects absolute paths (`/SOUL.md` fails; `SOUL.md` succeeds). +4. **Dual Access Redundancy**: If SSH reverse tunnels drop, Hatch WebSocket RPC is independent of SSH and can be used immediately to inspect logs, repair `authorized_keys`, or restart watchdog scripts.