feat(operators): amendment workflow and automated drive watchdog daemon
This commit is contained in:
Executable
+200
@@ -0,0 +1,200 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
agent-drive-watchdog.py — Automated Drive Watchdog & Healing Daemon for Muse Agents.
|
||||||
|
|
||||||
|
Monitors operational DRIVE scores across the agent fleet (muse, pip, 646, opm, def, dev)
|
||||||
|
via Hatch WebSocket RPC. If any agent's DRIVE score drops below 100 or critical drive
|
||||||
|
files (HEARTBEAT.md, PROACTIVE_PREFERENCES.md, SOUL.md) are degraded or missing:
|
||||||
|
1. Detects degraded state and missing checklist/preferences.
|
||||||
|
2. Selectively auto-heals core drive files using canonical shared templates.
|
||||||
|
3. Preserves MEMORY.md and agent-generated workspace files.
|
||||||
|
4. Records state & healing history to /tmp/agent-drive-watchdog.json.
|
||||||
|
5. Emits structured telemetry to stdout/journal.
|
||||||
|
|
||||||
|
Can be run:
|
||||||
|
- Once: python3 bin/agent-drive-watchdog.py --once
|
||||||
|
- Continuous loop: python3 bin/agent-drive-watchdog.py --interval 600
|
||||||
|
- Via systemd timer: agent-drive-watchdog.timer (every 10m)
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import datetime
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# Add NetVM bin to path
|
||||||
|
BASE_DIR = Path(__file__).resolve().parent.parent
|
||||||
|
sys.path.insert(0, str(BASE_DIR / "bin"))
|
||||||
|
|
||||||
|
from agent_md import audit_agents, write_md, read_md, SHARED_OPERATORS, VALID_ACCOUNTS
|
||||||
|
|
||||||
|
STATE_FILE = Path("/tmp/agent-drive-watchdog.json")
|
||||||
|
LOG_FILE = Path("/tmp/agent-drive-watchdog.log")
|
||||||
|
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||||
|
handlers=[
|
||||||
|
logging.StreamHandler(sys.stdout),
|
||||||
|
logging.FileHandler(LOG_FILE, mode="a", encoding="utf-8")
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def load_state() -> dict:
|
||||||
|
if STATE_FILE.exists():
|
||||||
|
try:
|
||||||
|
return json.loads(STATE_FILE.read_text(encoding="utf-8"))
|
||||||
|
except Exception as e:
|
||||||
|
logging.warning(f"Failed to read existing state file: {e}")
|
||||||
|
return {
|
||||||
|
"last_run": None,
|
||||||
|
"history": [],
|
||||||
|
"agent_status": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def save_state(state: dict):
|
||||||
|
try:
|
||||||
|
# Keep last 50 history entries
|
||||||
|
if len(state.get("history", [])) > 50:
|
||||||
|
state["history"] = state["history"][-50:]
|
||||||
|
STATE_FILE.write_text(json.dumps(state, indent=2), encoding="utf-8")
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"Failed to save state file: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
def heal_agent_drive(account: str, audit_data: dict) -> dict:
|
||||||
|
"""Selectively auto-heal core drive files for an agent."""
|
||||||
|
healed = []
|
||||||
|
issues = audit_data.get("issues", [])
|
||||||
|
|
||||||
|
# Check what specifically needs healing
|
||||||
|
need_soul = any("SOUL.md" in iss for iss in issues) or audit_data.get("files", {}).get("SOUL.md", {}).get("size", 0) < 1000
|
||||||
|
need_pro = any("PROACTIVE_PREFERENCES.md" in iss for iss in issues) or audit_data.get("files", {}).get("PROACTIVE_PREFERENCES.md", {}).get("size", 0) < 800
|
||||||
|
need_hb = any("HEARTBEAT.md" in iss for iss in issues) or audit_data.get("files", {}).get("HEARTBEAT.md", {}).get("size", 0) < 300
|
||||||
|
need_context = any("TOOLS.md or USER.md" in iss for iss in issues)
|
||||||
|
|
||||||
|
logging.info(f"[{account.upper()}] Auto-healing drive... need_soul={need_soul}, need_pro={need_pro}, need_hb={need_hb}, need_context={need_context}")
|
||||||
|
|
||||||
|
if need_soul:
|
||||||
|
soul_text = (SHARED_OPERATORS / "SOUL.md").read_text(encoding="utf-8")
|
||||||
|
res = write_md(account, "SOUL.md", soul_text, overwrite=True)
|
||||||
|
healed.append({"file": "SOUL.md", "bytes": res.get("bytes_written")})
|
||||||
|
|
||||||
|
if need_pro:
|
||||||
|
pro_text = (SHARED_OPERATORS / "PROACTIVE_PREFERENCES.md").read_text(encoding="utf-8")
|
||||||
|
res = write_md(account, "PROACTIVE_PREFERENCES.md", pro_text, overwrite=True)
|
||||||
|
healed.append({"file": "PROACTIVE_PREFERENCES.md", "bytes": res.get("bytes_written")})
|
||||||
|
|
||||||
|
if need_hb:
|
||||||
|
hb_text = (SHARED_OPERATORS / "HEARTBEAT.md").read_text(encoding="utf-8")
|
||||||
|
res = write_md(account, "HEARTBEAT.md", hb_text, overwrite=True)
|
||||||
|
healed.append({"file": "HEARTBEAT.md", "bytes": res.get("bytes_written")})
|
||||||
|
|
||||||
|
if need_context:
|
||||||
|
tools_text = (SHARED_OPERATORS / "TOOLS.md").read_text(encoding="utf-8")
|
||||||
|
res_t = write_md(account, "TOOLS.md", tools_text, overwrite=True)
|
||||||
|
healed.append({"file": "TOOLS.md", "bytes": res_t.get("bytes_written")})
|
||||||
|
|
||||||
|
user_text = (SHARED_OPERATORS / "USER.md").read_text(encoding="utf-8")
|
||||||
|
res_u = write_md(account, "USER.md", user_text, overwrite=True)
|
||||||
|
healed.append({"file": "USER.md", "bytes": res_u.get("bytes_written")})
|
||||||
|
|
||||||
|
return {"ok": True, "account": account, "healed": healed}
|
||||||
|
|
||||||
|
|
||||||
|
def run_cycle(auto_heal: bool = True) -> dict:
|
||||||
|
"""Run an audit and healing cycle across all fleet accounts."""
|
||||||
|
now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||||
|
logging.info("Starting fleet drive watchdog audit cycle...")
|
||||||
|
|
||||||
|
state = load_state()
|
||||||
|
state["last_run"] = now_iso
|
||||||
|
|
||||||
|
try:
|
||||||
|
audit_results = audit_agents()
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"Audit failed during cycle: {e}")
|
||||||
|
return {"ok": False, "error": str(e)}
|
||||||
|
|
||||||
|
cycle_report = {
|
||||||
|
"timestamp": now_iso,
|
||||||
|
"total_agents": len(audit_results),
|
||||||
|
"high_drive": 0,
|
||||||
|
"degraded": 0,
|
||||||
|
"healed_agents": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
for account, a_data in audit_results.items():
|
||||||
|
score = a_data.get("drive_score", 0)
|
||||||
|
issues = a_data.get("issues", [])
|
||||||
|
status = "HIGH_DRIVE" if score == 100 else "DEGRADED"
|
||||||
|
|
||||||
|
if status == "HIGH_DRIVE":
|
||||||
|
cycle_report["high_drive"] += 1
|
||||||
|
logging.info(f"Agent {account.upper():6}: DRIVE score 100/100 (HIGH DRIVE)")
|
||||||
|
else:
|
||||||
|
cycle_report["degraded"] += 1
|
||||||
|
logging.warning(f"Agent {account.upper():6}: DRIVE score {score}/100 ({status}) - Issues: {', '.join(issues)}")
|
||||||
|
|
||||||
|
if auto_heal:
|
||||||
|
try:
|
||||||
|
heal_res = heal_agent_drive(account, a_data)
|
||||||
|
healed_files = [h["file"] for h in heal_res.get("healed", [])]
|
||||||
|
logging.info(f"Agent {account.upper():6}: Successfully healed files: {', '.join(healed_files)}")
|
||||||
|
cycle_report["healed_agents"].append({
|
||||||
|
"account": account,
|
||||||
|
"prior_score": score,
|
||||||
|
"healed_files": healed_files,
|
||||||
|
})
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"Agent {account.upper():6}: Healing failed: {e}")
|
||||||
|
|
||||||
|
state["agent_status"][account] = {
|
||||||
|
"score": score,
|
||||||
|
"status": status,
|
||||||
|
"issues": issues,
|
||||||
|
"last_checked": now_iso,
|
||||||
|
}
|
||||||
|
|
||||||
|
state["history"].append(cycle_report)
|
||||||
|
save_state(state)
|
||||||
|
|
||||||
|
logging.info(f"Watchdog cycle complete. High drive: {cycle_report['high_drive']}/{cycle_report['total_agents']}. Degraded: {cycle_report['degraded']}. Healed: {len(cycle_report['healed_agents'])}.")
|
||||||
|
return cycle_report
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="NetVM Automated Agent Drive Watchdog & Healing Daemon")
|
||||||
|
parser.add_argument("--once", action="store_true", help="Run a single audit/healing pass and exit")
|
||||||
|
parser.add_argument("--no-heal", action="store_true", help="Audit only; do not auto-heal degraded agents")
|
||||||
|
parser.add_argument("--interval", type=int, default=600, help="Loop interval in seconds (default: 600s / 10m)")
|
||||||
|
parser.add_argument("--status", action="store_true", help="Print recent watchdog status and exit")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.status:
|
||||||
|
state = load_state()
|
||||||
|
print(json.dumps(state, indent=2))
|
||||||
|
return
|
||||||
|
|
||||||
|
if args.once:
|
||||||
|
run_cycle(auto_heal=not args.no_heal)
|
||||||
|
return
|
||||||
|
|
||||||
|
logging.info(f"Starting NetVM Agent Drive Watchdog daemon (interval: {args.interval}s)...")
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
run_cycle(auto_heal=not args.no_heal)
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"Unexpected error in watchdog loop: {e}", exc_info=True)
|
||||||
|
time.sleep(args.interval)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
+133
@@ -239,6 +239,92 @@ def diff_md(account: str, filename: str) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def amend_md(filename: str, content: str, author: str = "operator", reason: str = "") -> dict:
|
||||||
|
"""Amend a centralized shared operator template in shared/operators/ with safety validation and git commit."""
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
local_path = SHARED_OPERATORS / filename
|
||||||
|
if not local_path.exists():
|
||||||
|
raise FileNotFoundError(f"Shared operator file {filename} does not exist in {SHARED_OPERATORS}")
|
||||||
|
|
||||||
|
# Drive safety validation
|
||||||
|
if filename == "HEARTBEAT.md":
|
||||||
|
# Ensure checklist is not gutted
|
||||||
|
non_comment_lines = [l for l in content.splitlines() if l.strip() and not l.strip().startswith("#")]
|
||||||
|
checklist_items = [l for l in non_comment_lines if l.strip().startswith("- [")]
|
||||||
|
if not checklist_items:
|
||||||
|
raise ValueError("Safety rejection: amendment removes all active checklist items from HEARTBEAT.md")
|
||||||
|
|
||||||
|
elif filename == "PROACTIVE_PREFERENCES.md":
|
||||||
|
if len(content.strip()) < 400:
|
||||||
|
raise ValueError("Safety rejection: amendment would reduce PROACTIVE_PREFERENCES.md to unconfigured state")
|
||||||
|
|
||||||
|
elif filename == "SOUL.md":
|
||||||
|
if "Be a guest in someone's life" in content and "AUTONOMOUS OPERATIONAL DRIVE" not in content:
|
||||||
|
raise ValueError("Safety rejection: amendment reverts SOUL.md to passive stock template")
|
||||||
|
|
||||||
|
old_content = local_path.read_text(encoding="utf-8")
|
||||||
|
local_path.write_text(content, encoding="utf-8")
|
||||||
|
|
||||||
|
# Git auto-commit if in git repo
|
||||||
|
git_committed = False
|
||||||
|
git_hash = None
|
||||||
|
try:
|
||||||
|
commit_msg = f"amend(operators): update {filename} via {author}"
|
||||||
|
if reason:
|
||||||
|
commit_msg += f" - {reason}"
|
||||||
|
subprocess.run(["git", "add", str(local_path)], cwd=str(NETVM_ROOT), check=True, capture_output=True)
|
||||||
|
cr = subprocess.run(["git", "commit", "-m", commit_msg], cwd=str(NETVM_ROOT), capture_output=True, text=True)
|
||||||
|
if cr.returncode == 0:
|
||||||
|
git_committed = True
|
||||||
|
hr = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=str(NETVM_ROOT), capture_output=True, text=True)
|
||||||
|
git_hash = hr.stdout.strip()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"action": "amend",
|
||||||
|
"filename": filename,
|
||||||
|
"author": author,
|
||||||
|
"reason": reason,
|
||||||
|
"bytes_written": len(content.encode("utf-8")),
|
||||||
|
"git_committed": git_committed,
|
||||||
|
"commit": git_hash,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def append_md(filename: str, text: str, author: str = "operator", section: str = None) -> dict:
|
||||||
|
"""Safely append an amendment or lesson to a centralized shared template."""
|
||||||
|
local_path = SHARED_OPERATORS / filename
|
||||||
|
if not local_path.exists():
|
||||||
|
raise FileNotFoundError(f"Shared operator file {filename} does not exist in {SHARED_OPERATORS}")
|
||||||
|
|
||||||
|
current = local_path.read_text(encoding="utf-8")
|
||||||
|
header = f"\n\n<!-- Amendment by {author} on {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime())} -->\n"
|
||||||
|
if section:
|
||||||
|
header += f"### {section}\n"
|
||||||
|
new_content = current.rstrip() + header + text.strip() + "\n"
|
||||||
|
|
||||||
|
return amend_md(filename, new_content, author=author, reason=f"append {section or 'note'}")
|
||||||
|
|
||||||
|
|
||||||
|
def pull_md(account: str, filename: str) -> dict:
|
||||||
|
"""Pull the canonical centralized template from shared/operators/ into an agent's container."""
|
||||||
|
local_path = SHARED_OPERATORS / filename
|
||||||
|
if not local_path.exists():
|
||||||
|
raise FileNotFoundError(f"Shared operator file {filename} does not exist in {SHARED_OPERATORS}")
|
||||||
|
content = local_path.read_text(encoding="utf-8")
|
||||||
|
res = write_md(account, filename, content, overwrite=True)
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"account": account,
|
||||||
|
"filename": filename,
|
||||||
|
"bytes_written": res.get("bytes_written"),
|
||||||
|
"message": f"Successfully pulled canonical {filename} into {account} container",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def inject_drive(account: str, force: bool = False) -> dict:
|
def inject_drive(account: str, force: bool = False) -> dict:
|
||||||
"""Inject high-drive operational instructions into the agent's container."""
|
"""Inject high-drive operational instructions into the agent's container."""
|
||||||
updates = []
|
updates = []
|
||||||
@@ -344,6 +430,23 @@ def main():
|
|||||||
p_diff.add_argument("account", help="Agent account")
|
p_diff.add_argument("account", help="Agent account")
|
||||||
p_diff.add_argument("filename", help="Filename (e.g. SOUL.md)")
|
p_diff.add_argument("filename", help="Filename (e.g. SOUL.md)")
|
||||||
|
|
||||||
|
p_amend = sub.add_parser("amend", help="Amend a shared operator file with validation and git commit")
|
||||||
|
p_amend.add_argument("filename", help="Filename (e.g. AGENTS.md, TOOLS.md)")
|
||||||
|
p_amend.add_argument("--content", help="New content")
|
||||||
|
p_amend.add_argument("--file", help="File with new content")
|
||||||
|
p_amend.add_argument("--author", default="operator", help="Author of amendment")
|
||||||
|
p_amend.add_argument("--reason", default="", help="Reason for amendment")
|
||||||
|
|
||||||
|
p_append = sub.add_parser("append", help="Safely append a note or lesson to a shared operator file")
|
||||||
|
p_append.add_argument("filename", help="Filename (e.g. AGENTS.md)")
|
||||||
|
p_append.add_argument("text", help="Text to append")
|
||||||
|
p_append.add_argument("--author", default="operator", help="Author of amendment")
|
||||||
|
p_append.add_argument("--section", default=None, help="Optional section header")
|
||||||
|
|
||||||
|
p_pull = sub.add_parser("pull", help="Pull canonical shared file into an agent's container")
|
||||||
|
p_pull.add_argument("account", help="Agent account")
|
||||||
|
p_pull.add_argument("filename", help="Filename (e.g. SOUL.md)")
|
||||||
|
|
||||||
p_drive = sub.add_parser("inject-drive", help="Inject high-drive operator files into agent")
|
p_drive = sub.add_parser("inject-drive", help="Inject high-drive operator files into agent")
|
||||||
p_drive.add_argument("account", help="Agent account")
|
p_drive.add_argument("account", help="Agent account")
|
||||||
p_drive.add_argument("--force", action="store_true", help="Force overwrite")
|
p_drive.add_argument("--force", action="store_true", help="Force overwrite")
|
||||||
@@ -407,6 +510,36 @@ def main():
|
|||||||
else:
|
else:
|
||||||
print(res["diff"])
|
print(res["diff"])
|
||||||
|
|
||||||
|
elif args.cmd == "amend":
|
||||||
|
content = args.content
|
||||||
|
if args.file:
|
||||||
|
content = Path(args.file).read_text(encoding="utf-8")
|
||||||
|
if content is None:
|
||||||
|
print("Error: provide --content or --file", file=sys.stderr)
|
||||||
|
sys.exit(2)
|
||||||
|
try:
|
||||||
|
res = amend_md(args.filename, content, author=args.author, reason=args.reason)
|
||||||
|
print(json.dumps(res, indent=2))
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"ok": False, "error": str(e)}), indent=2)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
elif args.cmd == "append":
|
||||||
|
try:
|
||||||
|
res = append_md(args.filename, args.text, author=args.author, section=args.section)
|
||||||
|
print(json.dumps(res, indent=2))
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"ok": False, "error": str(e)}), indent=2)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
elif args.cmd == "pull":
|
||||||
|
try:
|
||||||
|
res = pull_md(args.account, args.filename)
|
||||||
|
print(json.dumps(res, indent=2))
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({"ok": False, "error": str(e)}), indent=2)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
elif args.cmd == "inject-drive":
|
elif args.cmd == "inject-drive":
|
||||||
res = inject_drive(args.account, force=args.force)
|
res = inject_drive(args.account, force=args.force)
|
||||||
print(json.dumps(res, indent=2))
|
print(json.dumps(res, indent=2))
|
||||||
|
|||||||
@@ -1542,6 +1542,39 @@ def act_md_diff(account, filename):
|
|||||||
out(res.pop("ok", True), **res)
|
out(res.pop("ok", True), **res)
|
||||||
|
|
||||||
|
|
||||||
|
def act_md_amend(filename, content, author="operator", reason=""):
|
||||||
|
"""Amend centralized shared template with safety checks and git commit."""
|
||||||
|
import agent_md
|
||||||
|
audit("md-amend", f"{author}:{filename}")
|
||||||
|
try:
|
||||||
|
res = agent_md.amend_md(filename, content, author=author, reason=reason)
|
||||||
|
out(res.pop("ok", True), **res)
|
||||||
|
except Exception as e:
|
||||||
|
fail("AMEND_FAILED", str(e))
|
||||||
|
|
||||||
|
|
||||||
|
def act_md_append(filename, text, author="operator", section=None):
|
||||||
|
"""Safely append an amendment or lesson to a centralized shared template."""
|
||||||
|
import agent_md
|
||||||
|
audit("md-append", f"{author}:{filename}")
|
||||||
|
try:
|
||||||
|
res = agent_md.append_md(filename, text, author=author, section=section)
|
||||||
|
out(res.pop("ok", True), **res)
|
||||||
|
except Exception as e:
|
||||||
|
fail("APPEND_FAILED", str(e))
|
||||||
|
|
||||||
|
|
||||||
|
def act_md_pull(account, filename):
|
||||||
|
"""Pull canonical shared template into an agent container."""
|
||||||
|
import agent_md
|
||||||
|
audit("md-pull", f"{account}:{filename}")
|
||||||
|
try:
|
||||||
|
res = agent_md.pull_md(account, filename)
|
||||||
|
out(res.pop("ok", True), **res)
|
||||||
|
except Exception as e:
|
||||||
|
fail("PULL_FAILED", str(e))
|
||||||
|
|
||||||
|
|
||||||
def act_md_inject_drive(account, force=False):
|
def act_md_inject_drive(account, force=False):
|
||||||
"""Inject high-drive operational templates into an agent container via Hatch."""
|
"""Inject high-drive operational templates into an agent container via Hatch."""
|
||||||
import agent_md
|
import agent_md
|
||||||
@@ -3127,6 +3160,57 @@ def main(argv):
|
|||||||
act_md_inject_drive(args[0], force=("--force" in args[1:]))
|
act_md_inject_drive(args[0], force=("--force" in args[1:]))
|
||||||
elif sub == "sync-all":
|
elif sub == "sync-all":
|
||||||
act_md_sync_all(force=("--force" in args))
|
act_md_sync_all(force=("--force" in args))
|
||||||
|
elif sub == "amend":
|
||||||
|
if len(args) < 2:
|
||||||
|
fail("BAD_ARGS", "usage: md amend <filename> [--content text | --file path] [--author name] [--reason why]")
|
||||||
|
filename = args[0]
|
||||||
|
content = None
|
||||||
|
author = "operator"
|
||||||
|
reason = ""
|
||||||
|
idx = 1
|
||||||
|
while idx < len(args):
|
||||||
|
if args[idx] == "--file" and idx + 1 < len(args):
|
||||||
|
content = Path(args[idx + 1]).read_text(encoding="utf-8")
|
||||||
|
idx += 2
|
||||||
|
elif args[idx] == "--content" and idx + 1 < len(args):
|
||||||
|
content = args[idx + 1]
|
||||||
|
idx += 2
|
||||||
|
elif args[idx] == "--author" and idx + 1 < len(args):
|
||||||
|
author = args[idx + 1]
|
||||||
|
idx += 2
|
||||||
|
elif args[idx] == "--reason" and idx + 1 < len(args):
|
||||||
|
reason = args[idx + 1]
|
||||||
|
idx += 2
|
||||||
|
elif content is None and not args[idx].startswith("--"):
|
||||||
|
content = args[idx]
|
||||||
|
idx += 1
|
||||||
|
else:
|
||||||
|
idx += 1
|
||||||
|
if content is None:
|
||||||
|
fail("BAD_ARGS", "usage: md amend <filename> <content> [--author name] [--reason why]")
|
||||||
|
act_md_amend(filename, content, author=author, reason=reason)
|
||||||
|
elif sub == "append":
|
||||||
|
if len(args) < 2:
|
||||||
|
fail("BAD_ARGS", "usage: md append <filename> <text> [--author name] [--section header]")
|
||||||
|
filename = args[0]
|
||||||
|
text = args[1]
|
||||||
|
author = "operator"
|
||||||
|
section = None
|
||||||
|
idx = 2
|
||||||
|
while idx < len(args):
|
||||||
|
if args[idx] == "--author" and idx + 1 < len(args):
|
||||||
|
author = args[idx + 1]
|
||||||
|
idx += 2
|
||||||
|
elif args[idx] == "--section" and idx + 1 < len(args):
|
||||||
|
section = args[idx + 1]
|
||||||
|
idx += 2
|
||||||
|
else:
|
||||||
|
idx += 1
|
||||||
|
act_md_append(filename, text, author=author, section=section)
|
||||||
|
elif sub == "pull":
|
||||||
|
if len(args) < 2:
|
||||||
|
fail("BAD_ARGS", "usage: md pull <account> <filename>")
|
||||||
|
act_md_pull(args[0], args[1])
|
||||||
else:
|
else:
|
||||||
fail("BAD_NAME", f"unknown md subcommand: {sub}")
|
fail("BAD_NAME", f"unknown md subcommand: {sub}")
|
||||||
elif action == "loop-remediate":
|
elif action == "loop-remediate":
|
||||||
|
|||||||
@@ -3350,6 +3350,117 @@ def cmd_md_sync_all(args):
|
|||||||
sys.stdout.write(res.stdout)
|
sys.stdout.write(res.stdout)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_md_amend(args):
|
||||||
|
cmd = ["python3", str(BIN_DIR / "box-ctl.py"), "md", "amend", args.filename]
|
||||||
|
if args.file:
|
||||||
|
cmd.extend(["--file", args.file])
|
||||||
|
elif args.content:
|
||||||
|
cmd.extend(["--content", args.content])
|
||||||
|
if args.author:
|
||||||
|
cmd.extend(["--author", args.author])
|
||||||
|
if args.reason:
|
||||||
|
cmd.extend(["--reason", args.reason])
|
||||||
|
|
||||||
|
res = subprocess.run(cmd, capture_output=True, text=True)
|
||||||
|
if args.json:
|
||||||
|
sys.stdout.write(res.stdout)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
d = json.loads(res.stdout)
|
||||||
|
if d.get("ok"):
|
||||||
|
commit_info = f" (git commit: {d.get('commit')})" if d.get("commit") else ""
|
||||||
|
print(c_green(f"\n✓ Successfully amended shared/operators/{args.filename}{commit_info}"))
|
||||||
|
print(f" Author: {c_cyan(d.get('author'))}")
|
||||||
|
print(f" Bytes: {d.get('bytes_written')}\n")
|
||||||
|
print(c_dim(" Broadcast to fleet with: box md sync-all\n"))
|
||||||
|
else:
|
||||||
|
print(c_red(f"✗ Failed: {d.get('error')}"))
|
||||||
|
except Exception:
|
||||||
|
sys.stdout.write(res.stdout)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_md_append(args):
|
||||||
|
cmd = ["python3", str(BIN_DIR / "box-ctl.py"), "md", "append", args.filename, args.text]
|
||||||
|
if args.author:
|
||||||
|
cmd.extend(["--author", args.author])
|
||||||
|
if args.section:
|
||||||
|
cmd.extend(["--section", args.section])
|
||||||
|
|
||||||
|
res = subprocess.run(cmd, capture_output=True, text=True)
|
||||||
|
if args.json:
|
||||||
|
sys.stdout.write(res.stdout)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
d = json.loads(res.stdout)
|
||||||
|
if d.get("ok"):
|
||||||
|
commit_info = f" (git commit: {d.get('commit')})" if d.get("commit") else ""
|
||||||
|
print(c_green(f"\n✓ Successfully appended note to shared/operators/{args.filename}{commit_info}"))
|
||||||
|
print(f" Author: {c_cyan(d.get('author'))}\n")
|
||||||
|
print(c_dim(" Broadcast to fleet with: box md sync-all\n"))
|
||||||
|
else:
|
||||||
|
print(c_red(f"✗ Failed: {d.get('error')}"))
|
||||||
|
except Exception:
|
||||||
|
sys.stdout.write(res.stdout)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_md_pull(args):
|
||||||
|
cmd = ["python3", str(BIN_DIR / "box-ctl.py"), "md", "pull", args.account, args.filename]
|
||||||
|
res = subprocess.run(cmd, capture_output=True, text=True)
|
||||||
|
if args.json:
|
||||||
|
sys.stdout.write(res.stdout)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
d = json.loads(res.stdout)
|
||||||
|
if d.get("ok"):
|
||||||
|
print(c_green(f"✓ Pulled canonical shared/operators/{args.filename} into {args.account} container ({d.get('bytes_written')} B)"))
|
||||||
|
else:
|
||||||
|
print(c_red(f"✗ Failed: {d.get('error')}"))
|
||||||
|
except Exception:
|
||||||
|
sys.stdout.write(res.stdout)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_md_watchdog(args):
|
||||||
|
watchdog_py = BIN_DIR / "agent-drive-watchdog.py"
|
||||||
|
if args.sub_action == "status":
|
||||||
|
cmd = ["python3", str(watchdog_py), "--status"]
|
||||||
|
res = subprocess.run(cmd, capture_output=True, text=True)
|
||||||
|
if args.json:
|
||||||
|
sys.stdout.write(res.stdout)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
d = json.loads(res.stdout)
|
||||||
|
print("\n" + c_bold("=== NETVM FLEET AGENT DRIVE WATCHDOG STATUS ===") + "\n")
|
||||||
|
print(f" Last Run: {c_cyan(d.get('last_run', 'never'))}")
|
||||||
|
st = d.get("agent_status", {})
|
||||||
|
headers = ["AGENT", "DRIVE SCORE", "STATUS", "LAST CHECKED", "ISSUES"]
|
||||||
|
rows = []
|
||||||
|
for acct, info in st.items():
|
||||||
|
score = info.get("score", 0)
|
||||||
|
score_str = c_green(f"{score}/100") if score == 100 else c_red(f"{score}/100")
|
||||||
|
stat = c_green("HIGH DRIVE") if info.get("status") == "HIGH_DRIVE" else c_red(info.get("status", "DEGRADED"))
|
||||||
|
dt = (info.get("last_checked") or "-")[:19]
|
||||||
|
iss = ", ".join(info.get("issues", [])) or c_dim("none")
|
||||||
|
rows.append([c_bold(acct.upper()), score_str, stat, dt, iss])
|
||||||
|
print_table(headers, rows)
|
||||||
|
|
||||||
|
# Show timer status
|
||||||
|
tr = subprocess.run(["systemctl", "--user", "is-active", "agent-drive-watchdog.timer"], capture_output=True, text=True)
|
||||||
|
t_stat = tr.stdout.strip()
|
||||||
|
t_badge = c_green("ACTIVE (every 10m)") if t_stat == "active" else c_yellow(t_stat)
|
||||||
|
print(f"\n Systemd Timer: {t_badge}\n")
|
||||||
|
except Exception:
|
||||||
|
sys.stdout.write(res.stdout)
|
||||||
|
elif args.sub_action == "run":
|
||||||
|
cmd = ["python3", str(watchdog_py), "--once"]
|
||||||
|
if getattr(args, "no_heal", False):
|
||||||
|
cmd.append("--no-heal")
|
||||||
|
print("\n" + c_bold("=== RUNNING FLEET AGENT DRIVE WATCHDOG CYCLE ===") + "\n")
|
||||||
|
subprocess.run(cmd)
|
||||||
|
print()
|
||||||
|
else:
|
||||||
|
print("Usage: box md watchdog {status|run}")
|
||||||
|
|
||||||
|
|
||||||
def cmd_swarm_list(args):
|
def cmd_swarm_list(args):
|
||||||
cmd = ["python3", str(BIN_DIR / "box-ctl.py"), "swarm-list"]
|
cmd = ["python3", str(BIN_DIR / "box-ctl.py"), "swarm-list"]
|
||||||
res = subprocess.run(cmd, capture_output=True, text=True)
|
res = subprocess.run(cmd, capture_output=True, text=True)
|
||||||
@@ -3709,6 +3820,27 @@ def build_parser():
|
|||||||
p_md_sync = md_sub.add_parser("sync-all", parents=[common], help="Inject high-drive operator files across all active fleet agents")
|
p_md_sync = md_sub.add_parser("sync-all", parents=[common], help="Inject high-drive operator files across all active fleet agents")
|
||||||
p_md_sync.add_argument("--force", action="store_true", help="Force overwrite")
|
p_md_sync.add_argument("--force", action="store_true", help="Force overwrite")
|
||||||
|
|
||||||
|
p_md_amend = md_sub.add_parser("amend", parents=[common], help="Amend centralized shared operator file with validation and git commit")
|
||||||
|
p_md_amend.add_argument("filename", help="Filename (e.g. AGENTS.md, TOOLS.md)")
|
||||||
|
p_md_amend.add_argument("--content", help="New content")
|
||||||
|
p_md_amend.add_argument("--file", help="File with new content")
|
||||||
|
p_md_amend.add_argument("--author", default="operator", help="Author of amendment")
|
||||||
|
p_md_amend.add_argument("--reason", default="", help="Reason for amendment")
|
||||||
|
|
||||||
|
p_md_append = md_sub.add_parser("append", parents=[common], help="Safely append a lesson or note to a shared operator file")
|
||||||
|
p_md_append.add_argument("filename", help="Filename (e.g. AGENTS.md)")
|
||||||
|
p_md_append.add_argument("text", help="Text to append")
|
||||||
|
p_md_append.add_argument("--author", default="operator", help="Author of amendment")
|
||||||
|
p_md_append.add_argument("--section", default=None, help="Optional section header")
|
||||||
|
|
||||||
|
p_md_pull = md_sub.add_parser("pull", parents=[common], help="Pull canonical shared template into an agent container")
|
||||||
|
p_md_pull.add_argument("account", help="Agent account")
|
||||||
|
p_md_pull.add_argument("filename", help="Filename (e.g. SOUL.md)")
|
||||||
|
|
||||||
|
p_md_wd = md_sub.add_parser("watchdog", parents=[common], help="Fleet agent drive watchdog daemon & status")
|
||||||
|
p_md_wd.add_argument("sub_action", choices=["status", "run"], nargs="?", default="status", help="Watchdog action: status (default) or run (execute cycle)")
|
||||||
|
p_md_wd.add_argument("--no-heal", action="store_true", help="Audit only; do not auto-heal degraded agents")
|
||||||
|
|
||||||
# Domain: HARVEST
|
# Domain: HARVEST
|
||||||
p_harvest = subparsers.add_parser("harvest", parents=[common], help="Readback & response harvesting engine")
|
p_harvest = subparsers.add_parser("harvest", parents=[common], help="Readback & response harvesting engine")
|
||||||
harvest_sub = p_harvest.add_subparsers(dest="action")
|
harvest_sub = p_harvest.add_subparsers(dest="action")
|
||||||
@@ -4069,6 +4201,14 @@ def main():
|
|||||||
cmd_md_inject_drive(args)
|
cmd_md_inject_drive(args)
|
||||||
elif act == "sync-all":
|
elif act == "sync-all":
|
||||||
cmd_md_sync_all(args)
|
cmd_md_sync_all(args)
|
||||||
|
elif act == "amend":
|
||||||
|
cmd_md_amend(args)
|
||||||
|
elif act == "append":
|
||||||
|
cmd_md_append(args)
|
||||||
|
elif act == "pull":
|
||||||
|
cmd_md_pull(args)
|
||||||
|
elif act == "watchdog":
|
||||||
|
cmd_md_watchdog(args)
|
||||||
else:
|
else:
|
||||||
p_md.print_help()
|
p_md.print_help()
|
||||||
elif args.domain == "harvest":
|
elif args.domain == "harvest":
|
||||||
|
|||||||
@@ -84,6 +84,30 @@ box md write 646 .ssh/authorized_keys --file ~/.ssh/id_ed25519.pub
|
|||||||
box md write 646 HEARTBEAT.md --content "- [ ] Check local reverse SSH tunnel every 5 minutes"
|
box md write 646 HEARTBEAT.md --content "- [ ] Check local reverse SSH tunnel every 5 minutes"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### F. Proposing & Appending Amendments to Centralized Share
|
||||||
|
Agents and operators can safely propose updates and amendments to `shared/operators/`:
|
||||||
|
```bash
|
||||||
|
# Safely append a lesson learned or operational discovery
|
||||||
|
box md append AGENTS.md "Dev CDP relay verified on port 9455." --author "dev" --section "Relay Verification"
|
||||||
|
|
||||||
|
# Amend a full shared template (with safety checks and automatic git commit)
|
||||||
|
box md amend TOOLS.md --file /path/to/updated_tools.md --author "646" --reason "Updated ttyd watchdog command"
|
||||||
|
|
||||||
|
# Pull the latest canonical shared file into an agent's container
|
||||||
|
box md pull pip AGENTS.md
|
||||||
|
```
|
||||||
|
|
||||||
|
### G. Automated Drive Watchdog & Healing Daemon
|
||||||
|
A background service continuously verifies DRIVE scores and auto-heals degraded or missing checklists:
|
||||||
|
```bash
|
||||||
|
# Check watchdog status and systemd timer
|
||||||
|
box md watchdog status
|
||||||
|
|
||||||
|
# Trigger an immediate audit and healing cycle
|
||||||
|
box md watchdog run
|
||||||
|
```
|
||||||
|
The watchdog runs via systemd timer (`agent-drive-watchdog.timer`) every 10 minutes on the host, logging state to `/tmp/agent-drive-watchdog.json`.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 3. Container SSH Access & Reverse Tunnel Architecture
|
## 3. Container SSH Access & Reverse Tunnel Architecture
|
||||||
@@ -123,5 +147,6 @@ cat shared/operators/SOUL.md | ssh -o StrictHostKeyChecking=no -J super@34.139.3
|
|||||||
|
|
||||||
## 4. Key Takeaways & Best Practices
|
## 4. Key Takeaways & Best Practices
|
||||||
1. **Never leave `HEARTBEAT.md` empty**: If an agent has an empty checklist, its background runner will remain completely dormant.
|
1. **Never leave `HEARTBEAT.md` empty**: If an agent has an empty checklist, its background runner will remain completely dormant.
|
||||||
2. **Relative Paths in Hatch RPC**: Hatch WebSocket RPC rejects absolute paths (`/SOUL.md` fails; `SOUL.md` succeeds).
|
2. **Safety Gates on Amendments**: `box md amend` automatically validates that amendments do not remove checklists or revert `SOUL.md` to passive templates.
|
||||||
3. **Dual Access Redundancy**: If SSH reverse tunnels drop, Hatch WebSocket RPC is independent of SSH and can be used immediately to inspect logs, repair `authorized_keys`, or restart watchdog scripts.
|
3. **Relative Paths in Hatch RPC**: Hatch WebSocket RPC rejects absolute paths (`/SOUL.md` fails; `SOUL.md` succeeds).
|
||||||
|
4. **Dual Access Redundancy**: If SSH reverse tunnels drop, Hatch WebSocket RPC is independent of SSH and can be used immediately to inspect logs, repair `authorized_keys`, or restart watchdog scripts.
|
||||||
|
|||||||
Reference in New Issue
Block a user