feat(watchers): add box stability watcher daemon, recovery guardrails, and CLI integration
- Add dedicated watchers/ project folder with box-stability-watcher.py supervisor - Monitor host load, memory, swap saturation, and crash-looping services - Implement tiered mitigations: yellow renicing, orange SIGSTOP pause with 60s grace, red shedding - Distinguish user-launched agents (allowed on desktop default socket) from automated box workloads - Wire first-class box stability CLI subcommand and top-line host status in fleet status - Harden tmux.service with cgroup memory limits to prevent OS freeze and OOM avalanches - Add 10-test unit test suite covering thresholds, safety whitelist, pause/resume, and isolation
This commit is contained in:
operator
+577
-22
@@ -348,6 +348,25 @@ def cmd_fleet_status(args):
|
||||
return
|
||||
|
||||
print("\n" + c_bold("=== NETVM FLEET STATUS ===") + c_dim(f" ({datetime.now().strftime('%H:%M:%S')} local)\n"))
|
||||
# Top-line host stability badge
|
||||
try:
|
||||
sys.path.insert(0, str(NETVM_ROOT / "watchers"))
|
||||
import importlib.util
|
||||
spec = importlib.util.spec_from_file_location("box_stability_watcher", str(NETVM_ROOT / "watchers" / "box-stability-watcher.py"))
|
||||
bsw = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(bsw)
|
||||
m = bsw.get_system_metrics()
|
||||
load_str = f"Load: {m['load_1m']} (1m) | {m['load_5m']} (5m) [Cores: {m['cpu_count']}]"
|
||||
ram_str = f"RAM: {m['ram_used_pct']}%"
|
||||
if m['load_1m'] < 20 and m['ram_used_pct'] < 80:
|
||||
stab_badge = badge_ok("STABLE")
|
||||
elif m['load_1m'] >= 60 or m['ram_used_pct'] >= 95:
|
||||
stab_badge = badge_err("EMERGENCY")
|
||||
else:
|
||||
stab_badge = badge_warn("ELEVATED")
|
||||
print(f" HOST {c_bold('bl')}: {stab_badge} {c_dim(load_str + ' ' + ram_str)}\n")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
headers = ["NODE", "STATUS", "PEER IP:PORT", "LATENCY", "QUEUE", "ACTIVE PAGE / THREAD"]
|
||||
rows = []
|
||||
@@ -931,6 +950,49 @@ def cmd_approvals(args):
|
||||
print(f" Reason: {c_cyan(reason)}")
|
||||
print(c_dim(f" Operator can approve with: box approvals allow {node}"))
|
||||
|
||||
elif action in ("gates", "gate"):
|
||||
scope = getattr(args, "scope", None)
|
||||
if scope:
|
||||
res = approvals.verify_coordinator_signoff(scope)
|
||||
if getattr(args, "json", False):
|
||||
print(json.dumps(res, indent=2))
|
||||
return
|
||||
if res.get("ok"):
|
||||
print(c_green(f"\n✔ Coordinator gate for scope '{scope}' is SIGNED-OFF."))
|
||||
print(f" Coordinator: {c_cyan(res.get('coordinator'))}")
|
||||
print(f" Accepted At: {c_dim(res.get('accepted_at'))}")
|
||||
print(f" Record: {res.get('doc_name')}\n")
|
||||
else:
|
||||
print(c_red(f"\n✖ Coordinator gate verification FAILED for scope '{scope}':"))
|
||||
print(f" Error: {res.get('error')}\n")
|
||||
sys.exit(1)
|
||||
return
|
||||
|
||||
gates = approvals.scan_coordinator_gates()
|
||||
if getattr(args, "json", False):
|
||||
print(json.dumps({"ok": True, "gates": gates}, indent=2))
|
||||
return
|
||||
print("\n" + c_bold("=== COORDINATOR GATES & DECISION RECORDS ===\n"))
|
||||
if not gates:
|
||||
print(c_dim(" (no coordinator decision records found in docs/)"))
|
||||
print()
|
||||
return
|
||||
headers = ["RECORD", "SCOPE", "STATUS", "COORDINATOR", "ACCEPTED AT", "TARGETS"]
|
||||
rows = []
|
||||
for g in gates:
|
||||
st = c_green("SIGNED-OFF") if g.get("is_signed_off") else c_yellow(str(g.get("status", "DRAFT")).upper())
|
||||
targets = ", ".join(g.get("signoff_targets") or []) if isinstance(g.get("signoff_targets"), list) else str(g.get("signoff_targets") or "-")
|
||||
rows.append([
|
||||
g.get("doc_name", "-"),
|
||||
c_cyan(g.get("scope", "-")),
|
||||
st,
|
||||
g.get("coordinator", "-"),
|
||||
str(g.get("accepted_at", "-"))[:19],
|
||||
targets or "-",
|
||||
])
|
||||
print_table(headers, rows)
|
||||
print()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Domain: RUNTIME (agentic management of Muse CLI tmux runtimes)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -939,18 +1001,24 @@ def cmd_runtime(args):
|
||||
import muse_choice_watcher as mcw
|
||||
action = getattr(args, "rt_action", None) or "list"
|
||||
as_json = getattr(args, "json", False)
|
||||
if getattr(args, "socket", None):
|
||||
args.socket = mcw.resolve_socket(args.socket)
|
||||
|
||||
if action == "list":
|
||||
sock = getattr(args, "socket", None)
|
||||
rows = mcw.all_runtime_rows([sock] if sock else None)
|
||||
errors = {}
|
||||
rows = mcw.all_runtime_rows([sock] if sock else None, errors=errors)
|
||||
if getattr(args, "muse_only", False):
|
||||
rows = [r for r in rows if r["is_muse"]]
|
||||
if as_json:
|
||||
print(json.dumps({"ok": True, "runtimes": rows}, indent=2))
|
||||
print(json.dumps({"ok": True, "runtimes": rows,
|
||||
"errors": errors}, indent=2))
|
||||
return
|
||||
print(c_bold("\n=== MUSE RUNTIMES ===\n"))
|
||||
if not rows:
|
||||
print(c_dim(" No panes found."))
|
||||
for s, e in errors.items():
|
||||
print(c_dim(" %s: %s" % (s, e)))
|
||||
print()
|
||||
return
|
||||
headers = ["SOCKET", "SESSION", "NODE", "PANE", "CMD",
|
||||
@@ -962,10 +1030,14 @@ def cmd_runtime(args):
|
||||
state = "%s(%s/%s)" % (state, r["prompt_kind"],
|
||||
r["prompt_key"] or "…")
|
||||
if r["is_muse"]:
|
||||
approve = (badge_ok("YES") if r["auto_approve"]
|
||||
eff_bypass = r.get("effective_bypass")
|
||||
if eff_bypass is None:
|
||||
eff_bypass = bool(r["auto_approve"])
|
||||
approve = (badge_ok("YES") if eff_bypass
|
||||
else badge_err("NO"))
|
||||
mode = r.get("permission_mode") or "default"
|
||||
if r.get("permission_bypass") and mode != "yolo":
|
||||
mode = (r.get("effective_mode")
|
||||
or r.get("permission_mode") or "default")
|
||||
if eff_bypass and mode != "yolo":
|
||||
mode += "!"
|
||||
mode = mode[:14]
|
||||
else:
|
||||
@@ -979,6 +1051,8 @@ def cmd_runtime(args):
|
||||
r["pane"], (r["cmd"] or "")[:26], state,
|
||||
approve, mode, watcher])
|
||||
print_table(headers, table)
|
||||
for s, e in errors.items():
|
||||
print(c_dim(" %s: %s" % (s, e)))
|
||||
print()
|
||||
|
||||
elif action == "send":
|
||||
@@ -990,10 +1064,16 @@ def cmd_runtime(args):
|
||||
if pre.get("error"):
|
||||
if as_json:
|
||||
print(json.dumps({"ok": False, "error": pre["error"],
|
||||
"socket": sock, "pane": pane}))
|
||||
"socket": sock, "pane": pane,
|
||||
"detail": pre.get("detail")}))
|
||||
return
|
||||
print(c_red("Error: no such pane %s on %s" % (pane, sock)),
|
||||
file=sys.stderr)
|
||||
if pre["error"] == "socket_unreachable":
|
||||
print(c_red("Error: cannot reach socket %s: %s"
|
||||
% (sock, pre.get("detail") or "tmux error")),
|
||||
file=sys.stderr)
|
||||
else:
|
||||
print(c_red("Error: no such pane %s on %s" % (pane, sock)),
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
# Verified send: keys + Enter in one tmux call arrive as a paste
|
||||
# burst, which the muse composer takes as a newline instead of a
|
||||
@@ -1022,19 +1102,89 @@ def cmd_runtime(args):
|
||||
sys.exit(1)
|
||||
print()
|
||||
|
||||
elif action == "launch":
|
||||
elif action == "open":
|
||||
sock = getattr(args, "socket", None) or mcw.KNOWN_SOCKETS[0]
|
||||
session = getattr(args, "session", None)
|
||||
dry_run = getattr(args, "dry_run", False)
|
||||
|
||||
def _fail_open(msg, candidates=None):
|
||||
if as_json:
|
||||
print(json.dumps({"ok": False, "error": msg,
|
||||
"socket": sock, "session": session,
|
||||
"sessions": candidates or []}))
|
||||
return
|
||||
print(c_red("Error: %s" % msg), file=sys.stderr)
|
||||
if candidates:
|
||||
print(" Sessions on %s: %s" % (sock, ", ".join(candidates)),
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
def _sessions():
|
||||
r = mcw._tmux(sock, "list-sessions", "-F", "#{session_name}",
|
||||
timeout=10)
|
||||
if r.returncode != 0:
|
||||
return []
|
||||
return [ln.strip() for ln in (r.stdout or "").split("\n")
|
||||
if ln.strip()]
|
||||
|
||||
if not os.path.exists(sock):
|
||||
_fail_open("no such socket %s" % sock)
|
||||
return
|
||||
if not session:
|
||||
names = _sessions()
|
||||
if len(names) == 1:
|
||||
session = names[0]
|
||||
else:
|
||||
_fail_open("no session given and %s on %s"
|
||||
% ("no sessions found" if not names
|
||||
else "multiple sessions", sock), names)
|
||||
return
|
||||
else:
|
||||
probe = mcw._tmux(sock, "has-session", "-t", session,
|
||||
timeout=10)
|
||||
if probe.returncode != 0:
|
||||
_fail_open("no such session %s on %s" % (session, sock),
|
||||
_sessions())
|
||||
return
|
||||
argv = ["tmux", "-S", sock, "attach-session", "-t", session]
|
||||
if dry_run or as_json:
|
||||
if as_json:
|
||||
print(json.dumps({"ok": True, "dry_run": bool(dry_run),
|
||||
"socket": sock, "session": session,
|
||||
"argv": argv}, indent=2))
|
||||
return
|
||||
print(c_bold("\n=== RUNTIME OPEN (dry-run) ===\n"))
|
||||
print(" Socket: %s" % sock)
|
||||
print(" Session: %s" % c_cyan(session))
|
||||
print(" Command: %s" % shlex.join(argv))
|
||||
print()
|
||||
return
|
||||
try:
|
||||
os.execvp("tmux", argv)
|
||||
except OSError as e:
|
||||
print(c_red("Error: cannot exec tmux: %s" % e),
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
elif action == "launch":
|
||||
sock = getattr(args, "socket", None) or mcw.FLEET_SOCKETS[0]
|
||||
# Track box-launched session
|
||||
try:
|
||||
box_state_file = NETVM_ROOT / ".state" / "box-launched-sessions.json"
|
||||
box_state_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
box_data = {}
|
||||
if box_state_file.exists():
|
||||
import json
|
||||
box_data = json.loads(box_state_file.read_text())
|
||||
box_data[args.session] = {"socket": sock, "launched_at": datetime.now(timezone.utc).isoformat(), "origin": "box-cli"}
|
||||
box_state_file.write_text(json.dumps(box_data, indent=2))
|
||||
except Exception:
|
||||
pass
|
||||
session = args.session
|
||||
window = getattr(args, "window", None)
|
||||
dry_run = getattr(args, "dry_run", False)
|
||||
muse_args = list(getattr(args, "muse_args", None) or [])
|
||||
if muse_args[:1] == ["--"]:
|
||||
muse_args = muse_args[1:]
|
||||
posture = mcw.muse_approval_flags(muse_args)
|
||||
injected = [] if posture["flags"] else ["--disable-approval"]
|
||||
launcher = (shutil.which("muse-code")
|
||||
or "/home/super/.local/bin/muse-code")
|
||||
cmdline = shlex.join([launcher] + injected + muse_args)
|
||||
cmdline, injected = mcw.muse_launch_cmdline(muse_args)
|
||||
if dry_run:
|
||||
if as_json:
|
||||
print(json.dumps({
|
||||
@@ -1047,7 +1197,7 @@ def cmd_runtime(args):
|
||||
print(" Session: %s" % c_cyan(session))
|
||||
print(" Command: %s" % cmdline)
|
||||
if injected:
|
||||
print(" %s auto-approve injected: %s" % (
|
||||
print(" %s approval trail injected: %s" % (
|
||||
c_green("✔"), " ".join(injected)))
|
||||
else:
|
||||
print(" %s caller already sets approval flags; "
|
||||
@@ -1077,6 +1227,7 @@ def cmd_runtime(args):
|
||||
print(c_red("Error: launch failed: %s" % err),
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
mcw.wait_for_session_pane(sock, session, timeout=10)
|
||||
rec = mcw.reconcile(sockets=[sock])
|
||||
if as_json:
|
||||
print(json.dumps({"ok": True, "socket": sock,
|
||||
@@ -1088,6 +1239,9 @@ def cmd_runtime(args):
|
||||
print(" Command: %s" % c_dim(cmdline))
|
||||
for s in rec.get("started") or []:
|
||||
print(" %s watcher %s" % (badge_ok("STARTED"), c_cyan(s)))
|
||||
for f in rec.get("failed") or []:
|
||||
print(" %s watcher %s (retry: box muse-choices reconcile)"
|
||||
% (badge_err("FAILED"), c_cyan(f)))
|
||||
print()
|
||||
|
||||
elif action == "layout":
|
||||
@@ -1184,6 +1338,50 @@ def cmd_runtime(args):
|
||||
badge_err("FAILED"), f["pane"], f["error"]))
|
||||
print()
|
||||
|
||||
elif action == "reconcile":
|
||||
import runtime_reconcile as rec
|
||||
manifest = (getattr(args, "manifest", None)
|
||||
or str(NETVM_ROOT / "fleet" / "agents.json"))
|
||||
tasks = getattr(args, "tasks", None)
|
||||
dry_run = getattr(args, "dry_run", False)
|
||||
adopt = getattr(args, "adopt", False)
|
||||
report = rec.run_reconcile(manifest, tasks_dir=tasks,
|
||||
dry_run=dry_run, adopt=adopt)
|
||||
if as_json:
|
||||
print(json.dumps(report, indent=2))
|
||||
return
|
||||
print(c_bold("\n=== RUNTIME RECONCILE%s ===\n" % (
|
||||
" (dry-run)" if dry_run else "")))
|
||||
if not report["agents"] and not report["errors"]:
|
||||
print(c_dim(" Manifest declares no agents."))
|
||||
for a in report["agents"]:
|
||||
if a["action"] in ("launched", "briefed", "adopted"):
|
||||
mark = badge_ok(a["action"].upper())
|
||||
elif a["action"] in ("failed",):
|
||||
mark = badge_err("FAILED")
|
||||
elif a["action"] in ("launch", "brief"):
|
||||
mark = c_cyan("WOULD " + a["action"].upper())
|
||||
else:
|
||||
mark = c_dim("• " + a["action"])
|
||||
print(" %s %s [%s]: %s" % (
|
||||
mark, c_cyan(a["session"]), a["hat"], a["detail"]))
|
||||
for c in report["claims"]["requeued"]:
|
||||
print(" %s task %s (%s)" % (
|
||||
badge_ok("REQUEUED"), c_cyan(c["task"]), c["reason"]))
|
||||
for n in report.get("nudges") or []:
|
||||
print(" %s %s nudge to %s: %s" % (
|
||||
badge_ok("NUDGED"), n["kind"],
|
||||
c_cyan(n["session"]), n["detail"]))
|
||||
for e in report["claims"]["errors"] + report["errors"]:
|
||||
print(" %s %s" % (badge_err("ERROR"), e))
|
||||
w = report.get("watchers") or {}
|
||||
if w.get("started"):
|
||||
print(" %s watchers: %s" % (
|
||||
badge_ok("STARTED"), ", ".join(w["started"])))
|
||||
print()
|
||||
if not report["ok"]:
|
||||
sys.exit(1)
|
||||
|
||||
else:
|
||||
if as_json:
|
||||
print(json.dumps({"ok": False,
|
||||
@@ -1202,6 +1400,8 @@ def cmd_muse_choices(args):
|
||||
import muse_choice_watcher as mcw
|
||||
action = getattr(args, "mc_action", None) or "status"
|
||||
as_json = getattr(args, "json", False)
|
||||
if getattr(args, "socket", None):
|
||||
args.socket = mcw.resolve_socket(args.socket)
|
||||
caller = os.environ.get("BOX_CALLER") or getattr(args, "from_agent", None) or "super"
|
||||
|
||||
if action == "on":
|
||||
@@ -1358,9 +1558,15 @@ def cmd_muse_choices(args):
|
||||
left = max(0, int(float(h.get("held_until", 0)) - time.time()))
|
||||
except (TypeError, ValueError):
|
||||
left = -1
|
||||
if mcw.hold_renews_forever(h, h.get("kind")):
|
||||
when = "gate (renews, never auto-approves)"
|
||||
elif left >= 0:
|
||||
when = f"expires in {left}s"
|
||||
else:
|
||||
when = "expiry unknown"
|
||||
print(f" • {badge_warn('HELD')} {c_bold(h.get('pane', '?'))} "
|
||||
f"{h.get('kind')}/{h.get('key')} rule={h.get('rule')} "
|
||||
f"expires in {left}s")
|
||||
f"{when}")
|
||||
print(f" {c_dim((h.get('reason') or '')[:100])}")
|
||||
print(f" {c_dim((h.get('text') or '')[:100])}")
|
||||
if answers:
|
||||
@@ -2218,8 +2424,12 @@ def cmd_dm_send(args):
|
||||
"--to", recipient,
|
||||
"--target", target,
|
||||
"--raw",
|
||||
signed_wire
|
||||
]
|
||||
if getattr(args, "expect_reply", False):
|
||||
cmd.append("--expect-reply")
|
||||
if getattr(args, "reply_timeout", None):
|
||||
cmd.extend(["--reply-timeout", str(args.reply_timeout)])
|
||||
cmd.append(signed_wire)
|
||||
print(f"Dispatching Cryptographically Signed DM [{c_green('verified from:' + sender)}] -> [{c_bold(recipient)}/{target}]...")
|
||||
res = subprocess.run(cmd)
|
||||
if res.returncode != 0:
|
||||
@@ -5623,6 +5833,15 @@ def cmd_tmux_dispatch(args):
|
||||
sys.exit(res.returncode)
|
||||
|
||||
|
||||
def cmd_flow_dispatch(args):
|
||||
"""Bridge 'box flow' commands directly to bin/flow_engine.py."""
|
||||
flow_bin = str(BIN_DIR / "flow_engine.py")
|
||||
f_args = getattr(args, "flow_args", []) or []
|
||||
cmd = [sys.executable, flow_bin] + f_args
|
||||
res = subprocess.run(cmd)
|
||||
sys.exit(res.returncode)
|
||||
|
||||
|
||||
def cmd_muse_dispatch(args):
|
||||
"""Bridge 'box muse' commands to muse-cli-node or interactive REPL / multi-node lookups."""
|
||||
m_args = getattr(args, "muse_args", []) or []
|
||||
@@ -5681,6 +5900,289 @@ def cmd_muse_dispatch(args):
|
||||
sys.exit(res.returncode)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Domain: SYSOP (one-shot fleet installer)
|
||||
# ---------------------------------------------------------------------------
|
||||
SYSOP_SYSTEMD_DIR = NETVM_ROOT / "systemd"
|
||||
|
||||
# `systemctl show -p NextElapseUSecRealtime --value` reports this when a
|
||||
# timer has no computed next elapse (UINT64_MAX = USEC_INFINITY).
|
||||
SYSOP_NO_NEXT = {"", "0", "n/a", "18446744073709551615"}
|
||||
|
||||
|
||||
def sysop_user_units_dir():
|
||||
return Path.home() / ".config" / "systemd" / "user"
|
||||
|
||||
|
||||
def sysop_unit_files(systemd_dir=None):
|
||||
"""Sorted unit filenames (*.timer + *.service) shipped in systemd/."""
|
||||
d = Path(systemd_dir) if systemd_dir else SYSOP_SYSTEMD_DIR
|
||||
if not d.is_dir():
|
||||
return []
|
||||
return sorted(p.name for p in d.iterdir()
|
||||
if p.is_file() and p.suffix in (".timer", ".service"))
|
||||
|
||||
|
||||
def sysop_timer_needs_anchor(timer_path):
|
||||
"""True when a timer uses relative triggers (OnBootSec/OnActiveSec/
|
||||
OnUnitActiveSec) whose NEXT stays empty until the service runs once."""
|
||||
try:
|
||||
text = Path(timer_path).read_text()
|
||||
except OSError:
|
||||
return False
|
||||
return any(k in text for k in ("OnUnitActiveSec=", "OnBootSec=",
|
||||
"OnActiveSec="))
|
||||
|
||||
|
||||
def sysop_is_daemon(service_path):
|
||||
"""True for long-lived daemons that must not be one-shot started."""
|
||||
try:
|
||||
text = Path(service_path).read_text()
|
||||
except OSError:
|
||||
return False
|
||||
return "Restart=always" in text
|
||||
|
||||
|
||||
def _sysop_next(run, timer):
|
||||
"""NEXT elapse for a timer, or None when absent/unparseable."""
|
||||
rc, out = run(["systemctl", "--user", "show", timer,
|
||||
"-p", "NextElapseUSecRealtime", "--value"])
|
||||
nxt = (out or "").strip()
|
||||
if rc == 0 and nxt not in SYSOP_NO_NEXT:
|
||||
return nxt
|
||||
return None
|
||||
|
||||
|
||||
def _sysop_service_state(run, service):
|
||||
"""ActiveState for a service ('' when unknown)."""
|
||||
rc, out = run(["systemctl", "--user", "show", service,
|
||||
"-p", "ActiveState", "--value"])
|
||||
return (out or "").strip() if rc == 0 else ""
|
||||
|
||||
|
||||
def sysop_install_units(systemd_dir=None, user_dir=None, dry_run=False,
|
||||
run=None, progress=None, anchor_timeout=90,
|
||||
poll_interval=3):
|
||||
"""Link fleet units, reload, enable timers, anchor + verify them.
|
||||
|
||||
Returns a result dict with per-unit reports and a failures list.
|
||||
Idempotent: correct symlinks are kept, systemctl calls are re-runnable.
|
||||
With dry_run=True nothing is changed and no command is executed.
|
||||
|
||||
progress, when given, is called with short status lines as work
|
||||
happens (the run is otherwise silent for minutes behind slow
|
||||
service starts). Anchors never block indefinitely: services are
|
||||
started --no-block and NEXT is polled up to anchor_timeout; a
|
||||
still-activating service is reported in-progress (it self-anchors
|
||||
on completion) rather than failed.
|
||||
"""
|
||||
run = run or _sh
|
||||
say = progress or (lambda line: None)
|
||||
src_dir = Path(systemd_dir) if systemd_dir else SYSOP_SYSTEMD_DIR
|
||||
dst_dir = Path(user_dir) if user_dir else sysop_user_units_dir()
|
||||
units = sysop_unit_files(src_dir)
|
||||
timers = [u for u in units if u.endswith(".timer")]
|
||||
services = {u for u in units if u.endswith(".service")}
|
||||
|
||||
result = {"ok": True, "dry_run": dry_run, "units": [],
|
||||
"daemon_reload": {"ok": True, "detail": ""},
|
||||
"timers": [], "failures": []}
|
||||
|
||||
def fail(msg):
|
||||
result["failures"].append(msg)
|
||||
result["ok"] = False
|
||||
|
||||
if not timers:
|
||||
fail("no *.timer units discovered in %s" % src_dir)
|
||||
return result
|
||||
|
||||
# (1) Symlink every shipped unit into the user systemd dir.
|
||||
if not dry_run:
|
||||
try:
|
||||
dst_dir.mkdir(parents=True, exist_ok=True)
|
||||
except OSError as e:
|
||||
fail("cannot create %s: %s" % (dst_dir, e))
|
||||
return result
|
||||
for name in units:
|
||||
src = src_dir / name
|
||||
dst = dst_dir / name
|
||||
if dry_run:
|
||||
result["units"].append(
|
||||
{"unit": name, "status": "would-link",
|
||||
"src": str(src), "dst": str(dst)})
|
||||
continue
|
||||
try:
|
||||
if dst.is_symlink() and dst.resolve() == src.resolve():
|
||||
result["units"].append(
|
||||
{"unit": name, "status": "already-linked",
|
||||
"dst": str(dst)})
|
||||
continue
|
||||
if dst.is_symlink() or dst.exists():
|
||||
dst.unlink()
|
||||
dst.symlink_to(src)
|
||||
result["units"].append(
|
||||
{"unit": name, "status": "linked", "dst": str(dst)})
|
||||
except OSError as e:
|
||||
result["units"].append(
|
||||
{"unit": name, "status": "failed", "error": str(e)})
|
||||
fail("link %s: %s" % (name, e))
|
||||
|
||||
if dry_run:
|
||||
for timer in timers:
|
||||
result["timers"].append(
|
||||
{"timer": timer, "enabled": None, "anchored": None,
|
||||
"anchor_skipped": None, "next": None,
|
||||
"note": "would enable --now, anchor, and verify"})
|
||||
return result
|
||||
|
||||
# (2) Reload the user manager.
|
||||
say("daemon-reload ...")
|
||||
rc, out = run(["systemctl", "--user", "daemon-reload"])
|
||||
result["daemon_reload"] = {"ok": rc == 0, "detail": out}
|
||||
if rc != 0:
|
||||
fail("daemon-reload: %s" % (out or ("exit %d" % rc)))
|
||||
|
||||
# (3)-(5) Enable, anchor, and verify each timer.
|
||||
for timer in timers:
|
||||
entry = {"timer": timer, "enabled": False, "anchored": False,
|
||||
"anchor_skipped": None, "next": None, "note": None}
|
||||
say("enable --now %s ..." % timer)
|
||||
rc, out = run(["systemctl", "--user", "enable", "--now", timer])
|
||||
entry["enabled"] = rc == 0
|
||||
if rc != 0:
|
||||
fail("enable --now %s: %s" % (timer, out or ("exit %d" % rc)))
|
||||
# (4) Anchor relative timers: start the matching oneshot
|
||||
# service once so OnUnitActiveSec gains a reference timestamp
|
||||
# (fresh-install-mid-boot otherwise leaves NEXT empty). The
|
||||
# start is --no-block: slow first runs (backlog scrapes)
|
||||
# would otherwise stall the whole install with no output.
|
||||
service = timer[:-len(".timer")] + ".service"
|
||||
if service not in services:
|
||||
entry["anchor_skipped"] = "no matching service shipped"
|
||||
elif not sysop_timer_needs_anchor(src_dir / timer):
|
||||
entry["anchor_skipped"] = "calendar timer needs no anchor"
|
||||
elif sysop_is_daemon(src_dir / service):
|
||||
entry["anchor_skipped"] = "long-lived daemon, not started"
|
||||
else:
|
||||
state = _sysop_service_state(run, service)
|
||||
if state == "activating":
|
||||
say("anchor %s: already running, waiting for NEXT ..."
|
||||
% service)
|
||||
else:
|
||||
say("anchor %s: starting ..." % service)
|
||||
rc, out = run(["systemctl", "--user", "start",
|
||||
"--no-block", service])
|
||||
if rc != 0:
|
||||
fail("start %s: %s"
|
||||
% (service, out or ("exit %d" % rc)))
|
||||
state = "start-failed"
|
||||
# (5) Poll for a real NEXT elapse (bounded).
|
||||
if state != "start-failed":
|
||||
polled = _sysop_poll_next(
|
||||
run, say, timer, service, anchor_timeout,
|
||||
poll_interval)
|
||||
msg = polled.pop("_fail", None)
|
||||
if msg:
|
||||
fail(msg)
|
||||
entry.update(polled)
|
||||
# Timers whose anchor was skipped still get one NEXT check.
|
||||
if entry["anchor_skipped"] and entry["next"] is None:
|
||||
nxt = _sysop_next(run, timer)
|
||||
if nxt is not None:
|
||||
entry["next"] = nxt
|
||||
else:
|
||||
fail("verify %s: no NEXT elapse" % timer)
|
||||
result["timers"].append(entry)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _sysop_poll_next(run, say, timer, service, timeout, interval):
|
||||
"""Poll until the timer shows NEXT, the service fails, or timeout.
|
||||
|
||||
Returns a partial timer entry (anchored/next/note). A service
|
||||
still activating at timeout is reported in-progress rather than
|
||||
failed: its running start job anchors the timer on completion.
|
||||
"""
|
||||
deadline = time.monotonic() + max(0, timeout)
|
||||
while True:
|
||||
nxt = _sysop_next(run, timer)
|
||||
if nxt is not None:
|
||||
return {"anchored": True, "next": nxt, "note": None}
|
||||
state = _sysop_service_state(run, service)
|
||||
if state == "failed":
|
||||
return {"anchored": False, "next": None,
|
||||
"note": None, "_fail":
|
||||
"anchor %s: service failed "
|
||||
"(journalctl --user -u %s)" % (service, service)}
|
||||
if time.monotonic() >= deadline:
|
||||
if state == "activating":
|
||||
say("anchor %s: still running, timer self-anchors "
|
||||
"on completion" % service)
|
||||
return {"anchored": "in-progress", "next": None,
|
||||
"note": "anchor running; verify NEXT shortly"}
|
||||
return {"anchored": False, "next": None,
|
||||
"note": None, "_fail":
|
||||
"verify %s: no NEXT elapse "
|
||||
"(service state: %s)" % (timer, state or "?")}
|
||||
say("anchor %s: waiting for NEXT ..." % service)
|
||||
time.sleep(max(0, interval))
|
||||
|
||||
|
||||
def cmd_sysop_install(args):
|
||||
dry_run = getattr(args, "dry_run", False)
|
||||
as_json = getattr(args, "json", False)
|
||||
# Live progress: stdout in text mode, stderr in --json mode so
|
||||
# stdout stays pure machine-readable JSON.
|
||||
progress = (lambda line: print(" ... %s" % line, flush=True,
|
||||
file=sys.stderr if as_json else sys.stdout))
|
||||
result = sysop_install_units(dry_run=dry_run, progress=progress)
|
||||
if as_json:
|
||||
print(json.dumps(result, indent=2))
|
||||
sys.exit(0 if result["ok"] else 1)
|
||||
print(c_bold("\n=== SYSOP INSTALL%s ===\n" % (
|
||||
" (dry-run)" if dry_run else "")))
|
||||
for u in result["units"]:
|
||||
st = u["status"]
|
||||
mark = (badge_ok("LINKED") if st == "linked"
|
||||
else badge_dim("KEPT") if st == "already-linked"
|
||||
else c_cyan("○ WOULD-LINK") if st == "would-link"
|
||||
else badge_err("FAILED"))
|
||||
print(" %s %s" % (mark, u["unit"]))
|
||||
if not dry_run:
|
||||
dr = result["daemon_reload"]
|
||||
print(" %s daemon-reload" % (
|
||||
badge_ok("OK") if dr["ok"] else badge_err("FAILED")))
|
||||
print()
|
||||
for t in result["timers"]:
|
||||
en = t["enabled"]
|
||||
emark = (badge_dim("?") if en is None
|
||||
else badge_ok("ON") if en else badge_err("OFF"))
|
||||
if t.get("anchor_skipped"):
|
||||
amark = c_dim("- %s" % t["anchor_skipped"])
|
||||
elif t.get("anchored") == "in-progress":
|
||||
amark = c_cyan("○ ANCHORING")
|
||||
elif t["anchored"]:
|
||||
amark = badge_ok("ANCHORED")
|
||||
elif t["anchored"] is None:
|
||||
amark = badge_dim("?")
|
||||
else:
|
||||
amark = badge_err("ANCHOR-FAILED")
|
||||
nxt = (t["next"] or c_dim(t["note"]) if t.get("note")
|
||||
else t["next"] or (c_dim("pending (dry-run)")
|
||||
if dry_run else badge_err("NO NEXT")))
|
||||
print(" %s %-32s %s NEXT=%s" % (emark, t["timer"], amark,
|
||||
nxt))
|
||||
print()
|
||||
if result["failures"]:
|
||||
for f in result["failures"]:
|
||||
print(" %s %s" % (c_red("✘"), f))
|
||||
print()
|
||||
sys.exit(1)
|
||||
print(" %s fleet units installed and verified.\n" % c_green("✔"))
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def build_parser():
|
||||
common = argparse.ArgumentParser(add_help=False)
|
||||
common.add_argument("--json", action="store_true", help="Output machine-readable JSON")
|
||||
@@ -5771,6 +6273,9 @@ def build_parser():
|
||||
p_app_req_key.add_argument("node", choices=VALID_NODES, help="Target node requesting key")
|
||||
p_app_req_key.add_argument("--reason", default="Passkey authentication required", help="Reason for key request")
|
||||
|
||||
p_app_gates = app_sub.add_parser("gates", aliases=["gate"], parents=[common], help="Inspect coordinator decision record gates")
|
||||
p_app_gates.add_argument("--scope", default=None, help="Check specific gate scope (e.g. role-layer, merges-to-main)")
|
||||
|
||||
# Domain: MUSE-CHOICES
|
||||
p_mc = subparsers.add_parser("muse-choices", parents=[common], help="Muse TUI A/B/C choice auto-answer daemon (on/off/status/logs)")
|
||||
mc_sub = p_mc.add_subparsers(dest="mc_action")
|
||||
@@ -5795,7 +6300,7 @@ def build_parser():
|
||||
p_mc_res.add_argument("decision", choices=["approve", "deny"], help="Release the hold to approve, or deny it (permission kinds only)")
|
||||
|
||||
# Domain: RUNTIME
|
||||
p_rt = subparsers.add_parser("runtime", parents=[common], help="Muse CLI tmux runtimes: list states, send input, launch auto-approved")
|
||||
p_rt = subparsers.add_parser("runtime", parents=[common], help="Muse CLI tmux runtimes: list states, send input, launch with approval trail")
|
||||
rt_sub = p_rt.add_subparsers(dest="rt_action")
|
||||
|
||||
p_rt_list = rt_sub.add_parser("list", parents=[common], help="List panes with runtime state + approval posture (default)")
|
||||
@@ -5808,8 +6313,13 @@ def build_parser():
|
||||
p_rt_send.add_argument("keys", help="Keys / text to send")
|
||||
p_rt_send.add_argument("--no-enter", action="store_true", help="Do not send Enter after keys")
|
||||
|
||||
p_rt_launch = rt_sub.add_parser("launch", parents=[common], help="Launch a Muse session with auto-approve injected")
|
||||
p_rt_launch.add_argument("--socket", default=None, help="Tmux socket (default: /tmp/tmux-1000/default)")
|
||||
p_rt_open = rt_sub.add_parser("open", parents=[common], help="Attach to a session on a socket (execs tmux attach)")
|
||||
p_rt_open.add_argument("--socket", default=None, help="Tmux socket (default: /tmp/tmux-1000/default)")
|
||||
p_rt_open.add_argument("--session", default=None, help="Session name (default: the only session)")
|
||||
p_rt_open.add_argument("--dry-run", action="store_true", help="Print the attach plan without attaching")
|
||||
|
||||
p_rt_launch = rt_sub.add_parser("launch", parents=[common], help="Launch a Muse session with approval trail injected (on-request)")
|
||||
p_rt_launch.add_argument("--socket", default=None, help="Tmux socket (default: /tmp/tmux-muse.sock for Box fleet)")
|
||||
p_rt_launch.add_argument("--session", required=True, help="New tmux session name")
|
||||
p_rt_launch.add_argument("--window", "-w", default=None, help="Initial window name")
|
||||
p_rt_launch.add_argument("--dry-run", action="store_true", help="Print the launch plan without creating")
|
||||
@@ -5823,6 +6333,12 @@ def build_parser():
|
||||
p_rt_spread.add_argument("--session", default=None, help="Only this tmux session")
|
||||
p_rt_spread.add_argument("--dry-run", action="store_true", help="Print the spread plan without moving panes")
|
||||
|
||||
p_rt_rec = rt_sub.add_parser("reconcile", parents=[common], help="Enforce fleet/agents.json: relaunch missing, brief fresh panes, requeue stale claims")
|
||||
p_rt_rec.add_argument("--manifest", default=None, help="Manifest path (default: fleet/agents.json)")
|
||||
p_rt_rec.add_argument("--tasks", default=None, help="Task queue dir (default: alongside manifest)")
|
||||
p_rt_rec.add_argument("--dry-run", action="store_true", help="Print the plan without changing anything")
|
||||
p_rt_rec.add_argument("--adopt", action="store_true", help="Record live sessions as briefed without sending")
|
||||
|
||||
# Domain: INVITE
|
||||
p_invite = subparsers.add_parser("invite", parents=[common], help="Muse.ai invite codes: find per-agent codes and redeem")
|
||||
p_invite.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node (status)")
|
||||
@@ -6398,6 +6914,10 @@ def build_parser():
|
||||
p_sub_spawn.add_argument("--wait", type=int, default=30, help="Seconds to wait for subagent response")
|
||||
|
||||
|
||||
# Domain: FLOW (Agentic workflows in persistent tmux panes with delta read-backs)
|
||||
p_flow = subparsers.add_parser("flow", parents=[common], help="Manage agentic flows in persistent tmux panes (start, read, send, list, stop)")
|
||||
p_flow.add_argument("flow_args", nargs=argparse.REMAINDER, help="Arguments passed directly to flow_engine.py")
|
||||
|
||||
# Domain: TMUX (Headless background tmux sessions with automatic logging)
|
||||
p_tmux = subparsers.add_parser("tmux", parents=[common], help="Manage headless background tmux sessions on /tmp/tmux-muse.sock")
|
||||
p_tmux.add_argument("tmux_args", nargs=argparse.REMAINDER, help="Arguments passed directly to muse-tmux.py")
|
||||
@@ -6423,6 +6943,12 @@ def build_parser():
|
||||
p_tui = subparsers.add_parser("tui", parents=[common], help="Interactive full-screen Muse TUI & Box fleet console")
|
||||
p_tui.add_argument("tui_args", nargs=argparse.REMAINDER, help="Arguments passed directly to muse-tui.py")
|
||||
|
||||
# Domain: SYSOP (one-shot fleet installer: link units, enable timers, anchor + verify)
|
||||
p_sysop = subparsers.add_parser("sysop", parents=[common], help="Fleet operations: one-shot systemd unit installer")
|
||||
sysop_sub = p_sysop.add_subparsers(dest="sysop_action")
|
||||
p_sysop_install = sysop_sub.add_parser("install", parents=[common], help="Link systemd units, enable timers, anchor and verify them")
|
||||
p_sysop_install.add_argument("--dry-run", action="store_true", help="Print actions without changing anything")
|
||||
|
||||
return parser
|
||||
|
||||
def main():
|
||||
@@ -6431,6 +6957,8 @@ def main():
|
||||
tui_args = sys.argv[2:]
|
||||
if tui_args and tui_args[0] in ("onboard", "tmux", "connects", "approvals"):
|
||||
cmd = [sys.executable, str(BIN_DIR / "box-onboard-tui.py")] + tui_args[1:]
|
||||
elif tui_args and tui_args[0] in ("fleet", "oversight"):
|
||||
cmd = [sys.executable, str(BIN_DIR / "box-fleet-tui.py")] + tui_args[1:]
|
||||
else:
|
||||
cmd = [sys.executable, str(BIN_DIR / "muse-tui.py"), "--mode", "box"] + tui_args
|
||||
res = subprocess.run(cmd)
|
||||
@@ -6439,6 +6967,10 @@ def main():
|
||||
cmd = [sys.executable, str(BIN_DIR / "box-onboard-tui.py")] + sys.argv[2:]
|
||||
res = subprocess.run(cmd)
|
||||
sys.exit(res.returncode)
|
||||
elif sys.argv[1] in ("fleet-tui",):
|
||||
cmd = [sys.executable, str(BIN_DIR / "box-fleet-tui.py")] + sys.argv[2:]
|
||||
res = subprocess.run(cmd)
|
||||
sys.exit(res.returncode)
|
||||
elif sys.argv[1] == "tmux":
|
||||
if len(sys.argv) > 2 and sys.argv[2] in ("tally", "auto", "watch", "once", "match", "rules", "status"):
|
||||
if sys.argv[2] == "auto":
|
||||
@@ -6459,6 +6991,21 @@ def main():
|
||||
cmd = [sys.executable, str(BIN_DIR / "docs-lookup.py")] + sys.argv[2:]
|
||||
res = subprocess.run(cmd)
|
||||
sys.exit(res.returncode)
|
||||
elif sys.argv[1] in ("stability", "stable"):
|
||||
cmd = [sys.executable, str(NETVM_ROOT / "watchers" / "box-stability-watcher.py")]
|
||||
sub = sys.argv[2:]
|
||||
if not sub or sub[0] == "status":
|
||||
cmd.append("--status")
|
||||
elif sub[0] == "check":
|
||||
cmd.append("--check")
|
||||
elif sub[0] == "json":
|
||||
cmd.extend(["--status", "--json"])
|
||||
elif sub[0] == "resume" and len(sub) > 1:
|
||||
cmd.extend(["--resume", sub[1]])
|
||||
else:
|
||||
cmd.extend(sub)
|
||||
res = subprocess.run(cmd)
|
||||
sys.exit(res.returncode)
|
||||
elif sys.argv[1] == "muse":
|
||||
m_args = sys.argv[2:]
|
||||
if not m_args:
|
||||
@@ -6759,12 +7306,20 @@ def main():
|
||||
cmd_lookup(args)
|
||||
elif args.domain in ("passkey", "key"):
|
||||
cmd_passkey_info(args)
|
||||
elif args.domain == "flow":
|
||||
cmd_flow_dispatch(args)
|
||||
elif args.domain == "tmux":
|
||||
cmd_tmux_dispatch(args)
|
||||
elif args.domain == "muse":
|
||||
cmd_muse_dispatch(args)
|
||||
elif args.domain in ("docs", "doc"):
|
||||
cmd_docs_dispatch(args)
|
||||
elif args.domain == "sysop":
|
||||
act = getattr(args, "sysop_action", None)
|
||||
if act == "install":
|
||||
cmd_sysop_install(args)
|
||||
else:
|
||||
parser.print_help()
|
||||
else:
|
||||
parser.print_help()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user