feat(kpi): expand KPI runtime monitoring, prompt advisory envelopes, and missing field resiliency

This commit is contained in:
operator
2026-10-06 20:02:48 +00:00
parent b824f6d405
commit ae4df2f29c
13 changed files with 1702 additions and 85 deletions
+370 -10
View File
@@ -930,6 +930,169 @@ def cmd_approvals(args):
print(f" Reason: {c_cyan(reason)}")
print(c_dim(f" Operator can approve with: box approvals allow {node}"))
# ---------------------------------------------------------------------------
# Domain: RUNTIME (agentic management of Muse CLI tmux runtimes)
# ---------------------------------------------------------------------------
def cmd_runtime(args):
import shlex
import muse_choice_watcher as mcw
action = getattr(args, "rt_action", None) or "list"
as_json = getattr(args, "json", False)
if action == "list":
sock = getattr(args, "socket", None)
rows = mcw.all_runtime_rows([sock] if sock else None)
if getattr(args, "muse_only", False):
rows = [r for r in rows if r["is_muse"]]
if as_json:
print(json.dumps({"ok": True, "runtimes": rows}, indent=2))
return
print(c_bold("\n=== MUSE RUNTIMES ===\n"))
if not rows:
print(c_dim(" No panes found."))
print()
return
headers = ["SOCKET", "SESSION", "PANE", "CMD", "STATE",
"APPROVE", "WATCHER"]
table = []
for r in rows:
state = r["state"]
if r["state"] == "approval-pending" and r["prompt_kind"]:
state = "%s(%s/%s)" % (state, r["prompt_kind"],
r["prompt_key"])
if r["is_muse"]:
approve = (badge_ok("YES") if r["auto_approve"]
else badge_err("NO"))
else:
approve = badge_dim("-")
watcher = (badge_ok("ALIVE %s" % r["watcher_pid"])
if r["watcher_alive"] else badge_dim("-"))
table.append([os.path.basename(r["socket"]),
"%s:%s" % (r["session"], r["window"]),
r["pane"], (r["cmd"] or "")[:26], state,
approve, watcher])
print_table(headers, table)
print()
elif action == "send":
sock = getattr(args, "socket", None) or mcw.KNOWN_SOCKETS[0]
pane = args.pane
keys = args.keys
enter = not getattr(args, "no_enter", False)
pre = mcw.pane_state(sock, pane)
if pre.get("error"):
if as_json:
print(json.dumps({"ok": False, "error": pre["error"],
"socket": sock, "pane": pane}))
return
print(c_red("Error: no such pane %s on %s" % (pane, sock)),
file=sys.stderr)
sys.exit(1)
t_args = ["send-keys", "-t", pane, keys]
if enter:
t_args.append("Enter")
r = mcw._tmux(sock, *t_args, timeout=10)
ok = r.returncode == 0
if as_json:
print(json.dumps({
"ok": ok, "socket": sock, "pane": pane,
"pre_state": pre["state"],
"prompt_kind": pre["prompt_kind"],
"sent": keys, "enter": enter,
"error": (r.stderr or r.stdout or "").strip() or None
if not ok else None}, indent=2))
return
print("\n Pane %s on %s [%s]" % (
c_bold(pane), sock, c_cyan(pre["state"])))
if ok:
print(" %s sent %r%s" % (c_green("✔"),
keys, " + Enter" if enter else ""))
else:
print(" %s send failed: %s" % (
c_red("✘"),
(r.stderr or r.stdout or "").strip() or "tmux error"))
sys.exit(1)
print()
elif action == "launch":
sock = getattr(args, "socket", None) or mcw.KNOWN_SOCKETS[0]
session = args.session
window = getattr(args, "window", None)
dry_run = getattr(args, "dry_run", False)
muse_args = list(getattr(args, "muse_args", None) or [])
if muse_args[:1] == ["--"]:
muse_args = muse_args[1:]
posture = mcw.muse_approval_flags(muse_args)
injected = [] if posture["flags"] else ["--disable-approval"]
launcher = (shutil.which("muse-code")
or "/home/super/.local/bin/muse-code")
cmdline = shlex.join([launcher] + injected + muse_args)
if dry_run:
if as_json:
print(json.dumps({
"ok": True, "dry_run": True, "socket": sock,
"session": session, "window": window,
"cmdline": cmdline, "injected": injected}, indent=2))
return
print(c_bold("\n=== RUNTIME LAUNCH (dry-run) ===\n"))
print(" Socket: %s" % sock)
print(" Session: %s" % c_cyan(session))
print(" Command: %s" % cmdline)
if injected:
print(" %s auto-approve injected: %s" % (
c_green("✔"), " ".join(injected)))
else:
print(" %s caller already sets approval flags; "
"nothing injected" % c_dim("•"))
print()
return
exists = mcw._tmux(sock, "has-session", "-t", session, timeout=10)
if exists.returncode == 0:
if as_json:
print(json.dumps({"ok": False, "error": "session_exists",
"socket": sock, "session": session}))
return
print(c_red("Error: session '%s' already exists on %s"
% (session, sock)), file=sys.stderr)
sys.exit(1)
t_args = ["new-session", "-d", "-s", session]
if window:
t_args.extend(["-n", window])
t_args.append(cmdline)
r = mcw._tmux(sock, *t_args, timeout=15)
if r.returncode != 0:
err = (r.stderr or r.stdout or "").strip() or "tmux error"
if as_json:
print(json.dumps({"ok": False, "error": err,
"socket": sock, "session": session}))
return
print(c_red("Error: launch failed: %s" % err),
file=sys.stderr)
sys.exit(1)
rec = mcw.reconcile(sockets=[sock])
if as_json:
print(json.dumps({"ok": True, "socket": sock,
"session": session, "window": window,
"cmdline": cmdline, "injected": injected,
"reconcile": rec}, indent=2))
return
print(c_green("\n✔ Launched '%s' on %s") % (session, sock))
print(" Command: %s" % c_dim(cmdline))
for s in rec.get("started") or []:
print(" %s watcher %s" % (badge_ok("STARTED"), c_cyan(s)))
print()
else:
if as_json:
print(json.dumps({"ok": False,
"error": "unknown_action",
"action": action}))
return
print(c_red("Error: unknown runtime action '%s'" % action),
file=sys.stderr)
sys.exit(1)
# ---------------------------------------------------------------------------
# Domain: MUSE-CHOICES (Muse TUI A/B/C auto-answer daemon)
# ---------------------------------------------------------------------------
@@ -1151,28 +1314,51 @@ def cmd_invite(args):
sys.exit(1)
elif action == "redeem":
code = getattr(args, "code", None)
method = getattr(args, "method", "auto")
notify_target = getattr(args, "notify", None) or getattr(args, "notify_target", None)
notify_agent = getattr(args, "notify_agent", None)
try:
res = invite.redeem_invite(node, code)
if method != "api" or notify_target or notify_agent:
from invite_handler import InviteHandler
with InviteHandler(node) as h:
res_obj = h.redeem_code(code, method=method, notify_target=notify_target, notify_agent=notify_agent)
res = res_obj.to_dict()
res["ok"] = res_obj.success
else:
res = invite.redeem_invite(node, code)
except invite.InviteError as e:
print(c_red(f" Error: {e}"), file=sys.stderr)
sys.exit(2)
except Exception as e:
print(c_red(f" Error during redemption: {e}"), file=sys.stderr)
sys.exit(1)
if as_json:
print(json.dumps(res, indent=2))
elif res.get("ok"):
msg = f" ✔ Redeemed {res['code']} on {node}."
msg = f" ✔ Redeemed {res.get('code') or code} on {node}."
if res.get("detail"):
msg += f" {res['detail']}"
print(c_green(msg))
else:
print(c_red(f" ✘ Redeem failed on {node}: "
f"{res.get('reason') or res.get('error')}"),
file=sys.stderr)
reason_str = res.get("reason") or res.get("error") or "failed"
print(c_red(f" ✘ Redeem failed on {node}: {reason_str}"), file=sys.stderr)
if res.get("detail"):
print(c_dim(f" {res['detail']}"), file=sys.stderr)
print(c_dim(f" Detail: {res['detail']}"), file=sys.stderr)
if res.get("field_missing"):
print(c_yellow(f" Note: Expected redemption input field was missing or hidden on @{node}."), file=sys.stderr)
if res.get("loopback_notified"):
print(c_cyan(f" Loopback: Dispatched alert to {notify_agent or 'coordinator'}/{notify_target}."), file=sys.stderr)
sys.exit(1)
elif action == "salvage":
target = getattr(args, "node", "646") or "646"
res = salvage_blocked_node(target, helper_node=getattr(args, "helper", None))
notify_target = getattr(args, "notify", None) or getattr(args, "notify_target", None)
notify_agent = getattr(args, "notify_agent", None)
res = salvage_blocked_node(
target,
helper_node=getattr(args, "helper", None),
notify_target=notify_target,
notify_agent=notify_agent,
)
if as_json:
print(json.dumps(res, indent=2))
return
@@ -1187,6 +1373,8 @@ def cmd_invite(args):
action_hint = res.get("share_instruction") or f"Redeem code '{code}' via onboarding pipeline to grant 1B tokens to {target}."
print(f" Action: {action_hint}")
print(f" Command: {c_cyan('box onboard start <new_node> --email <client_email> --for ' + target)}")
if res.get("loopback_notified"):
print(c_cyan(f" Loopback: Notice dispatched to {notify_agent or 'coordinator'}/{notify_target}."))
print()
@@ -1213,6 +1401,10 @@ def cmd_usage(args):
rows.append([c_bold(n), c_red("ERROR"), "-", "-",
(it.get("error") or "")[:30], "-"])
continue
if not it.get("stats_loaded", True):
rows.append([c_bold(n), c_yellow("UNLOADED"), "-", "-",
"Stats did not render", "-"])
continue
wu_val = it.get("weekly_used_pct")
wu = ("%d%%" % wu_val) if wu_val is not None else "-"
au_val = it.get("additional_used_pct")
@@ -1233,6 +1425,7 @@ def cmd_usage(args):
if has_blocked:
print("\n" + c_yellow(" ⚠ One or more agents have reached usage limits. Run 'box invite salvage <node>' to resolve.") + "\n")
else:
print()
@@ -1264,9 +1457,80 @@ def cmd_settings(args):
print(json.dumps(info, indent=2))
# ---------------------------------------------------------------------------
# Domain: KPI (Spend Monitoring, Performance & Runtime Preservation)
# ---------------------------------------------------------------------------
def cmd_kpi(args):
import kpi
act = getattr(args, "kpi_action", "status") or "status"
as_json = getattr(args, "json", False)
if act == "status":
nodes = [args.node] if getattr(args, "node", None) else kpi.VALID_NODES
kpis = kpi.fleet_kpi(nodes)
if as_json:
print(json.dumps({k: v.to_dict() for k, v in kpis.items()}, indent=2))
return
print("\n" + c_bold("=== NETVM FLEET KPI & RUNTIME PRESERVATION DASHBOARD ===") + "\n")
header = f"{'NODE':<6} {'QUOTA':<10} {'CALLS':<12} {'JOBS':<10} {'SUBAGENTS':<11} {'TMUX':<6} {'UPTIME':<8} {'ROUTE':<9} {'EFFICIENCY':<15}"
sep = f"{'────':<6} {'─────────':<10} {'───────────':<12} {'─────────':<10} {'──────────':<11} {'────':<6} {'──────':<8} {'───────':<9} {'──────────────':<15}"
print(c_bold(header))
print(sep)
for n in nodes:
k = kpis.get(n)
if not k:
continue
q_str = f"{k.weekly_used_pct}%" if k.weekly_used_pct is not None else "Active"
c_str = f"{k.calls_sent} ({k.calls_verified}v)"
j_str = f"{k.jobs_completed}/{k.jobs_assigned}"
sub_str = str(k.subagents_active)
tmux_str = str(k.tmux_workers_active)
up_str = f"{k.uptime_hours}h"
print(f"{c_bold(k.node):<15} {q_str:<10} {c_str:<12} {j_str:<10} {sub_str:<11} {tmux_str:<6} {up_str:<8} {k.route_status:<9} {k.efficiency_rating:<15}")
print("\n" + c_dim("Run 'box kpi report <node>' for prescriptive runtime preservation advisories.\n"))
elif act == "report":
node = getattr(args, "node", "646") or "646"
res = kpi.get_agent_kpi(node)
if as_json:
print(json.dumps(res.to_dict(), indent=2))
return
print(f"\n{c_bold('=== KPI & RUNTIME REPORT: @' + res.node.upper() + ' ===')}")
print(f" Weekly Quota: {res.weekly_used_pct}% used")
print(f" Extra Tokens: {res.extra_tokens_remaining}")
print(f" Blocked Status: {'YES (LIMIT REACHED)' if res.is_blocked else 'NO (HEALTHY)'}")
print(f" Messages / Calls: {res.calls_sent} sent ({res.calls_verified} verified delivered)")
print(f" Jobs Dispatched: {res.jobs_completed} completed / {res.jobs_assigned} assigned")
print(f" Active Subagents: {res.subagents_active}")
print(f" Active Tmux Workers:{res.tmux_workers_active}")
print(f" Process Uptime: {res.uptime_hours} hours")
print(f" Route Health: {res.route_status}")
print(f" Efficiency Index: {res.efficiency_index} ({res.efficiency_rating})")
print(f"\n {c_yellow('[RUNTIME PRESERVATION ADVISORY]')}\n {res.preservation_advisory}\n")
elif act == "routes":
routes = {n: kpi.check_node_routes(n) for n in kpi.VALID_NODES}
if as_json:
print(json.dumps(routes, indent=2))
else:
print("\n" + c_bold("=== NETVM ROUTE HEALTH ==="))
for n, st in routes.items():
print(f" @{c_bold(n):<15} : {st}")
print()
elif act == "spawn-worker":
res = kpi.spawn_tmux_worker(args.node, args.session, args.worker_command)
if as_json:
print(json.dumps(res, indent=2))
else:
if res.get("ok"):
print(c_green(f"✔ {res.get('message')}"))
else:
print(c_red(f"✘ Failed to spawn worker: {res.get('error')}"), file=sys.stderr)
sys.exit(1)
# ---------------------------------------------------------------------------
# Domain: ONBOARD (Agent-driven Onboarding & Invite Salvage Pipeline)
# ---------------------------------------------------------------------------
def cmd_onboard(args):
import onboard_pipeline
action = getattr(args, "onboard_action", "start") or "start"
@@ -1334,6 +1598,24 @@ def cmd_onboard(args):
print(f" Redemption: {r_st}")
print()
elif action == "connects":
conn_list = onboard_pipeline.get_all_connects()
if as_json:
print(json.dumps(conn_list, indent=2))
return
print(f"\n{c_bold('=== ACTIVE FLEET & CLIENT ONBOARD CONNECTS ===')}")
print(f" {'NODE':<8} {'TYPE':<16} {'STAGE / STATUS':<18} {'CDP':<8} {'INVITE':<10} {'ROLE / DETAIL'}")
print(" " + "─" * 78)
for c in conn_list:
node = c.get("node", "")
t_str = c.get("type", "")
st_str = c.get("stage", c.get("status", ""))
cdp = str(c.get("cdp_port") or "-")
code = c.get("invite_code") or "-"
role = c.get("role") or c.get("detail") or c.get("email") or ""
print(f" {node:<8} {t_str:<16} {st_str:<18} {cdp:<8} {code:<10} {role}")
print()
elif action == "salvage-wo":
node = getattr(args, "node", "646") or "646"
target_chat = getattr(args, "target", "646 tasks") or "646 tasks"
@@ -4874,9 +5156,18 @@ def cmd_docs_dispatch(args):
def cmd_tmux_dispatch(args):
"""Bridge 'box tmux' commands directly to bin/muse-tmux.py."""
tmux_bin = str(BIN_DIR / "muse-tmux.py")
"""Bridge 'box tmux' commands directly to bin/muse-tmux.py or tmux_auto_approver.py."""
t_args = getattr(args, "tmux_args", []) or []
if t_args and t_args[0] in ("tally", "auto", "watch", "once", "match", "rules", "status"):
approver_bin = str(BIN_DIR / "tmux_auto_approver.py")
if t_args[0] == "auto":
sub = t_args[1:] or ["status"]
else:
sub = t_args
cmd = [sys.executable, approver_bin] + sub
res = subprocess.run(cmd)
sys.exit(res.returncode)
tmux_bin = str(BIN_DIR / "muse-tmux.py")
if not t_args:
t_args = ["list"]
cmd = [sys.executable, tmux_bin] + t_args
@@ -5050,6 +5341,27 @@ def build_parser():
p_mc_rec = mc_sub.add_parser("reconcile", parents=[common], help="Enforce desired state now (start missing / stop excess)")
# Domain: RUNTIME
p_rt = subparsers.add_parser("runtime", parents=[common], help="Muse CLI tmux runtimes: list states, send input, launch auto-approved")
rt_sub = p_rt.add_subparsers(dest="rt_action")
p_rt_list = rt_sub.add_parser("list", parents=[common], help="List panes with runtime state + approval posture (default)")
p_rt_list.add_argument("--socket", default=None, help="Only this tmux socket")
p_rt_list.add_argument("--muse-only", action="store_true", help="Only Muse CLI panes")
p_rt_send = rt_sub.add_parser("send", parents=[common], help="Send keys to a pane (reports pre-send state)")
p_rt_send.add_argument("--socket", default=None, help="Tmux socket (default: /tmp/tmux-1000/default)")
p_rt_send.add_argument("pane", help="Pane id (e.g. %%37)")
p_rt_send.add_argument("keys", help="Keys / text to send")
p_rt_send.add_argument("--no-enter", action="store_true", help="Do not send Enter after keys")
p_rt_launch = rt_sub.add_parser("launch", parents=[common], help="Launch a Muse session with auto-approve injected")
p_rt_launch.add_argument("--socket", default=None, help="Tmux socket (default: /tmp/tmux-1000/default)")
p_rt_launch.add_argument("--session", required=True, help="New tmux session name")
p_rt_launch.add_argument("--window", "-w", default=None, help="Initial window name")
p_rt_launch.add_argument("--dry-run", action="store_true", help="Print the launch plan without creating")
p_rt_launch.add_argument("muse_args", nargs=argparse.REMAINDER, default=[], help="Extra muse args after --")
# Domain: INVITE
p_invite = subparsers.add_parser("invite", parents=[common], help="Muse.ai invite codes: find per-agent codes and redeem")
p_invite.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node (status)")
@@ -5070,10 +5382,15 @@ def build_parser():
p_inv_redeem = inv_sub.add_parser("redeem", parents=[common], help="Redeem an invite code on a node")
p_inv_redeem.add_argument("node", choices=VALID_NODES, help="Target node")
p_inv_redeem.add_argument("code", help="6-char invite code")
p_inv_redeem.add_argument("--method", choices=["auto", "dom", "api"], default="auto", help="Redemption method (default: auto)")
p_inv_redeem.add_argument("--notify", default=None, help="Sidechat to notify on loopback")
p_inv_redeem.add_argument("--notify-agent", default=None, help="Agent to notify on loopback")
p_inv_salvage = inv_sub.add_parser("salvage", parents=[common], help="Salvage an out-of-tokens agent (defaults to 646)")
p_inv_salvage.add_argument("node", nargs="?", default="646", choices=VALID_NODES, help="Blocked agent node (default: 646)")
p_inv_salvage.add_argument("--helper", choices=VALID_NODES, help="Specific helper agent to redeem code")
p_inv_salvage.add_argument("--notify", default=None, help="Sidechat to notify on loopback")
p_inv_salvage.add_argument("--notify-agent", default=None, help="Agent to notify on loopback")
# Domain: USAGE
p_usage = subparsers.add_parser("usage", parents=[common], help="Muse.ai usage limits per agent")
@@ -5112,8 +5429,28 @@ def build_parser():
p_onb_wo.add_argument("--target", default="646 tasks", help="Target sidechat (default: 646 tasks)")
onboard_sub.add_parser("feed-matrix", parents=[common], help="Display all agents ranked by feeding weight, job volume, role, and work done over time")
onboard_sub.add_parser("connects", parents=[common], help="Inventory of all active fleet nodes & client onboard connects")
# Domain: KPI
p_kpi = subparsers.add_parser("kpi", parents=[common], help="Fleet KPI, spend monitoring & runtime preservation")
p_kpi.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node")
kpi_sub = p_kpi.add_subparsers(dest="kpi_action")
p_kpi_status = kpi_sub.add_parser("status", parents=[common], help="Show fleet KPI dashboard (default)")
p_kpi_status.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node")
p_kpi_report = kpi_sub.add_parser("report", parents=[common], help="Detailed KPI & preservation report for an agent")
p_kpi_report.add_argument("node", choices=VALID_NODES, help="Target node")
p_kpi_routes = kpi_sub.add_parser("routes", parents=[common], help="Check route and CDP health across agents")
p_kpi_spawn = kpi_sub.add_parser("spawn-worker", parents=[common], help="Spawn background tmux worker to preserve runtime")
p_kpi_spawn.add_argument("node", choices=VALID_NODES, help="Target agent")
p_kpi_spawn.add_argument("session", help="Session label")
p_kpi_spawn.add_argument("worker_command", help="Command to run in background")
# Domain: CHROMEBOX
p_chromebox = subparsers.add_parser("chromebox", parents=[common], help="Agent browser settings-menu toggles (chromebox RPA)")
chrome_sub = p_chromebox.add_subparsers(dest="chrome_action")
p_chrome_perm = chrome_sub.add_parser("permissions", parents=[common], help="Settings menu toggles (permissions + related tabs)")
@@ -5617,13 +5954,32 @@ def main():
if len(sys.argv) > 1:
if sys.argv[1] == "tui":
tui_args = sys.argv[2:]
cmd = [sys.executable, str(BIN_DIR / "muse-tui.py"), "--mode", "box"] + tui_args
if tui_args and tui_args[0] in ("onboard", "tmux", "connects", "approvals"):
cmd = [sys.executable, str(BIN_DIR / "box-onboard-tui.py")] + tui_args[1:]
else:
cmd = [sys.executable, str(BIN_DIR / "muse-tui.py"), "--mode", "box"] + tui_args
res = subprocess.run(cmd)
sys.exit(res.returncode)
elif sys.argv[1] in ("onboard-tui", "dev-tui"):
cmd = [sys.executable, str(BIN_DIR / "box-onboard-tui.py")] + sys.argv[2:]
res = subprocess.run(cmd)
sys.exit(res.returncode)
elif sys.argv[1] == "tmux":
if len(sys.argv) > 2 and sys.argv[2] in ("tally", "auto", "watch", "once", "match", "rules", "status"):
if sys.argv[2] == "auto":
t_sub = sys.argv[3:] or ["status"]
else:
t_sub = sys.argv[2:]
cmd = [sys.executable, str(BIN_DIR / "tmux_auto_approver.py")] + t_sub
res = subprocess.run(cmd)
sys.exit(res.returncode)
cmd = [sys.executable, str(BIN_DIR / "muse-tmux.py")] + (sys.argv[2:] or ["list"])
res = subprocess.run(cmd)
sys.exit(res.returncode)
elif sys.argv[1] in ("tmux-auto", "auto-dev"):
cmd = [sys.executable, str(BIN_DIR / "tmux_auto_approver.py")] + sys.argv[2:]
res = subprocess.run(cmd)
sys.exit(res.returncode)
elif sys.argv[1] in ("docs", "doc"):
cmd = [sys.executable, str(BIN_DIR / "docs-lookup.py")] + sys.argv[2:]
res = subprocess.run(cmd)
@@ -5900,6 +6256,8 @@ def main():
cmd_approvals(args)
elif args.domain == "muse-choices":
cmd_muse_choices(args)
elif args.domain == "runtime":
cmd_runtime(args)
elif args.domain == "invite":
cmd_invite(args)
elif args.domain == "usage":
@@ -5908,6 +6266,8 @@ def main():
cmd_settings(args)
elif args.domain == "onboard":
cmd_onboard(args)
elif args.domain == "kpi":
cmd_kpi(args)
elif args.domain == "chromebox":
cmd_chromebox(args)
elif args.domain in ("deploy", "subagent"):