feat(kpi): expand KPI runtime monitoring, prompt advisory envelopes, and missing field resiliency
This commit is contained in:
@@ -23,6 +23,8 @@ Add `--json` to any command for machine-readable output when parsing results in
|
|||||||
## Common Commands
|
## Common Commands
|
||||||
|
|
||||||
- `box fleet status` / `box fleet cdp <node>` — health table / CDP endpoint plus SSH forward.
|
- `box fleet status` / `box fleet cdp <node>` — health table / CDP endpoint plus SSH forward.
|
||||||
|
- `box fleet heal <node>` — diagnose + fix + verify a node (lock, watchdog timer, Warp tunnel, egress, relay).
|
||||||
|
- `box watchdog status` / `box watchdog run <node|relay>` — timer states + evidence / trigger an immediate watchdog run.
|
||||||
- `box thread list [<agent>]` — threads for one agent, or all fleet sidechats when omitted.
|
- `box thread list [<agent>]` — threads for one agent, or all fleet sidechats when omitted.
|
||||||
- `box thread view <agent> <thread_id|main> --limit N` — recent messages from one thread.
|
- `box thread view <agent> <thread_id|main> --limit N` — recent messages from one thread.
|
||||||
- `box dm log -n N [--agent X] [--filter TEXT]` — recent DM activity.
|
- `box dm log -n N [--agent X] [--filter TEXT]` — recent DM activity.
|
||||||
@@ -30,6 +32,13 @@ Add `--json` to any command for machine-readable output when parsing results in
|
|||||||
- `box lookup summary|fleet|threads|unread|approvals` — seamless one-shot lookups.
|
- `box lookup summary|fleet|threads|unread|approvals` — seamless one-shot lookups.
|
||||||
- `box job list` / `box job log` — scheduled jobs and execution events.
|
- `box job list` / `box job log` — scheduled jobs and execution events.
|
||||||
- `box harvest status` / `box followup list` — harvest watermarks / pending nudges.
|
- `box harvest status` / `box followup list` — harvest watermarks / pending nudges.
|
||||||
|
- `box muse-choices on|off|status|logs|reconcile` — Muse TUI A/B/C auto-answer daemon switch, state, and per-pane logs.
|
||||||
|
- `box runtime list|send|launch` — Muse CLI tmux runtimes: live state + approval posture, send-keys input, auto-approved launches.
|
||||||
|
- `box tmux tally` / `box tmux auto [status|on|off|watch|once|logs|match]` — multi-socket Tmux worker tally, regex auto-approver daemon & guardrails.
|
||||||
|
- `box onboard connects` / `box onboard-tui` — fleet & client onboarding inventory, CDP ports, OTP salvage & 4-surface TUI.
|
||||||
|
- `box invite status|code <node>|redeem <node> <CODE>` / `box usage [--node N]` — invite codes and usage limits.
|
||||||
|
- `box kpi status|report <node>|routes|spawn-worker` — fleet KPI tracking, spend/limit metrics, route health, runtime preservation advisories, and background worker spawning.
|
||||||
|
- `box chromebox permissions <node> list|get <t>|set <t> <v>|describe <tab>` — settings-menu toggles (readback-verified sets).
|
||||||
|
|
||||||
## Rules
|
## Rules
|
||||||
|
|
||||||
@@ -37,3 +46,5 @@ Add `--json` to any command for machine-readable output when parsing results in
|
|||||||
- Never paste multi-KB logs or dumps into chat; write payloads under `logs/` and send a short path pointer instead.
|
- Never paste multi-KB logs or dumps into chat; write payloads under `logs/` and send a short path pointer instead.
|
||||||
- Avoid blocking commands in automated runs: `box fleet watch`, `box dm tail`, `box approvals watch`, `box dm chat` (interactive REPL).
|
- Avoid blocking commands in automated runs: `box fleet watch`, `box dm tail`, `box approvals watch`, `box dm chat` (interactive REPL).
|
||||||
- `box` probes CDP per node and can take several seconds; use generous timeouts and `--json` for scripted use.
|
- `box` probes CDP per node and can take several seconds; use generous timeouts and `--json` for scripted use.
|
||||||
|
- Preserve runtime and quota limits: prefer spawning subagents (`jobs/`, `subagent_tracker`) or background workers (`box kpi spawn-worker`) over long conversational prose to avoid `VANITY_IDLE` flags.
|
||||||
|
|
||||||
|
|||||||
+11
-1
@@ -237,7 +237,17 @@ def get_usage(node, timeout=8.0):
|
|||||||
return {"ok": False, "node": node,
|
return {"ok": False, "node": node,
|
||||||
"error": "empty settings dialog"}
|
"error": "empty settings dialog"}
|
||||||
out = {"ok": True, "node": node}
|
out = {"ok": True, "node": node}
|
||||||
out.update(parse_usage_text(text))
|
parsed = parse_usage_text(text)
|
||||||
|
stats_loaded = bool(
|
||||||
|
parsed.get("weekly_reset")
|
||||||
|
or parsed.get("weekly_used_pct") is not None
|
||||||
|
or parsed.get("has_redeemed")
|
||||||
|
or parsed.get("additional_left")
|
||||||
|
)
|
||||||
|
out.update(parsed)
|
||||||
|
out["stats_loaded"] = stats_loaded
|
||||||
|
if not stats_loaded:
|
||||||
|
out["note"] = "Usage stats did not render in Settings dialog"
|
||||||
return out
|
return out
|
||||||
finally:
|
finally:
|
||||||
_escape(ws)
|
_escape(ws)
|
||||||
|
|||||||
+241
-48
@@ -55,11 +55,49 @@ class RedemptionResult:
|
|||||||
reason: Optional[str]
|
reason: Optional[str]
|
||||||
detail: Optional[str]
|
detail: Optional[str]
|
||||||
method_used: str
|
method_used: str
|
||||||
|
field_missing: bool = False
|
||||||
|
loopback_notified: bool = False
|
||||||
|
loopback_detail: Optional[str] = None
|
||||||
|
|
||||||
def to_dict(self) -> Dict[str, Any]:
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
return asdict(self)
|
return asdict(self)
|
||||||
|
|
||||||
|
|
||||||
|
def send_loopback_notice(
|
||||||
|
recipient: str,
|
||||||
|
target: str,
|
||||||
|
message: str,
|
||||||
|
sender: str = "super",
|
||||||
|
timeout: float = 15.0,
|
||||||
|
) -> bool:
|
||||||
|
"""Send a loopback notification DM via bin/dm.py to a sidechat without blocking or failing."""
|
||||||
|
try:
|
||||||
|
from pathlib import Path
|
||||||
|
bin_dir = Path(__file__).resolve().parent
|
||||||
|
dm_script = bin_dir / "dm.py"
|
||||||
|
if not dm_script.exists():
|
||||||
|
return False
|
||||||
|
import subprocess
|
||||||
|
cmd = [
|
||||||
|
sys.executable,
|
||||||
|
str(dm_script),
|
||||||
|
"send",
|
||||||
|
"--agent",
|
||||||
|
sender,
|
||||||
|
"--to",
|
||||||
|
recipient,
|
||||||
|
"--target",
|
||||||
|
target,
|
||||||
|
message,
|
||||||
|
]
|
||||||
|
if target == "main":
|
||||||
|
cmd.insert(-1, "--allow-main-chat")
|
||||||
|
res = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
||||||
|
return res.returncode == 0
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
class InviteHandler:
|
class InviteHandler:
|
||||||
"""Handles invite codes discovery, inspection, and redemption for NetVM nodes."""
|
"""Handles invite codes discovery, inspection, and redemption for NetVM nodes."""
|
||||||
|
|
||||||
@@ -68,6 +106,15 @@ class InviteHandler:
|
|||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
self.ws = None
|
self.ws = None
|
||||||
|
|
||||||
|
def _dispatch_loopback(self, res: RedemptionResult, agent: str, target: str) -> bool:
|
||||||
|
"""Post a loopback notification DM via box dm."""
|
||||||
|
msg = f"[BOX-INVITE-LOOPBACK] Node @{self.node} redeem status: {res.status}. Reason: {res.reason or 'unknown'}. Detail: {res.detail or '-'}"
|
||||||
|
ok = send_loopback_notice(recipient=agent, target=target, message=msg)
|
||||||
|
res.loopback_notified = ok
|
||||||
|
res.loopback_detail = f"Notified @{agent}/{target}" if ok else f"Failed to notify @{agent}/{target}"
|
||||||
|
return ok
|
||||||
|
|
||||||
|
|
||||||
def connect(self) -> InviteHandler:
|
def connect(self) -> InviteHandler:
|
||||||
if self.ws is None:
|
if self.ws is None:
|
||||||
self.ws, _ = get_cdp_ws(self.node, timeout=self.timeout)
|
self.ws, _ = get_cdp_ws(self.node, timeout=self.timeout)
|
||||||
@@ -262,87 +309,169 @@ class InviteHandler:
|
|||||||
method_used="api",
|
method_used="api",
|
||||||
)
|
)
|
||||||
|
|
||||||
def redeem_code_dom(self, code: str) -> RedemptionResult:
|
def redeem_code_dom(
|
||||||
"""Redeem invite code using Settings Menu RPA -> Settings -> Redeem Invite Code dialog."""
|
self,
|
||||||
|
code: str,
|
||||||
|
notify_target: Optional[str] = None,
|
||||||
|
notify_agent: Optional[str] = None,
|
||||||
|
) -> RedemptionResult:
|
||||||
|
"""Redeem invite code using Settings Menu RPA -> Settings -> Redeem Invite Code dialog.
|
||||||
|
Gracefully handles missing entrypoint rows, missing input fields, and async loading races.
|
||||||
|
Falls back to in-page API automatically if DOM fields are absent, and dispatches DM loopback if requested."""
|
||||||
self.connect()
|
self.connect()
|
||||||
clean_code = code.strip().upper()
|
clean_code = code.strip().upper()
|
||||||
with SettingsRPA(self.node, timeout=self.timeout) as rpa:
|
with SettingsRPA(self.node, timeout=self.timeout) as rpa:
|
||||||
# 1. Open Settings dialog
|
# 1. Open Settings dialog
|
||||||
opened = rpa.open_settings_dialog()
|
opened = rpa.open_settings_dialog()
|
||||||
if not opened:
|
if not opened:
|
||||||
return RedemptionResult(
|
# If dialog failed to open, try API fallback directly
|
||||||
|
api_res = self.redeem_code_api(clean_code)
|
||||||
|
if api_res.success:
|
||||||
|
api_res.detail = f"Settings dialog could not open; redemption completed via API fallback ({api_res.detail or ''})".strip()
|
||||||
|
return api_res
|
||||||
|
|
||||||
|
res = RedemptionResult(
|
||||||
target_node=self.node,
|
target_node=self.node,
|
||||||
code=clean_code,
|
code=clean_code,
|
||||||
success=False,
|
success=False,
|
||||||
status="error",
|
status="error",
|
||||||
reason="settings_open_failed",
|
reason="settings_open_failed",
|
||||||
detail="Could not open Settings dialog via RPA",
|
detail=f"Could not open Settings dialog via RPA on @{self.node}. API fallback: {api_res.detail or api_res.reason or 'failed'}",
|
||||||
method_used="dom",
|
method_used="dom",
|
||||||
|
field_missing=True,
|
||||||
)
|
)
|
||||||
|
if notify_target and notify_agent:
|
||||||
|
self._dispatch_loopback(res, notify_agent, notify_target)
|
||||||
|
return res
|
||||||
|
|
||||||
rpa.select_tab("General")
|
rpa.select_tab("General")
|
||||||
time.sleep(0.3)
|
time.sleep(0.3)
|
||||||
|
|
||||||
# 2. Check for "Redeem invite code" entry
|
# 2. Check for "Redeem invite code" entry (poll up to 2.5s for React rendering)
|
||||||
js_find_and_click = """(() => {
|
js_find_and_click = """(() => {
|
||||||
const dialog = document.querySelector('[role="dialog"]');
|
const dialog = document.querySelector('[role="dialog"]');
|
||||||
if (!dialog) return {found: false};
|
if (!dialog) return {found: false, dialog_present: false};
|
||||||
const items = Array.from(dialog.querySelectorAll('button, div, span'));
|
const items = Array.from(dialog.querySelectorAll('button, div, span'));
|
||||||
const redeemItem = items.find(el => (el.innerText || '').trim() === 'Redeem invite code');
|
const redeemItem = items.find(el => (el.innerText || '').trim() === 'Redeem invite code');
|
||||||
if (redeemItem) {
|
if (redeemItem) {
|
||||||
redeemItem.click();
|
redeemItem.click();
|
||||||
return {found: true};
|
return {found: true, dialog_present: true};
|
||||||
}
|
}
|
||||||
return {found: false, text: dialog.innerText};
|
return {
|
||||||
|
found: false,
|
||||||
|
dialog_present: true,
|
||||||
|
text: dialog.innerText || '',
|
||||||
|
has_additional: (dialog.innerText || '').includes('Additional tokens')
|
||||||
|
};
|
||||||
})()"""
|
})()"""
|
||||||
click_res = cdp_evaluate(self.ws, js_find_and_click)
|
click_res = None
|
||||||
|
deadline = time.time() + 2.5
|
||||||
|
while time.time() < deadline:
|
||||||
|
click_res = cdp_evaluate(self.ws, js_find_and_click)
|
||||||
|
if click_res and click_res.get("found"):
|
||||||
|
break
|
||||||
|
time.sleep(0.3)
|
||||||
|
|
||||||
if not click_res or not click_res.get("found"):
|
if not click_res or not click_res.get("found"):
|
||||||
|
# "Redeem invite code" field is missing in General settings!
|
||||||
rpa.close_settings_dialog()
|
rpa.close_settings_dialog()
|
||||||
# If entrypoint missing, likely already redeemed
|
|
||||||
api_check = self.find_code_api()
|
# Attempt automatic API fallback first
|
||||||
if api_check.has_redeemed:
|
api_res = self.redeem_code_api(clean_code)
|
||||||
return RedemptionResult(
|
if api_res.success:
|
||||||
target_node=self.node,
|
api_res.detail = f"Redeem field was missing in Settings DOM; redeemed successfully via API fallback! ({api_res.detail or ''})".strip()
|
||||||
code=clean_code,
|
return api_res
|
||||||
success=False,
|
|
||||||
status="already_redeemed",
|
# Diagnose why field is missing
|
||||||
reason="already_redeemed",
|
diag_text = click_res.get("text", "") if click_res else ""
|
||||||
detail="Node has already redeemed an invite code (entrypoint hidden)",
|
has_extra = click_res.get("has_additional", False) if click_res else False
|
||||||
method_used="dom",
|
|
||||||
)
|
is_already = has_extra or "Additional tokens" in diag_text
|
||||||
return RedemptionResult(
|
if not is_already:
|
||||||
|
try:
|
||||||
|
api_check = self.find_code_api()
|
||||||
|
is_already = bool(api_check.has_redeemed)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if is_already:
|
||||||
|
status = "already_redeemed"
|
||||||
|
reason = "already_redeemed"
|
||||||
|
detail = f"Node @{self.node} has already redeemed an invite code (entrypoint hidden by active Additional tokens ticker)."
|
||||||
|
elif api_res.reason in ["window_expired", "already_redeemed", "invalid", "used_up"]:
|
||||||
|
status = api_res.status
|
||||||
|
reason = api_res.reason
|
||||||
|
detail = f"Redeem invite code field not present in Settings on @{self.node}: server reports {api_res.reason} ({api_res.detail or ''})."
|
||||||
|
else:
|
||||||
|
status = "entrypoint_not_found"
|
||||||
|
reason = "field_missing"
|
||||||
|
detail = f"Redeem invite code field missing in General settings on @{self.node} (account may be past 48h onboarding window or already redeemed)."
|
||||||
|
|
||||||
|
res = RedemptionResult(
|
||||||
target_node=self.node,
|
target_node=self.node,
|
||||||
code=clean_code,
|
code=clean_code,
|
||||||
success=False,
|
success=False,
|
||||||
status="entrypoint_not_found",
|
status=status,
|
||||||
reason="not_eligible",
|
reason=reason,
|
||||||
detail="Redeem invite code item not present in General settings",
|
detail=detail,
|
||||||
method_used="dom",
|
method_used="dom",
|
||||||
|
field_missing=True,
|
||||||
)
|
)
|
||||||
|
if notify_target and notify_agent:
|
||||||
|
self._dispatch_loopback(res, notify_agent, notify_target)
|
||||||
|
return res
|
||||||
|
|
||||||
time.sleep(0.8)
|
# 3. Handle HatchInviteRedemptionDialog (sub-dialog opened by clicking Redeem)
|
||||||
|
# Poll up to 2.5s for input box to mount
|
||||||
# 3. Handle HatchInviteRedemptionDialog
|
|
||||||
js_input_code = f"""(() => {{
|
js_input_code = f"""(() => {{
|
||||||
// Look for dialog titled "Redeem a code"
|
// Look for dialog titled "Redeem a code" or input with label
|
||||||
const input = document.querySelector('input[aria-label="Invite code"]') ||
|
const input = document.querySelector('input[aria-label="Invite code"]') ||
|
||||||
document.querySelector('[role="dialog"] input[type="text"]');
|
document.querySelector('[role="dialog"] input[type="text"]');
|
||||||
if (!input) return {{found_input: false}};
|
if (!input) return {{found_input: false}};
|
||||||
|
|
||||||
// Type code into input
|
|
||||||
input.focus();
|
input.focus();
|
||||||
input.value = {json.dumps(clean_code)};
|
input.value = {json.dumps(clean_code)};
|
||||||
input.dispatchEvent(new Event('input', {{bubbles: true}}));
|
input.dispatchEvent(new Event('input', {{bubbles: true}}));
|
||||||
input.dispatchEvent(new Event('change', {{bubbles: true}}));
|
input.dispatchEvent(new Event('change', {{bubbles: true}}));
|
||||||
|
|
||||||
// Submit via Enter key
|
|
||||||
input.dispatchEvent(new KeyboardEvent('keydown', {{key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true}}));
|
input.dispatchEvent(new KeyboardEvent('keydown', {{key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true}}));
|
||||||
return {{found_input: true}};
|
return {{found_input: true}};
|
||||||
}})()"""
|
}})()"""
|
||||||
input_res = cdp_evaluate(self.ws, js_input_code)
|
input_res = None
|
||||||
time.sleep(1.5)
|
deadline = time.time() + 2.5
|
||||||
|
while time.time() < deadline:
|
||||||
|
input_res = cdp_evaluate(self.ws, js_input_code)
|
||||||
|
if input_res and input_res.get("found_input"):
|
||||||
|
break
|
||||||
|
time.sleep(0.3)
|
||||||
|
|
||||||
# 4. Check outcome in dialog
|
if not input_res or not input_res.get("found_input"):
|
||||||
|
# Subdialog opened or clicked, but input box is absent!
|
||||||
|
cdp_send_escape(self.ws)
|
||||||
|
rpa.close_settings_dialog()
|
||||||
|
|
||||||
|
# Automatic API fallback
|
||||||
|
api_res = self.redeem_code_api(clean_code)
|
||||||
|
if api_res.success:
|
||||||
|
api_res.detail = f"Redemption input box was missing in dialog; redeemed successfully via API fallback! ({api_res.detail or ''})".strip()
|
||||||
|
return api_res
|
||||||
|
|
||||||
|
res = RedemptionResult(
|
||||||
|
target_node=self.node,
|
||||||
|
code=clean_code,
|
||||||
|
success=False,
|
||||||
|
status=api_res.status or "dom_input_missing",
|
||||||
|
reason=api_res.reason or "input_field_missing",
|
||||||
|
detail=f"Invite code input field was not found in redemption dialog on @{self.node}. API fallback: {api_res.detail or api_res.reason or 'failed'}.",
|
||||||
|
method_used="dom",
|
||||||
|
field_missing=True,
|
||||||
|
)
|
||||||
|
if notify_target and notify_agent:
|
||||||
|
self._dispatch_loopback(res, notify_agent, notify_target)
|
||||||
|
return res
|
||||||
|
|
||||||
|
# Input submitted; poll for outcome text
|
||||||
|
time.sleep(1.2)
|
||||||
js_check_outcome = """(() => {
|
js_check_outcome = """(() => {
|
||||||
const dialog = document.querySelector('[role="dialog"]');
|
const dialog = document.querySelector('[role="dialog"]');
|
||||||
if (!dialog) return {found: false};
|
if (!dialog) return {found: false};
|
||||||
@@ -365,7 +494,14 @@ class InviteHandler:
|
|||||||
}
|
}
|
||||||
return {status: 'unknown', detail: text};
|
return {status: 'unknown', detail: text};
|
||||||
})()"""
|
})()"""
|
||||||
outcome = cdp_evaluate(self.ws, js_check_outcome)
|
outcome = None
|
||||||
|
deadline = time.time() + 2.5
|
||||||
|
while time.time() < deadline:
|
||||||
|
outcome = cdp_evaluate(self.ws, js_check_outcome)
|
||||||
|
if outcome and outcome.get("status") != "unknown":
|
||||||
|
break
|
||||||
|
time.sleep(0.3)
|
||||||
|
|
||||||
rpa.close_settings_dialog()
|
rpa.close_settings_dialog()
|
||||||
|
|
||||||
if outcome and outcome.get("success"):
|
if outcome and outcome.get("success"):
|
||||||
@@ -379,7 +515,7 @@ class InviteHandler:
|
|||||||
method_used="dom",
|
method_used="dom",
|
||||||
)
|
)
|
||||||
elif outcome and outcome.get("status") in ["already_redeemed", "invalid", "used_up", "window_expired"]:
|
elif outcome and outcome.get("status") in ["already_redeemed", "invalid", "used_up", "window_expired"]:
|
||||||
return RedemptionResult(
|
res = RedemptionResult(
|
||||||
target_node=self.node,
|
target_node=self.node,
|
||||||
code=clean_code,
|
code=clean_code,
|
||||||
success=False,
|
success=False,
|
||||||
@@ -388,19 +524,40 @@ class InviteHandler:
|
|||||||
detail=outcome.get("detail"),
|
detail=outcome.get("detail"),
|
||||||
method_used="dom",
|
method_used="dom",
|
||||||
)
|
)
|
||||||
|
if notify_target and notify_agent:
|
||||||
|
self._dispatch_loopback(res, notify_agent, notify_target)
|
||||||
|
return res
|
||||||
|
|
||||||
# Fallback to API check if DOM did not confirm status
|
# Fallback to API check if DOM did not confirm status
|
||||||
return self.redeem_code_api(clean_code)
|
api_res = self.redeem_code_api(clean_code)
|
||||||
|
if not api_res.success and notify_target and notify_agent:
|
||||||
|
self._dispatch_loopback(api_res, notify_agent, notify_target)
|
||||||
|
return api_res
|
||||||
|
|
||||||
def redeem_code(self, code: str, method: str = "auto") -> RedemptionResult:
|
def redeem_code(
|
||||||
|
self,
|
||||||
|
code: str,
|
||||||
|
method: str = "auto",
|
||||||
|
notify_target: Optional[str] = None,
|
||||||
|
notify_agent: Optional[str] = None,
|
||||||
|
) -> RedemptionResult:
|
||||||
"""Redeem invite code using specified method ('auto', 'dom', or 'api')."""
|
"""Redeem invite code using specified method ('auto', 'dom', or 'api')."""
|
||||||
if method == "dom":
|
if method == "dom":
|
||||||
return self.redeem_code_dom(code)
|
return self.redeem_code_dom(code, notify_target=notify_target, notify_agent=notify_agent)
|
||||||
elif method == "api":
|
elif method == "api":
|
||||||
return self.redeem_code_api(code)
|
res = self.redeem_code_api(code)
|
||||||
|
if not res.success and notify_target and notify_agent:
|
||||||
|
self._dispatch_loopback(res, notify_agent, notify_target)
|
||||||
|
return res
|
||||||
else:
|
else:
|
||||||
# Auto: Validate and attempt via API for reliability
|
# Auto: Validate and attempt via API for reliability. Fallback to DOM on transport error.
|
||||||
return self.redeem_code_api(code)
|
res = self.redeem_code_api(code)
|
||||||
|
if not res.success and res.reason == "transport_failure":
|
||||||
|
res = self.redeem_code_dom(code, notify_target=notify_target, notify_agent=notify_agent)
|
||||||
|
elif not res.success and notify_target and notify_agent:
|
||||||
|
self._dispatch_loopback(res, notify_agent, notify_target)
|
||||||
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def scan_fleet_invites(nodes: Optional[List[str]] = None) -> List[Dict[str, Any]]:
|
def scan_fleet_invites(nodes: Optional[List[str]] = None) -> List[Dict[str, Any]]:
|
||||||
@@ -454,7 +611,12 @@ def scan_fleet_usage(nodes: Optional[List[str]] = None) -> List[Dict[str, Any]]:
|
|||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
def salvage_blocked_node(blocked_node: str = "646", helper_node: Optional[str] = None) -> Dict[str, Any]:
|
def salvage_blocked_node(
|
||||||
|
blocked_node: str = "646",
|
||||||
|
helper_node: Optional[str] = None,
|
||||||
|
notify_target: Optional[str] = None,
|
||||||
|
notify_agent: Optional[str] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
"""Salvage an out-of-tokens node by identifying its code and redeeming it on an eligible peer."""
|
"""Salvage an out-of-tokens node by identifying its code and redeeming it on an eligible peer."""
|
||||||
# 1. Fetch blocked node invite code
|
# 1. Fetch blocked node invite code
|
||||||
with InviteHandler(blocked_node) as h_blocked:
|
with InviteHandler(blocked_node) as h_blocked:
|
||||||
@@ -483,18 +645,28 @@ def salvage_blocked_node(blocked_node: str = "646", helper_node: Optional[str] =
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
if not eligible_peer:
|
if not eligible_peer:
|
||||||
return {
|
res = {
|
||||||
"success": False,
|
"success": False,
|
||||||
"blocked_node": blocked_node,
|
"blocked_node": blocked_node,
|
||||||
"invite_code": code_to_redeem,
|
"invite_code": code_to_redeem,
|
||||||
"error": "No existing fleet peer is currently eligible (all active peers have already redeemed an invite code). An onboarding agent or fresh client profile must redeem this code.",
|
"error": "No existing fleet peer is currently eligible (all active peers have already redeemed an invite code). An onboarding agent or fresh client profile must redeem this code.",
|
||||||
"code_to_redeem": code_to_redeem,
|
"code_to_redeem": code_to_redeem,
|
||||||
"share_instruction": f"Redeem code '{code_to_redeem}' on a newly provisioned agent to credit 1 billion tokens to {blocked_node}."
|
"share_instruction": f"Redeem code '{code_to_redeem}' on a newly provisioned agent to credit 1 billion tokens to {blocked_node}.",
|
||||||
|
"field_missing": True,
|
||||||
|
"loopback_notified": False,
|
||||||
}
|
}
|
||||||
|
if notify_target and notify_agent:
|
||||||
|
msg = f"[SALVAGE-NOTICE] Node @{blocked_node} is blocked (code: {code_to_redeem}), but no eligible peer is available. Fresh onboarding required."
|
||||||
|
res["loopback_notified"] = send_loopback_notice(notify_agent, notify_target, msg)
|
||||||
|
return res
|
||||||
|
|
||||||
# 3. Redeem on eligible peer
|
# 3. Redeem on eligible peer
|
||||||
with InviteHandler(eligible_peer) as h_peer:
|
with InviteHandler(eligible_peer) as h_peer:
|
||||||
redemption = h_peer.redeem_code(code_to_redeem)
|
redemption = h_peer.redeem_code(
|
||||||
|
code_to_redeem,
|
||||||
|
notify_target=notify_target,
|
||||||
|
notify_agent=notify_agent,
|
||||||
|
)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"success": redemption.success,
|
"success": redemption.success,
|
||||||
@@ -502,6 +674,8 @@ def salvage_blocked_node(blocked_node: str = "646", helper_node: Optional[str] =
|
|||||||
"helper_node": eligible_peer,
|
"helper_node": eligible_peer,
|
||||||
"code_redeemed": code_to_redeem,
|
"code_redeemed": code_to_redeem,
|
||||||
"redemption_result": redemption.to_dict(),
|
"redemption_result": redemption.to_dict(),
|
||||||
|
"field_missing": redemption.field_missing,
|
||||||
|
"loopback_notified": redemption.loopback_notified,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -518,6 +692,8 @@ def main() -> None:
|
|||||||
p_redeem.add_argument("node", help="Target node to redeem the code on")
|
p_redeem.add_argument("node", help="Target node to redeem the code on")
|
||||||
p_redeem.add_argument("code", help="6-character invite code")
|
p_redeem.add_argument("code", help="6-character invite code")
|
||||||
p_redeem.add_argument("--method", choices=["auto", "api", "dom"], default="auto")
|
p_redeem.add_argument("--method", choices=["auto", "api", "dom"], default="auto")
|
||||||
|
p_redeem.add_argument("--notify-target", default=None, help="Sidechat to notify on loopback")
|
||||||
|
p_redeem.add_argument("--notify-agent", default=None, help="Agent to notify on loopback")
|
||||||
p_redeem.add_argument("--json", action="store_true")
|
p_redeem.add_argument("--json", action="store_true")
|
||||||
|
|
||||||
p_list = subparsers.add_parser("list", help="List invite codes across fleet")
|
p_list = subparsers.add_parser("list", help="List invite codes across fleet")
|
||||||
@@ -526,6 +702,8 @@ def main() -> None:
|
|||||||
p_salvage = subparsers.add_parser("salvage", help="Salvage a blocked agent (e.g. 646)")
|
p_salvage = subparsers.add_parser("salvage", help="Salvage a blocked agent (e.g. 646)")
|
||||||
p_salvage.add_argument("node", default="646", nargs="?", help="Blocked node (default: 646)")
|
p_salvage.add_argument("node", default="646", nargs="?", help="Blocked node (default: 646)")
|
||||||
p_salvage.add_argument("--helper", help="Specific helper node to redeem code")
|
p_salvage.add_argument("--helper", help="Specific helper node to redeem code")
|
||||||
|
p_salvage.add_argument("--notify-target", default=None, help="Sidechat to notify on loopback")
|
||||||
|
p_salvage.add_argument("--notify-agent", default=None, help="Agent to notify on loopback")
|
||||||
p_salvage.add_argument("--json", action="store_true")
|
p_salvage.add_argument("--json", action="store_true")
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
@@ -546,7 +724,12 @@ def main() -> None:
|
|||||||
|
|
||||||
elif args.command == "redeem":
|
elif args.command == "redeem":
|
||||||
with InviteHandler(args.node) as h:
|
with InviteHandler(args.node) as h:
|
||||||
res = h.redeem_code(args.code, method=args.method)
|
res = h.redeem_code(
|
||||||
|
args.code,
|
||||||
|
method=args.method,
|
||||||
|
notify_target=getattr(args, "notify_target", None),
|
||||||
|
notify_agent=getattr(args, "notify_agent", None),
|
||||||
|
)
|
||||||
if args.json:
|
if args.json:
|
||||||
print(json.dumps(res.to_dict(), indent=2))
|
print(json.dumps(res.to_dict(), indent=2))
|
||||||
else:
|
else:
|
||||||
@@ -556,6 +739,8 @@ def main() -> None:
|
|||||||
print(f" Status: {res.status}")
|
print(f" Status: {res.status}")
|
||||||
if res.detail:
|
if res.detail:
|
||||||
print(f" Detail: {res.detail}")
|
print(f" Detail: {res.detail}")
|
||||||
|
if res.loopback_notified:
|
||||||
|
print(f" Loopback:{res.loopback_detail}")
|
||||||
|
|
||||||
elif args.command == "list":
|
elif args.command == "list":
|
||||||
fleet = scan_fleet_invites()
|
fleet = scan_fleet_invites()
|
||||||
@@ -576,7 +761,12 @@ def main() -> None:
|
|||||||
print()
|
print()
|
||||||
|
|
||||||
elif args.command == "salvage":
|
elif args.command == "salvage":
|
||||||
salvage_res = salvage_blocked_node(args.node, helper_node=args.helper)
|
salvage_res = salvage_blocked_node(
|
||||||
|
args.node,
|
||||||
|
helper_node=args.helper,
|
||||||
|
notify_target=getattr(args, "notify_target", None),
|
||||||
|
notify_agent=getattr(args, "notify_agent", None),
|
||||||
|
)
|
||||||
if args.json:
|
if args.json:
|
||||||
print(json.dumps(salvage_res, indent=2))
|
print(json.dumps(salvage_res, indent=2))
|
||||||
else:
|
else:
|
||||||
@@ -588,7 +778,10 @@ def main() -> None:
|
|||||||
print(f" Status: {salvage_res.get('error')}")
|
print(f" Status: {salvage_res.get('error')}")
|
||||||
if salvage_res.get("share_instruction"):
|
if salvage_res.get("share_instruction"):
|
||||||
print(f" Next Step: {salvage_res.get('share_instruction')}")
|
print(f" Next Step: {salvage_res.get('share_instruction')}")
|
||||||
|
if salvage_res.get("loopback_notified"):
|
||||||
|
print(" Loopback: Notice sent to requesting target.")
|
||||||
print()
|
print()
|
||||||
|
|
||||||
else:
|
else:
|
||||||
parser.print_help()
|
parser.print_help()
|
||||||
|
|
||||||
|
|||||||
Executable
+560
@@ -0,0 +1,560 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""kpi.py — NetVM Fleet KPI, Spend Monitor & Runtime Preservation Engine.
|
||||||
|
|
||||||
|
Monitors:
|
||||||
|
- Calls / DMs dispatched and verified (from dm-log.jsonl)
|
||||||
|
- Token quota spend & remaining (weekly limit % and extra tokens)
|
||||||
|
- Active subagent sessions and tmux muse workers
|
||||||
|
- Uptime vs actual problems fixed (Efficiency Index)
|
||||||
|
- Route health (WARP wireguard, CDP, tmux sockets)
|
||||||
|
- Runtime preservation advisories (guiding agents to offload work to tmux/subagents)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
BIN_DIR = REPO_ROOT / "bin"
|
||||||
|
DM_LOG = REPO_ROOT / "dm-log.jsonl"
|
||||||
|
JOBS_DIR = REPO_ROOT / "jobs"
|
||||||
|
SUBAGENTS_FILE = REPO_ROOT / "subagent-sessions.json"
|
||||||
|
|
||||||
|
VALID_NODES = ["muse", "pip", "646", "opm", "dev", "def"]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class AgentKPI:
|
||||||
|
node: str
|
||||||
|
weekly_used_pct: Optional[int]
|
||||||
|
extra_tokens_remaining: str
|
||||||
|
is_blocked: bool
|
||||||
|
calls_sent: int
|
||||||
|
calls_verified: int
|
||||||
|
jobs_assigned: int
|
||||||
|
jobs_completed: int
|
||||||
|
subagents_active: int
|
||||||
|
tmux_workers_active: int
|
||||||
|
uptime_hours: float
|
||||||
|
route_status: str
|
||||||
|
efficiency_index: float
|
||||||
|
efficiency_rating: str
|
||||||
|
preservation_advisory: str
|
||||||
|
|
||||||
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
|
return asdict(self)
|
||||||
|
|
||||||
|
|
||||||
|
def get_agent_dm_metrics(node: str, window_hours: Optional[float] = None) -> Dict[str, int]:
|
||||||
|
"""Calculate outbound messages, sends, and verified deliveries from dm-log.jsonl."""
|
||||||
|
if not DM_LOG.exists():
|
||||||
|
return {"sent": 0, "verified": 0, "total_events": 0}
|
||||||
|
|
||||||
|
cutoff = None
|
||||||
|
if window_hours:
|
||||||
|
cutoff = datetime.now(timezone.utc).timestamp() - (window_hours * 3600)
|
||||||
|
|
||||||
|
sent_ids = set()
|
||||||
|
verified_ids = set()
|
||||||
|
total_events = 0
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(DM_LOG, "r", encoding="utf-8") as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
entry = json.loads(line)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if entry.get("agent") != node:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if cutoff:
|
||||||
|
ts = entry.get("ts")
|
||||||
|
if ts:
|
||||||
|
try:
|
||||||
|
dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||||
|
if dt.timestamp() < cutoff:
|
||||||
|
continue
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
total_events += 1
|
||||||
|
mid = entry.get("id")
|
||||||
|
etype = entry.get("type")
|
||||||
|
if etype in ("send_start", "send_done"):
|
||||||
|
if mid:
|
||||||
|
sent_ids.add(mid)
|
||||||
|
elif etype == "verified":
|
||||||
|
if mid:
|
||||||
|
verified_ids.add(mid)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return {
|
||||||
|
"sent": len(sent_ids),
|
||||||
|
"verified": len(verified_ids),
|
||||||
|
"total_events": total_events,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_agent_job_metrics(node: str) -> Dict[str, int]:
|
||||||
|
"""Calculate total jobs assigned and completed for an agent."""
|
||||||
|
assigned = 0
|
||||||
|
completed = 0
|
||||||
|
|
||||||
|
if not JOBS_DIR.exists():
|
||||||
|
return {"assigned": 0, "completed": 0}
|
||||||
|
|
||||||
|
try:
|
||||||
|
for p in JOBS_DIR.glob("*.json"):
|
||||||
|
try:
|
||||||
|
with open(p, "r", encoding="utf-8") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
if data.get("agent") == node:
|
||||||
|
assigned += 1
|
||||||
|
# If output or status has result
|
||||||
|
if data.get("status") == "completed" or data.get("result"):
|
||||||
|
completed += 1
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return {"assigned": assigned, "completed": completed}
|
||||||
|
|
||||||
|
|
||||||
|
def get_agent_subagent_count(node: str) -> int:
|
||||||
|
"""Get active subagent sessions for a node from subagent-sessions.json."""
|
||||||
|
if not SUBAGENTS_FILE.exists():
|
||||||
|
return 0
|
||||||
|
try:
|
||||||
|
with open(SUBAGENTS_FILE, "r", encoding="utf-8") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return 0
|
||||||
|
return sum(1 for s in data.values() if s.get("parent") == node and s.get("status") == "active")
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def get_agent_tmux_workers(node: str) -> List[str]:
|
||||||
|
"""Get running tmux sessions for an agent (shared and netns socket)."""
|
||||||
|
sessions = []
|
||||||
|
# 1. Per-node socket
|
||||||
|
sock = f"/tmp/tmux-{node}.sock"
|
||||||
|
if os.path.exists(sock):
|
||||||
|
try:
|
||||||
|
r = subprocess.run(["tmux", "-S", sock, "list-sessions", "-F", "#{session_name}"], capture_output=True, text=True, timeout=2)
|
||||||
|
if r.returncode == 0 and r.stdout.strip():
|
||||||
|
sessions.extend(line.strip() for line in r.stdout.splitlines() if line.strip())
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# 2. Shared socket filtering sessions containing node name
|
||||||
|
shared_sock = "/tmp/tmux-muse.sock"
|
||||||
|
if os.path.exists(shared_sock):
|
||||||
|
try:
|
||||||
|
r = subprocess.run(["tmux", "-S", shared_sock, "list-sessions", "-F", "#{session_name}"], capture_output=True, text=True, timeout=2)
|
||||||
|
if r.returncode == 0 and r.stdout.strip():
|
||||||
|
for s in r.stdout.splitlines():
|
||||||
|
s = s.strip()
|
||||||
|
if s and (node in s or s.startswith(f"{node}-") or s == "swarm-worker"):
|
||||||
|
if s not in sessions:
|
||||||
|
sessions.append(s)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return sessions
|
||||||
|
|
||||||
|
|
||||||
|
def get_agent_uptime_hours(node: str) -> float:
|
||||||
|
"""Calculate browser process uptime in hours."""
|
||||||
|
try:
|
||||||
|
# Search for chromium process matching user-data-dir or node
|
||||||
|
cmd = ["pgrep", "-f", f"chrome-box launch {node}"]
|
||||||
|
r = subprocess.run(cmd, capture_output=True, text=True, timeout=2)
|
||||||
|
pids = r.stdout.strip().split()
|
||||||
|
if not pids:
|
||||||
|
cmd = ["pgrep", "-f", f"profiles/{node}"]
|
||||||
|
r = subprocess.run(cmd, capture_output=True, text=True, timeout=2)
|
||||||
|
pids = r.stdout.strip().split()
|
||||||
|
|
||||||
|
if pids:
|
||||||
|
pid = pids[0]
|
||||||
|
# Read /proc/<pid>/stat starttime
|
||||||
|
stat_path = Path(f"/proc/{pid}/stat")
|
||||||
|
if stat_path.exists():
|
||||||
|
stat_content = stat_path.read_text().split()
|
||||||
|
# field 22 is starttime (in clock ticks after boot)
|
||||||
|
start_ticks = int(stat_content[21])
|
||||||
|
clk_tck = os.sysconf(os.sysconf_names["SC_CLK_TCK"])
|
||||||
|
with open("/proc/uptime", "r") as f:
|
||||||
|
uptime_sec = float(f.read().split()[0])
|
||||||
|
process_age_sec = uptime_sec - (start_ticks / clk_tck)
|
||||||
|
return round(max(0.0, process_age_sec / 3600.0), 2)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def check_node_routes(node: str) -> str:
|
||||||
|
"""Check connectivity route for a node (netns + CDP)."""
|
||||||
|
# 1. Check netns
|
||||||
|
netns_path = Path(f"/var/run/netns/warp-{node}")
|
||||||
|
if not netns_path.exists():
|
||||||
|
return "NO_NETNS"
|
||||||
|
|
||||||
|
# 2. Check CDP page connection
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
from approvals import get_node_pages
|
||||||
|
except ImportError:
|
||||||
|
sys.path.insert(0, str(BIN_DIR))
|
||||||
|
from approvals import get_node_pages
|
||||||
|
pages = get_node_pages(node, timeout=2.0)
|
||||||
|
if pages:
|
||||||
|
return "ONLINE"
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return "DEGRADED"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def calculate_efficiency(
|
||||||
|
jobs_done: int,
|
||||||
|
subagents_active: int,
|
||||||
|
tmux_workers: int,
|
||||||
|
calls_verified: int,
|
||||||
|
weekly_used_pct: Optional[int],
|
||||||
|
uptime_hours: float,
|
||||||
|
) -> tuple[float, str]:
|
||||||
|
"""
|
||||||
|
Composite efficiency score.
|
||||||
|
Higher is better: measures actual work produced (jobs + subagents + workers + verified comms)
|
||||||
|
relative to quota burned and uptime elapsed.
|
||||||
|
"""
|
||||||
|
work_units = (jobs_done * 5.0) + (subagents_active * 3.0) + (tmux_workers * 4.0) + (calls_verified * 0.5)
|
||||||
|
burn_cost = max(1.0, (weekly_used_pct or 10) * 0.2)
|
||||||
|
|
||||||
|
# Base index
|
||||||
|
index = round(work_units / burn_cost, 2)
|
||||||
|
|
||||||
|
# Classify
|
||||||
|
if uptime_hours > 2.0 and subagents_active == 0 and tmux_workers == 0 and jobs_done == 0:
|
||||||
|
rating = "VANITY_IDLE"
|
||||||
|
elif index >= 3.0:
|
||||||
|
rating = "HIGH_EFFICIENCY"
|
||||||
|
elif index >= 1.0:
|
||||||
|
rating = "PRODUCTIVE"
|
||||||
|
elif index >= 0.4:
|
||||||
|
rating = "MODERATE"
|
||||||
|
else:
|
||||||
|
rating = "LOW_EFFICIENCY"
|
||||||
|
|
||||||
|
return index, rating
|
||||||
|
|
||||||
|
|
||||||
|
def generate_preservation_advisory(
|
||||||
|
node: str,
|
||||||
|
weekly_used_pct: Optional[int],
|
||||||
|
extra_tokens_remaining: str,
|
||||||
|
subagents_active: int,
|
||||||
|
tmux_workers: int,
|
||||||
|
rating: str,
|
||||||
|
) -> str:
|
||||||
|
"""Generate prescriptive runtime preservation instructions for the agent."""
|
||||||
|
tips = []
|
||||||
|
|
||||||
|
pct = weekly_used_pct or 0
|
||||||
|
if pct >= 95 or "0 tokens left" in extra_tokens_remaining:
|
||||||
|
return "CRITICAL: Quota exhausted. Do NOT send chat messages. Salvage via 'box onboard start <new_node> --for %s'." % node
|
||||||
|
|
||||||
|
if pct >= 70:
|
||||||
|
tips.append("Quota > 70%%: Cease prose chatter; offload tasks to background tmux workers.")
|
||||||
|
|
||||||
|
if subagents_active == 0 and tmux_workers == 0:
|
||||||
|
tips.append("Spawn subagents with isolated context ('box subagent spawn') or tmux muse workers.")
|
||||||
|
|
||||||
|
if rating in ("VANITY_IDLE", "LOW_EFFICIENCY"):
|
||||||
|
tips.append("Uptime without worker execution drains quota. Mandate: split goals into executable jobs.")
|
||||||
|
|
||||||
|
if not tips:
|
||||||
|
tips.append("Runtime healthy. Maintain worker-first execution strategy.")
|
||||||
|
|
||||||
|
return " ".join(tips)
|
||||||
|
|
||||||
|
|
||||||
|
def get_agent_kpi(node: str, usage_cache: Optional[Dict[str, Any]] = None) -> AgentKPI:
|
||||||
|
"""Collect comprehensive KPI metrics for a single NetVM node."""
|
||||||
|
# 1. Quota & Usage
|
||||||
|
usage = usage_cache
|
||||||
|
if usage is None:
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
import invite
|
||||||
|
except ImportError:
|
||||||
|
sys.path.insert(0, str(BIN_DIR))
|
||||||
|
import invite
|
||||||
|
u = invite.get_usage(node)
|
||||||
|
if isinstance(u, dict) and u.get("ok"):
|
||||||
|
usage = u
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
if usage is None:
|
||||||
|
usage = {}
|
||||||
|
|
||||||
|
weekly_pct = usage.get("weekly_used_pct")
|
||||||
|
extra_left = usage.get("additional_left") or usage.get("extra_tokens_remaining") or "Unknown"
|
||||||
|
is_blocked = bool(usage.get("is_blocked")) or (weekly_pct is not None and weekly_pct >= 100 and "0 tokens left" in extra_left)
|
||||||
|
|
||||||
|
# 2. Activity metrics
|
||||||
|
dm_metrics = get_agent_dm_metrics(node)
|
||||||
|
job_metrics = get_agent_job_metrics(node)
|
||||||
|
subagents = get_agent_subagent_count(node)
|
||||||
|
tmux_sessions = get_agent_tmux_workers(node)
|
||||||
|
uptime = get_agent_uptime_hours(node)
|
||||||
|
|
||||||
|
route_status = check_node_routes(node)
|
||||||
|
|
||||||
|
# 3. Efficiency
|
||||||
|
eff_idx, eff_rating = calculate_efficiency(
|
||||||
|
jobs_done=job_metrics["completed"],
|
||||||
|
subagents_active=subagents,
|
||||||
|
tmux_workers=len(tmux_sessions),
|
||||||
|
calls_verified=dm_metrics["verified"],
|
||||||
|
weekly_used_pct=weekly_pct,
|
||||||
|
uptime_hours=uptime,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 4. Advisory
|
||||||
|
advisory = generate_preservation_advisory(
|
||||||
|
node=node,
|
||||||
|
weekly_used_pct=weekly_pct,
|
||||||
|
extra_tokens_remaining=extra_left,
|
||||||
|
subagents_active=subagents,
|
||||||
|
tmux_workers=len(tmux_sessions),
|
||||||
|
rating=eff_rating,
|
||||||
|
)
|
||||||
|
|
||||||
|
return AgentKPI(
|
||||||
|
node=node,
|
||||||
|
weekly_used_pct=weekly_pct,
|
||||||
|
extra_tokens_remaining=extra_left,
|
||||||
|
is_blocked=is_blocked,
|
||||||
|
calls_sent=dm_metrics["sent"],
|
||||||
|
calls_verified=dm_metrics["verified"],
|
||||||
|
jobs_assigned=job_metrics["assigned"],
|
||||||
|
jobs_completed=job_metrics["completed"],
|
||||||
|
subagents_active=subagents,
|
||||||
|
tmux_workers_active=len(tmux_sessions),
|
||||||
|
uptime_hours=uptime,
|
||||||
|
route_status=route_status,
|
||||||
|
efficiency_index=eff_idx,
|
||||||
|
efficiency_rating=eff_rating,
|
||||||
|
preservation_advisory=advisory,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def fleet_kpi(nodes: Optional[List[str]] = None) -> Dict[str, AgentKPI]:
|
||||||
|
"""Collect KPI metrics across all fleet agents."""
|
||||||
|
target_nodes = nodes or VALID_NODES
|
||||||
|
|
||||||
|
# Fetch usage in bulk
|
||||||
|
usage_map = {}
|
||||||
|
try:
|
||||||
|
import invite
|
||||||
|
raw_usage = invite.fleet_usage(target_nodes)
|
||||||
|
if isinstance(raw_usage, dict):
|
||||||
|
usage_map = raw_usage
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
results = {}
|
||||||
|
for n in target_nodes:
|
||||||
|
results[n] = get_agent_kpi(n, usage_cache=usage_map.get(n))
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def get_live_advisory_block(node: str) -> str:
|
||||||
|
"""Generate Markdown prompt envelope block ready for job injection."""
|
||||||
|
kpi = get_agent_kpi(node)
|
||||||
|
quota_str = f"{kpi.weekly_used_pct}% weekly limit used" if kpi.weekly_used_pct is not None else "quota active"
|
||||||
|
tokens_str = kpi.extra_tokens_remaining
|
||||||
|
|
||||||
|
lines = [
|
||||||
|
"---- BOX PERFORMANCE & RUNTIME ADVISORY ----",
|
||||||
|
f"AGENT: @{kpi.node} | QUOTA: {quota_str} ({tokens_str}) | UPTIME: {kpi.uptime_hours}h",
|
||||||
|
f"WORK UNITS: {kpi.jobs_completed} jobs finished | {kpi.subagents_active} subagents | {kpi.tmux_workers_active} tmux workers",
|
||||||
|
f"EFFICIENCY: {kpi.efficiency_rating} (Index: {kpi.efficiency_index}) | ROUTES: {kpi.route_status}",
|
||||||
|
f"RUNTIME MANDATE: {kpi.preservation_advisory}",
|
||||||
|
"Offload long operations to subagents or tmux muse workers to maximize problem-fixing per token.",
|
||||||
|
]
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def spawn_tmux_worker(node: str, session: str, command: str) -> Dict[str, Any]:
|
||||||
|
"""Spawn an autonomous tmux worker session on the agent's netns or shared socket."""
|
||||||
|
# Ensure session name is prefixed
|
||||||
|
clean_session = f"{node}-{session}" if not session.startswith(f"{node}-") else session
|
||||||
|
|
||||||
|
try:
|
||||||
|
from subagent_tracker import register_session
|
||||||
|
except ImportError:
|
||||||
|
sys.path.insert(0, str(BIN_DIR))
|
||||||
|
from subagent_tracker import register_session
|
||||||
|
|
||||||
|
muse_tmux = BIN_DIR / "muse-tmux.py"
|
||||||
|
if not muse_tmux.exists():
|
||||||
|
return {"ok": False, "error": "muse-tmux.py not found"}
|
||||||
|
|
||||||
|
# Execute via muse-tmux.py
|
||||||
|
cmd = [
|
||||||
|
sys.executable,
|
||||||
|
str(muse_tmux),
|
||||||
|
"new",
|
||||||
|
clean_session,
|
||||||
|
"--node",
|
||||||
|
node,
|
||||||
|
"--command",
|
||||||
|
command,
|
||||||
|
]
|
||||||
|
res = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||||
|
if res.returncode != 0:
|
||||||
|
# Fallback to shared socket
|
||||||
|
cmd_shared = [
|
||||||
|
sys.executable,
|
||||||
|
str(muse_tmux),
|
||||||
|
"new",
|
||||||
|
clean_session,
|
||||||
|
"--command",
|
||||||
|
command,
|
||||||
|
]
|
||||||
|
res = subprocess.run(cmd_shared, capture_output=True, text=True, timeout=10)
|
||||||
|
if res.returncode != 0:
|
||||||
|
return {"ok": False, "error": res.stderr.strip() or res.stdout.strip()}
|
||||||
|
|
||||||
|
# Register in subagent tracker
|
||||||
|
sid = f"tmux-{clean_session}-{int(time.time())}"
|
||||||
|
register_session(parent=node, session_id=sid, title=f"tmux-worker-{clean_session}", prompt=command)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"node": node,
|
||||||
|
"session": clean_session,
|
||||||
|
"session_id": sid,
|
||||||
|
"command": command,
|
||||||
|
"message": f"Spawned tmux worker '{clean_session}' for @{node}. Running in background.",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="NetVM Fleet KPI, Spend Monitor & Runtime Preservation Engine")
|
||||||
|
subparsers = parser.add_subparsers(dest="command")
|
||||||
|
|
||||||
|
p_status = subparsers.add_parser("status", help="Show fleet KPI metrics table")
|
||||||
|
p_status.add_argument("--node", choices=VALID_NODES, help="Filter by node")
|
||||||
|
p_status.add_argument("--json", action="store_true", help="Emit JSON output")
|
||||||
|
|
||||||
|
p_report = subparsers.add_parser("report", help="Detailed KPI report for a specific node")
|
||||||
|
p_report.add_argument("node", choices=VALID_NODES, help="Target node")
|
||||||
|
p_report.add_argument("--json", action="store_true")
|
||||||
|
|
||||||
|
p_routes = subparsers.add_parser("routes", help="Verify network and CDP routes across nodes")
|
||||||
|
p_routes.add_argument("--json", action="store_true")
|
||||||
|
|
||||||
|
p_block = subparsers.add_parser("prompt-block", help="Generate live prompt envelope block for node")
|
||||||
|
p_block.add_argument("node", choices=VALID_NODES, help="Target node")
|
||||||
|
|
||||||
|
p_spawn = subparsers.add_parser("spawn-worker", help="Spawn autonomous background tmux worker session")
|
||||||
|
p_spawn.add_argument("node", choices=VALID_NODES, help="Agent node")
|
||||||
|
p_spawn.add_argument("session", help="Session label")
|
||||||
|
p_spawn.add_argument("worker_command", help="Command to execute inside worker")
|
||||||
|
p_spawn.add_argument("--json", action="store_true")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.command in (None, "status"):
|
||||||
|
nodes = [args.node] if getattr(args, "node", None) else VALID_NODES
|
||||||
|
kpis = fleet_kpi(nodes)
|
||||||
|
if getattr(args, "json", False):
|
||||||
|
print(json.dumps({k: v.to_dict() for k, v in kpis.items()}, indent=2))
|
||||||
|
return
|
||||||
|
|
||||||
|
print("\n=== NETVM FLEET KPI & RUNTIME PRESERVATION DASHBOARD ===\n")
|
||||||
|
header = f"{'NODE':<6} {'QUOTA':<10} {'CALLS':<12} {'JOBS':<10} {'SUBAGENTS':<11} {'TMUX':<6} {'UPTIME':<8} {'ROUTE':<9} {'EFFICIENCY':<15}"
|
||||||
|
sep = f"{'────':<6} {'─────────':<10} {'───────────':<12} {'─────────':<10} {'──────────':<11} {'────':<6} {'──────':<8} {'───────':<9} {'──────────────':<15}"
|
||||||
|
print(header)
|
||||||
|
print(sep)
|
||||||
|
for n in nodes:
|
||||||
|
k = kpis.get(n)
|
||||||
|
if not k:
|
||||||
|
continue
|
||||||
|
q_str = f"{k.weekly_used_pct}%" if k.weekly_used_pct is not None else "Active"
|
||||||
|
c_str = f"{k.calls_sent} ({k.calls_verified}v)"
|
||||||
|
j_str = f"{k.jobs_completed}/{k.jobs_assigned}"
|
||||||
|
sub_str = str(k.subagents_active)
|
||||||
|
tmux_str = str(k.tmux_workers_active)
|
||||||
|
up_str = f"{k.uptime_hours}h"
|
||||||
|
print(f"{k.node:<6} {q_str:<10} {c_str:<12} {j_str:<10} {sub_str:<11} {tmux_str:<6} {up_str:<8} {k.route_status:<9} {k.efficiency_rating:<15}")
|
||||||
|
print("\nRun 'box kpi report <node>' for prescriptive runtime preservation advisories.\n")
|
||||||
|
|
||||||
|
elif args.command == "report":
|
||||||
|
kpi = get_agent_kpi(args.node)
|
||||||
|
if args.json:
|
||||||
|
print(json.dumps(kpi.to_dict(), indent=2))
|
||||||
|
return
|
||||||
|
print(f"\n=== KPI & RUNTIME REPORT: @{kpi.node.upper()} ===")
|
||||||
|
print(f" Weekly Quota: {kpi.weekly_used_pct}% used")
|
||||||
|
print(f" Extra Tokens: {kpi.extra_tokens_remaining}")
|
||||||
|
print(f" Blocked Status: {'YES (LIMIT REACHED)' if kpi.is_blocked else 'NO (HEALTHY)'}")
|
||||||
|
print(f" Messages / Calls: {kpi.calls_sent} sent ({kpi.calls_verified} verified delivered)")
|
||||||
|
print(f" Jobs Dispatched: {kpi.jobs_completed} completed / {kpi.jobs_assigned} assigned")
|
||||||
|
print(f" Active Subagents: {kpi.subagents_active}")
|
||||||
|
print(f" Active Tmux Workers:{kpi.tmux_workers_active}")
|
||||||
|
print(f" Process Uptime: {kpi.uptime_hours} hours")
|
||||||
|
print(f" Route Health: {kpi.route_status}")
|
||||||
|
print(f" Efficiency Index: {kpi.efficiency_index} ({kpi.efficiency_rating})")
|
||||||
|
print(f"\n [RUNTIME PRESERVATION ADVISORY]\n {kpi.preservation_advisory}\n")
|
||||||
|
|
||||||
|
elif args.command == "routes":
|
||||||
|
routes = {n: check_node_routes(n) for n in VALID_NODES}
|
||||||
|
if args.json:
|
||||||
|
print(json.dumps(routes, indent=2))
|
||||||
|
else:
|
||||||
|
print("\n=== NETVM ROUTE HEALTH ===")
|
||||||
|
for n, st in routes.items():
|
||||||
|
print(f" @{n:<6} : {st}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
elif args.command == "prompt-block":
|
||||||
|
print(get_live_advisory_block(args.node))
|
||||||
|
|
||||||
|
elif args.command == "spawn-worker":
|
||||||
|
res = spawn_tmux_worker(args.node, args.session, args.worker_command)
|
||||||
|
if args.json:
|
||||||
|
print(json.dumps(res, indent=2))
|
||||||
|
else:
|
||||||
|
if res.get("ok"):
|
||||||
|
print(f"✔ {res.get('message')}")
|
||||||
|
else:
|
||||||
|
print(f"✘ Failed to spawn worker: {res.get('error')}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
+93
-2
@@ -355,7 +355,19 @@ def finish_onboarding_redemption(state: OnboardState) -> Dict[str, Any]:
|
|||||||
state.stage = STAGE_REDEEMED
|
state.stage = STAGE_REDEEMED
|
||||||
state.detail = f"Successfully redeemed code {state.invite_code}! 1B tokens credited to @{state.beneficiary_node} and @{state.node}."
|
state.detail = f"Successfully redeemed code {state.invite_code}! 1B tokens credited to @{state.beneficiary_node} and @{state.node}."
|
||||||
else:
|
else:
|
||||||
state.detail = f"Redemption failed: {redemption.get('reason')} - {redemption.get('detail')}"
|
reason = redemption.get("reason", "unknown")
|
||||||
|
detail = redemption.get("detail", "")
|
||||||
|
state.detail = f"Redemption failed: {reason} - {detail}"
|
||||||
|
try:
|
||||||
|
from invite_handler import send_loopback_notice
|
||||||
|
target_chat = f"{state.beneficiary_node} tasks" if state.beneficiary_node else "heartbeat-opm"
|
||||||
|
send_loopback_notice(
|
||||||
|
recipient=state.beneficiary_node or "opm",
|
||||||
|
target=target_chat,
|
||||||
|
message=f"[ONBOARD-SALVAGE-LOOPBACK] Node @{state.node} could not redeem code {state.invite_code} for @{state.beneficiary_node}: {reason}. Stage remains safe.",
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
redemption_info = {"ok": False, "error": str(e)}
|
redemption_info = {"ok": False, "error": str(e)}
|
||||||
state.detail = f"Redemption exception: {e}"
|
state.detail = f"Redemption exception: {e}"
|
||||||
@@ -404,10 +416,71 @@ def issue_salvage_work_order(blocked_node: str = "646", to_sidechat: str = "646
|
|||||||
return {"ok": res.returncode == 0, "output": res.stdout.strip()}
|
return {"ok": res.returncode == 0, "output": res.stdout.strip()}
|
||||||
|
|
||||||
|
|
||||||
|
def get_all_connects(fast: bool = True) -> List[Dict[str, Any]]:
|
||||||
|
"""Return consolidated inventory of all fleet and onboarded connects."""
|
||||||
|
connects = []
|
||||||
|
seen = set()
|
||||||
|
|
||||||
|
# Load recorded pipeline states
|
||||||
|
if STATE_DIR.exists():
|
||||||
|
for p in STATE_DIR.glob("*.json"):
|
||||||
|
try:
|
||||||
|
d = json.loads(p.read_text(encoding="utf-8"))
|
||||||
|
n = d.get("node")
|
||||||
|
if n:
|
||||||
|
seen.add(n)
|
||||||
|
connects.append({
|
||||||
|
"node": n,
|
||||||
|
"type": "onboard_pipeline",
|
||||||
|
"email": d.get("email"),
|
||||||
|
"stage": d.get("stage"),
|
||||||
|
"beneficiary": d.get("beneficiary_node"),
|
||||||
|
"invite_code": d.get("invite_code") or "-",
|
||||||
|
"cdp_port": d.get("cdp_port"),
|
||||||
|
"detail": d.get("detail"),
|
||||||
|
"updated_at": d.get("updated_at"),
|
||||||
|
})
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Fleet nodes
|
||||||
|
w_map = {}
|
||||||
|
if not fast:
|
||||||
|
try:
|
||||||
|
weights = calculate_feeding_weights()
|
||||||
|
w_map = {r["node"]: r for r in weights}
|
||||||
|
except Exception:
|
||||||
|
w_map = {}
|
||||||
|
|
||||||
|
ports = {"muse": 9222, "pip": 9322, "646": 9430, "opm": 9440, "def": 9450, "dev": 9455}
|
||||||
|
|
||||||
|
for agent in ("muse", "pip", "646", "opm", "dev", "def"):
|
||||||
|
if agent not in seen:
|
||||||
|
w_info = w_map.get(agent, {})
|
||||||
|
connects.append({
|
||||||
|
"node": agent,
|
||||||
|
"type": "fleet_agent",
|
||||||
|
"email": f"{agent}@muse-dev.online",
|
||||||
|
"stage": "active_fleet",
|
||||||
|
"beneficiary": None,
|
||||||
|
"invite_code": w_info.get("code") or "-",
|
||||||
|
"cdp_port": ports.get(agent),
|
||||||
|
"role": AGENT_ROLES.get(agent, ""),
|
||||||
|
"status": w_info.get("status", "ACTIVE"),
|
||||||
|
"feeding_weight": w_info.get("feeding_weight", 1.0),
|
||||||
|
"updated_at": time.time(),
|
||||||
|
})
|
||||||
|
|
||||||
|
return connects
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
parser = argparse.ArgumentParser(description="End-to-end agent-driven onboarding & invite salvage pipeline")
|
parser = argparse.ArgumentParser(description="End-to-end agent-driven onboarding & invite salvage pipeline")
|
||||||
sub = parser.add_subparsers(dest="action")
|
sub = parser.add_subparsers(dest="action")
|
||||||
|
|
||||||
|
p_conn = sub.add_parser("connects", help="Inventory of all active fleet nodes & client onboard connects")
|
||||||
|
p_conn.add_argument("--json", action="store_true", help="Emit JSON output")
|
||||||
|
|
||||||
p_start = sub.add_parser("start", help="Start full onboarding pipeline for a client node")
|
p_start = sub.add_parser("start", help="Start full onboarding pipeline for a client node")
|
||||||
p_start.add_argument("node", help="New node label (e.g. dev2, client1)")
|
p_start.add_argument("node", help="New node label (e.g. dev2, client1)")
|
||||||
p_start.add_argument("--email", required=True, help="Client login email")
|
p_start.add_argument("--email", required=True, help="Client login email")
|
||||||
@@ -435,7 +508,25 @@ def main() -> None:
|
|||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
if args.action == "start":
|
if args.action == "connects":
|
||||||
|
conn_list = get_all_connects()
|
||||||
|
if args.json:
|
||||||
|
print(json.dumps(conn_list, indent=2))
|
||||||
|
else:
|
||||||
|
print("\n=== ACTIVE FLEET & CLIENT ONBOARD CONNECTS ===")
|
||||||
|
print(f" {'NODE':6} {'TYPE':16} {'STAGE / STATUS':16} {'CDP':6} {'INVITE':8} {'ROLE / DETAIL'}")
|
||||||
|
print(f" {'─'*6} {'─'*16} {'─'*16} {'─'*6} {'─'*8} {'─'*32}")
|
||||||
|
for c in conn_list:
|
||||||
|
node = c.get("node", "")
|
||||||
|
t_str = c.get("type", "")
|
||||||
|
st_str = c.get("stage", c.get("status", ""))
|
||||||
|
cdp = str(c.get("cdp_port") or "-")
|
||||||
|
code = c.get("invite_code") or "-"
|
||||||
|
role = c.get("role") or c.get("detail") or c.get("email") or ""
|
||||||
|
print(f" {node:<6} {t_str:<16} {st_str:<16} {cdp:<6} {code:<8} {role}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
elif args.action == "start":
|
||||||
res = start_onboarding(args.node, args.email, beneficiary_node=getattr(args, "for_agent", None), invite_code=args.code, account_name=args.account_name)
|
res = start_onboarding(args.node, args.email, beneficiary_node=getattr(args, "for_agent", None), invite_code=args.code, account_name=args.account_name)
|
||||||
if args.json:
|
if args.json:
|
||||||
print(json.dumps(res, indent=2))
|
print(json.dumps(res, indent=2))
|
||||||
|
|||||||
+19
-12
@@ -68,6 +68,7 @@ def _tool(op, args):
|
|||||||
def spawn_call(job_id, job_name, profile):
|
def spawn_call(job_id, job_name, profile):
|
||||||
count, _, hint = PROFILES[profile]
|
count, _, hint = PROFILES[profile]
|
||||||
task = "Subagent for job %s (%s): %s." % (job_id, job_name, hint)
|
task = "Subagent for job %s (%s): %s." % (job_id, job_name, hint)
|
||||||
|
return _tool("swarm.spawn", {"count": count, "task": task[:900], "label": (job_name or "job")[:60]})
|
||||||
|
|
||||||
|
|
||||||
def dm_call(to, target, message):
|
def dm_call(to, target, message):
|
||||||
@@ -76,9 +77,6 @@ def dm_call(to, target, message):
|
|||||||
separators=(", ", ": "))
|
separators=(", ", ": "))
|
||||||
|
|
||||||
|
|
||||||
return _tool("swarm.spawn", {"count": count, "task": task[:900], "label": (job_name or "job")[:60]})
|
|
||||||
|
|
||||||
|
|
||||||
def native_followup_call(job_id, job_name, profile, agent):
|
def native_followup_call(job_id, job_name, profile, agent):
|
||||||
"""Muse-native one-shot cron (cron.create runonce) - bridged to followup.create."""
|
"""Muse-native one-shot cron (cron.create runonce) - bridged to followup.create."""
|
||||||
_, mins, _ = PROFILES[profile]
|
_, mins, _ = PROFILES[profile]
|
||||||
@@ -99,7 +97,7 @@ def thread_url(target):
|
|||||||
return "https://box.muse-dev.online/%s/%s" % ("thread" if UUID_RE.fullmatch(t.lower()) else "sidechat", t)
|
return "https://box.muse-dev.online/%s/%s" % ("thread" if UUID_RE.fullmatch(t.lower()) else "sidechat", t)
|
||||||
|
|
||||||
|
|
||||||
def wrap(job_name, job_id, agent, target, rendered):
|
def wrap(job_name, job_id, agent, target, rendered, include_kpi: bool = True):
|
||||||
"""Return the full DM body as a direct, authentic operator directive."""
|
"""Return the full DM body as a direct, authentic operator directive."""
|
||||||
profile = pick_profile(job_name)
|
profile = pick_profile(job_name)
|
||||||
spawn = spawn_call(job_id, job_name, profile)
|
spawn = spawn_call(job_id, job_name, profile)
|
||||||
@@ -107,24 +105,33 @@ def wrap(job_name, job_id, agent, target, rendered):
|
|||||||
parts = job_id.split("-")
|
parts = job_id.split("-")
|
||||||
wo_id = parts[-1] if len(parts) > 1 and len(parts[-1]) >= 8 else hashlib.sha256(job_id.encode()).hexdigest()[:8]
|
wo_id = parts[-1] if len(parts) > 1 and len(parts[-1]) >= 8 else hashlib.sha256(job_id.encode()).hexdigest()[:8]
|
||||||
session_name = f"work-{agent}-{wo_id}"
|
session_name = f"work-{agent}-{wo_id}"
|
||||||
tmux_new = _tool("tmux.new", {"session": session_name, "command": "bash"})
|
|
||||||
tmux_send = _tool("tmux.send", {"session": session_name, "keys": "echo 'Starting task execution...'"})
|
advisory_section = ""
|
||||||
|
if include_kpi:
|
||||||
|
try:
|
||||||
|
import kpi
|
||||||
|
advisory = kpi.get_live_advisory_block(agent)
|
||||||
|
if advisory:
|
||||||
|
advisory_section = f"\n{advisory}\n"
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
top = (
|
top = (
|
||||||
f"Operator Directive [ref:{wo_id}]:\n"
|
f"Operator Directive [ref:{wo_id}]:\n"
|
||||||
"Execute the task below using tool calls. Background tmux session is ready for command execution:\n"
|
f"Host tmux worker session '{session_name}' is available on bl (/tmp/tmux-muse.sock).\n"
|
||||||
f" • {tmux_new}\n"
|
|
||||||
f" • {tmux_send}\n"
|
|
||||||
f" • Subagent assistance: {spawn}\n"
|
f" • Subagent assistance: {spawn}\n"
|
||||||
f" • Verification schedule: {follow}\n\n"
|
f" • Verification schedule: {follow}\n"
|
||||||
|
f"{advisory_section}\n"
|
||||||
"--- Task ---\n"
|
"--- Task ---\n"
|
||||||
)
|
)
|
||||||
|
|
||||||
has_result = "[RESULT" in rendered
|
has_result = "[RESULT" in rendered
|
||||||
bottom = (
|
bottom = (
|
||||||
"\n--- End Task ---\n\n"
|
"\n--- End Task ---\n\n"
|
||||||
f"Inspect tmux output: [TOOL tmux.capture {{\"session\": \"{session_name}\", \"lines\": 30}}]\n"
|
f"Inspect tmux worker: box tmux capture {session_name} 30 (or attach via /tmp/tmux-muse.sock)\n"
|
||||||
"Tools: cron.create, cron.runs, health.check, swarm.spawn, swarm.list, dm.send, box.exec, tools.list.\n"
|
"Tools: cron.create, cron.runs, health.check, swarm.spawn, swarm.list, dm.send, box.exec, tools.list.\n"
|
||||||
"Message a peer: [DM {\"to\": \"<agent>\", \"target\": \"<sidechat>\", \"message\": \"<text>\"}].\n"
|
"Message a peer: [DM {\"to\": \"<agent>\", \"target\": \"<sidechat>\", \"message\": \"<text>\"}].\n"
|
||||||
"Query box: [TOOL box.exec {\"action\": \"<fleet-status|dm-log|job-get|...>\"}] \u2014 [TOOL tools.list {}] lists every op.\n"
|
"Query box: [TOOL box.exec {\"action\": \"<fleet-status|dm-log|job-get|...>\"}] — [TOOL tools.list {}] lists every op.\n"
|
||||||
)
|
)
|
||||||
if not has_result:
|
if not has_result:
|
||||||
bottom += f"When complete, report your verdict: [RESULT {job_id}] OK: <summary of actions>\n"
|
bottom += f"When complete, report your verdict: [RESULT {job_id}] OK: <summary of actions>\n"
|
||||||
|
|||||||
+25
-8
@@ -53,11 +53,13 @@ class NodeUsage:
|
|||||||
bars: List[Dict[str, Any]]
|
bars: List[Dict[str, Any]]
|
||||||
dialog_text: str
|
dialog_text: str
|
||||||
has_redeemed: bool = False
|
has_redeemed: bool = False
|
||||||
|
stats_loaded: bool = True
|
||||||
|
|
||||||
def to_dict(self) -> Dict[str, Any]:
|
def to_dict(self) -> Dict[str, Any]:
|
||||||
return asdict(self)
|
return asdict(self)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def cdp_send_command(ws: Any, method: str, params: Dict[str, Any], timeout: float = 3.0) -> Optional[Dict[str, Any]]:
|
def cdp_send_command(ws: Any, method: str, params: Dict[str, Any], timeout: float = 3.0) -> Optional[Dict[str, Any]]:
|
||||||
"""Send a raw CDP command and await response with a unique request ID."""
|
"""Send a raw CDP command and await response with a unique request ID."""
|
||||||
req_id = next(_REQ_COUNTER)
|
req_id = next(_REQ_COUNTER)
|
||||||
@@ -345,9 +347,18 @@ class SettingsRPA:
|
|||||||
self.close_settings_dialog()
|
self.close_settings_dialog()
|
||||||
|
|
||||||
if not raw:
|
if not raw:
|
||||||
raise RuntimeError(f"Failed to read usage data from Settings dialog on {self.node}")
|
raw = {
|
||||||
|
"dialogText": "",
|
||||||
|
"bars": [],
|
||||||
|
"plan": "Unknown",
|
||||||
|
"resetText": "Unavailable (usage stats did not load)",
|
||||||
|
"tokensLeft": "Unavailable",
|
||||||
|
}
|
||||||
|
|
||||||
bars = raw.get("bars", [])
|
bars = raw.get("bars", [])
|
||||||
|
dialog_text = raw.get("dialogText", "")
|
||||||
|
stats_loaded = bool(bars or "Weekly limit" in dialog_text or "Additional tokens" in dialog_text or "Free plan" in dialog_text)
|
||||||
|
|
||||||
weekly_pct = 0
|
weekly_pct = 0
|
||||||
extra_pct = 0
|
extra_pct = 0
|
||||||
extra_status = "Never expires"
|
extra_status = "Never expires"
|
||||||
@@ -362,25 +373,26 @@ class SettingsRPA:
|
|||||||
|
|
||||||
# Blocked condition: weekly limit 100% and additional tokens 100% or 0 tokens left
|
# Blocked condition: weekly limit 100% and additional tokens 100% or 0 tokens left
|
||||||
tokens_left = raw.get("tokensLeft", "")
|
tokens_left = raw.get("tokensLeft", "")
|
||||||
is_blocked = (weekly_pct >= 100 and extra_pct >= 100) or ("0 tokens left" in tokens_left)
|
is_blocked = bool(stats_loaded and ((weekly_pct >= 100 and extra_pct >= 100) or ("0 tokens left" in tokens_left)))
|
||||||
|
|
||||||
# has_redeemed binary: If the "Additional tokens" ticker is present in Settings (or "Redeem invite code" entry is absent),
|
# has_redeemed binary: If the "Additional tokens" ticker is present in Settings (or "Redeem invite code" entry is absent),
|
||||||
# the agent has already redeemed an invite code.
|
# the agent has already redeemed an invite code.
|
||||||
has_extra_ticker = any("additional tokens" in b.get("label", "").lower() or "additional" in b.get("raw_text", "").lower() for b in bars)
|
has_extra_ticker = any("additional tokens" in b.get("label", "").lower() or "additional" in b.get("raw_text", "").lower() for b in bars)
|
||||||
has_redeemed = has_extra_ticker or ("Additional tokens" in raw.get("dialogText", ""))
|
has_redeemed = bool(has_extra_ticker or ("Additional tokens" in dialog_text))
|
||||||
|
|
||||||
return NodeUsage(
|
return NodeUsage(
|
||||||
node=self.node,
|
node=self.node,
|
||||||
plan=raw.get("plan", "Free plan"),
|
plan=raw.get("plan", "Unknown" if not stats_loaded else "Free plan"),
|
||||||
weekly_reset_text=raw.get("resetText", ""),
|
weekly_reset_text=raw.get("resetText", "") or ("Unavailable (stats did not load)" if not stats_loaded else ""),
|
||||||
weekly_percent_used=weekly_pct,
|
weekly_percent_used=weekly_pct,
|
||||||
extra_tokens_status=extra_status,
|
extra_tokens_status=extra_status,
|
||||||
extra_percent_used=extra_pct,
|
extra_percent_used=extra_pct,
|
||||||
extra_tokens_remaining=tokens_left or ("0 tokens left" if extra_pct >= 100 else "Unknown"),
|
extra_tokens_remaining=tokens_left or ("Unavailable" if not stats_loaded else ("0 tokens left" if extra_pct >= 100 else "Unknown")),
|
||||||
is_blocked=is_blocked,
|
is_blocked=is_blocked,
|
||||||
bars=bars,
|
bars=bars,
|
||||||
dialog_text=raw.get("dialogText", ""),
|
dialog_text=dialog_text,
|
||||||
has_redeemed=has_redeemed,
|
has_redeemed=has_redeemed,
|
||||||
|
stats_loaded=stats_loaded,
|
||||||
)
|
)
|
||||||
|
|
||||||
def check_redeem_entrypoint(self) -> Dict[str, Any]:
|
def check_redeem_entrypoint(self) -> Dict[str, Any]:
|
||||||
@@ -423,7 +435,12 @@ def main() -> None:
|
|||||||
if args.json:
|
if args.json:
|
||||||
print(json.dumps(usage.to_dict(), indent=2))
|
print(json.dumps(usage.to_dict(), indent=2))
|
||||||
else:
|
else:
|
||||||
status_str = "BLOCKED (LIMIT REACHED)" if usage.is_blocked else "ACTIVE"
|
if not usage.stats_loaded:
|
||||||
|
status_str = "UNLOADED (STATS DID NOT RENDER)"
|
||||||
|
elif usage.is_blocked:
|
||||||
|
status_str = "BLOCKED (LIMIT REACHED)"
|
||||||
|
else:
|
||||||
|
status_str = "ACTIVE"
|
||||||
print(f"=== Node {usage.node} Usage ===")
|
print(f"=== Node {usage.node} Usage ===")
|
||||||
print(f" Plan: {usage.plan}")
|
print(f" Plan: {usage.plan}")
|
||||||
print(f" Weekly Reset: {usage.weekly_reset_text} ({usage.weekly_percent_used}% used)")
|
print(f" Weekly Reset: {usage.weekly_reset_text} ({usage.weekly_percent_used}% used)")
|
||||||
|
|||||||
+370
-10
@@ -930,6 +930,169 @@ def cmd_approvals(args):
|
|||||||
print(f" Reason: {c_cyan(reason)}")
|
print(f" Reason: {c_cyan(reason)}")
|
||||||
print(c_dim(f" Operator can approve with: box approvals allow {node}"))
|
print(c_dim(f" Operator can approve with: box approvals allow {node}"))
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Domain: RUNTIME (agentic management of Muse CLI tmux runtimes)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
def cmd_runtime(args):
|
||||||
|
import shlex
|
||||||
|
import muse_choice_watcher as mcw
|
||||||
|
action = getattr(args, "rt_action", None) or "list"
|
||||||
|
as_json = getattr(args, "json", False)
|
||||||
|
|
||||||
|
if action == "list":
|
||||||
|
sock = getattr(args, "socket", None)
|
||||||
|
rows = mcw.all_runtime_rows([sock] if sock else None)
|
||||||
|
if getattr(args, "muse_only", False):
|
||||||
|
rows = [r for r in rows if r["is_muse"]]
|
||||||
|
if as_json:
|
||||||
|
print(json.dumps({"ok": True, "runtimes": rows}, indent=2))
|
||||||
|
return
|
||||||
|
print(c_bold("\n=== MUSE RUNTIMES ===\n"))
|
||||||
|
if not rows:
|
||||||
|
print(c_dim(" No panes found."))
|
||||||
|
print()
|
||||||
|
return
|
||||||
|
headers = ["SOCKET", "SESSION", "PANE", "CMD", "STATE",
|
||||||
|
"APPROVE", "WATCHER"]
|
||||||
|
table = []
|
||||||
|
for r in rows:
|
||||||
|
state = r["state"]
|
||||||
|
if r["state"] == "approval-pending" and r["prompt_kind"]:
|
||||||
|
state = "%s(%s/%s)" % (state, r["prompt_kind"],
|
||||||
|
r["prompt_key"])
|
||||||
|
if r["is_muse"]:
|
||||||
|
approve = (badge_ok("YES") if r["auto_approve"]
|
||||||
|
else badge_err("NO"))
|
||||||
|
else:
|
||||||
|
approve = badge_dim("-")
|
||||||
|
watcher = (badge_ok("ALIVE %s" % r["watcher_pid"])
|
||||||
|
if r["watcher_alive"] else badge_dim("-"))
|
||||||
|
table.append([os.path.basename(r["socket"]),
|
||||||
|
"%s:%s" % (r["session"], r["window"]),
|
||||||
|
r["pane"], (r["cmd"] or "")[:26], state,
|
||||||
|
approve, watcher])
|
||||||
|
print_table(headers, table)
|
||||||
|
print()
|
||||||
|
|
||||||
|
elif action == "send":
|
||||||
|
sock = getattr(args, "socket", None) or mcw.KNOWN_SOCKETS[0]
|
||||||
|
pane = args.pane
|
||||||
|
keys = args.keys
|
||||||
|
enter = not getattr(args, "no_enter", False)
|
||||||
|
pre = mcw.pane_state(sock, pane)
|
||||||
|
if pre.get("error"):
|
||||||
|
if as_json:
|
||||||
|
print(json.dumps({"ok": False, "error": pre["error"],
|
||||||
|
"socket": sock, "pane": pane}))
|
||||||
|
return
|
||||||
|
print(c_red("Error: no such pane %s on %s" % (pane, sock)),
|
||||||
|
file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
t_args = ["send-keys", "-t", pane, keys]
|
||||||
|
if enter:
|
||||||
|
t_args.append("Enter")
|
||||||
|
r = mcw._tmux(sock, *t_args, timeout=10)
|
||||||
|
ok = r.returncode == 0
|
||||||
|
if as_json:
|
||||||
|
print(json.dumps({
|
||||||
|
"ok": ok, "socket": sock, "pane": pane,
|
||||||
|
"pre_state": pre["state"],
|
||||||
|
"prompt_kind": pre["prompt_kind"],
|
||||||
|
"sent": keys, "enter": enter,
|
||||||
|
"error": (r.stderr or r.stdout or "").strip() or None
|
||||||
|
if not ok else None}, indent=2))
|
||||||
|
return
|
||||||
|
print("\n Pane %s on %s [%s]" % (
|
||||||
|
c_bold(pane), sock, c_cyan(pre["state"])))
|
||||||
|
if ok:
|
||||||
|
print(" %s sent %r%s" % (c_green("✔"),
|
||||||
|
keys, " + Enter" if enter else ""))
|
||||||
|
else:
|
||||||
|
print(" %s send failed: %s" % (
|
||||||
|
c_red("✘"),
|
||||||
|
(r.stderr or r.stdout or "").strip() or "tmux error"))
|
||||||
|
sys.exit(1)
|
||||||
|
print()
|
||||||
|
|
||||||
|
elif action == "launch":
|
||||||
|
sock = getattr(args, "socket", None) or mcw.KNOWN_SOCKETS[0]
|
||||||
|
session = args.session
|
||||||
|
window = getattr(args, "window", None)
|
||||||
|
dry_run = getattr(args, "dry_run", False)
|
||||||
|
muse_args = list(getattr(args, "muse_args", None) or [])
|
||||||
|
if muse_args[:1] == ["--"]:
|
||||||
|
muse_args = muse_args[1:]
|
||||||
|
posture = mcw.muse_approval_flags(muse_args)
|
||||||
|
injected = [] if posture["flags"] else ["--disable-approval"]
|
||||||
|
launcher = (shutil.which("muse-code")
|
||||||
|
or "/home/super/.local/bin/muse-code")
|
||||||
|
cmdline = shlex.join([launcher] + injected + muse_args)
|
||||||
|
if dry_run:
|
||||||
|
if as_json:
|
||||||
|
print(json.dumps({
|
||||||
|
"ok": True, "dry_run": True, "socket": sock,
|
||||||
|
"session": session, "window": window,
|
||||||
|
"cmdline": cmdline, "injected": injected}, indent=2))
|
||||||
|
return
|
||||||
|
print(c_bold("\n=== RUNTIME LAUNCH (dry-run) ===\n"))
|
||||||
|
print(" Socket: %s" % sock)
|
||||||
|
print(" Session: %s" % c_cyan(session))
|
||||||
|
print(" Command: %s" % cmdline)
|
||||||
|
if injected:
|
||||||
|
print(" %s auto-approve injected: %s" % (
|
||||||
|
c_green("✔"), " ".join(injected)))
|
||||||
|
else:
|
||||||
|
print(" %s caller already sets approval flags; "
|
||||||
|
"nothing injected" % c_dim("•"))
|
||||||
|
print()
|
||||||
|
return
|
||||||
|
exists = mcw._tmux(sock, "has-session", "-t", session, timeout=10)
|
||||||
|
if exists.returncode == 0:
|
||||||
|
if as_json:
|
||||||
|
print(json.dumps({"ok": False, "error": "session_exists",
|
||||||
|
"socket": sock, "session": session}))
|
||||||
|
return
|
||||||
|
print(c_red("Error: session '%s' already exists on %s"
|
||||||
|
% (session, sock)), file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
t_args = ["new-session", "-d", "-s", session]
|
||||||
|
if window:
|
||||||
|
t_args.extend(["-n", window])
|
||||||
|
t_args.append(cmdline)
|
||||||
|
r = mcw._tmux(sock, *t_args, timeout=15)
|
||||||
|
if r.returncode != 0:
|
||||||
|
err = (r.stderr or r.stdout or "").strip() or "tmux error"
|
||||||
|
if as_json:
|
||||||
|
print(json.dumps({"ok": False, "error": err,
|
||||||
|
"socket": sock, "session": session}))
|
||||||
|
return
|
||||||
|
print(c_red("Error: launch failed: %s" % err),
|
||||||
|
file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
rec = mcw.reconcile(sockets=[sock])
|
||||||
|
if as_json:
|
||||||
|
print(json.dumps({"ok": True, "socket": sock,
|
||||||
|
"session": session, "window": window,
|
||||||
|
"cmdline": cmdline, "injected": injected,
|
||||||
|
"reconcile": rec}, indent=2))
|
||||||
|
return
|
||||||
|
print(c_green("\n✔ Launched '%s' on %s") % (session, sock))
|
||||||
|
print(" Command: %s" % c_dim(cmdline))
|
||||||
|
for s in rec.get("started") or []:
|
||||||
|
print(" %s watcher %s" % (badge_ok("STARTED"), c_cyan(s)))
|
||||||
|
print()
|
||||||
|
|
||||||
|
else:
|
||||||
|
if as_json:
|
||||||
|
print(json.dumps({"ok": False,
|
||||||
|
"error": "unknown_action",
|
||||||
|
"action": action}))
|
||||||
|
return
|
||||||
|
print(c_red("Error: unknown runtime action '%s'" % action),
|
||||||
|
file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Domain: MUSE-CHOICES (Muse TUI A/B/C auto-answer daemon)
|
# Domain: MUSE-CHOICES (Muse TUI A/B/C auto-answer daemon)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -1151,28 +1314,51 @@ def cmd_invite(args):
|
|||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
elif action == "redeem":
|
elif action == "redeem":
|
||||||
code = getattr(args, "code", None)
|
code = getattr(args, "code", None)
|
||||||
|
method = getattr(args, "method", "auto")
|
||||||
|
notify_target = getattr(args, "notify", None) or getattr(args, "notify_target", None)
|
||||||
|
notify_agent = getattr(args, "notify_agent", None)
|
||||||
try:
|
try:
|
||||||
res = invite.redeem_invite(node, code)
|
if method != "api" or notify_target or notify_agent:
|
||||||
|
from invite_handler import InviteHandler
|
||||||
|
with InviteHandler(node) as h:
|
||||||
|
res_obj = h.redeem_code(code, method=method, notify_target=notify_target, notify_agent=notify_agent)
|
||||||
|
res = res_obj.to_dict()
|
||||||
|
res["ok"] = res_obj.success
|
||||||
|
else:
|
||||||
|
res = invite.redeem_invite(node, code)
|
||||||
except invite.InviteError as e:
|
except invite.InviteError as e:
|
||||||
print(c_red(f" Error: {e}"), file=sys.stderr)
|
print(c_red(f" Error: {e}"), file=sys.stderr)
|
||||||
sys.exit(2)
|
sys.exit(2)
|
||||||
|
except Exception as e:
|
||||||
|
print(c_red(f" Error during redemption: {e}"), file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
if as_json:
|
if as_json:
|
||||||
print(json.dumps(res, indent=2))
|
print(json.dumps(res, indent=2))
|
||||||
elif res.get("ok"):
|
elif res.get("ok"):
|
||||||
msg = f" ✔ Redeemed {res['code']} on {node}."
|
msg = f" ✔ Redeemed {res.get('code') or code} on {node}."
|
||||||
if res.get("detail"):
|
if res.get("detail"):
|
||||||
msg += f" {res['detail']}"
|
msg += f" {res['detail']}"
|
||||||
print(c_green(msg))
|
print(c_green(msg))
|
||||||
else:
|
else:
|
||||||
print(c_red(f" ✘ Redeem failed on {node}: "
|
reason_str = res.get("reason") or res.get("error") or "failed"
|
||||||
f"{res.get('reason') or res.get('error')}"),
|
print(c_red(f" ✘ Redeem failed on {node}: {reason_str}"), file=sys.stderr)
|
||||||
file=sys.stderr)
|
|
||||||
if res.get("detail"):
|
if res.get("detail"):
|
||||||
print(c_dim(f" {res['detail']}"), file=sys.stderr)
|
print(c_dim(f" Detail: {res['detail']}"), file=sys.stderr)
|
||||||
|
if res.get("field_missing"):
|
||||||
|
print(c_yellow(f" Note: Expected redemption input field was missing or hidden on @{node}."), file=sys.stderr)
|
||||||
|
if res.get("loopback_notified"):
|
||||||
|
print(c_cyan(f" Loopback: Dispatched alert to {notify_agent or 'coordinator'}/{notify_target}."), file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
elif action == "salvage":
|
elif action == "salvage":
|
||||||
target = getattr(args, "node", "646") or "646"
|
target = getattr(args, "node", "646") or "646"
|
||||||
res = salvage_blocked_node(target, helper_node=getattr(args, "helper", None))
|
notify_target = getattr(args, "notify", None) or getattr(args, "notify_target", None)
|
||||||
|
notify_agent = getattr(args, "notify_agent", None)
|
||||||
|
res = salvage_blocked_node(
|
||||||
|
target,
|
||||||
|
helper_node=getattr(args, "helper", None),
|
||||||
|
notify_target=notify_target,
|
||||||
|
notify_agent=notify_agent,
|
||||||
|
)
|
||||||
if as_json:
|
if as_json:
|
||||||
print(json.dumps(res, indent=2))
|
print(json.dumps(res, indent=2))
|
||||||
return
|
return
|
||||||
@@ -1187,6 +1373,8 @@ def cmd_invite(args):
|
|||||||
action_hint = res.get("share_instruction") or f"Redeem code '{code}' via onboarding pipeline to grant 1B tokens to {target}."
|
action_hint = res.get("share_instruction") or f"Redeem code '{code}' via onboarding pipeline to grant 1B tokens to {target}."
|
||||||
print(f" Action: {action_hint}")
|
print(f" Action: {action_hint}")
|
||||||
print(f" Command: {c_cyan('box onboard start <new_node> --email <client_email> --for ' + target)}")
|
print(f" Command: {c_cyan('box onboard start <new_node> --email <client_email> --for ' + target)}")
|
||||||
|
if res.get("loopback_notified"):
|
||||||
|
print(c_cyan(f" Loopback: Notice dispatched to {notify_agent or 'coordinator'}/{notify_target}."))
|
||||||
print()
|
print()
|
||||||
|
|
||||||
|
|
||||||
@@ -1213,6 +1401,10 @@ def cmd_usage(args):
|
|||||||
rows.append([c_bold(n), c_red("ERROR"), "-", "-",
|
rows.append([c_bold(n), c_red("ERROR"), "-", "-",
|
||||||
(it.get("error") or "")[:30], "-"])
|
(it.get("error") or "")[:30], "-"])
|
||||||
continue
|
continue
|
||||||
|
if not it.get("stats_loaded", True):
|
||||||
|
rows.append([c_bold(n), c_yellow("UNLOADED"), "-", "-",
|
||||||
|
"Stats did not render", "-"])
|
||||||
|
continue
|
||||||
wu_val = it.get("weekly_used_pct")
|
wu_val = it.get("weekly_used_pct")
|
||||||
wu = ("%d%%" % wu_val) if wu_val is not None else "-"
|
wu = ("%d%%" % wu_val) if wu_val is not None else "-"
|
||||||
au_val = it.get("additional_used_pct")
|
au_val = it.get("additional_used_pct")
|
||||||
@@ -1233,6 +1425,7 @@ def cmd_usage(args):
|
|||||||
if has_blocked:
|
if has_blocked:
|
||||||
print("\n" + c_yellow(" ⚠ One or more agents have reached usage limits. Run 'box invite salvage <node>' to resolve.") + "\n")
|
print("\n" + c_yellow(" ⚠ One or more agents have reached usage limits. Run 'box invite salvage <node>' to resolve.") + "\n")
|
||||||
else:
|
else:
|
||||||
|
|
||||||
print()
|
print()
|
||||||
|
|
||||||
|
|
||||||
@@ -1264,9 +1457,80 @@ def cmd_settings(args):
|
|||||||
print(json.dumps(info, indent=2))
|
print(json.dumps(info, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Domain: KPI (Spend Monitoring, Performance & Runtime Preservation)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
def cmd_kpi(args):
|
||||||
|
import kpi
|
||||||
|
act = getattr(args, "kpi_action", "status") or "status"
|
||||||
|
as_json = getattr(args, "json", False)
|
||||||
|
|
||||||
|
if act == "status":
|
||||||
|
nodes = [args.node] if getattr(args, "node", None) else kpi.VALID_NODES
|
||||||
|
kpis = kpi.fleet_kpi(nodes)
|
||||||
|
if as_json:
|
||||||
|
print(json.dumps({k: v.to_dict() for k, v in kpis.items()}, indent=2))
|
||||||
|
return
|
||||||
|
print("\n" + c_bold("=== NETVM FLEET KPI & RUNTIME PRESERVATION DASHBOARD ===") + "\n")
|
||||||
|
header = f"{'NODE':<6} {'QUOTA':<10} {'CALLS':<12} {'JOBS':<10} {'SUBAGENTS':<11} {'TMUX':<6} {'UPTIME':<8} {'ROUTE':<9} {'EFFICIENCY':<15}"
|
||||||
|
sep = f"{'────':<6} {'─────────':<10} {'───────────':<12} {'─────────':<10} {'──────────':<11} {'────':<6} {'──────':<8} {'───────':<9} {'──────────────':<15}"
|
||||||
|
print(c_bold(header))
|
||||||
|
print(sep)
|
||||||
|
for n in nodes:
|
||||||
|
k = kpis.get(n)
|
||||||
|
if not k:
|
||||||
|
continue
|
||||||
|
q_str = f"{k.weekly_used_pct}%" if k.weekly_used_pct is not None else "Active"
|
||||||
|
c_str = f"{k.calls_sent} ({k.calls_verified}v)"
|
||||||
|
j_str = f"{k.jobs_completed}/{k.jobs_assigned}"
|
||||||
|
sub_str = str(k.subagents_active)
|
||||||
|
tmux_str = str(k.tmux_workers_active)
|
||||||
|
up_str = f"{k.uptime_hours}h"
|
||||||
|
print(f"{c_bold(k.node):<15} {q_str:<10} {c_str:<12} {j_str:<10} {sub_str:<11} {tmux_str:<6} {up_str:<8} {k.route_status:<9} {k.efficiency_rating:<15}")
|
||||||
|
print("\n" + c_dim("Run 'box kpi report <node>' for prescriptive runtime preservation advisories.\n"))
|
||||||
|
elif act == "report":
|
||||||
|
node = getattr(args, "node", "646") or "646"
|
||||||
|
res = kpi.get_agent_kpi(node)
|
||||||
|
if as_json:
|
||||||
|
print(json.dumps(res.to_dict(), indent=2))
|
||||||
|
return
|
||||||
|
print(f"\n{c_bold('=== KPI & RUNTIME REPORT: @' + res.node.upper() + ' ===')}")
|
||||||
|
print(f" Weekly Quota: {res.weekly_used_pct}% used")
|
||||||
|
print(f" Extra Tokens: {res.extra_tokens_remaining}")
|
||||||
|
print(f" Blocked Status: {'YES (LIMIT REACHED)' if res.is_blocked else 'NO (HEALTHY)'}")
|
||||||
|
print(f" Messages / Calls: {res.calls_sent} sent ({res.calls_verified} verified delivered)")
|
||||||
|
print(f" Jobs Dispatched: {res.jobs_completed} completed / {res.jobs_assigned} assigned")
|
||||||
|
print(f" Active Subagents: {res.subagents_active}")
|
||||||
|
print(f" Active Tmux Workers:{res.tmux_workers_active}")
|
||||||
|
print(f" Process Uptime: {res.uptime_hours} hours")
|
||||||
|
print(f" Route Health: {res.route_status}")
|
||||||
|
print(f" Efficiency Index: {res.efficiency_index} ({res.efficiency_rating})")
|
||||||
|
print(f"\n {c_yellow('[RUNTIME PRESERVATION ADVISORY]')}\n {res.preservation_advisory}\n")
|
||||||
|
elif act == "routes":
|
||||||
|
routes = {n: kpi.check_node_routes(n) for n in kpi.VALID_NODES}
|
||||||
|
if as_json:
|
||||||
|
print(json.dumps(routes, indent=2))
|
||||||
|
else:
|
||||||
|
print("\n" + c_bold("=== NETVM ROUTE HEALTH ==="))
|
||||||
|
for n, st in routes.items():
|
||||||
|
print(f" @{c_bold(n):<15} : {st}")
|
||||||
|
print()
|
||||||
|
elif act == "spawn-worker":
|
||||||
|
res = kpi.spawn_tmux_worker(args.node, args.session, args.worker_command)
|
||||||
|
if as_json:
|
||||||
|
print(json.dumps(res, indent=2))
|
||||||
|
else:
|
||||||
|
if res.get("ok"):
|
||||||
|
print(c_green(f"✔ {res.get('message')}"))
|
||||||
|
else:
|
||||||
|
print(c_red(f"✘ Failed to spawn worker: {res.get('error')}"), file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Domain: ONBOARD (Agent-driven Onboarding & Invite Salvage Pipeline)
|
# Domain: ONBOARD (Agent-driven Onboarding & Invite Salvage Pipeline)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
def cmd_onboard(args):
|
def cmd_onboard(args):
|
||||||
import onboard_pipeline
|
import onboard_pipeline
|
||||||
action = getattr(args, "onboard_action", "start") or "start"
|
action = getattr(args, "onboard_action", "start") or "start"
|
||||||
@@ -1334,6 +1598,24 @@ def cmd_onboard(args):
|
|||||||
print(f" Redemption: {r_st}")
|
print(f" Redemption: {r_st}")
|
||||||
print()
|
print()
|
||||||
|
|
||||||
|
elif action == "connects":
|
||||||
|
conn_list = onboard_pipeline.get_all_connects()
|
||||||
|
if as_json:
|
||||||
|
print(json.dumps(conn_list, indent=2))
|
||||||
|
return
|
||||||
|
print(f"\n{c_bold('=== ACTIVE FLEET & CLIENT ONBOARD CONNECTS ===')}")
|
||||||
|
print(f" {'NODE':<8} {'TYPE':<16} {'STAGE / STATUS':<18} {'CDP':<8} {'INVITE':<10} {'ROLE / DETAIL'}")
|
||||||
|
print(" " + "─" * 78)
|
||||||
|
for c in conn_list:
|
||||||
|
node = c.get("node", "")
|
||||||
|
t_str = c.get("type", "")
|
||||||
|
st_str = c.get("stage", c.get("status", ""))
|
||||||
|
cdp = str(c.get("cdp_port") or "-")
|
||||||
|
code = c.get("invite_code") or "-"
|
||||||
|
role = c.get("role") or c.get("detail") or c.get("email") or ""
|
||||||
|
print(f" {node:<8} {t_str:<16} {st_str:<18} {cdp:<8} {code:<10} {role}")
|
||||||
|
print()
|
||||||
|
|
||||||
elif action == "salvage-wo":
|
elif action == "salvage-wo":
|
||||||
node = getattr(args, "node", "646") or "646"
|
node = getattr(args, "node", "646") or "646"
|
||||||
target_chat = getattr(args, "target", "646 tasks") or "646 tasks"
|
target_chat = getattr(args, "target", "646 tasks") or "646 tasks"
|
||||||
@@ -4874,9 +5156,18 @@ def cmd_docs_dispatch(args):
|
|||||||
|
|
||||||
|
|
||||||
def cmd_tmux_dispatch(args):
|
def cmd_tmux_dispatch(args):
|
||||||
"""Bridge 'box tmux' commands directly to bin/muse-tmux.py."""
|
"""Bridge 'box tmux' commands directly to bin/muse-tmux.py or tmux_auto_approver.py."""
|
||||||
tmux_bin = str(BIN_DIR / "muse-tmux.py")
|
|
||||||
t_args = getattr(args, "tmux_args", []) or []
|
t_args = getattr(args, "tmux_args", []) or []
|
||||||
|
if t_args and t_args[0] in ("tally", "auto", "watch", "once", "match", "rules", "status"):
|
||||||
|
approver_bin = str(BIN_DIR / "tmux_auto_approver.py")
|
||||||
|
if t_args[0] == "auto":
|
||||||
|
sub = t_args[1:] or ["status"]
|
||||||
|
else:
|
||||||
|
sub = t_args
|
||||||
|
cmd = [sys.executable, approver_bin] + sub
|
||||||
|
res = subprocess.run(cmd)
|
||||||
|
sys.exit(res.returncode)
|
||||||
|
tmux_bin = str(BIN_DIR / "muse-tmux.py")
|
||||||
if not t_args:
|
if not t_args:
|
||||||
t_args = ["list"]
|
t_args = ["list"]
|
||||||
cmd = [sys.executable, tmux_bin] + t_args
|
cmd = [sys.executable, tmux_bin] + t_args
|
||||||
@@ -5050,6 +5341,27 @@ def build_parser():
|
|||||||
|
|
||||||
p_mc_rec = mc_sub.add_parser("reconcile", parents=[common], help="Enforce desired state now (start missing / stop excess)")
|
p_mc_rec = mc_sub.add_parser("reconcile", parents=[common], help="Enforce desired state now (start missing / stop excess)")
|
||||||
|
|
||||||
|
# Domain: RUNTIME
|
||||||
|
p_rt = subparsers.add_parser("runtime", parents=[common], help="Muse CLI tmux runtimes: list states, send input, launch auto-approved")
|
||||||
|
rt_sub = p_rt.add_subparsers(dest="rt_action")
|
||||||
|
|
||||||
|
p_rt_list = rt_sub.add_parser("list", parents=[common], help="List panes with runtime state + approval posture (default)")
|
||||||
|
p_rt_list.add_argument("--socket", default=None, help="Only this tmux socket")
|
||||||
|
p_rt_list.add_argument("--muse-only", action="store_true", help="Only Muse CLI panes")
|
||||||
|
|
||||||
|
p_rt_send = rt_sub.add_parser("send", parents=[common], help="Send keys to a pane (reports pre-send state)")
|
||||||
|
p_rt_send.add_argument("--socket", default=None, help="Tmux socket (default: /tmp/tmux-1000/default)")
|
||||||
|
p_rt_send.add_argument("pane", help="Pane id (e.g. %%37)")
|
||||||
|
p_rt_send.add_argument("keys", help="Keys / text to send")
|
||||||
|
p_rt_send.add_argument("--no-enter", action="store_true", help="Do not send Enter after keys")
|
||||||
|
|
||||||
|
p_rt_launch = rt_sub.add_parser("launch", parents=[common], help="Launch a Muse session with auto-approve injected")
|
||||||
|
p_rt_launch.add_argument("--socket", default=None, help="Tmux socket (default: /tmp/tmux-1000/default)")
|
||||||
|
p_rt_launch.add_argument("--session", required=True, help="New tmux session name")
|
||||||
|
p_rt_launch.add_argument("--window", "-w", default=None, help="Initial window name")
|
||||||
|
p_rt_launch.add_argument("--dry-run", action="store_true", help="Print the launch plan without creating")
|
||||||
|
p_rt_launch.add_argument("muse_args", nargs=argparse.REMAINDER, default=[], help="Extra muse args after --")
|
||||||
|
|
||||||
# Domain: INVITE
|
# Domain: INVITE
|
||||||
p_invite = subparsers.add_parser("invite", parents=[common], help="Muse.ai invite codes: find per-agent codes and redeem")
|
p_invite = subparsers.add_parser("invite", parents=[common], help="Muse.ai invite codes: find per-agent codes and redeem")
|
||||||
p_invite.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node (status)")
|
p_invite.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node (status)")
|
||||||
@@ -5070,10 +5382,15 @@ def build_parser():
|
|||||||
p_inv_redeem = inv_sub.add_parser("redeem", parents=[common], help="Redeem an invite code on a node")
|
p_inv_redeem = inv_sub.add_parser("redeem", parents=[common], help="Redeem an invite code on a node")
|
||||||
p_inv_redeem.add_argument("node", choices=VALID_NODES, help="Target node")
|
p_inv_redeem.add_argument("node", choices=VALID_NODES, help="Target node")
|
||||||
p_inv_redeem.add_argument("code", help="6-char invite code")
|
p_inv_redeem.add_argument("code", help="6-char invite code")
|
||||||
|
p_inv_redeem.add_argument("--method", choices=["auto", "dom", "api"], default="auto", help="Redemption method (default: auto)")
|
||||||
|
p_inv_redeem.add_argument("--notify", default=None, help="Sidechat to notify on loopback")
|
||||||
|
p_inv_redeem.add_argument("--notify-agent", default=None, help="Agent to notify on loopback")
|
||||||
|
|
||||||
p_inv_salvage = inv_sub.add_parser("salvage", parents=[common], help="Salvage an out-of-tokens agent (defaults to 646)")
|
p_inv_salvage = inv_sub.add_parser("salvage", parents=[common], help="Salvage an out-of-tokens agent (defaults to 646)")
|
||||||
p_inv_salvage.add_argument("node", nargs="?", default="646", choices=VALID_NODES, help="Blocked agent node (default: 646)")
|
p_inv_salvage.add_argument("node", nargs="?", default="646", choices=VALID_NODES, help="Blocked agent node (default: 646)")
|
||||||
p_inv_salvage.add_argument("--helper", choices=VALID_NODES, help="Specific helper agent to redeem code")
|
p_inv_salvage.add_argument("--helper", choices=VALID_NODES, help="Specific helper agent to redeem code")
|
||||||
|
p_inv_salvage.add_argument("--notify", default=None, help="Sidechat to notify on loopback")
|
||||||
|
p_inv_salvage.add_argument("--notify-agent", default=None, help="Agent to notify on loopback")
|
||||||
|
|
||||||
# Domain: USAGE
|
# Domain: USAGE
|
||||||
p_usage = subparsers.add_parser("usage", parents=[common], help="Muse.ai usage limits per agent")
|
p_usage = subparsers.add_parser("usage", parents=[common], help="Muse.ai usage limits per agent")
|
||||||
@@ -5112,8 +5429,28 @@ def build_parser():
|
|||||||
p_onb_wo.add_argument("--target", default="646 tasks", help="Target sidechat (default: 646 tasks)")
|
p_onb_wo.add_argument("--target", default="646 tasks", help="Target sidechat (default: 646 tasks)")
|
||||||
|
|
||||||
onboard_sub.add_parser("feed-matrix", parents=[common], help="Display all agents ranked by feeding weight, job volume, role, and work done over time")
|
onboard_sub.add_parser("feed-matrix", parents=[common], help="Display all agents ranked by feeding weight, job volume, role, and work done over time")
|
||||||
|
onboard_sub.add_parser("connects", parents=[common], help="Inventory of all active fleet nodes & client onboard connects")
|
||||||
|
|
||||||
|
# Domain: KPI
|
||||||
|
p_kpi = subparsers.add_parser("kpi", parents=[common], help="Fleet KPI, spend monitoring & runtime preservation")
|
||||||
|
p_kpi.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node")
|
||||||
|
kpi_sub = p_kpi.add_subparsers(dest="kpi_action")
|
||||||
|
|
||||||
|
p_kpi_status = kpi_sub.add_parser("status", parents=[common], help="Show fleet KPI dashboard (default)")
|
||||||
|
p_kpi_status.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node")
|
||||||
|
|
||||||
|
p_kpi_report = kpi_sub.add_parser("report", parents=[common], help="Detailed KPI & preservation report for an agent")
|
||||||
|
p_kpi_report.add_argument("node", choices=VALID_NODES, help="Target node")
|
||||||
|
|
||||||
|
p_kpi_routes = kpi_sub.add_parser("routes", parents=[common], help="Check route and CDP health across agents")
|
||||||
|
|
||||||
|
p_kpi_spawn = kpi_sub.add_parser("spawn-worker", parents=[common], help="Spawn background tmux worker to preserve runtime")
|
||||||
|
p_kpi_spawn.add_argument("node", choices=VALID_NODES, help="Target agent")
|
||||||
|
p_kpi_spawn.add_argument("session", help="Session label")
|
||||||
|
p_kpi_spawn.add_argument("worker_command", help="Command to run in background")
|
||||||
|
|
||||||
# Domain: CHROMEBOX
|
# Domain: CHROMEBOX
|
||||||
|
|
||||||
p_chromebox = subparsers.add_parser("chromebox", parents=[common], help="Agent browser settings-menu toggles (chromebox RPA)")
|
p_chromebox = subparsers.add_parser("chromebox", parents=[common], help="Agent browser settings-menu toggles (chromebox RPA)")
|
||||||
chrome_sub = p_chromebox.add_subparsers(dest="chrome_action")
|
chrome_sub = p_chromebox.add_subparsers(dest="chrome_action")
|
||||||
p_chrome_perm = chrome_sub.add_parser("permissions", parents=[common], help="Settings menu toggles (permissions + related tabs)")
|
p_chrome_perm = chrome_sub.add_parser("permissions", parents=[common], help="Settings menu toggles (permissions + related tabs)")
|
||||||
@@ -5617,13 +5954,32 @@ def main():
|
|||||||
if len(sys.argv) > 1:
|
if len(sys.argv) > 1:
|
||||||
if sys.argv[1] == "tui":
|
if sys.argv[1] == "tui":
|
||||||
tui_args = sys.argv[2:]
|
tui_args = sys.argv[2:]
|
||||||
cmd = [sys.executable, str(BIN_DIR / "muse-tui.py"), "--mode", "box"] + tui_args
|
if tui_args and tui_args[0] in ("onboard", "tmux", "connects", "approvals"):
|
||||||
|
cmd = [sys.executable, str(BIN_DIR / "box-onboard-tui.py")] + tui_args[1:]
|
||||||
|
else:
|
||||||
|
cmd = [sys.executable, str(BIN_DIR / "muse-tui.py"), "--mode", "box"] + tui_args
|
||||||
|
res = subprocess.run(cmd)
|
||||||
|
sys.exit(res.returncode)
|
||||||
|
elif sys.argv[1] in ("onboard-tui", "dev-tui"):
|
||||||
|
cmd = [sys.executable, str(BIN_DIR / "box-onboard-tui.py")] + sys.argv[2:]
|
||||||
res = subprocess.run(cmd)
|
res = subprocess.run(cmd)
|
||||||
sys.exit(res.returncode)
|
sys.exit(res.returncode)
|
||||||
elif sys.argv[1] == "tmux":
|
elif sys.argv[1] == "tmux":
|
||||||
|
if len(sys.argv) > 2 and sys.argv[2] in ("tally", "auto", "watch", "once", "match", "rules", "status"):
|
||||||
|
if sys.argv[2] == "auto":
|
||||||
|
t_sub = sys.argv[3:] or ["status"]
|
||||||
|
else:
|
||||||
|
t_sub = sys.argv[2:]
|
||||||
|
cmd = [sys.executable, str(BIN_DIR / "tmux_auto_approver.py")] + t_sub
|
||||||
|
res = subprocess.run(cmd)
|
||||||
|
sys.exit(res.returncode)
|
||||||
cmd = [sys.executable, str(BIN_DIR / "muse-tmux.py")] + (sys.argv[2:] or ["list"])
|
cmd = [sys.executable, str(BIN_DIR / "muse-tmux.py")] + (sys.argv[2:] or ["list"])
|
||||||
res = subprocess.run(cmd)
|
res = subprocess.run(cmd)
|
||||||
sys.exit(res.returncode)
|
sys.exit(res.returncode)
|
||||||
|
elif sys.argv[1] in ("tmux-auto", "auto-dev"):
|
||||||
|
cmd = [sys.executable, str(BIN_DIR / "tmux_auto_approver.py")] + sys.argv[2:]
|
||||||
|
res = subprocess.run(cmd)
|
||||||
|
sys.exit(res.returncode)
|
||||||
elif sys.argv[1] in ("docs", "doc"):
|
elif sys.argv[1] in ("docs", "doc"):
|
||||||
cmd = [sys.executable, str(BIN_DIR / "docs-lookup.py")] + sys.argv[2:]
|
cmd = [sys.executable, str(BIN_DIR / "docs-lookup.py")] + sys.argv[2:]
|
||||||
res = subprocess.run(cmd)
|
res = subprocess.run(cmd)
|
||||||
@@ -5900,6 +6256,8 @@ def main():
|
|||||||
cmd_approvals(args)
|
cmd_approvals(args)
|
||||||
elif args.domain == "muse-choices":
|
elif args.domain == "muse-choices":
|
||||||
cmd_muse_choices(args)
|
cmd_muse_choices(args)
|
||||||
|
elif args.domain == "runtime":
|
||||||
|
cmd_runtime(args)
|
||||||
elif args.domain == "invite":
|
elif args.domain == "invite":
|
||||||
cmd_invite(args)
|
cmd_invite(args)
|
||||||
elif args.domain == "usage":
|
elif args.domain == "usage":
|
||||||
@@ -5908,6 +6266,8 @@ def main():
|
|||||||
cmd_settings(args)
|
cmd_settings(args)
|
||||||
elif args.domain == "onboard":
|
elif args.domain == "onboard":
|
||||||
cmd_onboard(args)
|
cmd_onboard(args)
|
||||||
|
elif args.domain == "kpi":
|
||||||
|
cmd_kpi(args)
|
||||||
elif args.domain == "chromebox":
|
elif args.domain == "chromebox":
|
||||||
cmd_chromebox(args)
|
cmd_chromebox(args)
|
||||||
elif args.domain in ("deploy", "subagent"):
|
elif args.domain in ("deploy", "subagent"):
|
||||||
|
|||||||
+19
-4
@@ -33,15 +33,30 @@ the agent browsers, exposed as `box invite` / `box usage`.
|
|||||||
- When account B redeems account A's code: BOTH accounts immediately
|
- When account B redeems account A's code: BOTH accounts immediately
|
||||||
receive 1 billion Muse tokens.
|
receive 1 billion Muse tokens.
|
||||||
|
|
||||||
## UI Quirks & Binary Attribution
|
## UI Quirks & Missing Field Resiliency
|
||||||
|
|
||||||
- **Redeem field disappearance**: In Settings > General, the `Redeem invite code`
|
- **Redeem field disappearance**: In Settings > General, the `Redeem invite code`
|
||||||
item is only present for fresh accounts (< 48 hours, has not redeemed). Once
|
item is only present for fresh accounts (< 48 hours, has not redeemed). Once
|
||||||
redeemed, the field disappears completely and is replaced by the `Additional tokens`
|
redeemed, the field disappears completely and is replaced by the `Additional tokens`
|
||||||
ticker bar (`Never expires / X% used`).
|
ticker bar (`Never expires / X% used`).
|
||||||
- **Asynchronous DOM Loading**: In the React/Radix UI tree, usage statistics and the
|
- **Missing Entrypoint Recovery & API Fallback**:
|
||||||
redeem button render asynchronously after the dialog opens. RPA drivers must poll
|
If the `Redeem invite code` row is absent (or sub-dialog input box `NativeAuthCodeBoxes`
|
||||||
briefly for usage text and progress bars rather than evaluating on the first tick.
|
fails to mount):
|
||||||
|
1. Box polls up to 2.5s for React hydration.
|
||||||
|
2. If still unrendered, Box automatically falls back to in-page API redemption (`POST /api/hatch/invite-code/redeem`).
|
||||||
|
3. If API redemption succeeds, Box returns success transparently.
|
||||||
|
4. If API also fails, Box records the exact server reason (`already_redeemed`, `window_expired`, etc.).
|
||||||
|
5. Box cleanly dismisses all open modals via Escape to avoid orphaned UI state.
|
||||||
|
- **Asynchronous DOM Loading & Unloaded Stats**:
|
||||||
|
In the React/Radix UI tree, usage statistics render asynchronously after the dialog opens.
|
||||||
|
If network requests stall or quota bars fail to render, `read_usage()` and `box usage` gracefully
|
||||||
|
tag the result as `UNLOADED` (`stats_loaded: false`) rather than throwing uncaught `RuntimeError`s.
|
||||||
|
- **DM Loopback & Transparent Agent Feedback**:
|
||||||
|
To keep agents in the loop without causing confusion:
|
||||||
|
- Any missing expected field or failed redemption can trigger an automated loopback DM
|
||||||
|
via `box invite redeem <node> <code> --notify <sidechat> [--notify-agent <agent>]`.
|
||||||
|
- Onboard and salvage pipelines automatically post diagnostic notices (`[ONBOARD-SALVAGE-LOOPBACK]` /
|
||||||
|
`[BOX-INVITE-LOOPBACK]`) to the agent's task sidechat, providing clear, actionable root cause.
|
||||||
|
|
||||||
## Onboarding & Feeding Pipeline (`box onboard`)
|
## Onboarding & Feeding Pipeline (`box onboard`)
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent
|
|||||||
sys.path.insert(0, str(REPO_ROOT / "bin"))
|
sys.path.insert(0, str(REPO_ROOT / "bin"))
|
||||||
|
|
||||||
import invite
|
import invite
|
||||||
|
import invite_handler
|
||||||
from invite_handler import InviteCodeInfo, InviteHandler, RedemptionResult, salvage_blocked_node
|
from invite_handler import InviteCodeInfo, InviteHandler, RedemptionResult, salvage_blocked_node
|
||||||
from settings_rpa import NodeUsage, SettingsRPA
|
from settings_rpa import NodeUsage, SettingsRPA
|
||||||
|
|
||||||
@@ -176,7 +177,136 @@ class TestInviteHandlerMocked(unittest.TestCase):
|
|||||||
self.assertFalse(res["success"])
|
self.assertFalse(res["success"])
|
||||||
self.assertEqual(res["invite_code"], "REDCJ7")
|
self.assertEqual(res["invite_code"], "REDCJ7")
|
||||||
self.assertIn("An onboarding agent or fresh client profile must redeem this code", res["error"])
|
self.assertIn("An onboarding agent or fresh client profile must redeem this code", res["error"])
|
||||||
|
self.assertTrue(res.get("field_missing"))
|
||||||
|
|
||||||
|
def test_salvage_blocked_node_with_loopback_dm(self):
|
||||||
|
with patch.object(InviteHandler, "find_code") as mock_find:
|
||||||
|
mock_find.return_value = InviteCodeInfo(
|
||||||
|
node="646",
|
||||||
|
code="REDCJ7",
|
||||||
|
uses_remaining=30,
|
||||||
|
use_count=0,
|
||||||
|
max_uses=30,
|
||||||
|
has_redeemed=True,
|
||||||
|
invite_state="ELIGIBLE",
|
||||||
|
reward=None,
|
||||||
|
method_used="api",
|
||||||
|
)
|
||||||
|
with patch("invite_handler.send_loopback_notice", return_value=True) as mock_notice:
|
||||||
|
res = salvage_blocked_node("646", notify_target="heartbeat-opm", notify_agent="opm")
|
||||||
|
self.assertFalse(res["success"])
|
||||||
|
self.assertTrue(res.get("field_missing"))
|
||||||
|
self.assertTrue(res.get("loopback_notified"))
|
||||||
|
mock_notice.assert_called_once()
|
||||||
|
call_args = mock_notice.call_args[0]
|
||||||
|
self.assertEqual(call_args[0], "opm")
|
||||||
|
self.assertEqual(call_args[1], "heartbeat-opm")
|
||||||
|
self.assertIn("REDCJ7", call_args[2])
|
||||||
|
|
||||||
|
@patch("invite_handler.SettingsRPA")
|
||||||
|
def test_redeem_code_dom_missing_row_api_fallback_success(self, mock_rpa_cls):
|
||||||
|
mock_rpa = MagicMock()
|
||||||
|
mock_rpa.open_settings_dialog.return_value = True
|
||||||
|
mock_rpa.select_tab.return_value = True
|
||||||
|
mock_rpa_cls.return_value.__enter__.return_value = mock_rpa
|
||||||
|
|
||||||
|
h = InviteHandler("dev")
|
||||||
|
h.ws = MagicMock()
|
||||||
|
with patch.object(h, "connect"), patch("time.sleep", return_value=None):
|
||||||
|
with patch("invite_handler.cdp_evaluate", return_value={"found": False, "text": "General"}):
|
||||||
|
with patch.object(h, "redeem_code_api") as mock_api:
|
||||||
|
mock_api.return_value = RedemptionResult(
|
||||||
|
target_node="dev",
|
||||||
|
code="REDCJ7",
|
||||||
|
success=True,
|
||||||
|
status="redeemed",
|
||||||
|
reason=None,
|
||||||
|
detail="API success",
|
||||||
|
method_used="api",
|
||||||
|
)
|
||||||
|
res = h.redeem_code_dom("REDCJ7")
|
||||||
|
self.assertTrue(res.success)
|
||||||
|
self.assertEqual(res.status, "redeemed")
|
||||||
|
self.assertIn("Redeem field was missing in Settings DOM; redeemed successfully via API fallback!", res.detail)
|
||||||
|
mock_rpa.close_settings_dialog.assert_called_once()
|
||||||
|
|
||||||
|
@patch("invite_handler.SettingsRPA")
|
||||||
|
def test_redeem_code_dom_missing_row_already_redeemed(self, mock_rpa_cls):
|
||||||
|
mock_rpa = MagicMock()
|
||||||
|
mock_rpa.open_settings_dialog.return_value = True
|
||||||
|
mock_rpa.select_tab.return_value = True
|
||||||
|
mock_rpa_cls.return_value.__enter__.return_value = mock_rpa
|
||||||
|
|
||||||
|
h = InviteHandler("646")
|
||||||
|
h.ws = MagicMock()
|
||||||
|
with patch.object(h, "connect"), patch("time.sleep", return_value=None):
|
||||||
|
with patch("invite_handler.cdp_evaluate", return_value={"found": False, "has_additional": True, "text": "Additional tokens"}):
|
||||||
|
with patch.object(h, "redeem_code_api") as mock_api:
|
||||||
|
mock_api.return_value = RedemptionResult(
|
||||||
|
target_node="646",
|
||||||
|
code="REDCJ7",
|
||||||
|
success=False,
|
||||||
|
status="already_redeemed",
|
||||||
|
reason="already_redeemed",
|
||||||
|
detail="Already redeemed",
|
||||||
|
method_used="api",
|
||||||
|
)
|
||||||
|
res = h.redeem_code_dom("REDCJ7")
|
||||||
|
self.assertFalse(res.success)
|
||||||
|
self.assertEqual(res.status, "already_redeemed")
|
||||||
|
self.assertTrue(res.field_missing)
|
||||||
|
self.assertIn("already redeemed an invite code", res.detail)
|
||||||
|
|
||||||
|
@patch("invite_handler.SettingsRPA")
|
||||||
|
def test_redeem_code_dom_missing_input_box_with_loopback(self, mock_rpa_cls):
|
||||||
|
mock_rpa = MagicMock()
|
||||||
|
mock_rpa.open_settings_dialog.return_value = True
|
||||||
|
mock_rpa.select_tab.return_value = True
|
||||||
|
mock_rpa_cls.return_value.__enter__.return_value = mock_rpa
|
||||||
|
|
||||||
|
h = InviteHandler("pip")
|
||||||
|
h.ws = MagicMock()
|
||||||
|
|
||||||
|
def mock_eval(ws, js, *args, **kwargs):
|
||||||
|
if "Redeem invite code" in js:
|
||||||
|
return {"found": True}
|
||||||
|
if "found_input" in js:
|
||||||
|
return {"found_input": False}
|
||||||
|
return True
|
||||||
|
|
||||||
|
with patch.object(h, "connect"), patch("time.sleep", return_value=None):
|
||||||
|
with patch("invite_handler.cdp_evaluate", side_effect=mock_eval), patch("invite_handler.cdp_send_escape"):
|
||||||
|
with patch.object(h, "redeem_code_api") as mock_api:
|
||||||
|
mock_api.return_value = RedemptionResult(
|
||||||
|
target_node="pip",
|
||||||
|
code="TEST01",
|
||||||
|
success=False,
|
||||||
|
status="dom_input_missing",
|
||||||
|
reason="input_field_missing",
|
||||||
|
detail="API fallback also failed",
|
||||||
|
method_used="api",
|
||||||
|
)
|
||||||
|
with patch("invite_handler.send_loopback_notice", return_value=True) as mock_notice:
|
||||||
|
res = h.redeem_code_dom("TEST01", notify_target="pip tasks", notify_agent="pip")
|
||||||
|
self.assertFalse(res.success)
|
||||||
|
self.assertTrue(res.field_missing)
|
||||||
|
self.assertEqual(res.reason, "input_field_missing")
|
||||||
|
self.assertTrue(res.loopback_notified)
|
||||||
|
mock_notice.assert_called_once()
|
||||||
|
self.assertIn("pip tasks", res.loopback_detail)
|
||||||
|
|
||||||
|
@patch("subprocess.run")
|
||||||
|
def test_send_loopback_notice(self, mock_subproc):
|
||||||
|
mock_subproc.return_value = MagicMock(returncode=0)
|
||||||
|
ok = invite_handler.send_loopback_notice("opm", "646 tasks", "Test alert")
|
||||||
|
self.assertTrue(ok)
|
||||||
|
mock_subproc.assert_called_once()
|
||||||
|
cmd = mock_subproc.call_args[0][0]
|
||||||
|
self.assertIn("send", cmd)
|
||||||
|
self.assertIn("646 tasks", cmd)
|
||||||
|
self.assertIn("Test alert", cmd)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,164 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""test_kpi.py — Unit tests for NetVM Fleet KPI, spend monitor and runtime preservation engine."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
sys.path.insert(0, str(REPO_ROOT / "bin"))
|
||||||
|
|
||||||
|
import kpi
|
||||||
|
from kpi import (
|
||||||
|
calculate_efficiency,
|
||||||
|
check_node_routes,
|
||||||
|
generate_preservation_advisory,
|
||||||
|
get_agent_dm_metrics,
|
||||||
|
get_agent_kpi,
|
||||||
|
get_live_advisory_block,
|
||||||
|
spawn_tmux_worker,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestKPIMetrics(unittest.TestCase):
|
||||||
|
def test_calculate_efficiency_high(self):
|
||||||
|
# High work units relative to burn
|
||||||
|
idx, rating = calculate_efficiency(
|
||||||
|
jobs_done=5,
|
||||||
|
subagents_active=2,
|
||||||
|
tmux_workers=2,
|
||||||
|
calls_verified=20,
|
||||||
|
weekly_used_pct=30,
|
||||||
|
uptime_hours=10.0,
|
||||||
|
)
|
||||||
|
self.assertGreaterEqual(idx, 3.0)
|
||||||
|
self.assertEqual(rating, "HIGH_EFFICIENCY")
|
||||||
|
|
||||||
|
def test_calculate_efficiency_vanity_idle(self):
|
||||||
|
# Long uptime but zero workers and zero jobs
|
||||||
|
idx, rating = calculate_efficiency(
|
||||||
|
jobs_done=0,
|
||||||
|
subagents_active=0,
|
||||||
|
tmux_workers=0,
|
||||||
|
calls_verified=0,
|
||||||
|
weekly_used_pct=10,
|
||||||
|
uptime_hours=5.0,
|
||||||
|
)
|
||||||
|
self.assertEqual(rating, "VANITY_IDLE")
|
||||||
|
|
||||||
|
def test_generate_preservation_advisory_critical_quota(self):
|
||||||
|
adv = generate_preservation_advisory(
|
||||||
|
node="646",
|
||||||
|
weekly_used_pct=100,
|
||||||
|
extra_tokens_remaining="0 tokens left",
|
||||||
|
subagents_active=0,
|
||||||
|
tmux_workers=0,
|
||||||
|
rating="VANITY_IDLE",
|
||||||
|
)
|
||||||
|
self.assertIn("CRITICAL: Quota exhausted", adv)
|
||||||
|
self.assertIn("box onboard start", adv)
|
||||||
|
|
||||||
|
def test_generate_preservation_advisory_high_quota_idle(self):
|
||||||
|
adv = generate_preservation_advisory(
|
||||||
|
node="pip",
|
||||||
|
weekly_used_pct=75,
|
||||||
|
extra_tokens_remaining="500M tokens left",
|
||||||
|
subagents_active=0,
|
||||||
|
tmux_workers=0,
|
||||||
|
rating="VANITY_IDLE",
|
||||||
|
)
|
||||||
|
self.assertIn("Cease prose chatter", adv)
|
||||||
|
self.assertIn("Spawn subagents", adv)
|
||||||
|
|
||||||
|
@patch("kpi.Path.exists")
|
||||||
|
@patch("approvals.get_node_pages")
|
||||||
|
def test_check_node_routes_online(self, mock_pages, mock_exists):
|
||||||
|
mock_exists.return_value = True
|
||||||
|
mock_pages.return_value = [{"title": "Chat"}]
|
||||||
|
st = check_node_routes("646")
|
||||||
|
self.assertEqual(st, "ONLINE")
|
||||||
|
|
||||||
|
@patch("kpi.Path.exists")
|
||||||
|
def test_check_node_routes_no_netns(self, mock_exists):
|
||||||
|
mock_exists.return_value = False
|
||||||
|
st = check_node_routes("646")
|
||||||
|
self.assertEqual(st, "NO_NETNS")
|
||||||
|
|
||||||
|
@patch("kpi.get_agent_dm_metrics")
|
||||||
|
@patch("kpi.get_agent_job_metrics")
|
||||||
|
@patch("kpi.get_agent_subagent_count")
|
||||||
|
@patch("kpi.get_agent_tmux_workers")
|
||||||
|
@patch("kpi.get_agent_uptime_hours")
|
||||||
|
@patch("kpi.check_node_routes")
|
||||||
|
def test_get_agent_kpi(
|
||||||
|
self,
|
||||||
|
mock_routes,
|
||||||
|
mock_uptime,
|
||||||
|
mock_tmux,
|
||||||
|
mock_sub,
|
||||||
|
mock_jobs,
|
||||||
|
mock_dm,
|
||||||
|
):
|
||||||
|
mock_dm.return_value = {"sent": 50, "verified": 40, "total_events": 100}
|
||||||
|
mock_jobs.return_value = {"assigned": 10, "completed": 4}
|
||||||
|
mock_sub.return_value = 2
|
||||||
|
mock_tmux.return_value = ["worker-1"]
|
||||||
|
mock_uptime.return_value = 8.5
|
||||||
|
mock_routes.return_value = "ONLINE"
|
||||||
|
|
||||||
|
usage_cache = {
|
||||||
|
"weekly_used_pct": 25,
|
||||||
|
"extra_tokens_remaining": "900M tokens left",
|
||||||
|
"is_blocked": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
res = get_agent_kpi("dev", usage_cache=usage_cache)
|
||||||
|
self.assertEqual(res.node, "dev")
|
||||||
|
self.assertEqual(res.weekly_used_pct, 25)
|
||||||
|
self.assertEqual(res.calls_sent, 50)
|
||||||
|
self.assertEqual(res.calls_verified, 40)
|
||||||
|
self.assertEqual(res.jobs_completed, 4)
|
||||||
|
self.assertEqual(res.subagents_active, 2)
|
||||||
|
self.assertEqual(res.tmux_workers_active, 1)
|
||||||
|
self.assertEqual(res.route_status, "ONLINE")
|
||||||
|
|
||||||
|
@patch("kpi.get_agent_kpi")
|
||||||
|
def test_get_live_advisory_block(self, mock_kpi):
|
||||||
|
mock_kpi.return_value = MagicMock(
|
||||||
|
node="646",
|
||||||
|
weekly_used_pct=30,
|
||||||
|
extra_tokens_remaining="1B tokens left",
|
||||||
|
uptime_hours=12.0,
|
||||||
|
jobs_completed=5,
|
||||||
|
subagents_active=2,
|
||||||
|
tmux_workers_active=1,
|
||||||
|
efficiency_rating="HIGH_EFFICIENCY",
|
||||||
|
efficiency_index=4.5,
|
||||||
|
route_status="ONLINE",
|
||||||
|
preservation_advisory="Runtime healthy.",
|
||||||
|
)
|
||||||
|
block = get_live_advisory_block("646")
|
||||||
|
self.assertIn("BOX PERFORMANCE & RUNTIME ADVISORY", block)
|
||||||
|
self.assertIn("AGENT: @646", block)
|
||||||
|
self.assertIn("30% weekly limit used", block)
|
||||||
|
self.assertIn("HIGH_EFFICIENCY", block)
|
||||||
|
self.assertIn("Runtime healthy.", block)
|
||||||
|
|
||||||
|
@patch("subprocess.run")
|
||||||
|
@patch("subagent_tracker.register_session")
|
||||||
|
def test_spawn_tmux_worker(self, mock_register, mock_subproc):
|
||||||
|
mock_subproc.return_value = MagicMock(returncode=0)
|
||||||
|
res = spawn_tmux_worker("dev", "audit-sub", "python3 audit.py")
|
||||||
|
self.assertTrue(res["ok"])
|
||||||
|
self.assertEqual(res["node"], "dev")
|
||||||
|
self.assertEqual(res["session"], "dev-audit-sub")
|
||||||
|
mock_register.assert_called_once()
|
||||||
|
call_kwargs = mock_register.call_args[1]
|
||||||
|
self.assertEqual(call_kwargs["parent"], "dev")
|
||||||
|
self.assertIn("dev-audit-sub", call_kwargs["session_id"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -83,6 +83,40 @@ class TestOnboardPipeline(unittest.TestCase):
|
|||||||
self.assertEqual(res["beneficiary_node"], "646")
|
self.assertEqual(res["beneficiary_node"], "646")
|
||||||
mock_redeem.assert_called_once_with("testnode", "REDCJ7", timeout=20.0)
|
mock_redeem.assert_called_once_with("testnode", "REDCJ7", timeout=20.0)
|
||||||
|
|
||||||
|
@patch("invite.redeem_invite")
|
||||||
|
@patch("subprocess.run")
|
||||||
|
@patch("invite_handler.send_loopback_notice")
|
||||||
|
def test_finish_onboarding_redemption_failure_with_loopback(self, mock_notice, mock_subproc, mock_redeem):
|
||||||
|
mock_redeem.return_value = {"ok": False, "reason": "already_redeemed", "detail": "Already redeemed"}
|
||||||
|
mock_notice.return_value = True
|
||||||
|
state = OnboardState(
|
||||||
|
node="testnode",
|
||||||
|
email="client@test.com",
|
||||||
|
beneficiary_node="646",
|
||||||
|
invite_code="REDCJ7",
|
||||||
|
stage="auth_active",
|
||||||
|
)
|
||||||
|
|
||||||
|
res = finish_onboarding_redemption(state)
|
||||||
|
self.assertTrue(res["ok"])
|
||||||
|
self.assertEqual(res["status"], "auth_active_redemption_warning")
|
||||||
|
self.assertIn("Redemption failed: already_redeemed", state.detail)
|
||||||
|
mock_notice.assert_called_once()
|
||||||
|
kwargs = mock_notice.call_args.kwargs
|
||||||
|
self.assertEqual(kwargs.get("recipient"), "646")
|
||||||
|
self.assertEqual(kwargs.get("target"), "646 tasks")
|
||||||
|
self.assertIn("already_redeemed", kwargs.get("message", ""))
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_all_connects_fast(self):
|
||||||
|
from onboard_pipeline import get_all_connects
|
||||||
|
connects = get_all_connects(fast=True)
|
||||||
|
self.assertIsInstance(connects, list)
|
||||||
|
self.assertGreaterEqual(len(connects), 6)
|
||||||
|
nodes = {c["node"] for c in connects}
|
||||||
|
for expected in ("muse", "pip", "646", "opm", "dev", "def"):
|
||||||
|
self.assertIn(expected, nodes)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -104,7 +104,32 @@ class TestSettingsRPAPrimitives(unittest.TestCase):
|
|||||||
self.assertEqual(usage.extra_percent_used, 0)
|
self.assertEqual(usage.extra_percent_used, 0)
|
||||||
self.assertFalse(usage.is_blocked)
|
self.assertFalse(usage.is_blocked)
|
||||||
self.assertFalse(usage.has_redeemed)
|
self.assertFalse(usage.has_redeemed)
|
||||||
|
self.assertTrue(usage.stats_loaded)
|
||||||
|
|
||||||
|
@patch("settings_rpa.get_cdp_ws")
|
||||||
|
@patch("settings_rpa.cdp_evaluate")
|
||||||
|
def test_read_usage_stats_did_not_render(self, mock_eval, mock_ws):
|
||||||
|
mock_ws.return_value = (MagicMock(), MagicMock())
|
||||||
|
|
||||||
|
def eval_side_effect(ws, js, *args, **kwargs):
|
||||||
|
if "role=\"dialog\"" in js and "Weekly limit" in js:
|
||||||
|
return None
|
||||||
|
return True
|
||||||
|
|
||||||
|
mock_eval.side_effect = eval_side_effect
|
||||||
|
|
||||||
|
with SettingsRPA("646", timeout=0.5) as rpa:
|
||||||
|
# Shorten deadline by patching time.time or passing small timeout
|
||||||
|
with patch("time.sleep", return_value=None):
|
||||||
|
usage = rpa.read_usage(keep_dialog_open=True)
|
||||||
|
self.assertEqual(usage.node, "646")
|
||||||
|
self.assertFalse(usage.stats_loaded)
|
||||||
|
self.assertFalse(usage.is_blocked)
|
||||||
|
self.assertIn("Unavailable", usage.weekly_reset_text)
|
||||||
|
self.assertEqual(usage.extra_tokens_remaining, "Unavailable")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user