docs(netvm): document watchdog, swarm-worker, and alert relay in README; track approvals CLI

This commit is contained in:
operator
2026-10-05 17:42:01 +00:00
parent 4f4b8768b6
commit d72b63bfd1
6 changed files with 945 additions and 4 deletions
+3
View File
@@ -146,6 +146,9 @@ veth IPs aren't routable off the host and Warp forwards no inbound traffic.
- `bin/muse_hybrid.py` — programmatic hybrid bridge combining fast gateway calls with CDP fallbacks. - `bin/muse_hybrid.py` — programmatic hybrid bridge combining fast gateway calls with CDP fallbacks.
- `bin/muse-tmux.py` — shared tmux socket manager (`/tmp/tmux-muse.sock`) for agent background execution, pipe-pane logging, and 2h session pruning. - `bin/muse-tmux.py` — shared tmux socket manager (`/tmp/tmux-muse.sock`) for agent background execution, pipe-pane logging, and 2h session pruning.
- `bin/agent_md.py` — CLI & library for auditing, reading, writing, and synchronizing agent `.md` drive files (`SOUL.md`, `PROACTIVE_PREFERENCES.md`, `HEARTBEAT.md`, etc.) across containers via Hatch WebSocket RPC. - `bin/agent_md.py` — CLI & library for auditing, reading, writing, and synchronizing agent `.md` drive files (`SOUL.md`, `PROACTIVE_PREFERENCES.md`, `HEARTBEAT.md`, etc.) across containers via Hatch WebSocket RPC.
- `bin/agent-drive-watchdog.py` — background drive watchdog and auto-healing daemon (every 10m via `agent-drive-watchdog.timer`).
- `bin/swarm_worker/` & `bin/swarm-worker-supervise.sh` — supervised autonomous swarm worker daemon executing queued tasks in a hard sandbox.
- `bin/fleet-alert-relay.sh` — idempotent alert relay with 3-gate deduplication (watermark + 10m TTL hash + receipt verification) posting critical conditions to `#lobby`.
- `shared/operators/` — canonical operator drive markdown templates ensuring agents maintain autonomous loops, active supervision, and self-healing reflexes. - `shared/operators/` — canonical operator drive markdown templates ensuring agents maintain autonomous loops, active supervision, and self-healing reflexes.
- docs/OPERATOR-DRIVE-RUNBOOK.md — operator runbook for auditing and modifying agent `.md` files via Hatch WebSocket RPC and SSH reverse tunnels. - docs/OPERATOR-DRIVE-RUNBOOK.md — operator runbook for auditing and modifying agent `.md` files via Hatch WebSocket RPC and SSH reverse tunnels.
- docs/HYBRID-GATEWAY-ADAPTATION.md — architectural guide on the muse-cli fast gateway adaptation and per-node egress isolation. - docs/HYBRID-GATEWAY-ADAPTATION.md — architectural guide on the muse-cli fast gateway adaptation and per-node egress isolation.
+512
View File
@@ -0,0 +1,512 @@
#!/usr/bin/env python3
"""
approvals.py — Fleet approval detection, classification, and resolution engine.
Supports both:
1. Browser DOM element detection & interaction via CDP (the primary live surface):
- Confirmed selectors: [data-testid="approval-panel-header"],
button[data-hatch-approval-primary-action="true"] ("Allow once"),
and "Deny" / "Always allow this site" actions.
- Text fallback: "Allow <agent> to share information with <IP>?"
2. Auto-approval against TRUSTED_IPS (our infrastructure).
3. Manual operator resolution (allow once, always allow, deny).
4. Continuous watch & background integration into fleet status and loop health.
"""
import json
import os
import re
import sys
import time
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
try:
import websocket
except ImportError:
websocket = None
# Paths
REPO_ROOT = Path("/home/super/Projects/NetVM")
BIN_DIR = REPO_ROOT / "bin"
CTL_LOG = REPO_ROOT / "box-ctl.jsonl"
# Add BIN_DIR to sys.path
if str(BIN_DIR) not in sys.path:
sys.path.insert(0, str(BIN_DIR))
try:
import netvm_registry
except ImportError:
netvm_registry = None
VALID_NODES = ["muse", "pip", "646", "opm", "def", "dev"]
# Trusted infrastructure IPs safe for automated approval
TRUSTED_IPS = {
"34.139.37.135", # VM (gateway)
"100.123.153.75", # bl (main compute)
"100.81.31.9", # VM tailnet
}
def log_box_ctl(action: str, name: str = None, caller: str = "box-approvals", extra: dict = None):
"""Log an audit event to box-ctl.jsonl."""
try:
rec = {
"ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"action": action,
"name": name,
"caller": caller,
}
if extra:
rec.update(extra)
with open(CTL_LOG, "a") as f:
f.write(json.dumps(rec) + "\n")
except Exception:
pass
def get_node_connection_info(node: str) -> dict:
"""Return peer_ip, cdp_port, and netns for a given node."""
if netvm_registry:
nodes = netvm_registry.load()
if node in nodes:
rec = nodes[node]
return {
"node": node,
"peer_ip": rec.get("peer_ip", f"10.201.87.2"),
"cdp_port": rec.get("cdp_port", 9222),
"netns": rec.get("netns", f"warp-{node}"),
}
# Deterministic fallback
import hashlib
tag = hashlib.sha256(node.encode()).hexdigest()[:8]
idx = int(tag[:3], 16) % 200 + 10
peer_ip = f"10.201.{idx}.2"
pinned = {"muse": 9410, "pip": 9420, "646": 9430, "opm": 9440, "def": 9450, "dev": 9455}
return {
"node": node,
"peer_ip": peer_ip,
"cdp_port": pinned.get(node, 9222),
"netns": f"warp-{node}",
}
def get_cdp_ws(node: str, timeout: float = 3.0):
"""Connect to the node's active browser page over CDP WebSocket."""
if websocket is None:
raise RuntimeError("websocket-client library is required")
info = get_node_connection_info(node)
peer_ip = info["peer_ip"]
port = info["cdp_port"]
urls = [
f"http://{peer_ip}:{port}/json/list",
f"http://127.0.0.1:{port}/json/list",
]
tabs = None
last_err = None
for u in urls:
try:
req = urllib.request.Request(u, headers={"User-Agent": "box-approvals/1.0"})
with urllib.request.urlopen(req, timeout=timeout) as r:
tabs = json.load(r)
break
except Exception as e:
last_err = e
continue
if not tabs:
raise ConnectionError(f"Could not reach CDP for {node}: {last_err}")
pages = [t for t in tabs if t.get("type") == "page"]
if not pages:
raise ConnectionError(f"No active page found for node {node}")
ws_url = pages[0].get("webSocketDebuggerUrl")
if not ws_url:
raise ConnectionError(f"No webSocketDebuggerUrl for node {node}")
ws = websocket.create_connection(ws_url, timeout=timeout)
return ws, pages[0]
def cdp_evaluate(ws, js_expr: str, await_promise: bool = False, timeout: float = 3.0):
"""Evaluate a JavaScript expression via CDP Runtime.evaluate and return the result value."""
req_id = int(time.time() * 1000) % 100000
msg = {
"id": req_id,
"method": "Runtime.evaluate",
"params": {
"expression": js_expr,
"returnByValue": True,
"awaitPromise": await_promise,
},
}
ws.send(json.dumps(msg))
deadline = time.time() + timeout
while time.time() < deadline:
raw = ws.recv()
resp = json.loads(raw)
if resp.get("id") == req_id:
res = resp.get("result", {})
if "exceptionDetails" in res:
return {"error": res["exceptionDetails"].get("text", "JS exception")}
return res.get("result", {}).get("value")
return None
JS_INSPECT_APPROVALS = """(() => {
// 1. Locate active approval panels
const headers = Array.from(document.querySelectorAll('[data-testid="approval-panel-header"], [data-testid*="approval"]'));
let activeCard = null;
let cardText = '';
for (const h of headers) {
let curr = h;
for (let i = 0; i < 6 && curr && curr.parentElement && curr.parentElement !== document.body; i++) {
const hasPrimary = !!curr.querySelector('button[data-hatch-approval-primary-action="true"]');
const btns = Array.from(curr.querySelectorAll('button')).map(b => (b.innerText||'').trim().toLowerCase());
const hasAllow = btns.some(t => t.includes('allow once') || t === 'allow');
const hasDeny = btns.some(t => t === 'deny');
if (hasPrimary || (hasAllow && hasDeny)) {
activeCard = curr;
cardText = curr.innerText || '';
break;
}
curr = curr.parentElement;
}
if (activeCard) break;
}
// Fallback: look for "Allow ... to share" or buttons
if (!activeCard) {
const bodyText = document.body ? document.body.innerText : '';
if (bodyText.includes('Allow') && bodyText.includes('to share')) {
const btns = Array.from(document.querySelectorAll('button'));
const allowBtn = btns.find(b => (b.innerText||'').trim().toLowerCase().includes('allow once'));
if (allowBtn) {
let curr = allowBtn;
for (let i = 0; i < 5 && curr && curr.parentElement && curr.parentElement !== document.body; i++) {
if (curr.innerText && curr.innerText.includes('Allow') && curr.innerText.includes('to share')) {
activeCard = curr;
cardText = curr.innerText;
break;
}
curr = curr.parentElement;
}
}
}
}
// Inspect buttons in active card
const buttons = [];
let hasAllowOnce = false;
let hasAlwaysAllow = false;
let hasDeny = false;
if (activeCard) {
const btns = Array.from(activeCard.querySelectorAll('button'));
for (const b of btns) {
const t = (b.innerText || '').trim();
const low = t.toLowerCase();
if (low) buttons.push(t);
if (low === 'allow once' || b.getAttribute('data-hatch-approval-primary-action') === 'true') hasAllowOnce = true;
if (low.includes('always allow')) hasAlwaysAllow = true;
if (low === 'deny') hasDeny = true;
}
}
// Collect historical recent approval badges from chat stream
const historyBadges = [];
const allBtns = Array.from(document.querySelectorAll('button'));
for (const b of allBtns) {
const txt = (b.innerText || '').trim();
if (txt.includes('Allowed once ·') || txt.includes('Timed out ·') || txt.includes('Site always allowed ·') || txt.includes('Allowed for this scheduled task ·')) {
const lines = txt.split('\\n');
const summary = lines[0] || '';
const statusLine = lines[lines.length - 1] || '';
historyBadges.push({ summary, status: statusLine });
}
}
return JSON.stringify({
has_pending: !!activeCard && (hasAllowOnce || hasDeny),
card_text: cardText.slice(0, 1000),
buttons: buttons,
has_allow_once: hasAllowOnce,
has_always_allow: hasAlwaysAllow,
has_deny: hasDeny,
history: historyBadges.slice(0, 5)
});
})()"""
def inspect_node_approvals(node: str) -> dict:
"""Inspect a node for active browser approval prompts."""
try:
ws, page = get_cdp_ws(node, timeout=2.5)
except Exception as e:
return {
"node": node,
"status": "UNREACHABLE",
"error": str(e),
"has_pending": False,
}
try:
val_str = cdp_evaluate(ws, JS_INSPECT_APPROVALS, timeout=3.0)
ws.close()
if not val_str or not isinstance(val_str, str):
return {
"node": node,
"status": "CLEAR",
"has_pending": False,
"page_title": page.get("title", ""),
"page_url": page.get("url", ""),
}
data = json.loads(val_str)
has_pending = data.get("has_pending", False)
card_text = data.get("card_text", "")
# Parse details
ip = None
m_ip = re.search(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", card_text)
if m_ip:
ip = m_ip.group(0)
# Extract title and purpose summary
lines = [line.strip() for line in card_text.split("\n") if line.strip()]
title = lines[0] if lines else "Permission request"
purpose = lines[1] if len(lines) > 1 else ""
is_trusted = False
if ip:
is_trusted = ip in TRUSTED_IPS
return {
"node": node,
"status": "PENDING" if has_pending else "CLEAR",
"has_pending": has_pending,
"title": title,
"purpose": purpose,
"ip": ip,
"is_trusted": is_trusted,
"buttons": data.get("buttons", []),
"has_allow_once": data.get("has_allow_once", False),
"has_always_allow": data.get("has_always_allow", False),
"has_deny": data.get("has_deny", False),
"raw_text": card_text,
"history": data.get("history", []),
"page_title": page.get("title", ""),
"page_url": page.get("url", ""),
}
except Exception as e:
try:
ws.close()
except Exception:
pass
return {
"node": node,
"status": "ERROR",
"error": str(e),
"has_pending": False,
}
def check_fleet_approvals(nodes: list = None) -> list:
"""Check approval status across the fleet."""
target_nodes = nodes or VALID_NODES
results = []
for node in target_nodes:
results.append(inspect_node_approvals(node))
return results
def allow_node_approval(node: str, always: bool = False, force: bool = False, caller: str = "box-approvals") -> dict:
"""Approve a pending approval on a node (click 'Allow once' or 'Always allow this site')."""
info = inspect_node_approvals(node)
if not info.get("has_pending"):
return {"ok": False, "node": node, "error": "No pending approval dialog found on node"}
if not info.get("is_trusted") and not force:
target = info.get("ip") or "unrecognized target"
return {
"ok": False,
"node": node,
"error": f"Untrusted origin ({target}). Human review required. Use --force to override.",
"approval": info,
}
try:
ws, _ = get_cdp_ws(node, timeout=3.0)
except Exception as e:
return {"ok": False, "node": node, "error": f"Failed to connect to CDP: {e}"}
try:
if always:
js_click = """(() => {
const btns = Array.from(document.querySelectorAll('button'));
const btn = btns.find(b => (b.innerText||'').toLowerCase().includes('always allow'));
if (btn) {
btn.click();
return 'CLICKED_ALWAYS';
}
return 'NOT_FOUND';
})()"""
else:
js_click = """(() => {
const primary = document.querySelector('button[data-hatch-approval-primary-action="true"]');
if (primary) {
primary.click();
return 'CLICKED_PRIMARY';
}
const btns = Array.from(document.querySelectorAll('button'));
const btn = btns.find(b => {
const t = (b.innerText||'').trim().toLowerCase();
return t === 'allow once' || t === 'allow';
});
if (btn) {
btn.click();
return 'CLICKED_ALLOW';
}
return 'NOT_FOUND';
})()"""
click_res = cdp_evaluate(ws, js_click, timeout=3.0)
# Verify dismissal
time.sleep(0.8)
js_verify = """(() => {
const primary = document.querySelector('button[data-hatch-approval-primary-action="true"]');
if (primary) return 'STILL_PRESENT';
const headers = document.querySelectorAll('[data-testid="approval-panel-header"]');
return headers.length === 0 ? 'DISMISSED' : 'STILL_PRESENT';
})()"""
verify_res = cdp_evaluate(ws, js_verify, timeout=2.0)
ws.close()
dismissed = verify_res == "DISMISSED"
mode = "always" if always else "allow_once"
log_box_ctl(
"approval-allow",
name=node,
caller=caller,
extra={
"decision": mode,
"target_ip": info.get("ip"),
"dismissed": dismissed,
"forced": force,
},
)
return {
"ok": True,
"node": node,
"decision": mode,
"click_result": click_res,
"dismissed": dismissed,
"target_ip": info.get("ip"),
"title": info.get("title"),
}
except Exception as e:
try:
ws.close()
except Exception:
pass
return {"ok": False, "node": node, "error": str(e)}
def deny_node_approval(node: str, caller: str = "box-approvals") -> dict:
"""Deny a pending approval on a node (click 'Deny')."""
info = inspect_node_approvals(node)
if not info.get("has_pending"):
return {"ok": False, "node": node, "error": "No pending approval dialog found on node"}
try:
ws, _ = get_cdp_ws(node, timeout=3.0)
except Exception as e:
return {"ok": False, "node": node, "error": f"Failed to connect to CDP: {e}"}
try:
js_deny = """(() => {
const btns = Array.from(document.querySelectorAll('button'));
const btn = btns.find(b => (b.innerText||'').trim().toLowerCase() === 'deny');
if (btn) {
btn.click();
return 'CLICKED_DENY';
}
return 'NOT_FOUND';
})()"""
click_res = cdp_evaluate(ws, js_deny, timeout=3.0)
# Verify dismissal
time.sleep(0.8)
js_verify = """(() => {
const headers = document.querySelectorAll('[data-testid="approval-panel-header"]');
return headers.length === 0 ? 'DISMISSED' : 'STILL_PRESENT';
})()"""
verify_res = cdp_evaluate(ws, js_verify, timeout=2.0)
ws.close()
dismissed = verify_res == "DISMISSED"
log_box_ctl(
"approval-deny",
name=node,
caller=caller,
extra={
"decision": "deny",
"target_ip": info.get("ip"),
"dismissed": dismissed,
},
)
return {
"ok": True,
"node": node,
"decision": "deny",
"click_result": click_res,
"dismissed": dismissed,
"target_ip": info.get("ip"),
}
except Exception as e:
try:
ws.close()
except Exception:
pass
return {"ok": False, "node": node, "error": str(e)}
def auto_approve_fleet(nodes: list = None, caller: str = "box-approvals") -> dict:
"""Scan fleet nodes and automatically approve any requests to TRUSTED_IPS."""
fleet = check_fleet_approvals(nodes)
approved = []
untrusted = []
clear = []
for item in fleet:
node = item["node"]
if item.get("has_pending"):
if item.get("is_trusted"):
res = allow_node_approval(node, caller=caller)
approved.append({
"node": node,
"target_ip": item.get("ip"),
"title": item.get("title"),
"res": res,
})
else:
untrusted.append(item)
else:
clear.append(node)
return {
"ok": True,
"auto_approved": approved,
"untrusted_pending": untrusted,
"clear_nodes": clear,
}
+61
View File
@@ -981,6 +981,49 @@ def act_fleet_status():
fail("FLEET_ERROR", "failed to collect fleet data", {"stderr": r.stderr}) fail("FLEET_ERROR", "failed to collect fleet data", {"stderr": r.stderr})
def act_approval_check(node=None):
audit("approval-check", node)
import approvals
nodes = [node] if node else list(VALID_AGENTS)
res = approvals.check_fleet_approvals(nodes)
out(True, approvals=res)
def act_approval_allow(node, always=False, force=False):
audit("approval-allow", node)
if node not in VALID_AGENTS:
fail("BAD_NODE", f"unknown node: {node}")
import approvals
res = approvals.allow_node_approval(node, always=always, force=force, caller="box-ctl")
if res.get("ok"):
kw = {k: v for k, v in res.items() if k != "ok"}
out(True, **kw)
else:
fail("APPROVAL_FAILED", res.get("error", "approval failed"), res)
def act_approval_deny(node):
audit("approval-deny", node)
if node not in VALID_AGENTS:
fail("BAD_NODE", f"unknown node: {node}")
import approvals
res = approvals.deny_node_approval(node, caller="box-ctl")
if res.get("ok"):
kw = {k: v for k, v in res.items() if k != "ok"}
out(True, **kw)
else:
fail("APPROVAL_FAILED", res.get("error", "deny failed"), res)
def act_approval_auto(node=None):
audit("approval-auto", node)
import approvals
nodes = [node] if node else list(VALID_AGENTS)
res = approvals.auto_approve_fleet(nodes, caller="box-ctl")
kw = {k: v for k, v in res.items() if k != "ok"}
out(True, **kw)
def act_relay_health(): def act_relay_health():
"""Run relay-health-check.sh and return JSON results.""" """Run relay-health-check.sh and return JSON results."""
audit("relay-health") audit("relay-health")
@@ -3513,6 +3556,24 @@ def main(argv):
fail("BAD_ARGS", "usage: quality validate <action> [args...]") fail("BAD_ARGS", "usage: quality validate <action> [args...]")
audit("quality-validate", rest[1]) audit("quality-validate", rest[1])
act_quality_validate(rest[1], rest[2:]) act_quality_validate(rest[1], rest[2:])
elif action in ("approval-check", "approval-list"):
node = rest[0] if rest else None
act_approval_check(node)
elif action in ("approval-allow", "approval-approve"):
if not rest:
fail("BAD_ARGS", "usage: approval-allow <node> [--always] [--force]")
node = rest[0]
always = "--always" in rest[1:]
force = "--force" in rest[1:]
act_approval_allow(node, always=always, force=force)
elif action == "approval-deny":
if not rest:
fail("BAD_ARGS", "usage: approval-deny <node>")
node = rest[0]
act_approval_deny(node)
elif action == "approval-auto":
node = rest[0] if rest else None
act_approval_auto(node)
elif action == "dm-log": elif action == "dm-log":
limit = 50 limit = 50
if rest: if rest:
+37
View File
@@ -185,6 +185,43 @@ HEALTHY_AGENTS=""
esac esac
done done
# --- Agent approval blockage detection (catches agents held up on approvals) ---
"$BIN/netvm-registry.py" 2>/dev/null | while IFS=: read -r node port; do
[ -n "$node" ] || continue
cond="approval:$node"
pending_info=$(python3 -c "
import sys
sys.path.insert(0, '$BIN')
import approvals
info = approvals.inspect_node_approvals('$node')
if info.get('has_pending'):
print(f\"{info.get('ip') or 'unknown'}|{info.get('title') or ''}\")
" 2>/dev/null || true)
if [ -n "$pending_info" ]; then
failing=1
target="${pending_info%%|*}"
detail="Agent $node held up on browser approval for $target"
else
failing=0
detail="Agent $node approvals clear"
fi
injected "$cond" && failing=1
read -r action fails < <(state_machine "$cond" "$failing")
case "$action" in
ALERT_FIRST|ALERT_REALERT)
emit_record "ALERT" "$cond" "$detail" "$fails"
echo "$cond" >> "$STATE_DIR/.alerts.tmp"
;;
RECOVERY)
emit_record "RECOVERY" "$cond" "$detail" "$fails"
;;
SUPPRESSED)
log "$cond still critical x$fails — re-page suppressed"
;;
esac
done
# --- Warp partition detection (2026-10-04) --- # --- Warp partition detection (2026-10-04) ---
# A partitioned node has a live browser + CDP but no internet egress: the # A partitioned node has a live browser + CDP but no internet egress: the
# chromebox watchdog sees a healthy browser while all automation fails. # chromebox watchdog sees a healthy browser while all automation fails.
+204 -4
View File
@@ -261,6 +261,18 @@ def collect_fleet_data() -> list:
tab = fetch_active_tab(info["peer_ip"], info["cdp_port"]) if cdp["ok"] else {"title": "-", "url": "-"} tab = fetch_active_tab(info["peer_ip"], info["cdp_port"]) if cdp["ok"] else {"title": "-", "url": "-"}
q_depth = get_queue_depth(node) q_depth = get_queue_depth(node)
approval_pending = False
approval_detail = None
if cdp["ok"] and ("(1)" in tab.get("title", "") or "approval" in tab.get("title", "").lower()):
try:
import approvals
app_info = approvals.inspect_node_approvals(node)
if app_info.get("has_pending"):
approval_pending = True
approval_detail = app_info
except Exception:
pass
results.append({ results.append({
"node": node, "node": node,
"netns": info["netns"], "netns": info["netns"],
@@ -272,6 +284,8 @@ def collect_fleet_data() -> list:
"title": tab["title"], "title": tab["title"],
"url": tab["url"], "url": tab["url"],
"queue_depth": q_depth, "queue_depth": q_depth,
"approval_pending": approval_pending,
"approval_detail": approval_detail,
}) })
return results return results
@@ -285,9 +299,13 @@ def cmd_fleet_status(args):
headers = ["NODE", "STATUS", "PEER IP:PORT", "LATENCY", "QUEUE", "ACTIVE PAGE / THREAD"] headers = ["NODE", "STATUS", "PEER IP:PORT", "LATENCY", "QUEUE", "ACTIVE PAGE / THREAD"]
rows = [] rows = []
has_any_approval = False
for item in data: for item in data:
# Status calculation # Status calculation
if item["proc_alive"] and item["cdp_ok"]: if item.get("approval_pending"):
status = badge_warn("APPROVAL_REQ")
has_any_approval = True
elif item["proc_alive"] and item["cdp_ok"]:
status = badge_ok("ACTIVE") status = badge_ok("ACTIVE")
elif item["proc_alive"] and not item["cdp_ok"]: elif item["proc_alive"] and not item["cdp_ok"]:
status = badge_warn("CDP_DOWN") status = badge_warn("CDP_DOWN")
@@ -299,7 +317,10 @@ def cmd_fleet_status(args):
# Format title/URL nicely # Format title/URL nicely
title = item["title"] title = item["title"]
if "thread/" in item["url"]: if item.get("approval_pending"):
target = item.get("approval_detail", {}).get("ip") or "request"
title = f"{c_yellow('[APPROVAL: ' + target + ']')} {title}"
elif "thread/" in item["url"]:
m = re.search(r"thread/([0-9a-fA-F-]+)", item["url"]) m = re.search(r"thread/([0-9a-fA-F-]+)", item["url"])
if m: if m:
uuid_short = m.group(1)[:8] uuid_short = m.group(1)[:8]
@@ -313,11 +334,13 @@ def cmd_fleet_status(args):
f"{item['peer_ip']}:{item['cdp_port']}", f"{item['peer_ip']}:{item['cdp_port']}",
lat, lat,
q, q,
title[:45] title[:50]
]) ])
print_table(headers, rows) print_table(headers, rows)
print("\n" + c_dim(" Commands: super fleet watch | super fleet restart <node> | super fleet cdp <node>") + "\n") if has_any_approval:
print("\n" + c_yellow(" ⚠ Agent(s) held up on browser approval. Run 'box approvals' to inspect/allow."))
print("\n" + c_dim(" Commands: super fleet watch | super fleet restart <node> | box approvals [check|allow|auto]") + "\n")
def cmd_fleet_watch(args): def cmd_fleet_watch(args):
interval = getattr(args, "interval", 2) interval = getattr(args, "interval", 2)
@@ -372,6 +395,146 @@ def cmd_fleet_cdp(args):
print(f" SSH Forward : {c_yellow(f'ssh -L {port}:{peer}:{port} super@100.123.153.75')}") print(f" SSH Forward : {c_yellow(f'ssh -L {port}:{peer}:{port} super@100.123.153.75')}")
print(f" Local Connect : {c_dim(f'http://127.0.0.1:{port}')} (after forwarding)\n") print(f" Local Connect : {c_dim(f'http://127.0.0.1:{port}')} (after forwarding)\n")
# ---------------------------------------------------------------------------
# Domain: APPROVALS
# ---------------------------------------------------------------------------
def cmd_approvals(args):
import approvals
action = getattr(args, "app_action", None) or "check"
node = getattr(args, "node", None)
if action in ("check", "list"):
nodes = [node] if node else VALID_NODES
fleet = approvals.check_fleet_approvals(nodes)
if getattr(args, "json", False):
print(json.dumps({"ok": True, "approvals": fleet}, indent=2))
return
print("\n" + c_bold("=== FLEET APPROVALS STATUS ===") + c_dim(f" ({datetime.now().strftime('%H:%M:%S')} local)\n"))
headers = ["NODE", "STATUS", "TARGET / IP", "PURPOSE / DETAILS", "TRUST", "ACTIONS"]
rows = []
pending_count = 0
untrusted_count = 0
for it in fleet:
n = it["node"]
st = it["status"]
if st == "PENDING":
pending_count += 1
badge = badge_err("PENDING") if not it.get("is_trusted") else badge_warn("PENDING")
target = it.get("ip") or "-"
purp = (it.get("purpose") or it.get("title") or "-")[:50]
trust = c_green("TRUSTED") if it.get("is_trusted") else c_red("UNTRUSTED")
btns = " ".join([f"[{b}]" for b in it.get("buttons", [])])
if not it.get("is_trusted"):
untrusted_count += 1
elif st == "UNREACHABLE":
badge = badge_dim("OFFLINE")
target = "-"
purp = c_dim("CDP unreachable")
trust = "-"
btns = "-"
elif st == "ERROR":
badge = badge_err("ERROR")
target = "-"
purp = it.get("error", "-")[:40]
trust = "-"
btns = "-"
else:
badge = badge_ok("CLEAR")
target = "-"
purp = c_dim("No pending approvals")
trust = "-"
btns = "-"
rows.append([c_bold(n), badge, target, purp, trust, btns])
print_table(headers, rows)
if pending_count > 0:
print("\n" + c_yellow(f" ⚠ {pending_count} pending approval(s) detected across fleet."))
if untrusted_count > 0:
print(c_red(f" ⚠ {untrusted_count} UNTRUSTED approval(s) require manual review: 'box approvals allow <node> --force' or 'box approvals deny <node>'."))
else:
print(c_cyan(" All pending approvals are for trusted infrastructure. Run 'box approvals auto' to resolve."))
print()
else:
print("\n" + c_green(" ✔ All agent approval queues clear. No agents blocked.") + "\n")
elif action in ("allow", "approve"):
if not node:
print(c_red("Error: Must specify node for allow. e.g. 'box approvals allow 646'"), file=sys.stderr)
sys.exit(1)
always = getattr(args, "always", False)
force = getattr(args, "force", False)
res = approvals.allow_node_approval(node, always=always, force=force)
if getattr(args, "json", False):
print(json.dumps(res, indent=2))
return
if res.get("ok"):
decision_str = "Always allow this site" if always else "Allow once"
print(c_green(f"✔ Approved request on node '{node}' ({decision_str}). Dialog dismissed: {res.get('dismissed')}."))
else:
print(c_red(f"✖ Failed to approve on node '{node}': {res.get('error')}"))
sys.exit(2 if "Untrusted" in res.get("error", "") else 1)
elif action == "deny":
if not node:
print(c_red("Error: Must specify node for deny. e.g. 'box approvals deny 646'"), file=sys.stderr)
sys.exit(1)
res = approvals.deny_node_approval(node)
if getattr(args, "json", False):
print(json.dumps(res, indent=2))
return
if res.get("ok"):
print(c_green(f"✔ Denied request on node '{node}'. Dialog dismissed: {res.get('dismissed')}."))
else:
print(c_red(f"✖ Failed to deny on node '{node}': {res.get('error')}"))
sys.exit(1)
elif action == "auto":
nodes = [node] if node else VALID_NODES
res = approvals.auto_approve_fleet(nodes)
if getattr(args, "json", False):
print(json.dumps(res, indent=2))
return
approved = res.get("auto_approved", [])
untrusted = res.get("untrusted_pending", [])
if approved:
print("\n" + c_green(f"✔ Auto-approved {len(approved)} trusted request(s):"))
for a in approved:
print(f" • {c_bold(a['node'])}: {a.get('target_ip')} — {a.get('title')}")
if untrusted:
print("\n" + c_yellow(f"⚠ {len(untrusted)} untrusted request(s) require manual decision:"))
for u in untrusted:
print(f" • {c_bold(u['node'])}: {u.get('ip')} — {u.get('title')} (run: box approvals allow {u['node']} --force)")
if not approved and not untrusted:
print(c_green("✔ All nodes clear. No approvals pending."))
print()
if untrusted:
sys.exit(2)
elif action == "watch":
interval = getattr(args, "interval", 2)
auto_mode = getattr(args, "auto", False)
try:
while True:
sys.stdout.write("\033[2J\033[H")
sys.stdout.flush()
if auto_mode:
auto_res = approvals.auto_approve_fleet(nodes=[node] if node else VALID_NODES)
if auto_res.get("auto_approved"):
for a in auto_res["auto_approved"]:
print(c_green(f"[AUTO-APPROVED] {a['node']}: {a.get('target_ip')}"))
# Print status
setattr(args, "app_action", "check")
cmd_approvals(args)
auto_label = c_cyan(" [AUTO-APPROVE ENABLED]") if auto_mode else ""
print(c_dim(f" [Watching every {interval}s{auto_label}. Press Ctrl+C to exit]"))
time.sleep(interval)
except KeyboardInterrupt:
print("\n" + c_dim("Exited watch mode."))
def resolve_sender(args) -> str: def resolve_sender(args) -> str:
explicit = getattr(args, "from_agent", None) explicit = getattr(args, "from_agent", None)
if explicit and explicit != DEFAULT_SENDER: if explicit and explicit != DEFAULT_SENDER:
@@ -3602,6 +3765,41 @@ def build_parser():
p_fleet.add_argument("node", nargs="?", default=None, help="Target node (for restart / cdp)") p_fleet.add_argument("node", nargs="?", default=None, help="Target node (for restart / cdp)")
p_fleet.add_argument("--interval", type=int, default=2, help="Watch refresh interval in seconds") p_fleet.add_argument("--interval", type=int, default=2, help="Watch refresh interval in seconds")
# Domain: APPROVALS
p_approvals = subparsers.add_parser("approvals", parents=[common], help="Inspect and handle agent browser & gateway approvals")
p_approval = subparsers.add_parser("approval", parents=[common], help="Alias for 'approvals'")
for p_app in (p_approvals, p_approval):
p_app.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node")
app_sub = p_app.add_subparsers(dest="app_action")
p_app_check = app_sub.add_parser("check", parents=[common], help="Check fleet approval states")
p_app_check.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node")
p_app_list = app_sub.add_parser("list", parents=[common], help="Alias for 'check'")
p_app_list.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node")
p_app_allow = app_sub.add_parser("allow", parents=[common], help="Approve pending browser request")
p_app_allow.add_argument("node", choices=VALID_NODES, help="Target node to approve")
p_app_allow.add_argument("--always", action="store_true", help="Click 'Always allow this site' instead of 'Allow once'")
p_app_allow.add_argument("--force", action="store_true", help="Force approval even if target is untrusted")
p_app_approve = app_sub.add_parser("approve", parents=[common], help="Alias for 'allow'")
p_app_approve.add_argument("node", choices=VALID_NODES, help="Target node to approve")
p_app_approve.add_argument("--always", action="store_true", help="Click 'Always allow this site' instead of 'Allow once'")
p_app_approve.add_argument("--force", action="store_true", help="Force approval even if target is untrusted")
p_app_deny = app_sub.add_parser("deny", parents=[common], help="Deny pending browser request")
p_app_deny.add_argument("node", choices=VALID_NODES, help="Target node to deny")
p_app_auto = app_sub.add_parser("auto", parents=[common], help="Auto-approve all trusted requests across fleet")
p_app_auto.add_argument("--node", choices=VALID_NODES, default=None, help="Target node (or all nodes)")
p_app_watch = app_sub.add_parser("watch", parents=[common], help="Live watch pending approvals")
p_app_watch.add_argument("--interval", type=int, default=2, help="Watch refresh interval in seconds")
p_app_watch.add_argument("--auto", action="store_true", help="Automatically approve trusted requests as they appear")
p_app_watch.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node")
# Domain: DM # Domain: DM
p_dm = subparsers.add_parser("dm", parents=[common], help="Inter-agent DMs, work orders ([WO]), acks, live log tail") p_dm = subparsers.add_parser("dm", parents=[common], help="Inter-agent DMs, work orders ([WO]), acks, live log tail")
dm_sub = p_dm.add_subparsers(dest="action") dm_sub = p_dm.add_subparsers(dest="action")
@@ -4285,6 +4483,8 @@ def main():
cmd_swarm_prune(args) cmd_swarm_prune(args)
else: else:
p_swarm.print_help() p_swarm.print_help()
elif args.domain in ("approvals", "approval"):
cmd_approvals(args)
elif args.domain in ("deploy", "subagent"): elif args.domain in ("deploy", "subagent"):
if args.domain == "subagent": if args.domain == "subagent":
args.action = "subagent" args.action = "subagent"
+128
View File
@@ -0,0 +1,128 @@
#!/usr/bin/env python3
"""
test_approvals.py — Unit and integration tests for:
1. approvals.py module (inspect, check_fleet, auto_approve, allow/deny structure)
2. box approvals CLI command (check, list, auto, --json)
3. box-ctl.py RPC actions (approval-check, approval-auto)
4. gravity.py loop break detection (approval_blocked taxonomy)
5. muse-chat-api.py account loading and connection
"""
import json
import os
import subprocess
import sys
import unittest
from pathlib import Path
REPO_ROOT = Path("/home/super/Projects/NetVM")
BIN_DIR = REPO_ROOT / "bin"
sys.path.insert(0, str(BIN_DIR))
import approvals
import gravity
class TestApprovalsModule(unittest.TestCase):
"""Test approvals.py core module functionality."""
def test_trusted_ips_configuration(self):
self.assertIn("34.139.37.135", approvals.TRUSTED_IPS)
self.assertIn("100.123.153.75", approvals.TRUSTED_IPS)
def test_get_node_connection_info(self):
info = approvals.get_node_connection_info("pip")
self.assertEqual(info["node"], "pip")
self.assertEqual(info["cdp_port"], 9420)
self.assertTrue(info["peer_ip"].startswith("10.201."))
def test_check_fleet_approvals_structure(self):
res = approvals.check_fleet_approvals(nodes=["pip", "muse"])
self.assertIsInstance(res, list)
self.assertEqual(len(res), 2)
for it in res:
self.assertIn("node", it)
self.assertIn("status", it)
self.assertIn("has_pending", it)
self.assertIn("buttons", it)
self.assertIn("is_trusted", it)
def test_auto_approve_fleet_structure(self):
res = approvals.auto_approve_fleet(nodes=["pip"])
self.assertTrue(res.get("ok"))
self.assertIn("auto_approved", res)
self.assertIn("untrusted_pending", res)
self.assertIn("clear_nodes", res)
class TestBoxApprovalsCli(unittest.TestCase):
"""Test 'box approvals' and 'box approval' CLI commands."""
def test_box_approvals_check_json(self):
cmd = [sys.executable, str(BIN_DIR / "super-cli.py"), "approvals", "check", "--json"]
r = subprocess.run(cmd, capture_output=True, text=True)
self.assertEqual(r.returncode, 0)
data = json.loads(r.stdout)
self.assertTrue(data.get("ok"))
self.assertIn("approvals", data)
self.assertIsInstance(data["approvals"], list)
def test_box_approval_alias(self):
cmd = [sys.executable, str(BIN_DIR / "super-cli.py"), "approval", "--json"]
r = subprocess.run(cmd, capture_output=True, text=True)
self.assertEqual(r.returncode, 0)
data = json.loads(r.stdout)
self.assertTrue(data.get("ok"))
def test_box_approvals_auto_json(self):
cmd = [sys.executable, str(BIN_DIR / "super-cli.py"), "approvals", "auto", "--node", "pip", "--json"]
r = subprocess.run(cmd, capture_output=True, text=True)
self.assertEqual(r.returncode, 0)
data = json.loads(r.stdout)
self.assertTrue(data.get("ok"))
class TestBoxCtlApprovals(unittest.TestCase):
"""Test box-ctl.py allowlisted RPC actions."""
def test_box_ctl_approval_check(self):
cmd = [sys.executable, str(BIN_DIR / "box-ctl.py"), "approval-check"]
r = subprocess.run(cmd, capture_output=True, text=True)
self.assertEqual(r.returncode, 0)
data = json.loads(r.stdout)
self.assertTrue(data.get("ok"))
self.assertIn("approvals", data)
def test_box_ctl_approval_auto(self):
cmd = [sys.executable, str(BIN_DIR / "box-ctl.py"), "approval-auto", "pip"]
r = subprocess.run(cmd, capture_output=True, text=True)
self.assertEqual(r.returncode, 0)
data = json.loads(r.stdout)
self.assertTrue(data.get("ok"))
class TestGravityApprovalIntegration(unittest.TestCase):
"""Test loop break diagnostics for approvals."""
def test_diagnose_breaks_includes_approval_check(self):
breaks = gravity.diagnose_breaks()
self.assertIsInstance(breaks, list)
# Verify schema
for b in breaks:
self.assertIn("type", b)
self.assertIn("severity", b)
self.assertIn("detail", b)
class TestMuseChatApiConnection(unittest.TestCase):
"""Test muse-chat-api.py account mapping and connection."""
def test_muse_chat_api_approvals_command(self):
cmd = [sys.executable, str(BIN_DIR / "muse-chat-api.py"), "--account", "pip", "approvals"]
r = subprocess.run(cmd, capture_output=True, text=True)
self.assertEqual(r.returncode, 0)
self.assertIn("No pending approvals", r.stdout)
if __name__ == "__main__":
unittest.main()