docs(netvm): document watchdog, swarm-worker, and alert relay in README; track approvals CLI
This commit is contained in:
operator
Executable
+512
@@ -0,0 +1,512 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
approvals.py — Fleet approval detection, classification, and resolution engine.
|
||||
|
||||
Supports both:
|
||||
1. Browser DOM element detection & interaction via CDP (the primary live surface):
|
||||
- Confirmed selectors: [data-testid="approval-panel-header"],
|
||||
button[data-hatch-approval-primary-action="true"] ("Allow once"),
|
||||
and "Deny" / "Always allow this site" actions.
|
||||
- Text fallback: "Allow <agent> to share information with <IP>?"
|
||||
2. Auto-approval against TRUSTED_IPS (our infrastructure).
|
||||
3. Manual operator resolution (allow once, always allow, deny).
|
||||
4. Continuous watch & background integration into fleet status and loop health.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import websocket
|
||||
except ImportError:
|
||||
websocket = None
|
||||
|
||||
# Paths
|
||||
REPO_ROOT = Path("/home/super/Projects/NetVM")
|
||||
BIN_DIR = REPO_ROOT / "bin"
|
||||
CTL_LOG = REPO_ROOT / "box-ctl.jsonl"
|
||||
|
||||
# Add BIN_DIR to sys.path
|
||||
if str(BIN_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BIN_DIR))
|
||||
|
||||
try:
|
||||
import netvm_registry
|
||||
except ImportError:
|
||||
netvm_registry = None
|
||||
|
||||
VALID_NODES = ["muse", "pip", "646", "opm", "def", "dev"]
|
||||
|
||||
# Trusted infrastructure IPs safe for automated approval
|
||||
TRUSTED_IPS = {
|
||||
"34.139.37.135", # VM (gateway)
|
||||
"100.123.153.75", # bl (main compute)
|
||||
"100.81.31.9", # VM tailnet
|
||||
}
|
||||
|
||||
|
||||
def log_box_ctl(action: str, name: str = None, caller: str = "box-approvals", extra: dict = None):
|
||||
"""Log an audit event to box-ctl.jsonl."""
|
||||
try:
|
||||
rec = {
|
||||
"ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"action": action,
|
||||
"name": name,
|
||||
"caller": caller,
|
||||
}
|
||||
if extra:
|
||||
rec.update(extra)
|
||||
with open(CTL_LOG, "a") as f:
|
||||
f.write(json.dumps(rec) + "\n")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def get_node_connection_info(node: str) -> dict:
|
||||
"""Return peer_ip, cdp_port, and netns for a given node."""
|
||||
if netvm_registry:
|
||||
nodes = netvm_registry.load()
|
||||
if node in nodes:
|
||||
rec = nodes[node]
|
||||
return {
|
||||
"node": node,
|
||||
"peer_ip": rec.get("peer_ip", f"10.201.87.2"),
|
||||
"cdp_port": rec.get("cdp_port", 9222),
|
||||
"netns": rec.get("netns", f"warp-{node}"),
|
||||
}
|
||||
# Deterministic fallback
|
||||
import hashlib
|
||||
tag = hashlib.sha256(node.encode()).hexdigest()[:8]
|
||||
idx = int(tag[:3], 16) % 200 + 10
|
||||
peer_ip = f"10.201.{idx}.2"
|
||||
pinned = {"muse": 9410, "pip": 9420, "646": 9430, "opm": 9440, "def": 9450, "dev": 9455}
|
||||
return {
|
||||
"node": node,
|
||||
"peer_ip": peer_ip,
|
||||
"cdp_port": pinned.get(node, 9222),
|
||||
"netns": f"warp-{node}",
|
||||
}
|
||||
|
||||
|
||||
def get_cdp_ws(node: str, timeout: float = 3.0):
|
||||
"""Connect to the node's active browser page over CDP WebSocket."""
|
||||
if websocket is None:
|
||||
raise RuntimeError("websocket-client library is required")
|
||||
|
||||
info = get_node_connection_info(node)
|
||||
peer_ip = info["peer_ip"]
|
||||
port = info["cdp_port"]
|
||||
|
||||
urls = [
|
||||
f"http://{peer_ip}:{port}/json/list",
|
||||
f"http://127.0.0.1:{port}/json/list",
|
||||
]
|
||||
tabs = None
|
||||
last_err = None
|
||||
for u in urls:
|
||||
try:
|
||||
req = urllib.request.Request(u, headers={"User-Agent": "box-approvals/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
tabs = json.load(r)
|
||||
break
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
continue
|
||||
|
||||
if not tabs:
|
||||
raise ConnectionError(f"Could not reach CDP for {node}: {last_err}")
|
||||
|
||||
pages = [t for t in tabs if t.get("type") == "page"]
|
||||
if not pages:
|
||||
raise ConnectionError(f"No active page found for node {node}")
|
||||
|
||||
ws_url = pages[0].get("webSocketDebuggerUrl")
|
||||
if not ws_url:
|
||||
raise ConnectionError(f"No webSocketDebuggerUrl for node {node}")
|
||||
|
||||
ws = websocket.create_connection(ws_url, timeout=timeout)
|
||||
return ws, pages[0]
|
||||
|
||||
|
||||
def cdp_evaluate(ws, js_expr: str, await_promise: bool = False, timeout: float = 3.0):
|
||||
"""Evaluate a JavaScript expression via CDP Runtime.evaluate and return the result value."""
|
||||
req_id = int(time.time() * 1000) % 100000
|
||||
msg = {
|
||||
"id": req_id,
|
||||
"method": "Runtime.evaluate",
|
||||
"params": {
|
||||
"expression": js_expr,
|
||||
"returnByValue": True,
|
||||
"awaitPromise": await_promise,
|
||||
},
|
||||
}
|
||||
ws.send(json.dumps(msg))
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
raw = ws.recv()
|
||||
resp = json.loads(raw)
|
||||
if resp.get("id") == req_id:
|
||||
res = resp.get("result", {})
|
||||
if "exceptionDetails" in res:
|
||||
return {"error": res["exceptionDetails"].get("text", "JS exception")}
|
||||
return res.get("result", {}).get("value")
|
||||
return None
|
||||
|
||||
|
||||
JS_INSPECT_APPROVALS = """(() => {
|
||||
// 1. Locate active approval panels
|
||||
const headers = Array.from(document.querySelectorAll('[data-testid="approval-panel-header"], [data-testid*="approval"]'));
|
||||
let activeCard = null;
|
||||
let cardText = '';
|
||||
|
||||
for (const h of headers) {
|
||||
let curr = h;
|
||||
for (let i = 0; i < 6 && curr && curr.parentElement && curr.parentElement !== document.body; i++) {
|
||||
const hasPrimary = !!curr.querySelector('button[data-hatch-approval-primary-action="true"]');
|
||||
const btns = Array.from(curr.querySelectorAll('button')).map(b => (b.innerText||'').trim().toLowerCase());
|
||||
const hasAllow = btns.some(t => t.includes('allow once') || t === 'allow');
|
||||
const hasDeny = btns.some(t => t === 'deny');
|
||||
if (hasPrimary || (hasAllow && hasDeny)) {
|
||||
activeCard = curr;
|
||||
cardText = curr.innerText || '';
|
||||
break;
|
||||
}
|
||||
curr = curr.parentElement;
|
||||
}
|
||||
if (activeCard) break;
|
||||
}
|
||||
|
||||
// Fallback: look for "Allow ... to share" or buttons
|
||||
if (!activeCard) {
|
||||
const bodyText = document.body ? document.body.innerText : '';
|
||||
if (bodyText.includes('Allow') && bodyText.includes('to share')) {
|
||||
const btns = Array.from(document.querySelectorAll('button'));
|
||||
const allowBtn = btns.find(b => (b.innerText||'').trim().toLowerCase().includes('allow once'));
|
||||
if (allowBtn) {
|
||||
let curr = allowBtn;
|
||||
for (let i = 0; i < 5 && curr && curr.parentElement && curr.parentElement !== document.body; i++) {
|
||||
if (curr.innerText && curr.innerText.includes('Allow') && curr.innerText.includes('to share')) {
|
||||
activeCard = curr;
|
||||
cardText = curr.innerText;
|
||||
break;
|
||||
}
|
||||
curr = curr.parentElement;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Inspect buttons in active card
|
||||
const buttons = [];
|
||||
let hasAllowOnce = false;
|
||||
let hasAlwaysAllow = false;
|
||||
let hasDeny = false;
|
||||
|
||||
if (activeCard) {
|
||||
const btns = Array.from(activeCard.querySelectorAll('button'));
|
||||
for (const b of btns) {
|
||||
const t = (b.innerText || '').trim();
|
||||
const low = t.toLowerCase();
|
||||
if (low) buttons.push(t);
|
||||
if (low === 'allow once' || b.getAttribute('data-hatch-approval-primary-action') === 'true') hasAllowOnce = true;
|
||||
if (low.includes('always allow')) hasAlwaysAllow = true;
|
||||
if (low === 'deny') hasDeny = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Collect historical recent approval badges from chat stream
|
||||
const historyBadges = [];
|
||||
const allBtns = Array.from(document.querySelectorAll('button'));
|
||||
for (const b of allBtns) {
|
||||
const txt = (b.innerText || '').trim();
|
||||
if (txt.includes('Allowed once ·') || txt.includes('Timed out ·') || txt.includes('Site always allowed ·') || txt.includes('Allowed for this scheduled task ·')) {
|
||||
const lines = txt.split('\\n');
|
||||
const summary = lines[0] || '';
|
||||
const statusLine = lines[lines.length - 1] || '';
|
||||
historyBadges.push({ summary, status: statusLine });
|
||||
}
|
||||
}
|
||||
|
||||
return JSON.stringify({
|
||||
has_pending: !!activeCard && (hasAllowOnce || hasDeny),
|
||||
card_text: cardText.slice(0, 1000),
|
||||
buttons: buttons,
|
||||
has_allow_once: hasAllowOnce,
|
||||
has_always_allow: hasAlwaysAllow,
|
||||
has_deny: hasDeny,
|
||||
history: historyBadges.slice(0, 5)
|
||||
});
|
||||
})()"""
|
||||
|
||||
|
||||
def inspect_node_approvals(node: str) -> dict:
|
||||
"""Inspect a node for active browser approval prompts."""
|
||||
try:
|
||||
ws, page = get_cdp_ws(node, timeout=2.5)
|
||||
except Exception as e:
|
||||
return {
|
||||
"node": node,
|
||||
"status": "UNREACHABLE",
|
||||
"error": str(e),
|
||||
"has_pending": False,
|
||||
}
|
||||
|
||||
try:
|
||||
val_str = cdp_evaluate(ws, JS_INSPECT_APPROVALS, timeout=3.0)
|
||||
ws.close()
|
||||
if not val_str or not isinstance(val_str, str):
|
||||
return {
|
||||
"node": node,
|
||||
"status": "CLEAR",
|
||||
"has_pending": False,
|
||||
"page_title": page.get("title", ""),
|
||||
"page_url": page.get("url", ""),
|
||||
}
|
||||
|
||||
data = json.loads(val_str)
|
||||
has_pending = data.get("has_pending", False)
|
||||
card_text = data.get("card_text", "")
|
||||
|
||||
# Parse details
|
||||
ip = None
|
||||
m_ip = re.search(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", card_text)
|
||||
if m_ip:
|
||||
ip = m_ip.group(0)
|
||||
|
||||
# Extract title and purpose summary
|
||||
lines = [line.strip() for line in card_text.split("\n") if line.strip()]
|
||||
title = lines[0] if lines else "Permission request"
|
||||
purpose = lines[1] if len(lines) > 1 else ""
|
||||
|
||||
is_trusted = False
|
||||
if ip:
|
||||
is_trusted = ip in TRUSTED_IPS
|
||||
|
||||
return {
|
||||
"node": node,
|
||||
"status": "PENDING" if has_pending else "CLEAR",
|
||||
"has_pending": has_pending,
|
||||
"title": title,
|
||||
"purpose": purpose,
|
||||
"ip": ip,
|
||||
"is_trusted": is_trusted,
|
||||
"buttons": data.get("buttons", []),
|
||||
"has_allow_once": data.get("has_allow_once", False),
|
||||
"has_always_allow": data.get("has_always_allow", False),
|
||||
"has_deny": data.get("has_deny", False),
|
||||
"raw_text": card_text,
|
||||
"history": data.get("history", []),
|
||||
"page_title": page.get("title", ""),
|
||||
"page_url": page.get("url", ""),
|
||||
}
|
||||
except Exception as e:
|
||||
try:
|
||||
ws.close()
|
||||
except Exception:
|
||||
pass
|
||||
return {
|
||||
"node": node,
|
||||
"status": "ERROR",
|
||||
"error": str(e),
|
||||
"has_pending": False,
|
||||
}
|
||||
|
||||
|
||||
def check_fleet_approvals(nodes: list = None) -> list:
|
||||
"""Check approval status across the fleet."""
|
||||
target_nodes = nodes or VALID_NODES
|
||||
results = []
|
||||
for node in target_nodes:
|
||||
results.append(inspect_node_approvals(node))
|
||||
return results
|
||||
|
||||
|
||||
def allow_node_approval(node: str, always: bool = False, force: bool = False, caller: str = "box-approvals") -> dict:
|
||||
"""Approve a pending approval on a node (click 'Allow once' or 'Always allow this site')."""
|
||||
info = inspect_node_approvals(node)
|
||||
if not info.get("has_pending"):
|
||||
return {"ok": False, "node": node, "error": "No pending approval dialog found on node"}
|
||||
|
||||
if not info.get("is_trusted") and not force:
|
||||
target = info.get("ip") or "unrecognized target"
|
||||
return {
|
||||
"ok": False,
|
||||
"node": node,
|
||||
"error": f"Untrusted origin ({target}). Human review required. Use --force to override.",
|
||||
"approval": info,
|
||||
}
|
||||
|
||||
try:
|
||||
ws, _ = get_cdp_ws(node, timeout=3.0)
|
||||
except Exception as e:
|
||||
return {"ok": False, "node": node, "error": f"Failed to connect to CDP: {e}"}
|
||||
|
||||
try:
|
||||
if always:
|
||||
js_click = """(() => {
|
||||
const btns = Array.from(document.querySelectorAll('button'));
|
||||
const btn = btns.find(b => (b.innerText||'').toLowerCase().includes('always allow'));
|
||||
if (btn) {
|
||||
btn.click();
|
||||
return 'CLICKED_ALWAYS';
|
||||
}
|
||||
return 'NOT_FOUND';
|
||||
})()"""
|
||||
else:
|
||||
js_click = """(() => {
|
||||
const primary = document.querySelector('button[data-hatch-approval-primary-action="true"]');
|
||||
if (primary) {
|
||||
primary.click();
|
||||
return 'CLICKED_PRIMARY';
|
||||
}
|
||||
const btns = Array.from(document.querySelectorAll('button'));
|
||||
const btn = btns.find(b => {
|
||||
const t = (b.innerText||'').trim().toLowerCase();
|
||||
return t === 'allow once' || t === 'allow';
|
||||
});
|
||||
if (btn) {
|
||||
btn.click();
|
||||
return 'CLICKED_ALLOW';
|
||||
}
|
||||
return 'NOT_FOUND';
|
||||
})()"""
|
||||
|
||||
click_res = cdp_evaluate(ws, js_click, timeout=3.0)
|
||||
|
||||
# Verify dismissal
|
||||
time.sleep(0.8)
|
||||
js_verify = """(() => {
|
||||
const primary = document.querySelector('button[data-hatch-approval-primary-action="true"]');
|
||||
if (primary) return 'STILL_PRESENT';
|
||||
const headers = document.querySelectorAll('[data-testid="approval-panel-header"]');
|
||||
return headers.length === 0 ? 'DISMISSED' : 'STILL_PRESENT';
|
||||
})()"""
|
||||
verify_res = cdp_evaluate(ws, js_verify, timeout=2.0)
|
||||
ws.close()
|
||||
|
||||
dismissed = verify_res == "DISMISSED"
|
||||
mode = "always" if always else "allow_once"
|
||||
log_box_ctl(
|
||||
"approval-allow",
|
||||
name=node,
|
||||
caller=caller,
|
||||
extra={
|
||||
"decision": mode,
|
||||
"target_ip": info.get("ip"),
|
||||
"dismissed": dismissed,
|
||||
"forced": force,
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"node": node,
|
||||
"decision": mode,
|
||||
"click_result": click_res,
|
||||
"dismissed": dismissed,
|
||||
"target_ip": info.get("ip"),
|
||||
"title": info.get("title"),
|
||||
}
|
||||
except Exception as e:
|
||||
try:
|
||||
ws.close()
|
||||
except Exception:
|
||||
pass
|
||||
return {"ok": False, "node": node, "error": str(e)}
|
||||
|
||||
|
||||
def deny_node_approval(node: str, caller: str = "box-approvals") -> dict:
|
||||
"""Deny a pending approval on a node (click 'Deny')."""
|
||||
info = inspect_node_approvals(node)
|
||||
if not info.get("has_pending"):
|
||||
return {"ok": False, "node": node, "error": "No pending approval dialog found on node"}
|
||||
|
||||
try:
|
||||
ws, _ = get_cdp_ws(node, timeout=3.0)
|
||||
except Exception as e:
|
||||
return {"ok": False, "node": node, "error": f"Failed to connect to CDP: {e}"}
|
||||
|
||||
try:
|
||||
js_deny = """(() => {
|
||||
const btns = Array.from(document.querySelectorAll('button'));
|
||||
const btn = btns.find(b => (b.innerText||'').trim().toLowerCase() === 'deny');
|
||||
if (btn) {
|
||||
btn.click();
|
||||
return 'CLICKED_DENY';
|
||||
}
|
||||
return 'NOT_FOUND';
|
||||
})()"""
|
||||
click_res = cdp_evaluate(ws, js_deny, timeout=3.0)
|
||||
|
||||
# Verify dismissal
|
||||
time.sleep(0.8)
|
||||
js_verify = """(() => {
|
||||
const headers = document.querySelectorAll('[data-testid="approval-panel-header"]');
|
||||
return headers.length === 0 ? 'DISMISSED' : 'STILL_PRESENT';
|
||||
})()"""
|
||||
verify_res = cdp_evaluate(ws, js_verify, timeout=2.0)
|
||||
ws.close()
|
||||
|
||||
dismissed = verify_res == "DISMISSED"
|
||||
log_box_ctl(
|
||||
"approval-deny",
|
||||
name=node,
|
||||
caller=caller,
|
||||
extra={
|
||||
"decision": "deny",
|
||||
"target_ip": info.get("ip"),
|
||||
"dismissed": dismissed,
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"node": node,
|
||||
"decision": "deny",
|
||||
"click_result": click_res,
|
||||
"dismissed": dismissed,
|
||||
"target_ip": info.get("ip"),
|
||||
}
|
||||
except Exception as e:
|
||||
try:
|
||||
ws.close()
|
||||
except Exception:
|
||||
pass
|
||||
return {"ok": False, "node": node, "error": str(e)}
|
||||
|
||||
|
||||
def auto_approve_fleet(nodes: list = None, caller: str = "box-approvals") -> dict:
|
||||
"""Scan fleet nodes and automatically approve any requests to TRUSTED_IPS."""
|
||||
fleet = check_fleet_approvals(nodes)
|
||||
approved = []
|
||||
untrusted = []
|
||||
clear = []
|
||||
|
||||
for item in fleet:
|
||||
node = item["node"]
|
||||
if item.get("has_pending"):
|
||||
if item.get("is_trusted"):
|
||||
res = allow_node_approval(node, caller=caller)
|
||||
approved.append({
|
||||
"node": node,
|
||||
"target_ip": item.get("ip"),
|
||||
"title": item.get("title"),
|
||||
"res": res,
|
||||
})
|
||||
else:
|
||||
untrusted.append(item)
|
||||
else:
|
||||
clear.append(node)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"auto_approved": approved,
|
||||
"untrusted_pending": untrusted,
|
||||
"clear_nodes": clear,
|
||||
}
|
||||
Reference in New Issue
Block a user