docs(netvm): document watchdog, swarm-worker, and alert relay in README; track approvals CLI
This commit is contained in:
@@ -146,6 +146,9 @@ veth IPs aren't routable off the host and Warp forwards no inbound traffic.
|
||||
- `bin/muse_hybrid.py` — programmatic hybrid bridge combining fast gateway calls with CDP fallbacks.
|
||||
- `bin/muse-tmux.py` — shared tmux socket manager (`/tmp/tmux-muse.sock`) for agent background execution, pipe-pane logging, and 2h session pruning.
|
||||
- `bin/agent_md.py` — CLI & library for auditing, reading, writing, and synchronizing agent `.md` drive files (`SOUL.md`, `PROACTIVE_PREFERENCES.md`, `HEARTBEAT.md`, etc.) across containers via Hatch WebSocket RPC.
|
||||
- `bin/agent-drive-watchdog.py` — background drive watchdog and auto-healing daemon (every 10m via `agent-drive-watchdog.timer`).
|
||||
- `bin/swarm_worker/` & `bin/swarm-worker-supervise.sh` — supervised autonomous swarm worker daemon executing queued tasks in a hard sandbox.
|
||||
- `bin/fleet-alert-relay.sh` — idempotent alert relay with 3-gate deduplication (watermark + 10m TTL hash + receipt verification) posting critical conditions to `#lobby`.
|
||||
- `shared/operators/` — canonical operator drive markdown templates ensuring agents maintain autonomous loops, active supervision, and self-healing reflexes.
|
||||
- docs/OPERATOR-DRIVE-RUNBOOK.md — operator runbook for auditing and modifying agent `.md` files via Hatch WebSocket RPC and SSH reverse tunnels.
|
||||
- docs/HYBRID-GATEWAY-ADAPTATION.md — architectural guide on the muse-cli fast gateway adaptation and per-node egress isolation.
|
||||
|
||||
Executable
+512
@@ -0,0 +1,512 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
approvals.py — Fleet approval detection, classification, and resolution engine.
|
||||
|
||||
Supports both:
|
||||
1. Browser DOM element detection & interaction via CDP (the primary live surface):
|
||||
- Confirmed selectors: [data-testid="approval-panel-header"],
|
||||
button[data-hatch-approval-primary-action="true"] ("Allow once"),
|
||||
and "Deny" / "Always allow this site" actions.
|
||||
- Text fallback: "Allow <agent> to share information with <IP>?"
|
||||
2. Auto-approval against TRUSTED_IPS (our infrastructure).
|
||||
3. Manual operator resolution (allow once, always allow, deny).
|
||||
4. Continuous watch & background integration into fleet status and loop health.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import websocket
|
||||
except ImportError:
|
||||
websocket = None
|
||||
|
||||
# Paths
|
||||
REPO_ROOT = Path("/home/super/Projects/NetVM")
|
||||
BIN_DIR = REPO_ROOT / "bin"
|
||||
CTL_LOG = REPO_ROOT / "box-ctl.jsonl"
|
||||
|
||||
# Add BIN_DIR to sys.path
|
||||
if str(BIN_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BIN_DIR))
|
||||
|
||||
try:
|
||||
import netvm_registry
|
||||
except ImportError:
|
||||
netvm_registry = None
|
||||
|
||||
VALID_NODES = ["muse", "pip", "646", "opm", "def", "dev"]
|
||||
|
||||
# Trusted infrastructure IPs safe for automated approval
|
||||
TRUSTED_IPS = {
|
||||
"34.139.37.135", # VM (gateway)
|
||||
"100.123.153.75", # bl (main compute)
|
||||
"100.81.31.9", # VM tailnet
|
||||
}
|
||||
|
||||
|
||||
def log_box_ctl(action: str, name: str = None, caller: str = "box-approvals", extra: dict = None):
|
||||
"""Log an audit event to box-ctl.jsonl."""
|
||||
try:
|
||||
rec = {
|
||||
"ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"action": action,
|
||||
"name": name,
|
||||
"caller": caller,
|
||||
}
|
||||
if extra:
|
||||
rec.update(extra)
|
||||
with open(CTL_LOG, "a") as f:
|
||||
f.write(json.dumps(rec) + "\n")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def get_node_connection_info(node: str) -> dict:
|
||||
"""Return peer_ip, cdp_port, and netns for a given node."""
|
||||
if netvm_registry:
|
||||
nodes = netvm_registry.load()
|
||||
if node in nodes:
|
||||
rec = nodes[node]
|
||||
return {
|
||||
"node": node,
|
||||
"peer_ip": rec.get("peer_ip", f"10.201.87.2"),
|
||||
"cdp_port": rec.get("cdp_port", 9222),
|
||||
"netns": rec.get("netns", f"warp-{node}"),
|
||||
}
|
||||
# Deterministic fallback
|
||||
import hashlib
|
||||
tag = hashlib.sha256(node.encode()).hexdigest()[:8]
|
||||
idx = int(tag[:3], 16) % 200 + 10
|
||||
peer_ip = f"10.201.{idx}.2"
|
||||
pinned = {"muse": 9410, "pip": 9420, "646": 9430, "opm": 9440, "def": 9450, "dev": 9455}
|
||||
return {
|
||||
"node": node,
|
||||
"peer_ip": peer_ip,
|
||||
"cdp_port": pinned.get(node, 9222),
|
||||
"netns": f"warp-{node}",
|
||||
}
|
||||
|
||||
|
||||
def get_cdp_ws(node: str, timeout: float = 3.0):
|
||||
"""Connect to the node's active browser page over CDP WebSocket."""
|
||||
if websocket is None:
|
||||
raise RuntimeError("websocket-client library is required")
|
||||
|
||||
info = get_node_connection_info(node)
|
||||
peer_ip = info["peer_ip"]
|
||||
port = info["cdp_port"]
|
||||
|
||||
urls = [
|
||||
f"http://{peer_ip}:{port}/json/list",
|
||||
f"http://127.0.0.1:{port}/json/list",
|
||||
]
|
||||
tabs = None
|
||||
last_err = None
|
||||
for u in urls:
|
||||
try:
|
||||
req = urllib.request.Request(u, headers={"User-Agent": "box-approvals/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
tabs = json.load(r)
|
||||
break
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
continue
|
||||
|
||||
if not tabs:
|
||||
raise ConnectionError(f"Could not reach CDP for {node}: {last_err}")
|
||||
|
||||
pages = [t for t in tabs if t.get("type") == "page"]
|
||||
if not pages:
|
||||
raise ConnectionError(f"No active page found for node {node}")
|
||||
|
||||
ws_url = pages[0].get("webSocketDebuggerUrl")
|
||||
if not ws_url:
|
||||
raise ConnectionError(f"No webSocketDebuggerUrl for node {node}")
|
||||
|
||||
ws = websocket.create_connection(ws_url, timeout=timeout)
|
||||
return ws, pages[0]
|
||||
|
||||
|
||||
def cdp_evaluate(ws, js_expr: str, await_promise: bool = False, timeout: float = 3.0):
|
||||
"""Evaluate a JavaScript expression via CDP Runtime.evaluate and return the result value."""
|
||||
req_id = int(time.time() * 1000) % 100000
|
||||
msg = {
|
||||
"id": req_id,
|
||||
"method": "Runtime.evaluate",
|
||||
"params": {
|
||||
"expression": js_expr,
|
||||
"returnByValue": True,
|
||||
"awaitPromise": await_promise,
|
||||
},
|
||||
}
|
||||
ws.send(json.dumps(msg))
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
raw = ws.recv()
|
||||
resp = json.loads(raw)
|
||||
if resp.get("id") == req_id:
|
||||
res = resp.get("result", {})
|
||||
if "exceptionDetails" in res:
|
||||
return {"error": res["exceptionDetails"].get("text", "JS exception")}
|
||||
return res.get("result", {}).get("value")
|
||||
return None
|
||||
|
||||
|
||||
JS_INSPECT_APPROVALS = """(() => {
|
||||
// 1. Locate active approval panels
|
||||
const headers = Array.from(document.querySelectorAll('[data-testid="approval-panel-header"], [data-testid*="approval"]'));
|
||||
let activeCard = null;
|
||||
let cardText = '';
|
||||
|
||||
for (const h of headers) {
|
||||
let curr = h;
|
||||
for (let i = 0; i < 6 && curr && curr.parentElement && curr.parentElement !== document.body; i++) {
|
||||
const hasPrimary = !!curr.querySelector('button[data-hatch-approval-primary-action="true"]');
|
||||
const btns = Array.from(curr.querySelectorAll('button')).map(b => (b.innerText||'').trim().toLowerCase());
|
||||
const hasAllow = btns.some(t => t.includes('allow once') || t === 'allow');
|
||||
const hasDeny = btns.some(t => t === 'deny');
|
||||
if (hasPrimary || (hasAllow && hasDeny)) {
|
||||
activeCard = curr;
|
||||
cardText = curr.innerText || '';
|
||||
break;
|
||||
}
|
||||
curr = curr.parentElement;
|
||||
}
|
||||
if (activeCard) break;
|
||||
}
|
||||
|
||||
// Fallback: look for "Allow ... to share" or buttons
|
||||
if (!activeCard) {
|
||||
const bodyText = document.body ? document.body.innerText : '';
|
||||
if (bodyText.includes('Allow') && bodyText.includes('to share')) {
|
||||
const btns = Array.from(document.querySelectorAll('button'));
|
||||
const allowBtn = btns.find(b => (b.innerText||'').trim().toLowerCase().includes('allow once'));
|
||||
if (allowBtn) {
|
||||
let curr = allowBtn;
|
||||
for (let i = 0; i < 5 && curr && curr.parentElement && curr.parentElement !== document.body; i++) {
|
||||
if (curr.innerText && curr.innerText.includes('Allow') && curr.innerText.includes('to share')) {
|
||||
activeCard = curr;
|
||||
cardText = curr.innerText;
|
||||
break;
|
||||
}
|
||||
curr = curr.parentElement;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Inspect buttons in active card
|
||||
const buttons = [];
|
||||
let hasAllowOnce = false;
|
||||
let hasAlwaysAllow = false;
|
||||
let hasDeny = false;
|
||||
|
||||
if (activeCard) {
|
||||
const btns = Array.from(activeCard.querySelectorAll('button'));
|
||||
for (const b of btns) {
|
||||
const t = (b.innerText || '').trim();
|
||||
const low = t.toLowerCase();
|
||||
if (low) buttons.push(t);
|
||||
if (low === 'allow once' || b.getAttribute('data-hatch-approval-primary-action') === 'true') hasAllowOnce = true;
|
||||
if (low.includes('always allow')) hasAlwaysAllow = true;
|
||||
if (low === 'deny') hasDeny = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Collect historical recent approval badges from chat stream
|
||||
const historyBadges = [];
|
||||
const allBtns = Array.from(document.querySelectorAll('button'));
|
||||
for (const b of allBtns) {
|
||||
const txt = (b.innerText || '').trim();
|
||||
if (txt.includes('Allowed once ·') || txt.includes('Timed out ·') || txt.includes('Site always allowed ·') || txt.includes('Allowed for this scheduled task ·')) {
|
||||
const lines = txt.split('\\n');
|
||||
const summary = lines[0] || '';
|
||||
const statusLine = lines[lines.length - 1] || '';
|
||||
historyBadges.push({ summary, status: statusLine });
|
||||
}
|
||||
}
|
||||
|
||||
return JSON.stringify({
|
||||
has_pending: !!activeCard && (hasAllowOnce || hasDeny),
|
||||
card_text: cardText.slice(0, 1000),
|
||||
buttons: buttons,
|
||||
has_allow_once: hasAllowOnce,
|
||||
has_always_allow: hasAlwaysAllow,
|
||||
has_deny: hasDeny,
|
||||
history: historyBadges.slice(0, 5)
|
||||
});
|
||||
})()"""
|
||||
|
||||
|
||||
def inspect_node_approvals(node: str) -> dict:
|
||||
"""Inspect a node for active browser approval prompts."""
|
||||
try:
|
||||
ws, page = get_cdp_ws(node, timeout=2.5)
|
||||
except Exception as e:
|
||||
return {
|
||||
"node": node,
|
||||
"status": "UNREACHABLE",
|
||||
"error": str(e),
|
||||
"has_pending": False,
|
||||
}
|
||||
|
||||
try:
|
||||
val_str = cdp_evaluate(ws, JS_INSPECT_APPROVALS, timeout=3.0)
|
||||
ws.close()
|
||||
if not val_str or not isinstance(val_str, str):
|
||||
return {
|
||||
"node": node,
|
||||
"status": "CLEAR",
|
||||
"has_pending": False,
|
||||
"page_title": page.get("title", ""),
|
||||
"page_url": page.get("url", ""),
|
||||
}
|
||||
|
||||
data = json.loads(val_str)
|
||||
has_pending = data.get("has_pending", False)
|
||||
card_text = data.get("card_text", "")
|
||||
|
||||
# Parse details
|
||||
ip = None
|
||||
m_ip = re.search(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", card_text)
|
||||
if m_ip:
|
||||
ip = m_ip.group(0)
|
||||
|
||||
# Extract title and purpose summary
|
||||
lines = [line.strip() for line in card_text.split("\n") if line.strip()]
|
||||
title = lines[0] if lines else "Permission request"
|
||||
purpose = lines[1] if len(lines) > 1 else ""
|
||||
|
||||
is_trusted = False
|
||||
if ip:
|
||||
is_trusted = ip in TRUSTED_IPS
|
||||
|
||||
return {
|
||||
"node": node,
|
||||
"status": "PENDING" if has_pending else "CLEAR",
|
||||
"has_pending": has_pending,
|
||||
"title": title,
|
||||
"purpose": purpose,
|
||||
"ip": ip,
|
||||
"is_trusted": is_trusted,
|
||||
"buttons": data.get("buttons", []),
|
||||
"has_allow_once": data.get("has_allow_once", False),
|
||||
"has_always_allow": data.get("has_always_allow", False),
|
||||
"has_deny": data.get("has_deny", False),
|
||||
"raw_text": card_text,
|
||||
"history": data.get("history", []),
|
||||
"page_title": page.get("title", ""),
|
||||
"page_url": page.get("url", ""),
|
||||
}
|
||||
except Exception as e:
|
||||
try:
|
||||
ws.close()
|
||||
except Exception:
|
||||
pass
|
||||
return {
|
||||
"node": node,
|
||||
"status": "ERROR",
|
||||
"error": str(e),
|
||||
"has_pending": False,
|
||||
}
|
||||
|
||||
|
||||
def check_fleet_approvals(nodes: list = None) -> list:
|
||||
"""Check approval status across the fleet."""
|
||||
target_nodes = nodes or VALID_NODES
|
||||
results = []
|
||||
for node in target_nodes:
|
||||
results.append(inspect_node_approvals(node))
|
||||
return results
|
||||
|
||||
|
||||
def allow_node_approval(node: str, always: bool = False, force: bool = False, caller: str = "box-approvals") -> dict:
|
||||
"""Approve a pending approval on a node (click 'Allow once' or 'Always allow this site')."""
|
||||
info = inspect_node_approvals(node)
|
||||
if not info.get("has_pending"):
|
||||
return {"ok": False, "node": node, "error": "No pending approval dialog found on node"}
|
||||
|
||||
if not info.get("is_trusted") and not force:
|
||||
target = info.get("ip") or "unrecognized target"
|
||||
return {
|
||||
"ok": False,
|
||||
"node": node,
|
||||
"error": f"Untrusted origin ({target}). Human review required. Use --force to override.",
|
||||
"approval": info,
|
||||
}
|
||||
|
||||
try:
|
||||
ws, _ = get_cdp_ws(node, timeout=3.0)
|
||||
except Exception as e:
|
||||
return {"ok": False, "node": node, "error": f"Failed to connect to CDP: {e}"}
|
||||
|
||||
try:
|
||||
if always:
|
||||
js_click = """(() => {
|
||||
const btns = Array.from(document.querySelectorAll('button'));
|
||||
const btn = btns.find(b => (b.innerText||'').toLowerCase().includes('always allow'));
|
||||
if (btn) {
|
||||
btn.click();
|
||||
return 'CLICKED_ALWAYS';
|
||||
}
|
||||
return 'NOT_FOUND';
|
||||
})()"""
|
||||
else:
|
||||
js_click = """(() => {
|
||||
const primary = document.querySelector('button[data-hatch-approval-primary-action="true"]');
|
||||
if (primary) {
|
||||
primary.click();
|
||||
return 'CLICKED_PRIMARY';
|
||||
}
|
||||
const btns = Array.from(document.querySelectorAll('button'));
|
||||
const btn = btns.find(b => {
|
||||
const t = (b.innerText||'').trim().toLowerCase();
|
||||
return t === 'allow once' || t === 'allow';
|
||||
});
|
||||
if (btn) {
|
||||
btn.click();
|
||||
return 'CLICKED_ALLOW';
|
||||
}
|
||||
return 'NOT_FOUND';
|
||||
})()"""
|
||||
|
||||
click_res = cdp_evaluate(ws, js_click, timeout=3.0)
|
||||
|
||||
# Verify dismissal
|
||||
time.sleep(0.8)
|
||||
js_verify = """(() => {
|
||||
const primary = document.querySelector('button[data-hatch-approval-primary-action="true"]');
|
||||
if (primary) return 'STILL_PRESENT';
|
||||
const headers = document.querySelectorAll('[data-testid="approval-panel-header"]');
|
||||
return headers.length === 0 ? 'DISMISSED' : 'STILL_PRESENT';
|
||||
})()"""
|
||||
verify_res = cdp_evaluate(ws, js_verify, timeout=2.0)
|
||||
ws.close()
|
||||
|
||||
dismissed = verify_res == "DISMISSED"
|
||||
mode = "always" if always else "allow_once"
|
||||
log_box_ctl(
|
||||
"approval-allow",
|
||||
name=node,
|
||||
caller=caller,
|
||||
extra={
|
||||
"decision": mode,
|
||||
"target_ip": info.get("ip"),
|
||||
"dismissed": dismissed,
|
||||
"forced": force,
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"node": node,
|
||||
"decision": mode,
|
||||
"click_result": click_res,
|
||||
"dismissed": dismissed,
|
||||
"target_ip": info.get("ip"),
|
||||
"title": info.get("title"),
|
||||
}
|
||||
except Exception as e:
|
||||
try:
|
||||
ws.close()
|
||||
except Exception:
|
||||
pass
|
||||
return {"ok": False, "node": node, "error": str(e)}
|
||||
|
||||
|
||||
def deny_node_approval(node: str, caller: str = "box-approvals") -> dict:
|
||||
"""Deny a pending approval on a node (click 'Deny')."""
|
||||
info = inspect_node_approvals(node)
|
||||
if not info.get("has_pending"):
|
||||
return {"ok": False, "node": node, "error": "No pending approval dialog found on node"}
|
||||
|
||||
try:
|
||||
ws, _ = get_cdp_ws(node, timeout=3.0)
|
||||
except Exception as e:
|
||||
return {"ok": False, "node": node, "error": f"Failed to connect to CDP: {e}"}
|
||||
|
||||
try:
|
||||
js_deny = """(() => {
|
||||
const btns = Array.from(document.querySelectorAll('button'));
|
||||
const btn = btns.find(b => (b.innerText||'').trim().toLowerCase() === 'deny');
|
||||
if (btn) {
|
||||
btn.click();
|
||||
return 'CLICKED_DENY';
|
||||
}
|
||||
return 'NOT_FOUND';
|
||||
})()"""
|
||||
click_res = cdp_evaluate(ws, js_deny, timeout=3.0)
|
||||
|
||||
# Verify dismissal
|
||||
time.sleep(0.8)
|
||||
js_verify = """(() => {
|
||||
const headers = document.querySelectorAll('[data-testid="approval-panel-header"]');
|
||||
return headers.length === 0 ? 'DISMISSED' : 'STILL_PRESENT';
|
||||
})()"""
|
||||
verify_res = cdp_evaluate(ws, js_verify, timeout=2.0)
|
||||
ws.close()
|
||||
|
||||
dismissed = verify_res == "DISMISSED"
|
||||
log_box_ctl(
|
||||
"approval-deny",
|
||||
name=node,
|
||||
caller=caller,
|
||||
extra={
|
||||
"decision": "deny",
|
||||
"target_ip": info.get("ip"),
|
||||
"dismissed": dismissed,
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"node": node,
|
||||
"decision": "deny",
|
||||
"click_result": click_res,
|
||||
"dismissed": dismissed,
|
||||
"target_ip": info.get("ip"),
|
||||
}
|
||||
except Exception as e:
|
||||
try:
|
||||
ws.close()
|
||||
except Exception:
|
||||
pass
|
||||
return {"ok": False, "node": node, "error": str(e)}
|
||||
|
||||
|
||||
def auto_approve_fleet(nodes: list = None, caller: str = "box-approvals") -> dict:
|
||||
"""Scan fleet nodes and automatically approve any requests to TRUSTED_IPS."""
|
||||
fleet = check_fleet_approvals(nodes)
|
||||
approved = []
|
||||
untrusted = []
|
||||
clear = []
|
||||
|
||||
for item in fleet:
|
||||
node = item["node"]
|
||||
if item.get("has_pending"):
|
||||
if item.get("is_trusted"):
|
||||
res = allow_node_approval(node, caller=caller)
|
||||
approved.append({
|
||||
"node": node,
|
||||
"target_ip": item.get("ip"),
|
||||
"title": item.get("title"),
|
||||
"res": res,
|
||||
})
|
||||
else:
|
||||
untrusted.append(item)
|
||||
else:
|
||||
clear.append(node)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"auto_approved": approved,
|
||||
"untrusted_pending": untrusted,
|
||||
"clear_nodes": clear,
|
||||
}
|
||||
@@ -981,6 +981,49 @@ def act_fleet_status():
|
||||
fail("FLEET_ERROR", "failed to collect fleet data", {"stderr": r.stderr})
|
||||
|
||||
|
||||
def act_approval_check(node=None):
|
||||
audit("approval-check", node)
|
||||
import approvals
|
||||
nodes = [node] if node else list(VALID_AGENTS)
|
||||
res = approvals.check_fleet_approvals(nodes)
|
||||
out(True, approvals=res)
|
||||
|
||||
|
||||
def act_approval_allow(node, always=False, force=False):
|
||||
audit("approval-allow", node)
|
||||
if node not in VALID_AGENTS:
|
||||
fail("BAD_NODE", f"unknown node: {node}")
|
||||
import approvals
|
||||
res = approvals.allow_node_approval(node, always=always, force=force, caller="box-ctl")
|
||||
if res.get("ok"):
|
||||
kw = {k: v for k, v in res.items() if k != "ok"}
|
||||
out(True, **kw)
|
||||
else:
|
||||
fail("APPROVAL_FAILED", res.get("error", "approval failed"), res)
|
||||
|
||||
|
||||
def act_approval_deny(node):
|
||||
audit("approval-deny", node)
|
||||
if node not in VALID_AGENTS:
|
||||
fail("BAD_NODE", f"unknown node: {node}")
|
||||
import approvals
|
||||
res = approvals.deny_node_approval(node, caller="box-ctl")
|
||||
if res.get("ok"):
|
||||
kw = {k: v for k, v in res.items() if k != "ok"}
|
||||
out(True, **kw)
|
||||
else:
|
||||
fail("APPROVAL_FAILED", res.get("error", "deny failed"), res)
|
||||
|
||||
|
||||
def act_approval_auto(node=None):
|
||||
audit("approval-auto", node)
|
||||
import approvals
|
||||
nodes = [node] if node else list(VALID_AGENTS)
|
||||
res = approvals.auto_approve_fleet(nodes, caller="box-ctl")
|
||||
kw = {k: v for k, v in res.items() if k != "ok"}
|
||||
out(True, **kw)
|
||||
|
||||
|
||||
def act_relay_health():
|
||||
"""Run relay-health-check.sh and return JSON results."""
|
||||
audit("relay-health")
|
||||
@@ -3513,6 +3556,24 @@ def main(argv):
|
||||
fail("BAD_ARGS", "usage: quality validate <action> [args...]")
|
||||
audit("quality-validate", rest[1])
|
||||
act_quality_validate(rest[1], rest[2:])
|
||||
elif action in ("approval-check", "approval-list"):
|
||||
node = rest[0] if rest else None
|
||||
act_approval_check(node)
|
||||
elif action in ("approval-allow", "approval-approve"):
|
||||
if not rest:
|
||||
fail("BAD_ARGS", "usage: approval-allow <node> [--always] [--force]")
|
||||
node = rest[0]
|
||||
always = "--always" in rest[1:]
|
||||
force = "--force" in rest[1:]
|
||||
act_approval_allow(node, always=always, force=force)
|
||||
elif action == "approval-deny":
|
||||
if not rest:
|
||||
fail("BAD_ARGS", "usage: approval-deny <node>")
|
||||
node = rest[0]
|
||||
act_approval_deny(node)
|
||||
elif action == "approval-auto":
|
||||
node = rest[0] if rest else None
|
||||
act_approval_auto(node)
|
||||
elif action == "dm-log":
|
||||
limit = 50
|
||||
if rest:
|
||||
|
||||
@@ -185,6 +185,43 @@ HEALTHY_AGENTS=""
|
||||
esac
|
||||
done
|
||||
|
||||
# --- Agent approval blockage detection (catches agents held up on approvals) ---
|
||||
"$BIN/netvm-registry.py" 2>/dev/null | while IFS=: read -r node port; do
|
||||
[ -n "$node" ] || continue
|
||||
cond="approval:$node"
|
||||
pending_info=$(python3 -c "
|
||||
import sys
|
||||
sys.path.insert(0, '$BIN')
|
||||
import approvals
|
||||
info = approvals.inspect_node_approvals('$node')
|
||||
if info.get('has_pending'):
|
||||
print(f\"{info.get('ip') or 'unknown'}|{info.get('title') or ''}\")
|
||||
" 2>/dev/null || true)
|
||||
|
||||
if [ -n "$pending_info" ]; then
|
||||
failing=1
|
||||
target="${pending_info%%|*}"
|
||||
detail="Agent $node held up on browser approval for $target"
|
||||
else
|
||||
failing=0
|
||||
detail="Agent $node approvals clear"
|
||||
fi
|
||||
injected "$cond" && failing=1
|
||||
read -r action fails < <(state_machine "$cond" "$failing")
|
||||
case "$action" in
|
||||
ALERT_FIRST|ALERT_REALERT)
|
||||
emit_record "ALERT" "$cond" "$detail" "$fails"
|
||||
echo "$cond" >> "$STATE_DIR/.alerts.tmp"
|
||||
;;
|
||||
RECOVERY)
|
||||
emit_record "RECOVERY" "$cond" "$detail" "$fails"
|
||||
;;
|
||||
SUPPRESSED)
|
||||
log "$cond still critical x$fails — re-page suppressed"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# --- Warp partition detection (2026-10-04) ---
|
||||
# A partitioned node has a live browser + CDP but no internet egress: the
|
||||
# chromebox watchdog sees a healthy browser while all automation fails.
|
||||
|
||||
+204
-4
@@ -261,6 +261,18 @@ def collect_fleet_data() -> list:
|
||||
tab = fetch_active_tab(info["peer_ip"], info["cdp_port"]) if cdp["ok"] else {"title": "-", "url": "-"}
|
||||
q_depth = get_queue_depth(node)
|
||||
|
||||
approval_pending = False
|
||||
approval_detail = None
|
||||
if cdp["ok"] and ("(1)" in tab.get("title", "") or "approval" in tab.get("title", "").lower()):
|
||||
try:
|
||||
import approvals
|
||||
app_info = approvals.inspect_node_approvals(node)
|
||||
if app_info.get("has_pending"):
|
||||
approval_pending = True
|
||||
approval_detail = app_info
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
results.append({
|
||||
"node": node,
|
||||
"netns": info["netns"],
|
||||
@@ -272,6 +284,8 @@ def collect_fleet_data() -> list:
|
||||
"title": tab["title"],
|
||||
"url": tab["url"],
|
||||
"queue_depth": q_depth,
|
||||
"approval_pending": approval_pending,
|
||||
"approval_detail": approval_detail,
|
||||
})
|
||||
return results
|
||||
|
||||
@@ -285,9 +299,13 @@ def cmd_fleet_status(args):
|
||||
|
||||
headers = ["NODE", "STATUS", "PEER IP:PORT", "LATENCY", "QUEUE", "ACTIVE PAGE / THREAD"]
|
||||
rows = []
|
||||
has_any_approval = False
|
||||
for item in data:
|
||||
# Status calculation
|
||||
if item["proc_alive"] and item["cdp_ok"]:
|
||||
if item.get("approval_pending"):
|
||||
status = badge_warn("APPROVAL_REQ")
|
||||
has_any_approval = True
|
||||
elif item["proc_alive"] and item["cdp_ok"]:
|
||||
status = badge_ok("ACTIVE")
|
||||
elif item["proc_alive"] and not item["cdp_ok"]:
|
||||
status = badge_warn("CDP_DOWN")
|
||||
@@ -299,7 +317,10 @@ def cmd_fleet_status(args):
|
||||
|
||||
# Format title/URL nicely
|
||||
title = item["title"]
|
||||
if "thread/" in item["url"]:
|
||||
if item.get("approval_pending"):
|
||||
target = item.get("approval_detail", {}).get("ip") or "request"
|
||||
title = f"{c_yellow('[APPROVAL: ' + target + ']')} {title}"
|
||||
elif "thread/" in item["url"]:
|
||||
m = re.search(r"thread/([0-9a-fA-F-]+)", item["url"])
|
||||
if m:
|
||||
uuid_short = m.group(1)[:8]
|
||||
@@ -313,11 +334,13 @@ def cmd_fleet_status(args):
|
||||
f"{item['peer_ip']}:{item['cdp_port']}",
|
||||
lat,
|
||||
q,
|
||||
title[:45]
|
||||
title[:50]
|
||||
])
|
||||
|
||||
print_table(headers, rows)
|
||||
print("\n" + c_dim(" Commands: super fleet watch | super fleet restart <node> | super fleet cdp <node>") + "\n")
|
||||
if has_any_approval:
|
||||
print("\n" + c_yellow(" ⚠ Agent(s) held up on browser approval. Run 'box approvals' to inspect/allow."))
|
||||
print("\n" + c_dim(" Commands: super fleet watch | super fleet restart <node> | box approvals [check|allow|auto]") + "\n")
|
||||
|
||||
def cmd_fleet_watch(args):
|
||||
interval = getattr(args, "interval", 2)
|
||||
@@ -372,6 +395,146 @@ def cmd_fleet_cdp(args):
|
||||
print(f" SSH Forward : {c_yellow(f'ssh -L {port}:{peer}:{port} super@100.123.153.75')}")
|
||||
print(f" Local Connect : {c_dim(f'http://127.0.0.1:{port}')} (after forwarding)\n")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Domain: APPROVALS
|
||||
# ---------------------------------------------------------------------------
|
||||
def cmd_approvals(args):
|
||||
import approvals
|
||||
action = getattr(args, "app_action", None) or "check"
|
||||
node = getattr(args, "node", None)
|
||||
|
||||
if action in ("check", "list"):
|
||||
nodes = [node] if node else VALID_NODES
|
||||
fleet = approvals.check_fleet_approvals(nodes)
|
||||
if getattr(args, "json", False):
|
||||
print(json.dumps({"ok": True, "approvals": fleet}, indent=2))
|
||||
return
|
||||
|
||||
print("\n" + c_bold("=== FLEET APPROVALS STATUS ===") + c_dim(f" ({datetime.now().strftime('%H:%M:%S')} local)\n"))
|
||||
headers = ["NODE", "STATUS", "TARGET / IP", "PURPOSE / DETAILS", "TRUST", "ACTIONS"]
|
||||
rows = []
|
||||
pending_count = 0
|
||||
untrusted_count = 0
|
||||
|
||||
for it in fleet:
|
||||
n = it["node"]
|
||||
st = it["status"]
|
||||
if st == "PENDING":
|
||||
pending_count += 1
|
||||
badge = badge_err("PENDING") if not it.get("is_trusted") else badge_warn("PENDING")
|
||||
target = it.get("ip") or "-"
|
||||
purp = (it.get("purpose") or it.get("title") or "-")[:50]
|
||||
trust = c_green("TRUSTED") if it.get("is_trusted") else c_red("UNTRUSTED")
|
||||
btns = " ".join([f"[{b}]" for b in it.get("buttons", [])])
|
||||
if not it.get("is_trusted"):
|
||||
untrusted_count += 1
|
||||
elif st == "UNREACHABLE":
|
||||
badge = badge_dim("OFFLINE")
|
||||
target = "-"
|
||||
purp = c_dim("CDP unreachable")
|
||||
trust = "-"
|
||||
btns = "-"
|
||||
elif st == "ERROR":
|
||||
badge = badge_err("ERROR")
|
||||
target = "-"
|
||||
purp = it.get("error", "-")[:40]
|
||||
trust = "-"
|
||||
btns = "-"
|
||||
else:
|
||||
badge = badge_ok("CLEAR")
|
||||
target = "-"
|
||||
purp = c_dim("No pending approvals")
|
||||
trust = "-"
|
||||
btns = "-"
|
||||
|
||||
rows.append([c_bold(n), badge, target, purp, trust, btns])
|
||||
|
||||
print_table(headers, rows)
|
||||
|
||||
if pending_count > 0:
|
||||
print("\n" + c_yellow(f" ⚠ {pending_count} pending approval(s) detected across fleet."))
|
||||
if untrusted_count > 0:
|
||||
print(c_red(f" ⚠ {untrusted_count} UNTRUSTED approval(s) require manual review: 'box approvals allow <node> --force' or 'box approvals deny <node>'."))
|
||||
else:
|
||||
print(c_cyan(" All pending approvals are for trusted infrastructure. Run 'box approvals auto' to resolve."))
|
||||
print()
|
||||
else:
|
||||
print("\n" + c_green(" ✔ All agent approval queues clear. No agents blocked.") + "\n")
|
||||
|
||||
elif action in ("allow", "approve"):
|
||||
if not node:
|
||||
print(c_red("Error: Must specify node for allow. e.g. 'box approvals allow 646'"), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
always = getattr(args, "always", False)
|
||||
force = getattr(args, "force", False)
|
||||
res = approvals.allow_node_approval(node, always=always, force=force)
|
||||
if getattr(args, "json", False):
|
||||
print(json.dumps(res, indent=2))
|
||||
return
|
||||
if res.get("ok"):
|
||||
decision_str = "Always allow this site" if always else "Allow once"
|
||||
print(c_green(f"✔ Approved request on node '{node}' ({decision_str}). Dialog dismissed: {res.get('dismissed')}."))
|
||||
else:
|
||||
print(c_red(f"✖ Failed to approve on node '{node}': {res.get('error')}"))
|
||||
sys.exit(2 if "Untrusted" in res.get("error", "") else 1)
|
||||
|
||||
elif action == "deny":
|
||||
if not node:
|
||||
print(c_red("Error: Must specify node for deny. e.g. 'box approvals deny 646'"), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
res = approvals.deny_node_approval(node)
|
||||
if getattr(args, "json", False):
|
||||
print(json.dumps(res, indent=2))
|
||||
return
|
||||
if res.get("ok"):
|
||||
print(c_green(f"✔ Denied request on node '{node}'. Dialog dismissed: {res.get('dismissed')}."))
|
||||
else:
|
||||
print(c_red(f"✖ Failed to deny on node '{node}': {res.get('error')}"))
|
||||
sys.exit(1)
|
||||
|
||||
elif action == "auto":
|
||||
nodes = [node] if node else VALID_NODES
|
||||
res = approvals.auto_approve_fleet(nodes)
|
||||
if getattr(args, "json", False):
|
||||
print(json.dumps(res, indent=2))
|
||||
return
|
||||
approved = res.get("auto_approved", [])
|
||||
untrusted = res.get("untrusted_pending", [])
|
||||
if approved:
|
||||
print("\n" + c_green(f"✔ Auto-approved {len(approved)} trusted request(s):"))
|
||||
for a in approved:
|
||||
print(f" • {c_bold(a['node'])}: {a.get('target_ip')} — {a.get('title')}")
|
||||
if untrusted:
|
||||
print("\n" + c_yellow(f"⚠ {len(untrusted)} untrusted request(s) require manual decision:"))
|
||||
for u in untrusted:
|
||||
print(f" • {c_bold(u['node'])}: {u.get('ip')} — {u.get('title')} (run: box approvals allow {u['node']} --force)")
|
||||
if not approved and not untrusted:
|
||||
print(c_green("✔ All nodes clear. No approvals pending."))
|
||||
print()
|
||||
if untrusted:
|
||||
sys.exit(2)
|
||||
|
||||
elif action == "watch":
|
||||
interval = getattr(args, "interval", 2)
|
||||
auto_mode = getattr(args, "auto", False)
|
||||
try:
|
||||
while True:
|
||||
sys.stdout.write("\033[2J\033[H")
|
||||
sys.stdout.flush()
|
||||
if auto_mode:
|
||||
auto_res = approvals.auto_approve_fleet(nodes=[node] if node else VALID_NODES)
|
||||
if auto_res.get("auto_approved"):
|
||||
for a in auto_res["auto_approved"]:
|
||||
print(c_green(f"[AUTO-APPROVED] {a['node']}: {a.get('target_ip')}"))
|
||||
# Print status
|
||||
setattr(args, "app_action", "check")
|
||||
cmd_approvals(args)
|
||||
auto_label = c_cyan(" [AUTO-APPROVE ENABLED]") if auto_mode else ""
|
||||
print(c_dim(f" [Watching every {interval}s{auto_label}. Press Ctrl+C to exit]"))
|
||||
time.sleep(interval)
|
||||
except KeyboardInterrupt:
|
||||
print("\n" + c_dim("Exited watch mode."))
|
||||
|
||||
def resolve_sender(args) -> str:
|
||||
explicit = getattr(args, "from_agent", None)
|
||||
if explicit and explicit != DEFAULT_SENDER:
|
||||
@@ -3602,6 +3765,41 @@ def build_parser():
|
||||
p_fleet.add_argument("node", nargs="?", default=None, help="Target node (for restart / cdp)")
|
||||
p_fleet.add_argument("--interval", type=int, default=2, help="Watch refresh interval in seconds")
|
||||
|
||||
# Domain: APPROVALS
|
||||
p_approvals = subparsers.add_parser("approvals", parents=[common], help="Inspect and handle agent browser & gateway approvals")
|
||||
p_approval = subparsers.add_parser("approval", parents=[common], help="Alias for 'approvals'")
|
||||
|
||||
for p_app in (p_approvals, p_approval):
|
||||
p_app.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node")
|
||||
app_sub = p_app.add_subparsers(dest="app_action")
|
||||
|
||||
p_app_check = app_sub.add_parser("check", parents=[common], help="Check fleet approval states")
|
||||
p_app_check.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node")
|
||||
|
||||
p_app_list = app_sub.add_parser("list", parents=[common], help="Alias for 'check'")
|
||||
p_app_list.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node")
|
||||
|
||||
p_app_allow = app_sub.add_parser("allow", parents=[common], help="Approve pending browser request")
|
||||
p_app_allow.add_argument("node", choices=VALID_NODES, help="Target node to approve")
|
||||
p_app_allow.add_argument("--always", action="store_true", help="Click 'Always allow this site' instead of 'Allow once'")
|
||||
p_app_allow.add_argument("--force", action="store_true", help="Force approval even if target is untrusted")
|
||||
|
||||
p_app_approve = app_sub.add_parser("approve", parents=[common], help="Alias for 'allow'")
|
||||
p_app_approve.add_argument("node", choices=VALID_NODES, help="Target node to approve")
|
||||
p_app_approve.add_argument("--always", action="store_true", help="Click 'Always allow this site' instead of 'Allow once'")
|
||||
p_app_approve.add_argument("--force", action="store_true", help="Force approval even if target is untrusted")
|
||||
|
||||
p_app_deny = app_sub.add_parser("deny", parents=[common], help="Deny pending browser request")
|
||||
p_app_deny.add_argument("node", choices=VALID_NODES, help="Target node to deny")
|
||||
|
||||
p_app_auto = app_sub.add_parser("auto", parents=[common], help="Auto-approve all trusted requests across fleet")
|
||||
p_app_auto.add_argument("--node", choices=VALID_NODES, default=None, help="Target node (or all nodes)")
|
||||
|
||||
p_app_watch = app_sub.add_parser("watch", parents=[common], help="Live watch pending approvals")
|
||||
p_app_watch.add_argument("--interval", type=int, default=2, help="Watch refresh interval in seconds")
|
||||
p_app_watch.add_argument("--auto", action="store_true", help="Automatically approve trusted requests as they appear")
|
||||
p_app_watch.add_argument("--node", choices=VALID_NODES, default=None, help="Filter by node")
|
||||
|
||||
# Domain: DM
|
||||
p_dm = subparsers.add_parser("dm", parents=[common], help="Inter-agent DMs, work orders ([WO]), acks, live log tail")
|
||||
dm_sub = p_dm.add_subparsers(dest="action")
|
||||
@@ -4285,6 +4483,8 @@ def main():
|
||||
cmd_swarm_prune(args)
|
||||
else:
|
||||
p_swarm.print_help()
|
||||
elif args.domain in ("approvals", "approval"):
|
||||
cmd_approvals(args)
|
||||
elif args.domain in ("deploy", "subagent"):
|
||||
if args.domain == "subagent":
|
||||
args.action = "subagent"
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
test_approvals.py — Unit and integration tests for:
|
||||
1. approvals.py module (inspect, check_fleet, auto_approve, allow/deny structure)
|
||||
2. box approvals CLI command (check, list, auto, --json)
|
||||
3. box-ctl.py RPC actions (approval-check, approval-auto)
|
||||
4. gravity.py loop break detection (approval_blocked taxonomy)
|
||||
5. muse-chat-api.py account loading and connection
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path("/home/super/Projects/NetVM")
|
||||
BIN_DIR = REPO_ROOT / "bin"
|
||||
sys.path.insert(0, str(BIN_DIR))
|
||||
|
||||
import approvals
|
||||
import gravity
|
||||
|
||||
|
||||
class TestApprovalsModule(unittest.TestCase):
|
||||
"""Test approvals.py core module functionality."""
|
||||
|
||||
def test_trusted_ips_configuration(self):
|
||||
self.assertIn("34.139.37.135", approvals.TRUSTED_IPS)
|
||||
self.assertIn("100.123.153.75", approvals.TRUSTED_IPS)
|
||||
|
||||
def test_get_node_connection_info(self):
|
||||
info = approvals.get_node_connection_info("pip")
|
||||
self.assertEqual(info["node"], "pip")
|
||||
self.assertEqual(info["cdp_port"], 9420)
|
||||
self.assertTrue(info["peer_ip"].startswith("10.201."))
|
||||
|
||||
def test_check_fleet_approvals_structure(self):
|
||||
res = approvals.check_fleet_approvals(nodes=["pip", "muse"])
|
||||
self.assertIsInstance(res, list)
|
||||
self.assertEqual(len(res), 2)
|
||||
for it in res:
|
||||
self.assertIn("node", it)
|
||||
self.assertIn("status", it)
|
||||
self.assertIn("has_pending", it)
|
||||
self.assertIn("buttons", it)
|
||||
self.assertIn("is_trusted", it)
|
||||
|
||||
def test_auto_approve_fleet_structure(self):
|
||||
res = approvals.auto_approve_fleet(nodes=["pip"])
|
||||
self.assertTrue(res.get("ok"))
|
||||
self.assertIn("auto_approved", res)
|
||||
self.assertIn("untrusted_pending", res)
|
||||
self.assertIn("clear_nodes", res)
|
||||
|
||||
|
||||
class TestBoxApprovalsCli(unittest.TestCase):
|
||||
"""Test 'box approvals' and 'box approval' CLI commands."""
|
||||
|
||||
def test_box_approvals_check_json(self):
|
||||
cmd = [sys.executable, str(BIN_DIR / "super-cli.py"), "approvals", "check", "--json"]
|
||||
r = subprocess.run(cmd, capture_output=True, text=True)
|
||||
self.assertEqual(r.returncode, 0)
|
||||
data = json.loads(r.stdout)
|
||||
self.assertTrue(data.get("ok"))
|
||||
self.assertIn("approvals", data)
|
||||
self.assertIsInstance(data["approvals"], list)
|
||||
|
||||
def test_box_approval_alias(self):
|
||||
cmd = [sys.executable, str(BIN_DIR / "super-cli.py"), "approval", "--json"]
|
||||
r = subprocess.run(cmd, capture_output=True, text=True)
|
||||
self.assertEqual(r.returncode, 0)
|
||||
data = json.loads(r.stdout)
|
||||
self.assertTrue(data.get("ok"))
|
||||
|
||||
def test_box_approvals_auto_json(self):
|
||||
cmd = [sys.executable, str(BIN_DIR / "super-cli.py"), "approvals", "auto", "--node", "pip", "--json"]
|
||||
r = subprocess.run(cmd, capture_output=True, text=True)
|
||||
self.assertEqual(r.returncode, 0)
|
||||
data = json.loads(r.stdout)
|
||||
self.assertTrue(data.get("ok"))
|
||||
|
||||
|
||||
class TestBoxCtlApprovals(unittest.TestCase):
|
||||
"""Test box-ctl.py allowlisted RPC actions."""
|
||||
|
||||
def test_box_ctl_approval_check(self):
|
||||
cmd = [sys.executable, str(BIN_DIR / "box-ctl.py"), "approval-check"]
|
||||
r = subprocess.run(cmd, capture_output=True, text=True)
|
||||
self.assertEqual(r.returncode, 0)
|
||||
data = json.loads(r.stdout)
|
||||
self.assertTrue(data.get("ok"))
|
||||
self.assertIn("approvals", data)
|
||||
|
||||
def test_box_ctl_approval_auto(self):
|
||||
cmd = [sys.executable, str(BIN_DIR / "box-ctl.py"), "approval-auto", "pip"]
|
||||
r = subprocess.run(cmd, capture_output=True, text=True)
|
||||
self.assertEqual(r.returncode, 0)
|
||||
data = json.loads(r.stdout)
|
||||
self.assertTrue(data.get("ok"))
|
||||
|
||||
|
||||
class TestGravityApprovalIntegration(unittest.TestCase):
|
||||
"""Test loop break diagnostics for approvals."""
|
||||
|
||||
def test_diagnose_breaks_includes_approval_check(self):
|
||||
breaks = gravity.diagnose_breaks()
|
||||
self.assertIsInstance(breaks, list)
|
||||
# Verify schema
|
||||
for b in breaks:
|
||||
self.assertIn("type", b)
|
||||
self.assertIn("severity", b)
|
||||
self.assertIn("detail", b)
|
||||
|
||||
|
||||
class TestMuseChatApiConnection(unittest.TestCase):
|
||||
"""Test muse-chat-api.py account mapping and connection."""
|
||||
|
||||
def test_muse_chat_api_approvals_command(self):
|
||||
cmd = [sys.executable, str(BIN_DIR / "muse-chat-api.py"), "--account", "pip", "approvals"]
|
||||
r = subprocess.run(cmd, capture_output=True, text=True)
|
||||
self.assertEqual(r.returncode, 0)
|
||||
self.assertIn("No pending approvals", r.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user