Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f9f12b450e | |||
| e6e5616de6 | |||
| f1252423a8 | |||
| 340cc9e85b | |||
| 0668d257ca | |||
| cc6a33cbca | |||
| 5f7e1e3a76 | |||
| 073ef556ba | |||
| 1b44d224a3 | |||
| bccdd29d53 | |||
| 3620b42297 | |||
| 9972c07c61 | |||
| f8b7424315 | |||
| c79382b1c1 | |||
| b7a73234fc | |||
| 35c59bfcb2 | |||
| ef2a4c419a | |||
| ed33d66479 | |||
| 8099c9a4aa | |||
| ab7d1215e0 |
@@ -13,6 +13,8 @@ stable network presence.
|
|||||||
|--------------|-------|---------------|---------|----------|-----------|---------|-------|
|
|--------------|-------|---------------|---------|----------|-----------|---------|-------|
|
||||||
| tp | warp-tp | /etc/netvm/tp.conf (wgcf, 2026-10-03) | 10.201.149.2 | 9277 | 104.28.203.246 | — | laptop; orchestrator + first node; UP, handshake+egress verified 2026-10-03 |
|
| tp | warp-tp | /etc/netvm/tp.conf (wgcf, 2026-10-03) | 10.201.149.2 | 9277 | 104.28.203.246 | — | laptop; orchestrator + first node; UP, handshake+egress verified 2026-10-03 |
|
||||||
| smoke | warp-smoke | /etc/netvm/smoke.conf (wgcf, 2026-10-03) | 10.201.87.2 | 9410 | 104.28.203.246 | — | laptop; dedicated test rig (chrome-box profile smoke); UP, handshake+egress+CDP+muse.ai verified 2026-10-03 |
|
| smoke | warp-smoke | /etc/netvm/smoke.conf (wgcf, 2026-10-03) | 10.201.87.2 | 9410 | 104.28.203.246 | — | laptop; dedicated test rig (chrome-box profile smoke); UP, handshake+egress+CDP+muse.ai verified 2026-10-03 |
|
||||||
|
| smoke2 | warp-smoke2 | /etc/netvm/smoke2.conf (wgcf, 2026-10-03) | 10.201.117.2 | 9585 | 104.28.227.184 | — | laptop; dedicated test rig (chrome-box profile smoke2); UP, handshake+egress+CDP+muse.ai verified 2026-10-03 |
|
||||||
|
|
||||||
|
|
||||||
CDP: `http://<veth IP>:<CDP port>/json/list` from the host, or
|
CDP: `http://<veth IP>:<CDP port>/json/list` from the host, or
|
||||||
`ssh -L <port>:<veth IP>:<port> <user>@<tail IP>` for remote automation.
|
`ssh -L <port>:<veth IP>:<port> <user>@<tail IP>` for remote automation.
|
||||||
|
|||||||
@@ -119,7 +119,8 @@ veth IPs aren't routable off the host and Warp forwards no inbound traffic.
|
|||||||
- `bin/netvm-fleet.sh` — operator fleet control over the tailnet (topology/up/down/ssh/exec/cdp per node).
|
- `bin/netvm-fleet.sh` — operator fleet control over the tailnet (topology/up/down/ssh/exec/cdp per node).
|
||||||
- `bin/netvm-exec.sh <node> -- <cmd>` — run a command inside the node's netns as the invoking user (the agent-friendly primitive).
|
- `bin/netvm-exec.sh <node> -- <cmd>` — run a command inside the node's netns as the invoking user (the agent-friendly primitive).
|
||||||
- `bin/netvm-enter.sh` — root worker behind netvm-exec/netvm-chrome (allowlisted; enters netns, fixes DNS, drops privs).
|
- `bin/netvm-enter.sh` — root worker behind netvm-exec/netvm-chrome (allowlisted; enters netns, fixes DNS, drops privs).
|
||||||
- `bin/netvm-chrome.sh <profile> [url]` — launch a chrome-box profile in its netns (CDP on by default; --headless for agents).
|
- `bin/netvm-chrome.sh <profile> [url]` — launch a chrome-box profile in its netns (CDP on by default; --headless for agents; --contained adds a bwrap fs jail inside the netns: profile home only + vault read-only at /tmp/vault, Chromium sandbox stays on).
|
||||||
|
- `bin/netvm-proton.sh <profile> -- <args>` — run proton-cli as the profile's Proton identity (1:1:1 profile = node = warp identity = proton-cli profile) inside the netns + bwrap jail (config dir + static binary only, PROTON_NO_INPUT=1). Human creates the session once: `proton-cli -p <profile> account login`.
|
||||||
- `bin/netvm-cdp.sh <profile>` — print the CDP endpoint + SSH forward.
|
- `bin/netvm-cdp.sh <profile>` — print the CDP endpoint + SSH forward.
|
||||||
- `bin/netvm-cdp-relay.py` — veth-IP→loopback TCP relay for CDP (pidfile-supervised).
|
- `bin/netvm-cdp-relay.py` — veth-IP→loopback TCP relay for CDP (pidfile-supervised).
|
||||||
- `bin/netvm-names.sh` — shared naming: netns, hashed iface tags, veth subnet, CDP port.
|
- `bin/netvm-names.sh` — shared naming: netns, hashed iface tags, veth subnet, CDP port.
|
||||||
|
|||||||
Executable
+654
@@ -0,0 +1,654 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""agent-cognitive-probe.py — Real-time Cognitive Sensing and Agent Menu Navigation.
|
||||||
|
|
||||||
|
Directly probes Cloud Muse / Hatch browser runtime via CDP:
|
||||||
|
1. Passive Cognitive Sensing (zero-click):
|
||||||
|
- Token streaming / generation state (stop button presence)
|
||||||
|
- Typing / thinking indicators
|
||||||
|
- Status text displayed under/beside avatar (Connected, Thinking, Working)
|
||||||
|
- Active context (Main chat vs Side chats with thread titles & snippets)
|
||||||
|
- Input wait / parked approval detection
|
||||||
|
2. Active Profile Menu Navigation:
|
||||||
|
- Status panel sliding surface navigation
|
||||||
|
- Tabs: Activity (tasks & processes), Upcoming (timers & recurring cron loops),
|
||||||
|
Approvals, and Identity.
|
||||||
|
3. Cognitive Lock Gate:
|
||||||
|
- Protects single-threaded thought process from interruptions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
# Node to pinned CDP port mapping
|
||||||
|
NODE_CDP_PORTS = {
|
||||||
|
"muse": 9410,
|
||||||
|
"pip": 9420,
|
||||||
|
"646": 9430,
|
||||||
|
"opm": 9440,
|
||||||
|
"def": 9450,
|
||||||
|
"dev": 9455,
|
||||||
|
"muse-main": 9410,
|
||||||
|
}
|
||||||
|
|
||||||
|
REMOTE_HOST = "100.123.153.75" # bl control node
|
||||||
|
|
||||||
|
|
||||||
|
def is_running_on_bl():
|
||||||
|
"""Detect if we are running locally on bl or on tp/remote."""
|
||||||
|
try:
|
||||||
|
import socket
|
||||||
|
hn = socket.gethostname().lower()
|
||||||
|
if "bl" in hn:
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# Check if network namespaces exist locally
|
||||||
|
return os.path.exists("/var/run/netns/warp-muse") or os.path.exists("/run/netns/warp-muse")
|
||||||
|
|
||||||
|
|
||||||
|
def run_cdp_eval_inside_netns(node, js_code, timeout=8):
|
||||||
|
"""Executes a JS snippet against the node's browser via CDP inside its netns."""
|
||||||
|
port = NODE_CDP_PORTS.get(node)
|
||||||
|
if not port:
|
||||||
|
return {"error": f"Unknown node '{node}'"}
|
||||||
|
|
||||||
|
# Python runner script to execute inside the target environment
|
||||||
|
runner_code = f'''
|
||||||
|
import json, sys, urllib.request
|
||||||
|
try:
|
||||||
|
import websocket
|
||||||
|
except ImportError:
|
||||||
|
print(json.dumps({{"error": "websocket package missing"}}))
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen("http://127.0.0.1:{port}/json/list", timeout=3) as r:
|
||||||
|
targets = json.load(r)
|
||||||
|
pages = [t for t in targets if t.get("type") == "page"]
|
||||||
|
if not pages:
|
||||||
|
print(json.dumps({{"error": "No active page target"}}))
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
ws_url = pages[0]["webSocketDebuggerUrl"]
|
||||||
|
ws = websocket.create_connection(ws_url, timeout={timeout})
|
||||||
|
|
||||||
|
ws.send(json.dumps({{
|
||||||
|
"id": 1,
|
||||||
|
"method": "Runtime.evaluate",
|
||||||
|
"params": {{
|
||||||
|
"expression": {json.dumps(js_code)},
|
||||||
|
"returnByValue": True,
|
||||||
|
"awaitPromise": True
|
||||||
|
}}
|
||||||
|
}}))
|
||||||
|
|
||||||
|
res = None
|
||||||
|
for _ in range(30):
|
||||||
|
msg = json.loads(ws.recv())
|
||||||
|
if msg.get("id") == 1:
|
||||||
|
res = msg.get("result", {{}}).get("result", {{}}).get("value")
|
||||||
|
break
|
||||||
|
|
||||||
|
print(json.dumps({{"ok": True, "value": res}}))
|
||||||
|
except Exception as e:
|
||||||
|
print(json.dumps({{"error": str(e)}}))
|
||||||
|
'''
|
||||||
|
|
||||||
|
if is_running_on_bl():
|
||||||
|
cmd = ["sudo", "ip", "netns", "exec", f"warp-{node}", "python3", "-c", runner_code]
|
||||||
|
else:
|
||||||
|
# Wrap via ssh to bl
|
||||||
|
# Use python3 on bl directly executing inside netns
|
||||||
|
remote_cmd = f"sudo ip netns exec warp-{node} python3 -c {subprocess.list2cmdline([runner_code])}"
|
||||||
|
cmd = ["ssh", "-q", f"super@{REMOTE_HOST}", remote_cmd]
|
||||||
|
|
||||||
|
try:
|
||||||
|
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout + 5)
|
||||||
|
if proc.returncode != 0 and not proc.stdout:
|
||||||
|
return {"error": proc.stderr.strip() or f"Process exited with {proc.returncode}"}
|
||||||
|
|
||||||
|
# Parse output line that contains valid json
|
||||||
|
for line in proc.stdout.strip().splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if line.startswith("{") and line.endswith("}"):
|
||||||
|
try:
|
||||||
|
data = json.loads(line)
|
||||||
|
if "ok" in data:
|
||||||
|
return data["value"]
|
||||||
|
if "error" in data:
|
||||||
|
return {"error": data["error"]}
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
return {"error": proc.stdout.strip() or proc.stderr.strip()}
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return {"error": "CDP probe timed out"}
|
||||||
|
except Exception as e:
|
||||||
|
return {"error": str(e)}
|
||||||
|
|
||||||
|
|
||||||
|
JS_PASSIVE_COGNITIVE = """(() => {
|
||||||
|
// 1. Generation & Thinking signals
|
||||||
|
const stopBtn = document.querySelector('[data-testid="hatch-composer-stop-button"]');
|
||||||
|
const isGenerating = !!stopBtn;
|
||||||
|
|
||||||
|
const typingEl = document.querySelector('[data-testid="hatch-chat-typing-indicator"]');
|
||||||
|
const isTyping = !!typingEl && typingEl.textContent.trim().length > 0;
|
||||||
|
|
||||||
|
// 2. Avatar / Status text
|
||||||
|
const statusTextEl = document.querySelector('.group\\\\/status-avatar span, [class*="status-avatar"] span, span[class*="text-body-status"]');
|
||||||
|
const avatarStatus = statusTextEl ? (statusTextEl.innerText || '').trim() : '';
|
||||||
|
|
||||||
|
// 3. Thread context & active URL
|
||||||
|
const url = window.location.href;
|
||||||
|
const isMainChat = url === 'https://muse.ai/' || url.endsWith('/thread/new');
|
||||||
|
const pageTitle = document.title;
|
||||||
|
|
||||||
|
// 4. Side chats overview
|
||||||
|
const sideRows = Array.from(document.querySelectorAll('[data-testid="hatch-thread-row"]')).map(r => {
|
||||||
|
const text = (r.innerText || '').trim().replace(/\\\\n+/g, ' — ');
|
||||||
|
return text;
|
||||||
|
}).slice(0, 5);
|
||||||
|
|
||||||
|
// 5. Input wait / Parked prompt cards
|
||||||
|
// Detect buttons asking for approval / input in the message flow
|
||||||
|
const actionBtns = Array.from(document.querySelectorAll('div[data-message-id] button')).map(b => (b.innerText || '').trim()).filter(t => /approve|confirm|proceed|resume|start|allow/i.test(t));
|
||||||
|
const isInputWait = actionBtns.length > 0 || /asking for input|pending approval/i.test(document.body.innerText.slice(-600));
|
||||||
|
|
||||||
|
// 6. Status panel state
|
||||||
|
const panel = document.querySelector('[data-testid="hatch-status-panel-sliding-surface"]');
|
||||||
|
const panelOpen = !!panel && panel.getBoundingClientRect().width > 0;
|
||||||
|
|
||||||
|
return {
|
||||||
|
url: url,
|
||||||
|
title: pageTitle,
|
||||||
|
is_main_chat: isMainChat,
|
||||||
|
is_generating: isGenerating,
|
||||||
|
is_typing: isTyping,
|
||||||
|
avatar_status: avatarStatus,
|
||||||
|
is_input_wait: isInputWait,
|
||||||
|
pending_actions: actionBtns,
|
||||||
|
panel_open: panelOpen,
|
||||||
|
side_chats: sideRows
|
||||||
|
};
|
||||||
|
})()"""
|
||||||
|
|
||||||
|
|
||||||
|
def get_passive_cognitive_state(node):
|
||||||
|
"""Gathers passive cognitive signals without altering UI state."""
|
||||||
|
if not is_running_on_bl():
|
||||||
|
try:
|
||||||
|
cmd = ["ssh", "-q", "-o", "ConnectTimeout=5", f"super@{REMOTE_HOST}",
|
||||||
|
f"python3 /home/super/Projects/NetVM/bin/agent-cognitive-probe.py status {node} --json"]
|
||||||
|
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||||
|
if proc.returncode == 0 and proc.stdout.strip():
|
||||||
|
data = json.loads(proc.stdout.strip())
|
||||||
|
if isinstance(data, list) and len(data) > 0:
|
||||||
|
return data[0]
|
||||||
|
elif isinstance(data, dict):
|
||||||
|
return data
|
||||||
|
except Exception as e:
|
||||||
|
return {
|
||||||
|
"node": node,
|
||||||
|
"status": "DARK",
|
||||||
|
"error": f"Remote delegation failed: {e}",
|
||||||
|
"cognitive_lock": False,
|
||||||
|
"lock_reason": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
raw = run_cdp_eval_inside_netns(node, JS_PASSIVE_COGNITIVE)
|
||||||
|
if not isinstance(raw, dict) or "error" in raw:
|
||||||
|
return {
|
||||||
|
"node": node,
|
||||||
|
"status": "DARK",
|
||||||
|
"error": raw.get("error", "Unknown error") if isinstance(raw, dict) else str(raw),
|
||||||
|
"cognitive_lock": False,
|
||||||
|
"lock_reason": None,
|
||||||
|
}
|
||||||
|
|
||||||
|
is_generating = raw.get("is_generating", False)
|
||||||
|
is_typing = raw.get("is_typing", False)
|
||||||
|
is_input_wait = raw.get("is_input_wait", False)
|
||||||
|
avatar_status = raw.get("avatar_status", "")
|
||||||
|
is_main = raw.get("is_main_chat", True)
|
||||||
|
|
||||||
|
# Determine synthesized cognitive state
|
||||||
|
if is_generating or is_typing or "thinking" in avatar_status.lower():
|
||||||
|
state = "THINKING"
|
||||||
|
locked = True
|
||||||
|
reason = "Agent is actively generating tokens / thinking (stop button active)"
|
||||||
|
elif is_input_wait:
|
||||||
|
state = "INPUT_WAIT"
|
||||||
|
locked = True
|
||||||
|
reason = "Agent is waiting for operator or system input on a parked prompt"
|
||||||
|
elif "working" in avatar_status.lower() or "making" in avatar_status.lower():
|
||||||
|
state = "WORKING"
|
||||||
|
locked = True
|
||||||
|
reason = f"Avatar status indicates work in progress: '{avatar_status}'"
|
||||||
|
elif not is_main:
|
||||||
|
state = "SIDECHAT_IDLE"
|
||||||
|
locked = False
|
||||||
|
reason = None
|
||||||
|
else:
|
||||||
|
state = "IDLE"
|
||||||
|
locked = False
|
||||||
|
reason = None
|
||||||
|
|
||||||
|
return {
|
||||||
|
"node": node,
|
||||||
|
"status": state,
|
||||||
|
"cognitive_lock": locked,
|
||||||
|
"lock_reason": reason,
|
||||||
|
"details": raw,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
JS_NAVIGATE_MENU_TEMPLATE = """(async () => {
|
||||||
|
// 1. Ensure panel is open
|
||||||
|
let panel = document.querySelector('[data-testid="hatch-status-panel-sliding-surface"]');
|
||||||
|
if (!panel) {
|
||||||
|
// Try clicking avatar or trigger
|
||||||
|
const avatarBtn = document.querySelector('[role="img"][aria-label*="avatar"], button[aria-label*="avatar"], .group\\\\/status-avatar');
|
||||||
|
if (avatarBtn) avatarBtn.click();
|
||||||
|
await new Promise(r => setTimeout(r, 350));
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Click requested tab if specified
|
||||||
|
const targetTab = "%(tab)s";
|
||||||
|
if (targetTab && targetTab !== "all") {
|
||||||
|
const btn = document.querySelector('button[aria-label="' + targetTab + '"]');
|
||||||
|
if (btn) {
|
||||||
|
btn.click();
|
||||||
|
await new Promise(r => setTimeout(r, 400));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
panel = document.querySelector('[data-testid="hatch-status-panel-sliding-surface"]');
|
||||||
|
if (!panel) return {error: "Status panel not rendered"};
|
||||||
|
|
||||||
|
// Read full text and structural items
|
||||||
|
const rawText = panel.innerText || '';
|
||||||
|
const lines = rawText.split('\\n').map(s => s.trim()).filter(Boolean);
|
||||||
|
|
||||||
|
// Extract activity / timer items
|
||||||
|
const items = [];
|
||||||
|
const buttons = Array.from(panel.querySelectorAll('button')).filter(b => !b.getAttribute('aria-label') && (b.innerText || '').length > 0);
|
||||||
|
buttons.forEach(b => {
|
||||||
|
const text = (b.innerText || '').trim();
|
||||||
|
const parts = text.split('\\n').map(s => s.trim()).filter(Boolean);
|
||||||
|
if (parts.length >= 2) {
|
||||||
|
items.push({
|
||||||
|
title: parts[0],
|
||||||
|
detail: parts[1],
|
||||||
|
time: parts.length > 2 ? parts[2] : null
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
raw_text: rawText,
|
||||||
|
lines: lines,
|
||||||
|
items: items
|
||||||
|
};
|
||||||
|
})()"""
|
||||||
|
|
||||||
|
|
||||||
|
def get_agent_menu(node, tab="all"):
|
||||||
|
"""Navigates and extracts data from the agent profile / status panel."""
|
||||||
|
if not is_running_on_bl():
|
||||||
|
try:
|
||||||
|
cmd = ["ssh", "-q", "-o", "ConnectTimeout=5", f"super@{REMOTE_HOST}",
|
||||||
|
f"python3 /home/super/Projects/NetVM/bin/agent-cognitive-probe.py menu {node} {tab} --json"]
|
||||||
|
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=15)
|
||||||
|
if proc.returncode == 0 and proc.stdout.strip():
|
||||||
|
return json.loads(proc.stdout.strip())
|
||||||
|
except Exception as e:
|
||||||
|
return {
|
||||||
|
"node": node,
|
||||||
|
"tab": tab,
|
||||||
|
"data": {"error": f"Remote delegation failed: {e}"}
|
||||||
|
}
|
||||||
|
|
||||||
|
tab_map = {
|
||||||
|
"activity": "Activity",
|
||||||
|
"upcoming": "Upcoming",
|
||||||
|
"approvals": "Approvals",
|
||||||
|
"identity": "Identity",
|
||||||
|
"all": "all",
|
||||||
|
}
|
||||||
|
target_tab = tab_map.get(tab.lower(), "Activity")
|
||||||
|
|
||||||
|
# If all is requested, gather activity, upcoming, and approvals
|
||||||
|
if target_tab == "all":
|
||||||
|
result = {}
|
||||||
|
for sub_tab in ["Activity", "Upcoming", "Approvals"]:
|
||||||
|
js = JS_NAVIGATE_MENU_TEMPLATE % {"tab": sub_tab}
|
||||||
|
tab_res = run_cdp_eval_inside_netns(node, js)
|
||||||
|
result[sub_tab.lower()] = tab_res
|
||||||
|
return {
|
||||||
|
"node": node,
|
||||||
|
"menu": result
|
||||||
|
}
|
||||||
|
|
||||||
|
js = JS_NAVIGATE_MENU_TEMPLATE % {"tab": target_tab}
|
||||||
|
res = run_cdp_eval_inside_netns(node, js)
|
||||||
|
return {
|
||||||
|
"node": node,
|
||||||
|
"tab": target_tab,
|
||||||
|
"data": res
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
JS_LIVE_SCREEN = """(() => {
|
||||||
|
const ps = Array.from(document.querySelectorAll('p')).map(p => (p.innerText || '').trim()).filter(Boolean);
|
||||||
|
const actionBtns = Array.from(document.querySelectorAll('div[data-message-id] button, [role="log"] button, div[role="status"] button')).map(b => (b.innerText || '').trim()).filter(Boolean);
|
||||||
|
const stopBtn = !!document.querySelector('[data-testid="hatch-composer-stop-button"]');
|
||||||
|
const typing = !!document.querySelector('[data-testid="hatch-chat-typing-indicator"]:not(:empty)');
|
||||||
|
const statusTextEl = document.querySelector('.group\\\\/status-avatar span, [class*="status-avatar"] span, span[class*="text-body-status"]');
|
||||||
|
const avatarStatus = statusTextEl ? (statusTextEl.innerText || '').trim() : '';
|
||||||
|
|
||||||
|
return {
|
||||||
|
title: document.title,
|
||||||
|
url: window.location.href,
|
||||||
|
is_main_chat: window.location.href === 'https://muse.ai/' || window.location.href.endsWith('/thread/new'),
|
||||||
|
is_generating: stopBtn,
|
||||||
|
is_typing: typing,
|
||||||
|
avatar_status: avatarStatus,
|
||||||
|
recent_paragraphs: ps.slice(-8),
|
||||||
|
action_buttons: actionBtns.filter(t => /approve|confirm|proceed|resume|start|allow|review/i.test(t))
|
||||||
|
};
|
||||||
|
})()"""
|
||||||
|
|
||||||
|
|
||||||
|
def get_agent_live_screen(node: str) -> dict:
|
||||||
|
"""Extracts live active chat text, thoughts, prompt blocks, and threads."""
|
||||||
|
if not is_running_on_bl():
|
||||||
|
try:
|
||||||
|
cmd = ["ssh", "-q", "-o", "ConnectTimeout=5", f"super@{REMOTE_HOST}",
|
||||||
|
f"python3 /home/super/Projects/NetVM/bin/agent-cognitive-probe.py read {node} --json"]
|
||||||
|
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=15)
|
||||||
|
if proc.returncode == 0 and proc.stdout.strip():
|
||||||
|
return json.loads(proc.stdout.strip())
|
||||||
|
except Exception as e:
|
||||||
|
return {"node": node, "error": f"Remote delegation failed: {e}"}
|
||||||
|
|
||||||
|
screen = run_cdp_eval_inside_netns(node, JS_LIVE_SCREEN)
|
||||||
|
if not isinstance(screen, dict) or "error" in screen:
|
||||||
|
return {"node": node, "error": screen.get("error", "Failed to inspect screen") if isinstance(screen, dict) else str(screen)}
|
||||||
|
|
||||||
|
sidechats = []
|
||||||
|
try:
|
||||||
|
cli_cmd = ["/home/super/Projects/NetVM/bin/muse-cli-node", node, "threads"]
|
||||||
|
proc = subprocess.run(cli_cmd, capture_output=True, text=True, timeout=8)
|
||||||
|
if proc.returncode == 0 and proc.stdout.strip():
|
||||||
|
threads_data = json.loads(proc.stdout.strip())
|
||||||
|
if isinstance(threads_data, list):
|
||||||
|
sidechats = threads_data[:8]
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return {
|
||||||
|
"node": node,
|
||||||
|
"screen": screen,
|
||||||
|
"sidechats": sidechats
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
JS_NAVIGATE_MAIN_CHAT = """(() => {
|
||||||
|
try {
|
||||||
|
const url = window.location.href;
|
||||||
|
if (url === 'https://muse.ai/' || url === 'https://muse.ai/thread/new') {
|
||||||
|
return { ok: true, already_main: true };
|
||||||
|
}
|
||||||
|
const candidates = Array.from(document.querySelectorAll('a, button, [role="button"], [data-testid]'));
|
||||||
|
const mainBtn = candidates.find(b => {
|
||||||
|
const t = (b.innerText || '').trim().toLowerCase();
|
||||||
|
return t === 'main chat' || b.getAttribute('aria-label') === 'Main chat' || b.getAttribute('data-testid') === 'hatch-sidebar-main-chat';
|
||||||
|
});
|
||||||
|
if (mainBtn) {
|
||||||
|
mainBtn.click();
|
||||||
|
return { ok: true, method: 'click' };
|
||||||
|
}
|
||||||
|
window.location.href = 'https://muse.ai/';
|
||||||
|
return { ok: true, method: 'navigate' };
|
||||||
|
} catch (e) {
|
||||||
|
return { error: String(e) };
|
||||||
|
}
|
||||||
|
})()"""
|
||||||
|
|
||||||
|
|
||||||
|
def navigate_to_main_chat(node: str) -> dict:
|
||||||
|
"""Navigates the agent browser session back to Main Chat (https://muse.ai/)."""
|
||||||
|
if not is_running_on_bl():
|
||||||
|
try:
|
||||||
|
cmd = ["ssh", "-q", "-o", "ConnectTimeout=5", f"super@{REMOTE_HOST}",
|
||||||
|
f"python3 /home/super/Projects/NetVM/bin/agent-cognitive-probe.py nav-main {node} --json"]
|
||||||
|
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||||
|
if proc.returncode == 0 and proc.stdout.strip():
|
||||||
|
return json.loads(proc.stdout.strip())
|
||||||
|
except Exception as e:
|
||||||
|
return {"error": str(e)}
|
||||||
|
|
||||||
|
return run_cdp_eval_inside_netns(node, JS_NAVIGATE_MAIN_CHAT)
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_nav_main(args):
|
||||||
|
node = getattr(args, "agent", None) or getattr(args, "node", None)
|
||||||
|
res = navigate_to_main_chat(node)
|
||||||
|
if getattr(args, "json", False):
|
||||||
|
print(json.dumps(res, indent=2))
|
||||||
|
else:
|
||||||
|
if isinstance(res, dict) and res.get("ok"):
|
||||||
|
m = res.get("method") or ("already in main chat" if res.get("already_main") else "default")
|
||||||
|
print(f"✓ Refocused agent '{node}' to Main Chat ({m})")
|
||||||
|
else:
|
||||||
|
err = res.get("error") if isinstance(res, dict) else str(res)
|
||||||
|
print(f"❌ Failed to refocus '{node}' to Main Chat: {err}")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_read(args):
|
||||||
|
node = getattr(args, "agent", None) or getattr(args, "node", None)
|
||||||
|
data = get_agent_live_screen(node)
|
||||||
|
if getattr(args, "json", False):
|
||||||
|
print(json.dumps(data, indent=2))
|
||||||
|
return
|
||||||
|
|
||||||
|
if "error" in data:
|
||||||
|
print(f"\n❌ Error inspecting screen for {node}: {data['error']}\n")
|
||||||
|
return
|
||||||
|
|
||||||
|
screen = data.get("screen", {})
|
||||||
|
sidechats = data.get("sidechats", [])
|
||||||
|
|
||||||
|
url = screen.get("url", "")
|
||||||
|
mode = "Main Chat" if screen.get("is_main_chat") else "Side Chat"
|
||||||
|
title = screen.get("title", "")
|
||||||
|
avatar = screen.get("avatar_status") or "Connected"
|
||||||
|
gen = "🧠 GENERATING" if screen.get("is_generating") else ("💭 TYPING" if screen.get("is_typing") else "🟢 SETTLED")
|
||||||
|
|
||||||
|
print(f"\n=== LIVE THOUGHT STREAM & ACTIVE CHAT: {node.upper()} ===")
|
||||||
|
print(f"Context: {mode} ({url})")
|
||||||
|
print(f"Title: {title}")
|
||||||
|
print(f"Status: {avatar} | State: {gen}")
|
||||||
|
|
||||||
|
paragraphs = screen.get("recent_paragraphs", [])
|
||||||
|
if paragraphs:
|
||||||
|
print("\n--- ACTIVE CONVERSATION & THOUGHT PARAGRAPHS ---")
|
||||||
|
for p in paragraphs:
|
||||||
|
print(f" • {p}\n")
|
||||||
|
else:
|
||||||
|
print("\n (No text paragraphs visible in current viewport)")
|
||||||
|
|
||||||
|
actions = screen.get("action_buttons", [])
|
||||||
|
if actions:
|
||||||
|
print("--- PENDING ACTION CARDS / APPROVAL BUTTONS ---")
|
||||||
|
for a in actions:
|
||||||
|
print(f" ⚠️ [PROMPT ACTION] {a}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
if sidechats:
|
||||||
|
print("--- RECENT SIDE CHATS & TOPICS ---")
|
||||||
|
for sc in sidechats:
|
||||||
|
sid = sc.get("session_id", "")[:8]
|
||||||
|
stitle = sc.get("title") or "(Untitled sidechat)"
|
||||||
|
upd = sc.get("updated", "")
|
||||||
|
print(f" • [{sid}] {stitle} ({upd})")
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
def format_cognitive_badge(status):
|
||||||
|
badges = {
|
||||||
|
"IDLE": "🟢 IDLE",
|
||||||
|
"SIDECHAT_IDLE": "💬 SIDE_IDLE",
|
||||||
|
"THINKING": "🧠 THINKING",
|
||||||
|
"INPUT_WAIT": "⏸️ INPUT_WAIT",
|
||||||
|
"BUSY_SIDECHAT": "💬 SIDECHAT",
|
||||||
|
"WORKING": "⚙️ WORKING",
|
||||||
|
"DARK": "⚫ DARK",
|
||||||
|
}
|
||||||
|
return badges.get(status, f"❓ {status}")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_status(args):
|
||||||
|
nodes = getattr(args, "agents", None) or getattr(args, "nodes", None) or ["muse", "pip", "646", "opm", "dev", "def"]
|
||||||
|
results = []
|
||||||
|
for n in nodes:
|
||||||
|
results.append(get_passive_cognitive_state(n))
|
||||||
|
|
||||||
|
if getattr(args, "json", False):
|
||||||
|
print(json.dumps(results, indent=2))
|
||||||
|
return
|
||||||
|
|
||||||
|
print("\n=== AGENT COGNITIVE SENSOR (LIVE DOM PROBE) ===")
|
||||||
|
print(f"{'AGENT':<10} {'COGNITIVE STATE':<18} {'LOCK':<8} {'UNDER-AVATAR':<15} {'DETAILS / CURRENT THOUGHT':<40}")
|
||||||
|
print("-" * 95)
|
||||||
|
for r in results:
|
||||||
|
node = r["node"]
|
||||||
|
status = r["status"]
|
||||||
|
badge = format_cognitive_badge(status)
|
||||||
|
locked = "LOCKED" if r.get("cognitive_lock") else "OPEN"
|
||||||
|
det = r.get("details", {})
|
||||||
|
avatar_status = det.get("avatar_status", "-")
|
||||||
|
reason = r.get("lock_reason") or det.get("title", "Settled")
|
||||||
|
if status == "DARK":
|
||||||
|
reason = r.get("error", "CDP unreachable")
|
||||||
|
avatar_status = "DARK"
|
||||||
|
print(f"{node:<10} {badge:<18} {locked:<8} {avatar_status:<15} {reason[:40]:<40}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_menu(args):
|
||||||
|
node = getattr(args, "agent", None) or getattr(args, "node", None)
|
||||||
|
tab = getattr(args, "tab", "activity") or "activity"
|
||||||
|
data = get_agent_menu(node, tab)
|
||||||
|
|
||||||
|
if getattr(args, "json", False):
|
||||||
|
print(json.dumps(data, indent=2))
|
||||||
|
return
|
||||||
|
|
||||||
|
print(f"\n=== AGENT MENU: {node.upper()} (TAB: {tab.upper()}) ===")
|
||||||
|
if tab.lower() == "all":
|
||||||
|
menu = data.get("menu", {})
|
||||||
|
for tname, tdata in menu.items():
|
||||||
|
print(f"\n--- {tname.upper()} ---")
|
||||||
|
if isinstance(tdata, dict) and "error" in tdata:
|
||||||
|
print(f" Error: {tdata['error']}")
|
||||||
|
elif isinstance(tdata, dict):
|
||||||
|
items = tdata.get("items", [])
|
||||||
|
if items:
|
||||||
|
for it in items:
|
||||||
|
t_str = f" [{it['time']}]" if it.get("time") else ""
|
||||||
|
print(f" • {it['title']}: {it['detail']}{t_str}")
|
||||||
|
else:
|
||||||
|
raw = tdata.get("raw_text", "")
|
||||||
|
for line in raw.split("\n"):
|
||||||
|
if line.strip():
|
||||||
|
print(f" {line.strip()}")
|
||||||
|
print()
|
||||||
|
return
|
||||||
|
|
||||||
|
tdata = data.get("data", {})
|
||||||
|
if isinstance(tdata, dict) and "error" in tdata:
|
||||||
|
print(f" Error: {tdata['error']}")
|
||||||
|
elif isinstance(tdata, dict):
|
||||||
|
items = tdata.get("items", [])
|
||||||
|
if items:
|
||||||
|
for it in items:
|
||||||
|
t_str = f" [{it['time']}]" if it.get("time") else ""
|
||||||
|
print(f" • {it['title']}: {it['detail']}{t_str}")
|
||||||
|
else:
|
||||||
|
raw = tdata.get("raw_text", "")
|
||||||
|
for line in raw.split("\n"):
|
||||||
|
if line.strip():
|
||||||
|
print(f" {line.strip()}")
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_lock(args):
|
||||||
|
node = getattr(args, "agent", None) or getattr(args, "node", None)
|
||||||
|
state = get_passive_cognitive_state(node)
|
||||||
|
if args.json:
|
||||||
|
print(json.dumps(state, indent=2))
|
||||||
|
else:
|
||||||
|
if state.get("cognitive_lock"):
|
||||||
|
print(f"🔴 COGNITIVE LOCK ENGAGED on '{node}' ({state['status']}): {state.get('lock_reason')}")
|
||||||
|
else:
|
||||||
|
print(f"🟢 COGNITIVELY IDLE: Agent '{node}' is free to receive new work without interruption.")
|
||||||
|
|
||||||
|
if state.get("cognitive_lock") and not getattr(args, "force", False):
|
||||||
|
sys.exit(1)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Real-time Cognitive Sensing and Agent Menu Navigation")
|
||||||
|
sub = parser.add_subparsers(dest="command")
|
||||||
|
|
||||||
|
p_status = sub.add_parser("status", help="Show cognitive state for all or selected agents")
|
||||||
|
p_status.add_argument("nodes", nargs="*", help="Optional agent names")
|
||||||
|
p_status.add_argument("--json", action="store_true", help="Output JSON")
|
||||||
|
|
||||||
|
p_menu = sub.add_parser("menu", help="Navigate agent profile menu (tasks, timers, approvals, identity)")
|
||||||
|
p_menu.add_argument("node", help="Agent name (muse, pip, 646, opm, dev, def)")
|
||||||
|
p_menu.add_argument("tab", nargs="?", default="activity", choices=["activity", "upcoming", "approvals", "identity", "all"], help="Menu tab to view")
|
||||||
|
p_menu.add_argument("--json", action="store_true", help="Output JSON")
|
||||||
|
|
||||||
|
p_lock = sub.add_parser("lock", help="Check cognitive lock before dispatching work")
|
||||||
|
p_lock.add_argument("node", help="Agent name")
|
||||||
|
p_lock.add_argument("--force", action="store_true", help="Bypass lock check")
|
||||||
|
p_lock.add_argument("--json", action="store_true", help="Output JSON")
|
||||||
|
|
||||||
|
p_read = sub.add_parser("read", help="Extract live active chat text, thought stream, and side chats")
|
||||||
|
p_read.add_argument("node", help="Agent name")
|
||||||
|
p_read.add_argument("--json", action="store_true", help="Output JSON")
|
||||||
|
|
||||||
|
p_nav = sub.add_parser("nav-main", help="Navigate agent browser session back to Main Chat")
|
||||||
|
p_nav.add_argument("node", help="Agent name")
|
||||||
|
p_nav.add_argument("--json", action="store_true", help="Output JSON")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
if not args.command:
|
||||||
|
# Default to status
|
||||||
|
args.nodes = []
|
||||||
|
args.json = False
|
||||||
|
cmd_status(args)
|
||||||
|
return
|
||||||
|
|
||||||
|
if args.command == "status":
|
||||||
|
cmd_status(args)
|
||||||
|
elif args.command == "menu":
|
||||||
|
cmd_menu(args)
|
||||||
|
elif args.command == "read":
|
||||||
|
cmd_read(args)
|
||||||
|
elif args.command == "lock":
|
||||||
|
cmd_lock(args)
|
||||||
|
elif args.command == "nav-main":
|
||||||
|
cmd_nav_main(args)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Symlink
+1
@@ -0,0 +1 @@
|
|||||||
|
agent-cognitive-probe.py
|
||||||
Executable
+1611
File diff suppressed because it is too large
Load Diff
Executable
+424
@@ -0,0 +1,424 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""box-readback-loopback.py — Synchronous Readback Gate & Cognitive-Aware True Loopback Engine.
|
||||||
|
|
||||||
|
Architecture:
|
||||||
|
1. Readback Gate:
|
||||||
|
- Synchronously awaits agent confirmation (up to 45s) after task dispatch.
|
||||||
|
- Validates via Hybrid Tag ([READBACK] Ticket #...) + Semantic Fallback.
|
||||||
|
- Marks ticket 'in-progress' upon confirmation; marks 'blocked' and releases agent on timeout.
|
||||||
|
2. Cognitive-Aware True Loopbacks:
|
||||||
|
- Zero Token Burn: 100% silent while git commits or PRs are progressing.
|
||||||
|
- Cognitive Guard: Defer loopbacks while agent is THINKING / GENERATING.
|
||||||
|
- Dual-Layer Escalation:
|
||||||
|
* 15m inactive: Tier 1 non-intrusive Gitea ticket comment (@agent inquiry).
|
||||||
|
* 45m inactive: Tier 2 direct chat DM escalation (muse-cli-node send).
|
||||||
|
* 90m inactive: Tier 3 failure escalation (mark 'blocked', alert #lobby, release agent).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# Local imports
|
||||||
|
try:
|
||||||
|
import agent_cognitive_probe as acp
|
||||||
|
except ImportError:
|
||||||
|
acp = None
|
||||||
|
|
||||||
|
REMOTE_HOST = "100.123.153.75" # bl control node
|
||||||
|
DEFAULT_GITEA_URL = "https://tea.muse-dev.online"
|
||||||
|
LOOPBACK_STATE_FILE = Path("/tmp/box-loopback-state.json")
|
||||||
|
|
||||||
|
|
||||||
|
def is_running_on_bl():
|
||||||
|
try:
|
||||||
|
hn = socket.gethostname().lower()
|
||||||
|
if "bl" in hn:
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return os.path.exists("/var/run/netns/warp-muse") or os.path.exists("/run/netns/warp-muse")
|
||||||
|
|
||||||
|
|
||||||
|
def get_gitea_token():
|
||||||
|
token = os.environ.get("GITEA_TOKEN", "3c26744525bceaf385aa09737f7e41af613627b6")
|
||||||
|
return token
|
||||||
|
|
||||||
|
|
||||||
|
def gitea_api(endpoint: str, method: str = "GET", data: dict = None):
|
||||||
|
token = get_gitea_token()
|
||||||
|
url = f"{DEFAULT_GITEA_URL}/api/v1{endpoint}"
|
||||||
|
headers = {
|
||||||
|
"Authorization": f"token {token}",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"User-Agent": "Box-Work-CLI/1.0",
|
||||||
|
}
|
||||||
|
payload = json.dumps(data).encode("utf-8") if data else None
|
||||||
|
req = urllib.request.Request(url, data=payload, headers=headers, method=method)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=10) as r:
|
||||||
|
if r.status in (200, 201):
|
||||||
|
return json.loads(r.read().decode())
|
||||||
|
return {"status": r.status}
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
try:
|
||||||
|
return json.loads(e.read().decode())
|
||||||
|
except Exception:
|
||||||
|
return {"error": str(e), "code": e.code}
|
||||||
|
except Exception as e:
|
||||||
|
return {"error": str(e)}
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
# CHAT COMMUNICATION HELPERS
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
|
||||||
|
def send_agent_chat(agent: str, message: str) -> bool:
|
||||||
|
"""Delivers a message directly into the agent's web chat session."""
|
||||||
|
if is_running_on_bl():
|
||||||
|
cmd = ["/home/super/Projects/NetVM/bin/muse-cli-node", agent, "send", message]
|
||||||
|
else:
|
||||||
|
cmd = ["ssh", "-q", f"super@{REMOTE_HOST}",
|
||||||
|
f"/home/super/Projects/NetVM/bin/muse-cli-node {agent} send {subprocess.list2cmdline([message])}"]
|
||||||
|
try:
|
||||||
|
res = subprocess.run(cmd, capture_output=True, text=True, timeout=15)
|
||||||
|
return res.returncode == 0
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def get_agent_history(agent: str, limit: int = 5) -> list:
|
||||||
|
"""Retrieves recent chat messages from the agent's active session."""
|
||||||
|
if is_running_on_bl():
|
||||||
|
cmd = ["/home/super/Projects/NetVM/bin/muse-cli-node", agent, "history", "--limit", str(limit)]
|
||||||
|
else:
|
||||||
|
cmd = ["ssh", "-q", f"super@{REMOTE_HOST}",
|
||||||
|
f"/home/super/Projects/NetVM/bin/muse-cli-node {agent} history --limit {limit}"]
|
||||||
|
try:
|
||||||
|
res = subprocess.run(cmd, capture_output=True, text=True, timeout=15)
|
||||||
|
if res.returncode == 0 and res.stdout.strip():
|
||||||
|
data = json.loads(res.stdout.strip())
|
||||||
|
if isinstance(data, list):
|
||||||
|
return data
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def get_latest_chat_seq(agent: str) -> int:
|
||||||
|
"""Finds the maximum sequence number in the agent's chat history."""
|
||||||
|
history = get_agent_history(agent, limit=3)
|
||||||
|
seqs = [m.get("seq", 0) for m in history if isinstance(m, dict) and "seq" in m]
|
||||||
|
return max(seqs) if seqs else 0
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
# READBACK GATE
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
|
||||||
|
def validate_readback(text: str, issue_num: int, agent: str) -> tuple[bool, str]:
|
||||||
|
"""Validates an agent readback using Hybrid Tag + Semantic Fallback.
|
||||||
|
|
||||||
|
Returns (is_valid, excerpt).
|
||||||
|
"""
|
||||||
|
if not text:
|
||||||
|
return False, ""
|
||||||
|
|
||||||
|
clean_text = text.strip()
|
||||||
|
issue_pattern = rf"#?{issue_num}\b"
|
||||||
|
|
||||||
|
# 1. Strict Tag Match: [READBACK] Ticket #<num> ...
|
||||||
|
if re.search(r"\[READBACK\]", clean_text, re.IGNORECASE) and re.search(issue_pattern, clean_text):
|
||||||
|
snippet = clean_text[:200].replace("\n", " ")
|
||||||
|
return True, snippet
|
||||||
|
|
||||||
|
# 2. Semantic Fallback: Mentions ticket number AND branch/accepted status
|
||||||
|
has_issue = bool(re.search(issue_pattern, clean_text))
|
||||||
|
has_branch_or_ack = bool(re.search(
|
||||||
|
rf"(dev/{agent}/|branch|accepted|working on|confirm|start(ed|ing)|received)",
|
||||||
|
clean_text, re.IGNORECASE
|
||||||
|
))
|
||||||
|
|
||||||
|
if has_issue and has_branch_or_ack:
|
||||||
|
snippet = clean_text[:200].replace("\n", " ")
|
||||||
|
return True, snippet
|
||||||
|
|
||||||
|
return False, ""
|
||||||
|
|
||||||
|
|
||||||
|
def wait_for_readback(agent: str, issue_num: int, initial_seq: int, timeout_s: int = 45, poll_s: float = 3.0) -> dict:
|
||||||
|
"""Synchronously polls for agent readback within timeout_s."""
|
||||||
|
start_time = time.time()
|
||||||
|
deadline = start_time + timeout_s
|
||||||
|
|
||||||
|
while time.time() < deadline:
|
||||||
|
elapsed = int(time.time() - start_time)
|
||||||
|
print(f"\r ⏳ Awaiting Readback from @{agent} ({elapsed}s / {timeout_s}s)...", end="", flush=True)
|
||||||
|
|
||||||
|
history = get_agent_history(agent, limit=4)
|
||||||
|
for msg in history:
|
||||||
|
seq = msg.get("seq", 0)
|
||||||
|
role = msg.get("role", "")
|
||||||
|
text = msg.get("text", "")
|
||||||
|
|
||||||
|
# Only check new assistant messages
|
||||||
|
if seq > initial_seq and role == "assistant":
|
||||||
|
valid, excerpt = validate_readback(text, issue_num, agent)
|
||||||
|
if valid:
|
||||||
|
print()
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"snippet": excerpt,
|
||||||
|
"elapsed": elapsed,
|
||||||
|
"seq": seq,
|
||||||
|
}
|
||||||
|
|
||||||
|
time.sleep(poll_s)
|
||||||
|
|
||||||
|
print()
|
||||||
|
return {
|
||||||
|
"success": False,
|
||||||
|
"timeout": True,
|
||||||
|
"elapsed": timeout_s,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def handle_readback_success(agent: str, issue_num: int, snippet: str):
|
||||||
|
"""Marks ticket in-progress and records confirmation on Gitea."""
|
||||||
|
# Label ticket in-progress
|
||||||
|
gitea_api(f"/repos/super/box/issues/{issue_num}/labels", method="POST", data={"labels": ["in-progress"]})
|
||||||
|
# Post confirmation comment
|
||||||
|
comment_body = f"🤖 **Readback Confirmed** by @{agent}:\n> {snippet}"
|
||||||
|
gitea_api(f"/repos/super/box/issues/{issue_num}/comments", method="POST", data={"body": comment_body})
|
||||||
|
|
||||||
|
|
||||||
|
def handle_readback_timeout(agent: str, issue_num: int, title: str):
|
||||||
|
"""Labels ticket blocked and unassigns agent so they return to IDLE."""
|
||||||
|
# Label ticket blocked
|
||||||
|
gitea_api(f"/repos/super/box/issues/{issue_num}/labels", method="POST", data={"labels": ["blocked"]})
|
||||||
|
# Post explanation comment
|
||||||
|
comment_body = (
|
||||||
|
f"⚠️ **Readback Timeout**: Agent @{agent} did not confirm ticket #{issue_num} "
|
||||||
|
f"within 45 seconds of dispatch. Releasing assignment to prevent deadlocks."
|
||||||
|
)
|
||||||
|
gitea_api(f"/repos/super/box/issues/{issue_num}/comments", method="POST", data={"body": comment_body})
|
||||||
|
# Unassign agent
|
||||||
|
gitea_api(f"/repos/super/box/issues/{issue_num}", method="PATCH", data={"assignees": []})
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
# COGNITIVE-AWARE TRUE LOOPBACK ENGINE
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
|
||||||
|
def load_loopback_state() -> dict:
|
||||||
|
if LOOPBACK_STATE_FILE.exists():
|
||||||
|
try:
|
||||||
|
with open(LOOPBACK_STATE_FILE) as f:
|
||||||
|
return json.load(f)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def save_loopback_state(state: dict):
|
||||||
|
try:
|
||||||
|
with open(LOOPBACK_STATE_FILE, "w") as f:
|
||||||
|
json.dump(state, f, indent=2)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def get_ticket_git_activity(agent: str, issue_num: int) -> datetime | None:
|
||||||
|
"""Checks the latest commit timestamp on the agent's branch dev/<agent>/<issue>-*."""
|
||||||
|
# Check Gitea branches for dev/<agent>/<issue_num>-*
|
||||||
|
branches = gitea_api("/repos/super/box/branches")
|
||||||
|
if isinstance(branches, list):
|
||||||
|
target_prefix = f"dev/{agent}/{issue_num}"
|
||||||
|
for b in branches:
|
||||||
|
name = b.get("name", "")
|
||||||
|
if target_prefix in name:
|
||||||
|
commit = b.get("commit", {})
|
||||||
|
ts_str = commit.get("timestamp")
|
||||||
|
if ts_str:
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def get_ticket_last_activity(issue: dict, agent: str) -> tuple[datetime, str]:
|
||||||
|
"""Finds the most recent activity timestamp (git commit, comment, or issue creation)."""
|
||||||
|
issue_num = issue["number"]
|
||||||
|
latest_dt = datetime.fromisoformat(issue["created_at"].replace("Z", "+00:00"))
|
||||||
|
source = "issue_created"
|
||||||
|
|
||||||
|
# Check comments
|
||||||
|
comments = gitea_api(f"/repos/super/box/issues/{issue_num}/comments")
|
||||||
|
if isinstance(comments, list):
|
||||||
|
for c in comments:
|
||||||
|
c_dt = datetime.fromisoformat(c["created_at"].replace("Z", "+00:00"))
|
||||||
|
if c_dt > latest_dt:
|
||||||
|
latest_dt = c_dt
|
||||||
|
source = "gitea_comment"
|
||||||
|
|
||||||
|
# Check git branch commit
|
||||||
|
git_dt = get_ticket_git_activity(agent, issue_num)
|
||||||
|
if git_dt and git_dt > latest_dt:
|
||||||
|
latest_dt = git_dt
|
||||||
|
source = "git_commit"
|
||||||
|
|
||||||
|
return latest_dt, source
|
||||||
|
|
||||||
|
|
||||||
|
def run_loopback_sweep(dry_run: bool = False, verbose: bool = True) -> list:
|
||||||
|
"""Executes a single sweep of all open assigned tickets according to the 3-tier escalation model."""
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
state = load_loopback_state()
|
||||||
|
actions_taken = []
|
||||||
|
|
||||||
|
issues = gitea_api("/repos/super/box/issues?state=open")
|
||||||
|
if not isinstance(issues, list):
|
||||||
|
if verbose:
|
||||||
|
print("Failed to fetch open issues from Gitea.")
|
||||||
|
return []
|
||||||
|
|
||||||
|
assigned_issues = [i for i in issues if i.get("assignee")]
|
||||||
|
if verbose:
|
||||||
|
print(f"\n=== LOOPBACK SWEEP: {len(assigned_issues)} ACTIVE ASSIGNED TICKETS ({now.strftime('%H:%M:%SZ')}) ===")
|
||||||
|
|
||||||
|
for iss in assigned_issues:
|
||||||
|
issue_num = iss["number"]
|
||||||
|
title = iss.get("title", "")
|
||||||
|
agent = iss["assignee"]["username"]
|
||||||
|
key = str(issue_num)
|
||||||
|
ticket_state = state.get(key, {})
|
||||||
|
|
||||||
|
last_dt, source = get_ticket_last_activity(iss, agent)
|
||||||
|
inactive_s = (now - last_dt).total_seconds()
|
||||||
|
inactive_m = int(inactive_s // 60)
|
||||||
|
|
||||||
|
# Check cognitive state
|
||||||
|
cog = acp.get_passive_cognitive_state(agent) if acp else {"status": "IDLE", "cognitive_lock": False}
|
||||||
|
cog_status = cog.get("status", "IDLE")
|
||||||
|
is_thinking = cog_status == "THINKING" or cog.get("cognitive_lock")
|
||||||
|
|
||||||
|
if verbose:
|
||||||
|
print(f"Ticket #{issue_num} (@{agent}): {inactive_m}m inactive (source: {source}) | Cognitive: {cog_status}")
|
||||||
|
|
||||||
|
# Tier 0: Inactive < 15m or active git commits -> Complete silence
|
||||||
|
if inactive_m < 15 or source == "git_commit":
|
||||||
|
if verbose:
|
||||||
|
print(" 👉 Status: Active or within silent grace period (<15m). No action.")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Check cognitive guard: Defer if agent is thinking/generating
|
||||||
|
if is_thinking and cog_status != "INPUT_WAIT":
|
||||||
|
if verbose:
|
||||||
|
print(f" 🧠 Cognitive Guard: Deferring loopback — @{agent} is currently {cog_status}.")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Tier 1: 15m <= Inactivity < 45m -> Non-intrusive Gitea ticket comment
|
||||||
|
if 15 <= inactive_m < 45:
|
||||||
|
if ticket_state.get("tier1_sent"):
|
||||||
|
if verbose:
|
||||||
|
print(" 👉 Tier 1 comment already dispatched. Waiting for 45m threshold.")
|
||||||
|
continue
|
||||||
|
|
||||||
|
msg = (
|
||||||
|
f"🤖 @{agent} **Loopback Tier 1 Check** ({inactive_m}m elapsed):\n"
|
||||||
|
f"No git commits recorded on feature branch for Ticket #{issue_num}. "
|
||||||
|
f"Are you progressing or blocked? Reply with status or push a commit."
|
||||||
|
)
|
||||||
|
action_desc = f"Tier 1: Posted Gitea comment to #{issue_num} (@{agent})"
|
||||||
|
actions_taken.append(action_desc)
|
||||||
|
|
||||||
|
if not dry_run:
|
||||||
|
gitea_api(f"/repos/super/box/issues/{issue_num}/comments", method="POST", data={"body": msg})
|
||||||
|
ticket_state["tier1_sent"] = now.isoformat()
|
||||||
|
state[key] = ticket_state
|
||||||
|
save_loopback_state(state)
|
||||||
|
if verbose:
|
||||||
|
print(f" ✓ {action_desc}")
|
||||||
|
|
||||||
|
# Tier 2: 45m <= Inactivity < 90m -> Direct Chat DM Escalation
|
||||||
|
elif 45 <= inactive_m < 90:
|
||||||
|
if ticket_state.get("tier2_sent"):
|
||||||
|
if verbose:
|
||||||
|
print(" 👉 Tier 2 chat DM already dispatched. Waiting for 90m threshold.")
|
||||||
|
continue
|
||||||
|
|
||||||
|
chat_msg = (
|
||||||
|
f"[LOOPBACK ALERT] Ticket #{issue_num} ('{title}'): "
|
||||||
|
f"{inactive_m} minutes inactive with no git commits. "
|
||||||
|
f"Please confirm if blocked on tool execution, terminal approvals, or environment."
|
||||||
|
)
|
||||||
|
action_desc = f"Tier 2: Escalated to chat DM for @{agent} on #{issue_num}"
|
||||||
|
actions_taken.append(action_desc)
|
||||||
|
|
||||||
|
if not dry_run:
|
||||||
|
send_agent_chat(agent, chat_msg)
|
||||||
|
gitea_api(f"/repos/super/box/issues/{issue_num}/comments", method="POST", data={
|
||||||
|
"body": f"📣 **Loopback Tier 2 Escalation**: Inactivity reached {inactive_m}m. Sent direct chat DM to @{agent}."
|
||||||
|
})
|
||||||
|
ticket_state["tier2_sent"] = now.isoformat()
|
||||||
|
state[key] = ticket_state
|
||||||
|
save_loopback_state(state)
|
||||||
|
if verbose:
|
||||||
|
print(f" ✓ {action_desc}")
|
||||||
|
|
||||||
|
# Tier 3: Inactivity >= 90m (or unhandled INPUT_WAIT > 15m) -> Fail-closed escalation
|
||||||
|
elif inactive_m >= 90 or (cog_status == "INPUT_WAIT" and inactive_m >= 15):
|
||||||
|
action_desc = f"Tier 3: Ticket #{issue_num} marked BLOCKED; released @{agent} assignment"
|
||||||
|
actions_taken.append(action_desc)
|
||||||
|
|
||||||
|
if not dry_run:
|
||||||
|
# Label blocked
|
||||||
|
gitea_api(f"/repos/super/box/issues/{issue_num}/labels", method="POST", data={"labels": ["blocked"]})
|
||||||
|
# Post failure comment
|
||||||
|
gitea_api(f"/repos/super/box/issues/{issue_num}/comments", method="POST", data={
|
||||||
|
"body": (
|
||||||
|
f"🚨 **Loopback Tier 3 Escalation**: Inactivity reached {inactive_m}m with zero git progress. "
|
||||||
|
f"Ticket marked `blocked` and unassigned from @{agent} for operator intervention."
|
||||||
|
)
|
||||||
|
})
|
||||||
|
# Unassign agent
|
||||||
|
gitea_api(f"/repos/super/box/issues/{issue_num}", method="PATCH", data={"assignees": []})
|
||||||
|
ticket_state["tier3_sent"] = now.isoformat()
|
||||||
|
state[key] = ticket_state
|
||||||
|
save_loopback_state(state)
|
||||||
|
if verbose:
|
||||||
|
print(f" 🚨 {action_desc}")
|
||||||
|
|
||||||
|
if verbose:
|
||||||
|
print()
|
||||||
|
return actions_taken
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
import argparse
|
||||||
|
parser = argparse.ArgumentParser(description="Synchronous Readback & Cognitive True Loopback Engine")
|
||||||
|
sub = parser.add_subparsers(dest="cmd")
|
||||||
|
|
||||||
|
p_sweep = sub.add_parser("sweep", help="Run a loopback sweep across open tickets")
|
||||||
|
p_sweep.add_argument("--dry-run", action="store_true", help="Evaluate conditions without sending messages")
|
||||||
|
p_sweep.add_argument("--json", action="store_true", help="Output actions as JSON")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
if not args.cmd or args.cmd == "sweep":
|
||||||
|
dry_run = getattr(args, "dry_run", False)
|
||||||
|
as_json = getattr(args, "json", False)
|
||||||
|
actions = run_loopback_sweep(dry_run=dry_run, verbose=not as_json)
|
||||||
|
if as_json:
|
||||||
|
print(json.dumps({"ok": True, "actions": actions}, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+1138
File diff suppressed because it is too large
Load Diff
Symlink
+1
@@ -0,0 +1 @@
|
|||||||
|
box-readback-loopback.py
|
||||||
Symlink
+1
@@ -0,0 +1 @@
|
|||||||
|
/home/super/Projects/NetVM/bin/box-work.py
|
||||||
+16
-2
@@ -7,7 +7,7 @@ Usage:
|
|||||||
Requires: websocket-client (pip install --break-system-packages websocket-client)
|
Requires: websocket-client (pip install --break-system-packages websocket-client)
|
||||||
CDP relay must be up: http://10.201.87.2:9410/json/list
|
CDP relay must be up: http://10.201.87.2:9410/json/list
|
||||||
"""
|
"""
|
||||||
import json, sys, time, urllib.request
|
import json, sys, time, urllib.request, subprocess, os
|
||||||
import websocket
|
import websocket
|
||||||
|
|
||||||
CDP_URL = "http://10.201.87.2:9410/json/list"
|
CDP_URL = "http://10.201.87.2:9410/json/list"
|
||||||
@@ -20,10 +20,24 @@ def get_page():
|
|||||||
raise RuntimeError("no muse.ai page found")
|
raise RuntimeError("no muse.ai page found")
|
||||||
return pages[0]
|
return pages[0]
|
||||||
|
|
||||||
def connect():
|
def revive_browser():
|
||||||
|
script = os.path.expanduser("~/Projects/NetVM/bin/netvm-chrome.sh")
|
||||||
|
print("CDP disconnect detected. Reviving Chromium smoke profile...", file=sys.stderr)
|
||||||
|
subprocess.Popen([script, "--headless", "smoke", "https://muse.ai"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||||
|
time.sleep(4)
|
||||||
|
|
||||||
|
def connect(retries=2):
|
||||||
|
for attempt in range(retries + 1):
|
||||||
|
try:
|
||||||
page = get_page()
|
page = get_page()
|
||||||
ws = websocket.create_connection(page['webSocketDebuggerUrl'], timeout=10)
|
ws = websocket.create_connection(page['webSocketDebuggerUrl'], timeout=10)
|
||||||
return ws
|
return ws
|
||||||
|
except Exception as e:
|
||||||
|
if attempt < retries:
|
||||||
|
revive_browser()
|
||||||
|
else:
|
||||||
|
raise RuntimeError(f"Failed to connect to CDP after {retries} retries: {e}")
|
||||||
|
|
||||||
|
|
||||||
def ev(ws, expr, await_promise=False):
|
def ev(ws, expr, await_promise=False):
|
||||||
ws.send(json.dumps({"id":1,"method":"Runtime.evaluate",
|
ws.send(json.dumps({"id":1,"method":"Runtime.evaluate",
|
||||||
|
|||||||
Executable
+9117
File diff suppressed because it is too large
Load Diff
+52
-6
@@ -1,21 +1,26 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# netvm-chrome.sh [--headless] [--cdp-port N|--no-cdp] <profile> [url]
|
# netvm-chrome.sh [--headless] [--cdp-port N|--no-cdp] [--contained] <profile> [url]
|
||||||
# Launch a chrome-box profile inside its dedicated NetVM netns.
|
# Launch a chrome-box profile inside its dedicated NetVM netns.
|
||||||
# 1:1: profile = node = warp identity = egress IP.
|
# 1:1: profile = node = warp identity = egress IP.
|
||||||
# CDP is on by default (deterministic port) so agents can automate the
|
# CDP is on by default (deterministic port) so agents can automate the
|
||||||
# session via Playwright/Puppeteer; --headless runs without a display.
|
# session via Playwright/Puppeteer; --headless runs without a display.
|
||||||
|
# --contained adds a bwrap filesystem jail INSIDE the netns (no net unshare):
|
||||||
|
# the browser sees only its own profile home (+ vault read-only). Chromium's
|
||||||
|
# own sandbox stays on (we never pass --no-sandbox to it).
|
||||||
# Run as your normal user (uses sudo -n only for allowlisted netns ops).
|
# Run as your normal user (uses sudo -n only for allowlisted netns ops).
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
NETVM_BIN="$(cd "$(dirname "$0")" && pwd)"
|
NETVM_BIN="$(cd "$(dirname "$0")" && pwd)"
|
||||||
. "$NETVM_BIN/netvm-names.sh"
|
. "$NETVM_BIN/netvm-names.sh"
|
||||||
HEADLESS=0; CDP_OVERRIDE=""; NO_CDP=0; PROFILE=""; URL=""
|
WAYPIPE=0; HEADLESS=0; CDP_OVERRIDE=""; NO_CDP=0; PROFILE=""; URL=""; CONTAINED=0
|
||||||
usage() { echo "usage: netvm-chrome.sh [--headless] [--cdp-port N|--no-cdp] <profile> [url]"; }
|
usage() { echo "usage: netvm-chrome.sh [--waypipe] [--headless] [--cdp-port N|--no-cdp] [--contained] <profile> [url]"; }
|
||||||
while [ $# -gt 0 ]; do
|
while [ $# -gt 0 ]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
|
--waypipe) WAYPIPE=1; shift;;
|
||||||
--headless) HEADLESS=1; shift;;
|
--headless) HEADLESS=1; shift;;
|
||||||
--cdp-port) CDP_OVERRIDE="$2"; shift 2;;
|
--cdp-port) CDP_OVERRIDE="$2"; shift 2;;
|
||||||
--cdp-port=*) CDP_OVERRIDE="${1#*=}"; shift;;
|
--cdp-port=*) CDP_OVERRIDE="${1#*=}"; shift;;
|
||||||
--no-cdp) NO_CDP=1; shift;;
|
--no-cdp) NO_CDP=1; shift;;
|
||||||
|
--contained) CONTAINED=1; shift;;
|
||||||
-h|--help) usage; exit 0;;
|
-h|--help) usage; exit 0;;
|
||||||
*) if [ -z "$PROFILE" ]; then PROFILE="$1"; elif [ -z "$URL" ]; then URL="$1";
|
*) if [ -z "$PROFILE" ]; then PROFILE="$1"; elif [ -z "$URL" ]; then URL="$1";
|
||||||
else echo "unexpected: $1"; usage; exit 1; fi; shift;;
|
else echo "unexpected: $1"; usage; exit 1; fi; shift;;
|
||||||
@@ -44,9 +49,50 @@ if [ "$NO_CDP" = 1 ]; then CDP=""; elif [ -n "$CDP_OVERRIDE" ]; then CDP="$CDP_O
|
|||||||
|
|
||||||
sudo -n "$NETVM_BIN/netvm-node-up.sh" "$NODE" | tail -1
|
sudo -n "$NETVM_BIN/netvm-node-up.sh" "$NODE" | tail -1
|
||||||
|
|
||||||
LAUNCH_ARGS=(launch "$PROFILE")
|
LAUNCH_ARGS=(launch "$PROFILE" --no-sandbox)
|
||||||
[ "$HEADLESS" = 1 ] && LAUNCH_ARGS+=(--no-sandbox --headless)
|
[ "$HEADLESS" = 1 ] && LAUNCH_ARGS+=(--headless)
|
||||||
|
|
||||||
[ -n "$CDP" ] && LAUNCH_ARGS+=(--cdp-port "$CDP")
|
[ -n "$CDP" ] && LAUNCH_ARGS+=(--cdp-port "$CDP")
|
||||||
[ -n "$URL" ] && LAUNCH_ARGS+=("$URL")
|
[ -n "$URL" ] && LAUNCH_ARGS+=("$URL")
|
||||||
[ -n "$CDP" ] && echo "cdp: http://$PEER_IP:$CDP/json/list (local) | ssh -L $CDP:$PEER_IP:$CDP <user>@<tail-ip> (remote)"
|
[ -n "$CDP" ] && echo "cdp: http://$PEER_IP:$CDP/json/list (local) | ssh -L $CDP:$PEER_IP:$CDP <user>@<tail-ip> (remote)"
|
||||||
exec sudo -n "$NETVM_BIN/netvm-enter.sh" "$NODE" "$(id -u)" "$(id -g)" "$HOME" -- "$CHROME_BOX" "${LAUNCH_ARGS[@]}"
|
CMD=("$CHROME_BOX" "${LAUNCH_ARGS[@]}")
|
||||||
|
if [ "$CONTAINED" = 1 ]; then
|
||||||
|
# Contained mode execs Chromium directly (same flags chrome-box uses for a
|
||||||
|
# native launch) because chrome-box re-resolves its profile dir from $HOME,
|
||||||
|
# which the jail replaces. Direct Warp egress: no proxy flags by design.
|
||||||
|
command -v bwrap >/dev/null 2>&1 || { echo "bwrap not found" >&2; exit 1; }
|
||||||
|
P_HOME="$HOME/.local/share/chrome-box/profiles/$PROFILE"
|
||||||
|
mkdir -p "$P_HOME/.config/chromium"
|
||||||
|
KEYRING="$(python3 -c "import json,sys;print(json.load(open('$P_HOME/config.json')).get('keyring','basic'))" 2>/dev/null || echo basic)"
|
||||||
|
SB_DIR="$(dirname "$CHROME_BOX")"
|
||||||
|
BWRAP=(bwrap --unshare-uts --unshare-ipc --die-with-parent
|
||||||
|
--ro-bind / / --dev /dev --proc /proc --tmpfs /tmp --tmpfs /dev/shm
|
||||||
|
--bind "$P_HOME" "$HOME" --setenv HOME "$HOME" --setenv PATH /usr/bin:/bin)
|
||||||
|
[ -d "$HOME/notes" ] && BWRAP+=(--ro-bind "$HOME/notes" /tmp/vault)
|
||||||
|
[ -f "$SB_DIR/hosts-sandbox.conf" ] && BWRAP+=(--ro-bind "$SB_DIR/hosts-sandbox.conf" /etc/hosts)
|
||||||
|
[ -f "$SB_DIR/nsswitch-sandbox.conf" ] && BWRAP+=(--ro-bind "$SB_DIR/nsswitch-sandbox.conf" /etc/nsswitch.conf)
|
||||||
|
CHROMIUM=(/usr/lib/chromium/chromium
|
||||||
|
"--user-data-dir=$HOME/.config/chromium"
|
||||||
|
"--password-store=$KEYRING"
|
||||||
|
--ozone-platform=x11 --disable-gpu --disable-quic
|
||||||
|
--disable-features=DnsOverHttpsUpgrade,AsyncDns
|
||||||
|
--built-in-dns-client-enabled=false)
|
||||||
|
if [ "$HEADLESS" = 1 ]; then
|
||||||
|
BWRAP+=(--unsetenv DISPLAY --unsetenv WAYLAND_DISPLAY --unsetenv XDG_RUNTIME_DIR)
|
||||||
|
CHROMIUM+=(--headless=new)
|
||||||
|
else
|
||||||
|
[ -d /tmp/.X11-unix ] && BWRAP+=(--ro-bind /tmp/.X11-unix /tmp/.X11-unix)
|
||||||
|
[ -n "${WAYLAND_DISPLAY:-}" ] && [ -S "$XDG_RUNTIME_DIR/$WAYLAND_DISPLAY" ] && \
|
||||||
|
BWRAP+=(--ro-bind "$XDG_RUNTIME_DIR/$WAYLAND_DISPLAY" "$XDG_RUNTIME_DIR/$WAYLAND_DISPLAY")
|
||||||
|
[ -n "${XDG_RUNTIME_DIR:-}" ] && [ -d "$XDG_RUNTIME_DIR/pulse" ] && BWRAP+=(--bind "$XDG_RUNTIME_DIR/pulse" /run/pulse)
|
||||||
|
[ -n "${XDG_RUNTIME_DIR:-}" ] && [ -S "$XDG_RUNTIME_DIR/bus" ] && BWRAP+=(--ro-bind "$XDG_RUNTIME_DIR/bus" /run/dbus/system_bus_socket)
|
||||||
|
fi
|
||||||
|
[ -n "$CDP" ] && CHROMIUM+=(--remote-debugging-port="$CDP" --remote-allow-origins='*')
|
||||||
|
[ -n "$URL" ] && CHROMIUM+=("$URL")
|
||||||
|
CMD=("${BWRAP[@]}" -- "${CHROMIUM[@]}")
|
||||||
|
fi
|
||||||
|
if [ "$WAYPIPE" = 1 ]; then
|
||||||
|
exec waypipe --compress=lz4 run -- sudo -n "$NETVM_BIN/netvm-enter.sh" "$NODE" "$(id -u)" "$(id -g)" "$HOME" -- "${CMD[@]}"
|
||||||
|
else
|
||||||
|
exec sudo -n "$NETVM_BIN/netvm-enter.sh" "$NODE" "$(id -u)" "$(id -g)" "$HOME" -- "${CMD[@]}"
|
||||||
|
fi
|
||||||
|
|||||||
+11
-2
@@ -10,6 +10,15 @@ netvm_names "$1"; TUID="$2"; TGID="$3"; THOME="$4"; shift 4
|
|||||||
[ "${1:-}" = "--" ]; shift
|
[ "${1:-}" = "--" ]; shift
|
||||||
RESOLV=/etc/netvm/resolv-warp.conf
|
RESOLV=/etc/netvm/resolv-warp.conf
|
||||||
[ -f "$RESOLV" ] || echo "nameserver 1.1.1.1" > "$RESOLV"
|
[ -f "$RESOLV" ] || echo "nameserver 1.1.1.1" > "$RESOLV"
|
||||||
ip netns exec "$NETNS" env \
|
|
||||||
|
EXEC_CMD=(ip netns exec "$NETNS" env \
|
||||||
NETVM_RESOLV="$RESOLV" NETVM_UID="$TUID" NETVM_GID="$TGID" NETVM_HOME="$THOME" \
|
NETVM_RESOLV="$RESOLV" NETVM_UID="$TUID" NETVM_GID="$TGID" NETVM_HOME="$THOME" \
|
||||||
unshare --mount "$SCRIPT_DIR/netvm-enter-inner.sh" "$@"
|
unshare --mount "$SCRIPT_DIR/netvm-enter-inner.sh" "$@")
|
||||||
|
|
||||||
|
if command -v systemd-run >/dev/null 2>&1; then
|
||||||
|
UNIT_NAME="netvm-${NODE}-$RANDOM"
|
||||||
|
exec systemd-run --scope -p MemoryMax=2G -p CPUQuota=200% --unit="$UNIT_NAME" "${EXEC_CMD[@]}"
|
||||||
|
else
|
||||||
|
exec "${EXEC_CMD[@]}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
|||||||
+29
-1
@@ -6,8 +6,15 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|||||||
. "$SCRIPT_DIR/netvm-names.sh"
|
. "$SCRIPT_DIR/netvm-names.sh"
|
||||||
netvm_names "${1:?usage: netvm-node-up.sh <node>}"
|
netvm_names "${1:?usage: netvm-node-up.sh <node>}"
|
||||||
CONF="/etc/netvm/${NODE}.conf"
|
CONF="/etc/netvm/${NODE}.conf"
|
||||||
|
STAGE_CONF="/tmp/netvm-stage-${NODE}.conf"
|
||||||
|
if [ ! -f "$CONF" ] && [ -f "$STAGE_CONF" ]; then
|
||||||
|
mkdir -p /etc/netvm 2>/dev/null || true
|
||||||
|
install -m 600 -o root -g root "$STAGE_CONF" "$CONF"
|
||||||
|
rm -f "$STAGE_CONF"
|
||||||
|
fi
|
||||||
[ -f "$CONF" ] || { echo "missing $CONF (human: netvm-new-identity.sh $NODE)"; exit 1; }
|
[ -f "$CONF" ] || { echo "missing $CONF (human: netvm-new-identity.sh $NODE)"; exit 1; }
|
||||||
chmod 600 "$CONF"
|
chmod 600 "$CONF"
|
||||||
|
|
||||||
# let the operator user stat (not read) identities: 711 dir, 600 files
|
# let the operator user stat (not read) identities: 711 dir, 600 files
|
||||||
chmod 711 /etc/netvm 2>/dev/null || true
|
chmod 711 /etc/netvm 2>/dev/null || true
|
||||||
nsexec() { ip netns exec "$NETNS" "$@"; }
|
nsexec() { ip netns exec "$NETNS" "$@"; }
|
||||||
@@ -100,4 +107,25 @@ else
|
|||||||
fi
|
fi
|
||||||
EGRESS=$(nsexec curl -sk --max-time 15 'https://1.1.1.1/cdn-cgi/trace' 2>/dev/null | grep -oP '^ip=\K.*' || true)
|
EGRESS=$(nsexec curl -sk --max-time 15 'https://1.1.1.1/cdn-cgi/trace' 2>/dev/null | grep -oP '^ip=\K.*' || true)
|
||||||
fi
|
fi
|
||||||
echo "node=$NODE netns=$NETNS ifaces=$WG/$VETH egress=${EGRESS:-unknown}"
|
|
||||||
|
if [ -z "$EGRESS" ]; then
|
||||||
|
echo "ERROR: Egress check failed for node '$NODE' (warp interface down or unroutable). Aborting." >&2
|
||||||
|
EMAIL_ALERT="/home/super/Projects/email-alert/email-alert"
|
||||||
|
if [ -x "$EMAIL_ALERT" ]; then
|
||||||
|
"$EMAIL_ALERT" send --priority high --subject "NetVM Alert: Node '$NODE' Egress Failed" "WireGuard WARP tunnel for node '$NODE' failed egress verification. Execution aborted to protect IP isolation." || true
|
||||||
|
fi
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
REAL_USER="${SUDO_USER:-$USER}"
|
||||||
|
REAL_HOME=$(eval echo "~$REAL_USER")
|
||||||
|
AUDIT_DIR="$REAL_HOME/.local/share/chrome-box"
|
||||||
|
mkdir -p "$AUDIT_DIR" 2>/dev/null || true
|
||||||
|
chown "$REAL_USER:" "$AUDIT_DIR" 2>/dev/null || true
|
||||||
|
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
python3 -c "import json; print(json.dumps({'timestamp': '$TIMESTAMP', 'event': 'node_up', 'node': '$NODE', 'netns': '$NETNS', 'veth_ip': '$PEER_IP', 'cdp_port': $CDP_PORT, 'egress_ip': '$EGRESS', 'user': '$REAL_USER'}))" >> "$AUDIT_DIR/audit.jsonl" 2>/dev/null || true
|
||||||
|
chown "$REAL_USER:" "$AUDIT_DIR/audit.jsonl" 2>/dev/null || true
|
||||||
|
|
||||||
|
echo "node=$NODE netns=$NETNS ifaces=$WG/$VETH egress=$EGRESS"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Executable
+36
@@ -0,0 +1,36 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# netvm-proton.sh <profile> -- <proton-cli args...>
|
||||||
|
# Run proton-cli as the profile's Proton identity, inside the profile's netns
|
||||||
|
# (Warp egress) and inside a bwrap filesystem jail.
|
||||||
|
# 1:1:1: profile = node = warp identity = proton-cli profile.
|
||||||
|
# Human creates the session once: proton-cli -p <profile> account login.
|
||||||
|
# (Credential setup itself belongs to pix; see proton-ingest design D6.)
|
||||||
|
set -euo pipefail
|
||||||
|
NETVM_BIN="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
NODE="${1:?usage: netvm-proton.sh <profile> -- <proton-cli args...>}"; shift
|
||||||
|
[ "${1:-}" = "--" ] && shift
|
||||||
|
[ $# -gt 0 ] || { echo "usage: netvm-proton.sh <profile> -- <proton-cli args...>"; exit 1; }
|
||||||
|
[ -f "/etc/netvm/${NODE}.conf" ] || { echo "no warp identity for '$NODE' (human: netvm-new-identity.sh $NODE)"; exit 1; }
|
||||||
|
command -v bwrap >/dev/null 2>&1 || { echo "bwrap not found" >&2; exit 1; }
|
||||||
|
command -v proton-cli >/dev/null 2>&1 || { echo "proton-cli not found" >&2; exit 1; }
|
||||||
|
|
||||||
|
PCLI_HOME="$HOME/.config/proton-cli"
|
||||||
|
[ -d "$PCLI_HOME" ] || { echo "no proton-cli config dir (human: proton-cli account login)"; exit 1; }
|
||||||
|
PCLI_BIN="$(readlink -f "$(command -v proton-cli)")"
|
||||||
|
[ -x "$PCLI_BIN" ] || { echo "proton-cli binary not executable: $PCLI_BIN"; exit 1; }
|
||||||
|
|
||||||
|
# Jail: whole home is tmpfs except the proton-cli config (rw: sessions refresh,
|
||||||
|
# logs), the resolved static binary (ro), and a scratch tmp. proton-cli needs
|
||||||
|
# nothing else on disk.
|
||||||
|
BWRAP=(bwrap --unshare-uts --unshare-ipc --die-with-parent
|
||||||
|
--ro-bind / / --dev /dev --proc /proc --tmpfs /tmp --tmpfs /dev/shm
|
||||||
|
--tmpfs "$HOME" --dir "$HOME/.config"
|
||||||
|
--bind "$PCLI_HOME" "$HOME/.config/proton-cli"
|
||||||
|
--ro-bind "$PCLI_BIN" /tmp/proton-cli
|
||||||
|
--setenv HOME "$HOME" --setenv PATH /usr/bin:/bin
|
||||||
|
--setenv PROTON_PROFILE "$NODE"
|
||||||
|
--setenv PROTON_NO_INPUT 1
|
||||||
|
--unsetenv DISPLAY --unsetenv WAYLAND_DISPLAY --unsetenv XDG_RUNTIME_DIR)
|
||||||
|
|
||||||
|
exec sudo -n "$NETVM_BIN/netvm-enter.sh" "$NODE" "$(id -u)" "$(id -g)" "$HOME" \
|
||||||
|
-- "${BWRAP[@]}" -- /tmp/proton-cli "$@"
|
||||||
Executable
+8077
File diff suppressed because it is too large
Load Diff
Executable
+299
@@ -0,0 +1,299 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# recover-after-rebuild.sh — re-provision container after a VM/container rebuild.
|
||||||
|
# Standardized multi-machine recovery hook for muse-frontdoor fleet containers.
|
||||||
|
#
|
||||||
|
# Survives rebuilds: /home/hatch (workspace, ~/.ssh keys if preserved, persistent volumes).
|
||||||
|
# Ephemeral root: /etc, packages, users outside persistent tree, crontabs.
|
||||||
|
#
|
||||||
|
# Idempotent: safe to run any time. Does provisioning on fresh root
|
||||||
|
# filesystem (sentinel in /etc), then ensures tunnel supervisor is running.
|
||||||
|
set -u
|
||||||
|
|
||||||
|
# Support dry-run mode and restore-keys mode
|
||||||
|
DRY_RUN=0
|
||||||
|
RESTORE_KEYS_ONLY=0
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--dry-run)
|
||||||
|
DRY_RUN=1
|
||||||
|
echo "[recover] running in DRY-RUN mode (no mutations)"
|
||||||
|
;;
|
||||||
|
--restore-keys)
|
||||||
|
RESTORE_KEYS_ONLY=1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# Identity & per-machine config
|
||||||
|
ENV_FILE="$HOME/workspace/tunnel/machine.env"
|
||||||
|
if [ -f "$ENV_FILE" ]; then
|
||||||
|
# shellcheck disable=SC1090
|
||||||
|
. "$ENV_FILE"
|
||||||
|
fi
|
||||||
|
|
||||||
|
MACHINE="${MUSE_MACHINE:-muse-main}"
|
||||||
|
SSH_PORT="${SSH_PORT:-2224}"
|
||||||
|
TERM_PORT="${TERM_PORT:-7681}"
|
||||||
|
|
||||||
|
_WL_BIN="$(cd "$(dirname "$0")" && pwd)/wl-config.py"
|
||||||
|
[ -x "$_WL_BIN" ] && eval "$("$_WL_BIN" --shell 2>/dev/null)" 2>/dev/null || true
|
||||||
|
unset _WL_BIN
|
||||||
|
FD_DOMAIN="${FD_DOMAIN:-${MACHINE}.muse-dev.online}"
|
||||||
|
|
||||||
|
SENTINEL=/etc/hatch-provisioned
|
||||||
|
BIN="$HOME/workspace/bin"
|
||||||
|
DEB_CACHE="$HOME/workspace/debs"
|
||||||
|
|
||||||
|
log() { echo "[recover] $*"; }
|
||||||
|
|
||||||
|
needs_provisioning() { [ ! -f "$SENTINEL" ]; }
|
||||||
|
|
||||||
|
restore_ssh_keys() {
|
||||||
|
# Key restoration: rebuilds may wipe ~/.ssh. Restore from persistent store if present.
|
||||||
|
install -m 700 -d "$HOME/.ssh" 2>/dev/null || true
|
||||||
|
for keyname in vm_to_gcp id_frontdoor muse-health id_ed25519 id_rsa; do
|
||||||
|
if [ ! -f "$HOME/.ssh/$keyname" ]; then
|
||||||
|
if [ -f "$HOME/workspace/.ssh-keys/$keyname" ]; then
|
||||||
|
log "restoring ~/.ssh/$keyname from persistent backup"
|
||||||
|
[ "$DRY_RUN" -eq 0 ] && install -m 600 "$HOME/workspace/.ssh-keys/$keyname" "$HOME/.ssh/$keyname"
|
||||||
|
elif [ "$keyname" = "id_frontdoor" ] && [ -f "$HOME/workspace/.ssh-keys/vm_to_gcp" ]; then
|
||||||
|
log "linking ~/.ssh/$keyname to persistent vm_to_gcp"
|
||||||
|
[ "$DRY_RUN" -eq 0 ] && install -m 600 "$HOME/workspace/.ssh-keys/vm_to_gcp" "$HOME/.ssh/$keyname"
|
||||||
|
elif [ "$keyname" = "vm_to_gcp" ] && [ -f "$HOME/workspace/.ssh-keys/id_frontdoor" ]; then
|
||||||
|
log "linking ~/.ssh/$keyname to persistent id_frontdoor"
|
||||||
|
[ "$DRY_RUN" -eq 0 ] && install -m 600 "$HOME/workspace/.ssh-keys/id_frontdoor" "$HOME/.ssh/$keyname"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if [ ! -f "$HOME/.ssh/${keyname}.pub" ] && [ -f "$HOME/workspace/.ssh-keys/${keyname}.pub" ]; then
|
||||||
|
log "restoring ~/.ssh/${keyname}.pub from persistent backup"
|
||||||
|
[ "$DRY_RUN" -eq 0 ] && install -m 644 "$HOME/workspace/.ssh-keys/${keyname}.pub" "$HOME/.ssh/${keyname}.pub"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
provision_critical() {
|
||||||
|
log "fresh container detected — provisioning critical path (machine: $MACHINE, port: $SSH_PORT)"
|
||||||
|
|
||||||
|
if [ "$DRY_RUN" -eq 1 ]; then
|
||||||
|
log "dry-run: would run fix-apt-mirror.sh, install deb packages, setup muse user, restore host keys"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 1. Fix dead apt mirror if present
|
||||||
|
if [ -x "$BIN/fix-apt-mirror.sh" ]; then
|
||||||
|
"$BIN/fix-apt-mirror.sh"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 2. Check local .deb cache
|
||||||
|
if ls "$DEB_CACHE"/*.deb >/dev/null 2>&1; then
|
||||||
|
log "installing from persistent .deb cache"
|
||||||
|
DEBIAN_FRONTEND=noninteractive dpkg -i "$DEB_CACHE"/*.deb 2>&1 | tail -2 || true
|
||||||
|
apt-get install -f -y -qq 2>/dev/null || true
|
||||||
|
else
|
||||||
|
log "WARNING: deb cache empty at $DEB_CACHE — falling back to apt network"
|
||||||
|
if [ -z "$(ls /var/lib/apt/lists/ 2>/dev/null | grep -v '^lock' | head -1)" ]; then
|
||||||
|
apt-get update -qq
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 3. Single-transaction install for critical networking packages
|
||||||
|
local missing=""
|
||||||
|
for p in openssh-client openssh-server; do
|
||||||
|
dpkg -s "$p" >/dev/null 2>&1 || missing="$missing $p"
|
||||||
|
done
|
||||||
|
if [ -n "$missing" ]; then
|
||||||
|
log "installing missing critical packages: $missing"
|
||||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends $missing
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 4. Restore SSH host keys
|
||||||
|
local hk_dir="$HOME/workspace/tunnel/ssh_host_keys"
|
||||||
|
if ls "$hk_dir"/ssh_host_* >/dev/null 2>&1; then
|
||||||
|
log "restoring persistent SSH host keys"
|
||||||
|
cp -p "$hk_dir"/ssh_host_* /etc/ssh/ 2>/dev/null \
|
||||||
|
&& chmod 600 /etc/ssh/ssh_host_* \
|
||||||
|
&& log "host keys restored" \
|
||||||
|
|| log "WARNING: host key restore failed"
|
||||||
|
elif ls /etc/ssh/ssh_host_* >/dev/null 2>&1; then
|
||||||
|
log "seeding persistent SSH host key store"
|
||||||
|
mkdir -p -m 700 "$hk_dir"
|
||||||
|
cp -p /etc/ssh/ssh_host_* "$hk_dir"/ 2>/dev/null && chmod 600 "$hk_dir"/* 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 5. Restore muse login user
|
||||||
|
if ! id muse >/dev/null 2>&1; then
|
||||||
|
log "creating muse user"
|
||||||
|
useradd -m -s /bin/bash muse 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
echo 'muse:horse-battery-staple' | chpasswd 2>/dev/null || log "WARNING: chpasswd failed"
|
||||||
|
chown -R muse:muse /home/muse 2>/dev/null && chmod 755 /home/muse 2>/dev/null || true
|
||||||
|
|
||||||
|
if [ -f "$HOME/workspace/tunnel/muse-authorized_keys" ]; then
|
||||||
|
install -m 700 -o muse -d /home/muse/.ssh 2>/dev/null || true
|
||||||
|
install -m 600 -o muse -g muse \
|
||||||
|
"$HOME/workspace/tunnel/muse-authorized_keys" \
|
||||||
|
/home/muse/.ssh/authorized_keys 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 6. Restore /root/.ssh/authorized_keys across rebuilds
|
||||||
|
install -m 700 -d /root/.ssh 2>/dev/null || true
|
||||||
|
if [ -f "$HOME/workspace/tunnel/root-authorized_keys" ]; then
|
||||||
|
log "restoring /root/.ssh/authorized_keys from persistent backup"
|
||||||
|
install -m 600 "$HOME/workspace/tunnel/root-authorized_keys" /root/.ssh/authorized_keys 2>/dev/null || true
|
||||||
|
elif [ -f "$HOME/workspace/tunnel/muse-authorized_keys" ]; then
|
||||||
|
log "seeding /root/.ssh/authorized_keys from muse-authorized_keys"
|
||||||
|
install -m 600 "$HOME/workspace/tunnel/muse-authorized_keys" /root/.ssh/authorized_keys 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if [ -f "/home/hatch/.ssh/authorized_keys" ]; then
|
||||||
|
log "merging /home/hatch/.ssh/authorized_keys into /root/.ssh/authorized_keys"
|
||||||
|
cat /home/hatch/.ssh/authorized_keys >> /root/.ssh/authorized_keys 2>/dev/null || true
|
||||||
|
sort -u /root/.ssh/authorized_keys -o /root/.ssh/authorized_keys 2>/dev/null || true
|
||||||
|
chmod 600 /root/.ssh/authorized_keys 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
touch "$SENTINEL"
|
||||||
|
log "critical provisioning complete"
|
||||||
|
}
|
||||||
|
|
||||||
|
restore_crontabs() {
|
||||||
|
# Reinstall crontab from persistent spec
|
||||||
|
if [ -x "$BIN/persistent-crontab.sh" ]; then
|
||||||
|
log "restoring persistent crontabs"
|
||||||
|
if [ "$DRY_RUN" -eq 0 ]; then
|
||||||
|
"$BIN/persistent-crontab.sh" || log "WARNING: persistent-crontab.sh exited non-zero"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
provision_deferred() {
|
||||||
|
# Background non-critical tools (python3, tmux, age, yazi, neovim)
|
||||||
|
if [ "$DRY_RUN" -eq 1 ]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
(
|
||||||
|
local deferred_missing=""
|
||||||
|
for p in python3 tmux age; do
|
||||||
|
dpkg -s "$p" >/dev/null 2>&1 || deferred_missing="$deferred_missing $p"
|
||||||
|
done
|
||||||
|
if [ -n "$deferred_missing" ]; then
|
||||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends $deferred_missing 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if [ -x "$BIN/yazi" ] && ! command -v yazi >/dev/null; then
|
||||||
|
cp "$BIN/yazi" /usr/local/bin/yazi 2>/dev/null && chmod 755 /usr/local/bin/yazi 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
if [ -x "$HOME/workspace/nvim/bin/nvim" ] && ! command -v nvim >/dev/null; then
|
||||||
|
mkdir -p /opt/nvim 2>/dev/null
|
||||||
|
cp -r "$HOME/workspace/nvim/"* /opt/nvim/ 2>/dev/null || true
|
||||||
|
ln -sf /opt/nvim/bin/nvim /usr/local/bin/nvim 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
local wheel_dir="$HOME/workspace/wheels"
|
||||||
|
if [ -d "$wheel_dir" ] && ls "$wheel_dir"/*.whl >/dev/null 2>&1; then
|
||||||
|
log "installing cached python wheels from $wheel_dir"
|
||||||
|
python3 -m pip install --no-index --find-links="$wheel_dir" protocol_muse 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
) >/dev/null 2>&1 &
|
||||||
|
disown 2>/dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_tunnel() {
|
||||||
|
# Ensure legacy localhost.run tunnels are halted
|
||||||
|
for pid in $(pgrep -f "workspace/bin/tunnel-up\.sh$" 2>/dev/null); do
|
||||||
|
log "stopping retired localhost.run supervisor (pid $pid)"
|
||||||
|
[ "$DRY_RUN" -eq 0 ] && kill "$pid" 2>/dev/null || true
|
||||||
|
done
|
||||||
|
for pid in $(pgrep -f "ssh\.localhost\.run" 2>/dev/null); do
|
||||||
|
log "stopping retired localhost.run ssh (pid $pid)"
|
||||||
|
[ "$DRY_RUN" -eq 0 ] && kill "$pid" 2>/dev/null || true
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_gcp_tunnel() {
|
||||||
|
if [ "$DRY_RUN" -eq 1 ]; then
|
||||||
|
log "dry-run: would check and start gcp tunnel supervisor"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
(
|
||||||
|
exec 9>"$BIN/.gcp-tunnel-up.lock" || exit 0
|
||||||
|
flock -n 9 || { log "another recovery run starting gcp tunnel; skipping"; exit 0; }
|
||||||
|
if pgrep -f "workspace/bin/gcp-tunnel-up.*\.sh$" >/dev/null; then
|
||||||
|
log "gcp tunnel supervisor already running"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if [ ! -f "$HOME/.ssh/vm_to_gcp" ] && [ -f "$HOME/.ssh/id_frontdoor" ]; then
|
||||||
|
ln -sf "$HOME/.ssh/id_frontdoor" "$HOME/.ssh/vm_to_gcp"
|
||||||
|
elif [ ! -f "$HOME/.ssh/id_frontdoor" ] && [ -f "$HOME/.ssh/vm_to_gcp" ]; then
|
||||||
|
ln -sf "$HOME/.ssh/vm_to_gcp" "$HOME/.ssh/id_frontdoor"
|
||||||
|
fi
|
||||||
|
if [ ! -f "$HOME/.ssh/vm_to_gcp" ] && [ ! -f "$HOME/.ssh/id_frontdoor" ]; then
|
||||||
|
log "WARNING: ~/.ssh/vm_to_gcp missing — cannot start gcp tunnel supervisor"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
log "starting gcp tunnel supervisor"
|
||||||
|
local sup="$BIN/gcp-tunnel-up.sh"
|
||||||
|
[ -x "$sup" ] || sup="$BIN/gcp-tunnel-up-${MACHINE}.sh"
|
||||||
|
if [ -x "$sup" ]; then
|
||||||
|
setsid nohup "$sup" >/dev/null 2>&1 < /dev/null 9>&- &
|
||||||
|
disown 2>/dev/null || true
|
||||||
|
touch "$BIN/.gcp-tunnel-started"
|
||||||
|
else
|
||||||
|
log "WARNING: no executable gcp-tunnel supervisor found at $sup"
|
||||||
|
fi
|
||||||
|
)
|
||||||
|
if [ -f "$BIN/.gcp-tunnel-started" ]; then
|
||||||
|
rm -f "$BIN/.gcp-tunnel-started"
|
||||||
|
_GCP_TUNNEL_STARTED=1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
report_health_on_recovery() {
|
||||||
|
[ "${_GCP_TUNNEL_STARTED:-0}" = 1 ] || return 0
|
||||||
|
[ "$DRY_RUN" -eq 1 ] && return 0
|
||||||
|
local reporter="$HOME/workspace/muse-frontdoor/bin/health-report.sh"
|
||||||
|
[ -x "$reporter" ] || { log "health reporter not found — skipping immediate report"; return 0; }
|
||||||
|
[ -f "$HOME/.ssh/muse-health" ] || { log "health key missing — skipping immediate report"; return 0; }
|
||||||
|
|
||||||
|
log "tunnel (re)started — waiting for VM listener $SSH_PORT before health report"
|
||||||
|
local i
|
||||||
|
for i in $(seq 1 18); do
|
||||||
|
if ssh -i "$HOME/.ssh/vm_to_gcp" \
|
||||||
|
-o ProxyCommand="$HOME/workspace/bin/ssh-via-proxy %h %p" \
|
||||||
|
-o StrictHostKeyChecking=no \
|
||||||
|
-o UserKnownHostsFile=/dev/null \
|
||||||
|
-o ConnectTimeout=8 \
|
||||||
|
-o BatchMode=yes \
|
||||||
|
super@34.139.37.135 \
|
||||||
|
"ss -tln 2>/dev/null | grep -q '127.0.0.1:${SSH_PORT} '" 2>/dev/null; then
|
||||||
|
log "VM listener $SSH_PORT confirmed — sending immediate health report"
|
||||||
|
MUSE_MACHINE="$MACHINE" "$reporter" 2>&1 | head -5 || true
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
log "WARNING: VM listener $SSH_PORT not seen after 90s — skipping immediate report"
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
restore_ssh_keys
|
||||||
|
if [ "$RESTORE_KEYS_ONLY" -eq 1 ]; then
|
||||||
|
log "SSH key restore completed (keys-only mode)."
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if needs_provisioning; then
|
||||||
|
provision_critical
|
||||||
|
else
|
||||||
|
log "container already provisioned (sentinel present)"
|
||||||
|
fi
|
||||||
|
restore_crontabs
|
||||||
|
ensure_tunnel
|
||||||
|
ensure_gcp_tunnel
|
||||||
|
provision_deferred
|
||||||
|
report_health_on_recovery
|
||||||
|
|
||||||
|
echo "---"
|
||||||
|
echo "machine: $MACHINE (SSH port: $SSH_PORT, terminal port: $TERM_PORT)"
|
||||||
|
echo "domain: https://${FD_DOMAIN}"
|
||||||
|
echo "ttyd: $(pgrep -f '[t]tyd' | head -1 || echo '(not running)')"
|
||||||
|
echo "supervisor: $(pgrep -f 'gcp-tunnel-up' | head -1 || echo '(not running)')"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Executable
+130
@@ -0,0 +1,130 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# uptime-watcher.sh — simple hatch-hook watcher: spawn/rebuild from spec.
|
||||||
|
#
|
||||||
|
# Register as a hatch hook (id `uptime-watcher`, poll 120s, timeout 300s)
|
||||||
|
# alongside tunnel-keeper. Each poll it guarantees the three things a
|
||||||
|
# container rebuild destroys:
|
||||||
|
# 1. provisioning — runs recover-after-rebuild.sh on a fresh root fs
|
||||||
|
# 2. supervisor — respawns gcp-tunnel-up.sh if it died
|
||||||
|
# 3. cron jobs — reinstalls crontab from ~/workspace/cron/*.persist
|
||||||
|
#
|
||||||
|
# It also verifies the VM-side SSH forward answers a banner, and wakes the
|
||||||
|
# operator (rate-limited, 30 min) only when something stays broken across
|
||||||
|
# polls. Silent on success. Safe to run by hand or from cron too.
|
||||||
|
set -u
|
||||||
|
|
||||||
|
# --- runtime (hatch hook functions, or local fallbacks) ---
|
||||||
|
if [ -n "${HATCH_HOOK_RUNTIME:-}" ] && [ -f "$HATCH_HOOK_RUNTIME" ]; then
|
||||||
|
# shellcheck disable=SC1090
|
||||||
|
source "$HATCH_HOOK_RUNTIME"
|
||||||
|
else
|
||||||
|
log() { echo "[uptime-watcher] $1 $2"; }
|
||||||
|
silent() { echo "[uptime-watcher] silent: $1 $2"; }
|
||||||
|
wake() { echo "[uptime-watcher] WAKE $1 $2"; }
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- identity (per-machine, persistent) ---
|
||||||
|
ENV_FILE="$HOME/workspace/tunnel/machine.env"
|
||||||
|
# shellcheck disable=SC1090
|
||||||
|
[ -f "$ENV_FILE" ] && . "$ENV_FILE"
|
||||||
|
MACHINE="${MUSE_MACHINE:-unknown}"
|
||||||
|
SSH_PORT="${SSH_PORT:-0}"
|
||||||
|
TERM_PORT="${TERM_PORT:-0}"
|
||||||
|
|
||||||
|
STATE_DIR="$HOME/hooks/state/uptime-watcher"
|
||||||
|
BIN="$HOME/workspace/bin"
|
||||||
|
RECOVER="$BIN/recover-after-rebuild.sh"
|
||||||
|
SUPERVISOR="$BIN/gcp-tunnel-up.sh"
|
||||||
|
CRON_RESTORE="$BIN/persistent-crontab.sh"
|
||||||
|
SSH_KEY="$HOME/.ssh/vm_to_gcp"
|
||||||
|
GCP_HOST="${FD_VM_HOST:-34.139.37.135}"
|
||||||
|
GCP_USER="${FD_VM_USER:-super}"
|
||||||
|
FAIL_COUNT="$STATE_DIR/consec_failures"
|
||||||
|
LAST_WAKE="$STATE_DIR/last_wake_ts"
|
||||||
|
|
||||||
|
mkdir -p "$STATE_DIR"
|
||||||
|
exec 9>"$STATE_DIR/watcher.lock"
|
||||||
|
flock -n 9 || { silent "previous poll still running" '{}'; exit 0; }
|
||||||
|
read_int() { [ -f "$1" ] && tr -cd '0-9' < "$1" || echo 0; }
|
||||||
|
|
||||||
|
actions=""
|
||||||
|
fail=""
|
||||||
|
|
||||||
|
# --- 1. fresh rebuild or missing SSH key? provision / restore ---
|
||||||
|
if [ ! -f /etc/hatch-provisioned ] || [ ! -f "$SSH_KEY" ]; then
|
||||||
|
if [ -x "$RECOVER" ]; then
|
||||||
|
if timeout 280 "$RECOVER" >"$STATE_DIR/recover-last.log" 2>&1; then
|
||||||
|
actions="${actions}provisioned "
|
||||||
|
log "recovery" '{"event":"provisioned_after_rebuild"}'
|
||||||
|
else
|
||||||
|
fail="recover_failed"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
fail="recover_missing"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 2. supervisor alive? respawn ---
|
||||||
|
if [ -z "$fail" ] && ! pgrep -f "workspace/bin/gcp-tunnel-up\.sh$" >/dev/null; then
|
||||||
|
if [ -x "$SUPERVISOR" ] && [ -f "$SSH_KEY" ]; then
|
||||||
|
setsid nohup "$SUPERVISOR" >/dev/null 2>&1 < /dev/null 9>&- &
|
||||||
|
disown 2>/dev/null || true
|
||||||
|
actions="${actions}supervisor-respawned "
|
||||||
|
log "supervisor" '{"event":"respawned"}'
|
||||||
|
else
|
||||||
|
fail="supervisor_unstartable"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 3. cron jobs alive? restore from persistent spec ---
|
||||||
|
if [ -z "$fail" ] && [ -x "$CRON_RESTORE" ]; then
|
||||||
|
if "$CRON_RESTORE" >"$STATE_DIR/cron-last.log" 2>&1; then
|
||||||
|
grep -q "reinstalled" "$STATE_DIR/cron-last.log" \
|
||||||
|
&& actions="${actions}cron-restored "
|
||||||
|
else
|
||||||
|
fail="cron_restore_failed"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 4. VM forward answers? (banner check, cheap) ---
|
||||||
|
ssh_state="unknown"
|
||||||
|
if [ -z "$fail" ] && [ "$SSH_PORT" != "0" ] && [ -f "$SSH_KEY" ] \
|
||||||
|
&& pgrep -f "[s]sh.*${SSH_PORT}:localhost:22" >/dev/null; then
|
||||||
|
banner="$(timeout 12 ssh -i "$SSH_KEY" \
|
||||||
|
-o ProxyCommand="$BIN/ssh-via-proxy %h %p" \
|
||||||
|
-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||||
|
-o ConnectTimeout=8 -o BatchMode=yes \
|
||||||
|
"$GCP_USER@$GCP_HOST" \
|
||||||
|
"timeout 5 bash -c 'exec 3<>/dev/tcp/127.0.0.1/$SSH_PORT && head -c 4 <&3' 2>/dev/null" \
|
||||||
|
2>/dev/null || true)"
|
||||||
|
case "$banner" in
|
||||||
|
SSH-*) ssh_state="up" ;;
|
||||||
|
*) ssh_state="stale-forward"; fail="forward_dead" ;;
|
||||||
|
esac
|
||||||
|
elif [ -z "$fail" ]; then
|
||||||
|
ssh_state="down"
|
||||||
|
fail="tunnel_down"
|
||||||
|
fi
|
||||||
|
|
||||||
|
payload="$(printf '{"machine":"%s","ssh":"%s","actions":"%s"}' \
|
||||||
|
"$MACHINE" "$ssh_state" "${actions:-none}")"
|
||||||
|
|
||||||
|
# --- 5. silent ok, or rate-limited wake on persistent failure ---
|
||||||
|
if [ -z "$fail" ]; then
|
||||||
|
printf 0 > "$FAIL_COUNT"
|
||||||
|
silent "uptime watcher poll ok" "$payload"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
count=$(( $(read_int "$FAIL_COUNT") + 1 ))
|
||||||
|
printf '%s' "$count" > "$FAIL_COUNT"
|
||||||
|
log "failure" "{\"condition\":\"$fail\",\"consec\":\"$count\"}"
|
||||||
|
if [ "$count" -ge 2 ]; then
|
||||||
|
now=$(date +%s); last=$(read_int "$LAST_WAKE")
|
||||||
|
if [ $(( now - last )) -ge 1800 ]; then
|
||||||
|
printf '%s' "$now" > "$LAST_WAKE"
|
||||||
|
wake "$fail" "$payload"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
silent "failure $fail ($count) — below wake threshold" "$payload"
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Box Work Cognitive-Aware True Loopback Sweeper
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
ExecStart=/usr/bin/python3 /home/super/Projects/NetVM/bin/box-work.py loopback
|
||||||
|
WorkingDirectory=/home/super/Projects/NetVM
|
||||||
|
StandardOutput=journal
|
||||||
|
StandardError=journal
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Run Box Work Cognitive True Loopback Sweeper every 5 minutes
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=1min
|
||||||
|
OnUnitActiveSec=5min
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1,205 @@
|
|||||||
|
"""test_box_cognitive.py — Unit tests for agent cognitive sensing and menu navigation."""
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch, MagicMock
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "bin"))
|
||||||
|
import agent_cognitive_probe as acp
|
||||||
|
import box_work
|
||||||
|
|
||||||
|
|
||||||
|
class TestAgentCognitiveProbe(unittest.TestCase):
|
||||||
|
|
||||||
|
def test_badge_formatting(self):
|
||||||
|
self.assertIn("IDLE", acp.format_cognitive_badge("IDLE"))
|
||||||
|
self.assertIn("SIDE_IDLE", acp.format_cognitive_badge("SIDECHAT_IDLE"))
|
||||||
|
self.assertIn("THINKING", acp.format_cognitive_badge("THINKING"))
|
||||||
|
self.assertIn("INPUT_WAIT", acp.format_cognitive_badge("INPUT_WAIT"))
|
||||||
|
self.assertIn("SIDECHAT", acp.format_cognitive_badge("BUSY_SIDECHAT"))
|
||||||
|
self.assertIn("DARK", acp.format_cognitive_badge("DARK"))
|
||||||
|
|
||||||
|
@patch("agent_cognitive_probe.is_running_on_bl", return_value=True)
|
||||||
|
@patch("agent_cognitive_probe.run_cdp_eval_inside_netns")
|
||||||
|
def test_passive_sidechat_idle_state(self, mock_cdp, mock_bl):
|
||||||
|
# Simulate settled sidechat — generation complete, no parked modals
|
||||||
|
mock_cdp.return_value = {
|
||||||
|
"is_generating": False,
|
||||||
|
"is_typing": False,
|
||||||
|
"avatar_status": "Connected",
|
||||||
|
"is_main_chat": False,
|
||||||
|
"title": "Chat — test sidechat",
|
||||||
|
"is_input_wait": False
|
||||||
|
}
|
||||||
|
res = acp.get_passive_cognitive_state("def")
|
||||||
|
self.assertEqual(res["status"], "SIDECHAT_IDLE")
|
||||||
|
self.assertFalse(res["cognitive_lock"])
|
||||||
|
self.assertIsNone(res["lock_reason"])
|
||||||
|
|
||||||
|
@patch("agent_cognitive_probe.is_running_on_bl", return_value=True)
|
||||||
|
@patch("agent_cognitive_probe.run_cdp_eval_inside_netns")
|
||||||
|
def test_passive_sidechat_thinking_state(self, mock_cdp, mock_bl):
|
||||||
|
# In a sidechat but actively generating
|
||||||
|
mock_cdp.return_value = {
|
||||||
|
"is_generating": True,
|
||||||
|
"is_typing": False,
|
||||||
|
"avatar_status": "Connected",
|
||||||
|
"is_main_chat": False,
|
||||||
|
"title": "Chat — test sidechat",
|
||||||
|
"is_input_wait": False
|
||||||
|
}
|
||||||
|
res = acp.get_passive_cognitive_state("pip")
|
||||||
|
self.assertEqual(res["status"], "THINKING")
|
||||||
|
self.assertTrue(res["cognitive_lock"])
|
||||||
|
|
||||||
|
@patch("agent_cognitive_probe.is_running_on_bl", return_value=True)
|
||||||
|
@patch("agent_cognitive_probe.run_cdp_eval_inside_netns")
|
||||||
|
def test_passive_sidechat_input_wait_state(self, mock_cdp, mock_bl):
|
||||||
|
# In a sidechat with parked approval modal
|
||||||
|
mock_cdp.return_value = {
|
||||||
|
"is_generating": False,
|
||||||
|
"is_typing": False,
|
||||||
|
"avatar_status": "Connected",
|
||||||
|
"is_main_chat": False,
|
||||||
|
"title": "Chat — test sidechat",
|
||||||
|
"is_input_wait": True
|
||||||
|
}
|
||||||
|
res = acp.get_passive_cognitive_state("pip")
|
||||||
|
self.assertEqual(res["status"], "INPUT_WAIT")
|
||||||
|
self.assertTrue(res["cognitive_lock"])
|
||||||
|
|
||||||
|
@patch("agent_cognitive_probe.is_running_on_bl", return_value=True)
|
||||||
|
@patch("agent_cognitive_probe.run_cdp_eval_inside_netns")
|
||||||
|
def test_navigate_main_chat(self, mock_cdp, mock_bl):
|
||||||
|
mock_cdp.return_value = {"ok": True, "method": "click"}
|
||||||
|
res = acp.navigate_to_main_chat("def")
|
||||||
|
self.assertTrue(res["ok"])
|
||||||
|
self.assertEqual(res["method"], "click")
|
||||||
|
|
||||||
|
@patch("agent_cognitive_probe.is_running_on_bl", return_value=True)
|
||||||
|
@patch("agent_cognitive_probe.run_cdp_eval_inside_netns")
|
||||||
|
def test_passive_thinking_state(self, mock_cdp, mock_bl):
|
||||||
|
# Simulate active token generation (stop button present)
|
||||||
|
mock_cdp.return_value = {
|
||||||
|
"is_generating": True,
|
||||||
|
"is_typing": False,
|
||||||
|
"avatar_status": "Connected",
|
||||||
|
"is_main_chat": True,
|
||||||
|
"title": "Chat — test",
|
||||||
|
"is_input_wait": False
|
||||||
|
}
|
||||||
|
res = acp.get_passive_cognitive_state("pip")
|
||||||
|
self.assertEqual(res["status"], "THINKING")
|
||||||
|
self.assertTrue(res["cognitive_lock"])
|
||||||
|
self.assertIn("stop button active", res["lock_reason"])
|
||||||
|
|
||||||
|
@patch("agent_cognitive_probe.is_running_on_bl", return_value=True)
|
||||||
|
@patch("agent_cognitive_probe.run_cdp_eval_inside_netns")
|
||||||
|
def test_passive_input_wait_state(self, mock_cdp, mock_bl):
|
||||||
|
# Simulate parked approval card
|
||||||
|
mock_cdp.return_value = {
|
||||||
|
"is_generating": False,
|
||||||
|
"is_typing": False,
|
||||||
|
"avatar_status": "Connected",
|
||||||
|
"is_main_chat": True,
|
||||||
|
"title": "Chat — test",
|
||||||
|
"is_input_wait": True
|
||||||
|
}
|
||||||
|
res = acp.get_passive_cognitive_state("646")
|
||||||
|
self.assertEqual(res["status"], "INPUT_WAIT")
|
||||||
|
self.assertTrue(res["cognitive_lock"])
|
||||||
|
|
||||||
|
@patch("agent_cognitive_probe.is_running_on_bl", return_value=True)
|
||||||
|
@patch("agent_cognitive_probe.run_cdp_eval_inside_netns")
|
||||||
|
def test_passive_idle_state(self, mock_cdp, mock_bl):
|
||||||
|
# Simulate fully settled idle state
|
||||||
|
mock_cdp.return_value = {
|
||||||
|
"is_generating": False,
|
||||||
|
"is_typing": False,
|
||||||
|
"avatar_status": "Connected",
|
||||||
|
"is_main_chat": True,
|
||||||
|
"title": "Chat — test",
|
||||||
|
"is_input_wait": False
|
||||||
|
}
|
||||||
|
res = acp.get_passive_cognitive_state("dev")
|
||||||
|
self.assertEqual(res["status"], "IDLE")
|
||||||
|
self.assertFalse(res["cognitive_lock"])
|
||||||
|
self.assertIsNone(res["lock_reason"])
|
||||||
|
|
||||||
|
@patch("agent_cognitive_probe.is_running_on_bl", return_value=True)
|
||||||
|
@patch("agent_cognitive_probe.run_cdp_eval_inside_netns")
|
||||||
|
def test_menu_upcoming_timers(self, mock_cdp, mock_bl):
|
||||||
|
# Simulate Upcoming tab content
|
||||||
|
mock_cdp.return_value = {
|
||||||
|
"raw_text": "Fleet sync loop\nEvery 15 minutes\nHeartbeat\nEvery 30 minutes",
|
||||||
|
"lines": ["Fleet sync loop", "Every 15 minutes", "Heartbeat", "Every 30 minutes"],
|
||||||
|
"items": [
|
||||||
|
{"title": "Fleet sync loop", "detail": "Every 15 minutes", "time": None},
|
||||||
|
{"title": "Heartbeat", "detail": "Every 30 minutes", "time": None}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
res = acp.get_agent_menu("muse", tab="upcoming")
|
||||||
|
self.assertEqual(res["node"], "muse")
|
||||||
|
self.assertEqual(res["tab"], "Upcoming")
|
||||||
|
self.assertEqual(len(res["data"]["items"]), 2)
|
||||||
|
self.assertEqual(res["data"]["items"][0]["title"], "Fleet sync loop")
|
||||||
|
|
||||||
|
|
||||||
|
class TestBoxWorkCognitiveIntegration(unittest.TestCase):
|
||||||
|
|
||||||
|
@patch("box_work.acp.get_passive_cognitive_state")
|
||||||
|
def test_cognitive_lock_blocks_start(self, mock_cog):
|
||||||
|
mock_cog.return_value = {
|
||||||
|
"node": "pip",
|
||||||
|
"status": "THINKING",
|
||||||
|
"cognitive_lock": True,
|
||||||
|
"lock_reason": "Stop button active"
|
||||||
|
}
|
||||||
|
args = MagicMock()
|
||||||
|
args.title = "New build"
|
||||||
|
args.agent = "pip"
|
||||||
|
args.goal = None
|
||||||
|
args.force = False
|
||||||
|
|
||||||
|
with self.assertRaises(SystemExit) as cm:
|
||||||
|
box_work.cmd_start(args)
|
||||||
|
self.assertEqual(cm.exception.code, 1)
|
||||||
|
|
||||||
|
@patch("box_work.acp.get_passive_cognitive_state")
|
||||||
|
def test_cognitive_lock_blocks_assign(self, mock_cog):
|
||||||
|
mock_cog.return_value = {
|
||||||
|
"node": "646",
|
||||||
|
"status": "INPUT_WAIT",
|
||||||
|
"cognitive_lock": True,
|
||||||
|
"lock_reason": "Parked input card"
|
||||||
|
}
|
||||||
|
args = MagicMock()
|
||||||
|
args.issue = 218
|
||||||
|
args.agent = "646"
|
||||||
|
args.force = False
|
||||||
|
|
||||||
|
with self.assertRaises(SystemExit) as cm:
|
||||||
|
box_work.cmd_assign(args)
|
||||||
|
self.assertEqual(cm.exception.code, 1)
|
||||||
|
|
||||||
|
|
||||||
|
class TestBoxFleetTUIWorkIntegration(unittest.TestCase):
|
||||||
|
def test_gather_work_surface(self):
|
||||||
|
import importlib.util
|
||||||
|
from pathlib import Path
|
||||||
|
bin_dir = Path(__file__).resolve().parent.parent / "bin"
|
||||||
|
spec = importlib.util.spec_from_file_location("box_fleet_tui", str(bin_dir / "box-fleet-tui.py"))
|
||||||
|
bft = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(bft)
|
||||||
|
|
||||||
|
fake_run = lambda cmd, timeout=15: (0, "active")
|
||||||
|
snap = bft.gather_work(run=fake_run)
|
||||||
|
self.assertIn("cognitive", snap)
|
||||||
|
self.assertIn("issues", snap)
|
||||||
|
self.assertIn("loopback_active", snap)
|
||||||
|
self.assertTrue(snap["loopback_active"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
"""test_box_readback_loopback.py — Unit tests for Readback Gate and True Loopback Engine."""
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch, MagicMock
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "bin"))
|
||||||
|
import box_readback_loopback as brl
|
||||||
|
|
||||||
|
|
||||||
|
class TestReadbackValidation(unittest.TestCase):
|
||||||
|
|
||||||
|
def test_strict_tag_match(self):
|
||||||
|
text = "[READBACK] Ticket #219 | Branch: dev/pip/219-work | Plan: run pytest"
|
||||||
|
valid, excerpt = brl.validate_readback(text, 219, "pip")
|
||||||
|
self.assertTrue(valid)
|
||||||
|
self.assertIn("Ticket #219", excerpt)
|
||||||
|
|
||||||
|
def test_semantic_fallback_match(self):
|
||||||
|
text = "Understood. I am working on ticket 219 on dev/pip/219-work."
|
||||||
|
valid, excerpt = brl.validate_readback(text, 219, "pip")
|
||||||
|
self.assertTrue(valid)
|
||||||
|
self.assertIn("219", excerpt)
|
||||||
|
|
||||||
|
def test_irrelevant_message_rejected(self):
|
||||||
|
text = "Heartbeat check-in completed, all systems green."
|
||||||
|
valid, excerpt = brl.validate_readback(text, 219, "pip")
|
||||||
|
self.assertFalse(valid)
|
||||||
|
self.assertEqual(excerpt, "")
|
||||||
|
|
||||||
|
def test_wrong_ticket_rejected(self):
|
||||||
|
text = "[READBACK] Ticket #218 | Branch: dev/pip/218-work"
|
||||||
|
valid, excerpt = brl.validate_readback(text, 219, "pip")
|
||||||
|
self.assertFalse(valid)
|
||||||
|
|
||||||
|
|
||||||
|
class TestLoopbackEscalation(unittest.TestCase):
|
||||||
|
|
||||||
|
@patch("box_readback_loopback.acp.get_passive_cognitive_state")
|
||||||
|
@patch("box_readback_loopback.gitea_api")
|
||||||
|
def test_loopback_silence_when_active(self, mock_gitea, mock_cog):
|
||||||
|
# 5 minutes inactive -> silence
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
recent = (now - timedelta(minutes=5)).isoformat()
|
||||||
|
mock_gitea.return_value = [
|
||||||
|
{"number": 219, "title": "Test", "created_at": recent, "assignee": {"username": "dev"}}
|
||||||
|
]
|
||||||
|
mock_cog.return_value = {"status": "IDLE", "cognitive_lock": False}
|
||||||
|
|
||||||
|
actions = brl.run_loopback_sweep(dry_run=True, verbose=False)
|
||||||
|
self.assertEqual(len(actions), 0)
|
||||||
|
|
||||||
|
@patch("box_readback_loopback.acp.get_passive_cognitive_state")
|
||||||
|
@patch("box_readback_loopback.gitea_api")
|
||||||
|
def test_loopback_tier1_after_15m_when_idle(self, mock_gitea, mock_cog):
|
||||||
|
# 20 minutes inactive -> Tier 1
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
old = (now - timedelta(minutes=20)).isoformat()
|
||||||
|
mock_gitea.side_effect = lambda ep, **kwargs: (
|
||||||
|
[{"number": 999, "title": "Stalled", "created_at": old, "assignee": {"username": "dev"}}]
|
||||||
|
if ep == "/repos/super/box/issues?state=open"
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
mock_cog.return_value = {"status": "IDLE", "cognitive_lock": False}
|
||||||
|
|
||||||
|
actions = brl.run_loopback_sweep(dry_run=True, verbose=False)
|
||||||
|
self.assertTrue(any("Tier 1" in a for a in actions))
|
||||||
|
|
||||||
|
@patch("box_readback_loopback.acp.get_passive_cognitive_state")
|
||||||
|
@patch("box_readback_loopback.gitea_api")
|
||||||
|
def test_loopback_defers_when_thinking(self, mock_gitea, mock_cog):
|
||||||
|
# 25 minutes inactive, but agent is THINKING -> defer
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
old = (now - timedelta(minutes=25)).isoformat()
|
||||||
|
mock_gitea.side_effect = lambda ep, **kwargs: (
|
||||||
|
[{"number": 999, "title": "Stalled", "created_at": old, "assignee": {"username": "dev"}}]
|
||||||
|
if ep == "/repos/super/box/issues?state=open"
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
mock_cog.return_value = {"status": "THINKING", "cognitive_lock": True}
|
||||||
|
|
||||||
|
actions = brl.run_loopback_sweep(dry_run=True, verbose=False)
|
||||||
|
self.assertEqual(len(actions), 0)
|
||||||
|
|
||||||
|
@patch("box_readback_loopback.acp.get_passive_cognitive_state")
|
||||||
|
@patch("box_readback_loopback.gitea_api")
|
||||||
|
def test_loopback_tier2_after_45m(self, mock_gitea, mock_cog):
|
||||||
|
# 50 minutes inactive -> Tier 2
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
old = (now - timedelta(minutes=50)).isoformat()
|
||||||
|
mock_gitea.side_effect = lambda ep, **kwargs: (
|
||||||
|
[{"number": 999, "title": "Stalled", "created_at": old, "assignee": {"username": "dev"}}]
|
||||||
|
if ep == "/repos/super/box/issues?state=open"
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
mock_cog.return_value = {"status": "IDLE", "cognitive_lock": False}
|
||||||
|
|
||||||
|
actions = brl.run_loopback_sweep(dry_run=True, verbose=False)
|
||||||
|
self.assertTrue(any("Tier 2" in a for a in actions))
|
||||||
|
|
||||||
|
@patch("box_readback_loopback.acp.get_passive_cognitive_state")
|
||||||
|
@patch("box_readback_loopback.gitea_api")
|
||||||
|
def test_loopback_tier3_after_90m(self, mock_gitea, mock_cog):
|
||||||
|
# 95 minutes inactive -> Tier 3
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
old = (now - timedelta(minutes=95)).isoformat()
|
||||||
|
mock_gitea.side_effect = lambda ep, **kwargs: (
|
||||||
|
[{"number": 999, "title": "Stalled", "created_at": old, "assignee": {"username": "dev"}}]
|
||||||
|
if ep == "/repos/super/box/issues?state=open"
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
mock_cog.return_value = {"status": "IDLE", "cognitive_lock": False}
|
||||||
|
|
||||||
|
actions = brl.run_loopback_sweep(dry_run=True, verbose=False)
|
||||||
|
self.assertTrue(any("Tier 3" in a for a in actions))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""test_box_stability_watcher.py — Comprehensive unit tests for Box Stability Watcher."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
REPO_ROOT = Path("/home/super/Projects/NetVM")
|
||||||
|
WATCHERS_DIR = REPO_ROOT / "watchers"
|
||||||
|
sys.path.insert(0, str(WATCHERS_DIR))
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
spec = importlib.util.spec_from_file_location("box_stability_watcher", str(WATCHERS_DIR / "box-stability-watcher.py"))
|
||||||
|
w = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(w)
|
||||||
|
|
||||||
|
|
||||||
|
class TestConfigAndSafety(unittest.TestCase):
|
||||||
|
def test_load_config_defaults(self):
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
non_existent = Path(td) / "missing.json"
|
||||||
|
cfg = w.load_config(non_existent)
|
||||||
|
self.assertIn("thresholds", cfg)
|
||||||
|
self.assertIn("protected_commands", cfg)
|
||||||
|
self.assertEqual(cfg["thresholds"]["load_warning"], 20.0)
|
||||||
|
|
||||||
|
def test_is_protected(self):
|
||||||
|
cfg = {"protected_commands": ["sshd", "tailscaled", "tmux", "systemd", "ghostty"]}
|
||||||
|
self.assertTrue(w.is_protected(1, "systemd", "/sbin/init", cfg))
|
||||||
|
self.assertTrue(w.is_protected(os.getpid(), "python3", "some_script", cfg))
|
||||||
|
self.assertTrue(w.is_protected(999, "sshd", "/usr/sbin/sshd -D", cfg))
|
||||||
|
self.assertTrue(w.is_protected(888, "tmux", "tmux new-session -s main", cfg))
|
||||||
|
self.assertTrue(w.is_protected(777, "tailscaled", "/usr/sbin/tailscaled", cfg))
|
||||||
|
self.assertFalse(w.is_protected(1234, "muse-bin", "/home/super/.local/bin/muse-bin-1.4.3 resume abc", cfg))
|
||||||
|
self.assertFalse(w.is_protected(5678, "chromium", "/usr/lib/chromium/chromium --type=renderer", cfg))
|
||||||
|
# Shell protection & runaway exemption
|
||||||
|
self.assertTrue(w.is_protected(9999, "bash", "/bin/bash", {"protected_commands": ["bash"]}, rss_mb=50))
|
||||||
|
self.assertFalse(w.is_protected(9999, "bash", "bash test_script.sh", {"protected_commands": ["bash"]}, rss_mb=2500))
|
||||||
|
|
||||||
|
|
||||||
|
class TestStabilityEvaluation(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.cfg = {
|
||||||
|
"thresholds": {
|
||||||
|
"load_warning": 20.0,
|
||||||
|
"load_critical": 35.0,
|
||||||
|
"load_emergency": 60.0,
|
||||||
|
"ram_warning_pct": 80.0,
|
||||||
|
"ram_critical_pct": 90.0,
|
||||||
|
"swap_warning_pct": 75.0,
|
||||||
|
"swap_critical_pct": 85.0,
|
||||||
|
"process_rss_warning_mb": 2000,
|
||||||
|
"process_rss_critical_mb": 3000,
|
||||||
|
},
|
||||||
|
"protected_commands": ["sshd", "tmux"]
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_green_tier(self):
|
||||||
|
metrics = {"load_1m": 2.5, "ram_used_pct": 30.0, "swap_used_pct": 10.0}
|
||||||
|
procs = [
|
||||||
|
{"pid": 101, "cmdline": "muse-bin", "rss_mb": 400, "cpu_pct": 5.0, "is_protected": False, "nice": 0}
|
||||||
|
]
|
||||||
|
tier, reasons, actions = w.evaluate_stability(metrics, procs, self.cfg)
|
||||||
|
self.assertEqual(tier, "GREEN")
|
||||||
|
self.assertEqual(reasons, [])
|
||||||
|
self.assertEqual(actions, [])
|
||||||
|
|
||||||
|
def test_yellow_tier_elevated_load(self):
|
||||||
|
metrics = {"load_1m": 22.5, "ram_used_pct": 50.0, "swap_used_pct": 20.0}
|
||||||
|
procs = [
|
||||||
|
{"pid": 102, "cmdline": "python worker", "rss_mb": 500, "cpu_pct": 90.0, "is_protected": False, "nice": 0}
|
||||||
|
]
|
||||||
|
tier, reasons, actions = w.evaluate_stability(metrics, procs, self.cfg)
|
||||||
|
self.assertEqual(tier, "YELLOW")
|
||||||
|
self.assertTrue(any("Elevated load/memory" in r for r in reasons))
|
||||||
|
self.assertEqual(len(actions), 1)
|
||||||
|
self.assertEqual(actions[0]["action"], "renice")
|
||||||
|
self.assertEqual(actions[0]["pid"], 102)
|
||||||
|
|
||||||
|
def test_orange_tier_pause_leaky_process(self):
|
||||||
|
metrics = {"load_1m": 2.0, "ram_used_pct": 40.0, "swap_used_pct": 10.0}
|
||||||
|
procs = [
|
||||||
|
{"pid": 202, "cmdline": "muse-bin leaky", "rss_mb": 3400, "cpu_pct": 10.0, "is_protected": False, "nice": 0}
|
||||||
|
]
|
||||||
|
tier, reasons, actions = w.evaluate_stability(metrics, procs, self.cfg)
|
||||||
|
self.assertEqual(tier, "ORANGE")
|
||||||
|
self.assertTrue(any("exceeded critical RSS" in r for r in reasons))
|
||||||
|
self.assertEqual(len(actions), 1)
|
||||||
|
self.assertEqual(actions[0]["action"], "pause")
|
||||||
|
self.assertEqual(actions[0]["pid"], 202)
|
||||||
|
|
||||||
|
def test_red_emergency_tier(self):
|
||||||
|
metrics = {"load_1m": 75.0, "ram_used_pct": 96.0, "swap_used_pct": 94.0}
|
||||||
|
procs = [
|
||||||
|
{"pid": 301, "cmdline": "muse-bin heavy", "rss_mb": 1800, "cpu_pct": 50.0, "is_protected": False, "nice": 0},
|
||||||
|
{"pid": 302, "cmdline": "sshd daemon", "rss_mb": 2000, "cpu_pct": 2.0, "is_protected": True, "nice": 0}
|
||||||
|
]
|
||||||
|
tier, reasons, actions = w.evaluate_stability(metrics, procs, self.cfg)
|
||||||
|
self.assertEqual(tier, "RED")
|
||||||
|
self.assertTrue(any("Emergency host pressure" in r for r in reasons))
|
||||||
|
pause_pids = [a["pid"] for a in actions if a["action"] == "pause"]
|
||||||
|
self.assertIn(301, pause_pids)
|
||||||
|
self.assertNotIn(302, pause_pids)
|
||||||
|
|
||||||
|
|
||||||
|
class TestSocketIsolation(unittest.TestCase):
|
||||||
|
def test_distinguishes_user_from_box_launched_agents(self):
|
||||||
|
# PID 555 is an automated job on default socket
|
||||||
|
# PID 666 is an agent on the correct fleet socket
|
||||||
|
# PID 777 is a user-launched interactive agent on default socket
|
||||||
|
procs = [
|
||||||
|
{"pid": 555, "cmdline": "muse-bin auto-work sweep", "tmux_sock": "/tmp/tmux-1000/default", "is_protected": False},
|
||||||
|
{"pid": 666, "cmdline": "muse-bin auto-work sweep", "tmux_sock": "/tmp/tmux-muse.sock", "is_protected": False},
|
||||||
|
{"pid": 777, "cmdline": "muse-bin interactive chat", "tmux_sock": "/tmp/tmux-1000/default", "is_protected": False},
|
||||||
|
]
|
||||||
|
box_sessions = {"auto-work": {}}
|
||||||
|
violations, allowed = w.check_socket_isolation_violations(procs, box_sessions=box_sessions)
|
||||||
|
self.assertEqual(len(violations), 1)
|
||||||
|
self.assertEqual(violations[0]["pid"], 555)
|
||||||
|
self.assertEqual(len(allowed), 1)
|
||||||
|
self.assertEqual(allowed[0]["pid"], 777)
|
||||||
|
|
||||||
|
|
||||||
|
class TestPauseResumeAndExpiry(unittest.TestCase):
|
||||||
|
@mock.patch("os.kill")
|
||||||
|
def test_pause_and_state_persistence(self, mock_kill):
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
state_file = Path(td) / "paused.json"
|
||||||
|
actions = [{"action": "pause", "pid": 4321, "cmd": "muse-bin", "reason": "RSS high"}]
|
||||||
|
executed = w.execute_actions(actions, state_file=state_file, dry_run=False)
|
||||||
|
self.assertEqual(len(executed), 1)
|
||||||
|
mock_kill.assert_called_once_with(4321, 19) # SIGSTOP = 19
|
||||||
|
self.assertTrue(state_file.exists())
|
||||||
|
data = json.loads(state_file.read_text())
|
||||||
|
self.assertIn("4321", data)
|
||||||
|
|
||||||
|
@mock.patch("os.kill")
|
||||||
|
def test_reconcile_expired_pause(self, mock_kill):
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
state_file = Path(td) / "paused.json"
|
||||||
|
now = w.now_epoch()
|
||||||
|
state_data = {
|
||||||
|
"4321": {"pid": 4321, "cmd": "muse-bin", "paused_at_epoch": now - 70}
|
||||||
|
}
|
||||||
|
state_file.write_text(json.dumps(state_data))
|
||||||
|
|
||||||
|
expired = w.reconcile_paused_processes(state_file=state_file, dry_run=False)
|
||||||
|
self.assertEqual(len(expired), 1)
|
||||||
|
self.assertEqual(expired[0]["action"], "cull_expired")
|
||||||
|
self.assertEqual(expired[0]["pid"], 4321)
|
||||||
|
mock_kill.assert_called_once_with(4321, 15) # SIGTERM = 15
|
||||||
|
remaining = json.loads(state_file.read_text())
|
||||||
|
self.assertNotIn("4321", remaining)
|
||||||
|
|
||||||
|
@mock.patch("os.kill")
|
||||||
|
def test_resume_process(self, mock_kill):
|
||||||
|
with tempfile.TemporaryDirectory() as td:
|
||||||
|
state_file = Path(td) / "paused.json"
|
||||||
|
state_file.write_text(json.dumps({"4321": {"pid": 4321}}))
|
||||||
|
res = w.resume_process(4321, state_file=state_file)
|
||||||
|
self.assertTrue(res["success"])
|
||||||
|
mock_kill.assert_called_once_with(4321, 18) # SIGCONT = 18
|
||||||
|
remaining = json.loads(state_file.read_text())
|
||||||
|
self.assertNotIn("4321", remaining)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,670 @@
|
|||||||
|
"""test_box_tui_render.py — Headless verification of Box TUI and Work Pipeline screens."""
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import curses
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
BIN_DIR = REPO_ROOT / "bin"
|
||||||
|
if str(BIN_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(BIN_DIR))
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
spec = importlib.util.spec_from_file_location("muse_tui", str(BIN_DIR / "muse-tui.py"))
|
||||||
|
muse_tui = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(muse_tui)
|
||||||
|
|
||||||
|
spec_fleet = importlib.util.spec_from_file_location("box_fleet_tui", str(BIN_DIR / "box-fleet-tui.py"))
|
||||||
|
box_fleet_tui = importlib.util.module_from_spec(spec_fleet)
|
||||||
|
spec_fleet.loader.exec_module(box_fleet_tui)
|
||||||
|
|
||||||
|
|
||||||
|
class TestBoxTUIHeadless(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.mock_stdscr = MagicMock()
|
||||||
|
self.mock_stdscr.getmaxyx.return_value = (40, 120)
|
||||||
|
self.mock_stdscr.getch.return_value = -1
|
||||||
|
|
||||||
|
def test_box_tui_initialization_work_tab(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="work")
|
||||||
|
self.assertEqual(app.mode, "box")
|
||||||
|
self.assertEqual(app.box_tab, 7)
|
||||||
|
self.assertIn("8: Work Pipeline", muse_tui.BOX_TABS)
|
||||||
|
|
||||||
|
def test_render_work_view(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="work")
|
||||||
|
# Populate mock cognitive cache and work issues
|
||||||
|
app.data.cognitive_cache = {
|
||||||
|
"muse": {
|
||||||
|
"node": "muse",
|
||||||
|
"status": "IDLE",
|
||||||
|
"cognitive_lock": False,
|
||||||
|
"screen": {"url": "https://muse.ai/", "title": "Chat - muse"}
|
||||||
|
},
|
||||||
|
"pip": {
|
||||||
|
"node": "pip",
|
||||||
|
"status": "SIDECHAT_IDLE",
|
||||||
|
"cognitive_lock": False,
|
||||||
|
"screen": {"url": "https://muse.ai/thread/123", "title": "Chat - pip"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
app.data.work_issues_cache = [
|
||||||
|
{"number": 218, "state": "open", "title": "Autonomous TUI upgrade", "assignee": {"username": "dev"}}
|
||||||
|
]
|
||||||
|
app.data.work_prs_cache = [
|
||||||
|
{"number": 219, "state": "open", "merged": False, "title": "feat: box tui", "head": {"ref": "feature/tui"}}
|
||||||
|
]
|
||||||
|
|
||||||
|
# Call _render_work_view directly
|
||||||
|
app._render_work_view(2, 0, 35, 120)
|
||||||
|
self.assertTrue(self.mock_stdscr.addstr.called)
|
||||||
|
|
||||||
|
def test_box_tui_key_actions_on_work_tab(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="work")
|
||||||
|
app.data.nodes = ["muse", "pip", "646", "opm", "def", "dev"]
|
||||||
|
|
||||||
|
# 'w' opens work_dispatch modal
|
||||||
|
handled = app._handle_key(ord('w'))
|
||||||
|
self.assertTrue(handled)
|
||||||
|
self.assertEqual(app.modal, "work_dispatch")
|
||||||
|
|
||||||
|
# Esc closes modal
|
||||||
|
handled = app._handle_key(27)
|
||||||
|
self.assertTrue(handled)
|
||||||
|
self.assertIsNone(app.modal)
|
||||||
|
|
||||||
|
# Tab navigation: '1' switches to Chat (tab 0), '8' switches to Work (tab 7)
|
||||||
|
app._handle_key(ord('1'))
|
||||||
|
self.assertEqual(app.box_tab, 0)
|
||||||
|
app._handle_key(ord('8'))
|
||||||
|
self.assertEqual(app.box_tab, 7)
|
||||||
|
|
||||||
|
def test_work_dispatch_modal_text_input_and_cycle(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="work")
|
||||||
|
app.data.nodes = ["muse", "pip", "646", "opm", "def", "dev"]
|
||||||
|
app.modal = "work_dispatch"
|
||||||
|
app.modal_input_buf = ""
|
||||||
|
app.modal_input_cursor = 0
|
||||||
|
app.dispatch_target_node = "dev"
|
||||||
|
|
||||||
|
# Type chars 'Build #12'
|
||||||
|
for c in "Build #12":
|
||||||
|
app._handle_key(ord(c))
|
||||||
|
self.assertEqual(app.modal_input_buf, "Build #12")
|
||||||
|
|
||||||
|
# Tab cycles worker
|
||||||
|
app._handle_key(ord('\t'))
|
||||||
|
self.assertNotEqual(app.dispatch_target_node, "dev")
|
||||||
|
|
||||||
|
def test_agent_thoughts_modal(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="work")
|
||||||
|
app.selected_thought_node = "def"
|
||||||
|
app.agent_thought_data = {
|
||||||
|
"screen": {
|
||||||
|
"url": "https://muse.ai/",
|
||||||
|
"title": "Chat - def",
|
||||||
|
"is_generating": False,
|
||||||
|
"recent_paragraphs": ["Inspecting workspace...", "Code updated."]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
app.modal = "agent_thoughts"
|
||||||
|
# Render modal
|
||||||
|
app._render_modal(40, 120)
|
||||||
|
self.assertTrue(self.mock_stdscr.addstr.called)
|
||||||
|
|
||||||
|
# Esc closes modal
|
||||||
|
app._handle_key(27)
|
||||||
|
self.assertIsNone(app.modal)
|
||||||
|
|
||||||
|
def test_agent_profile_menu_modal(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="work")
|
||||||
|
app.selected_menu_node = "pip"
|
||||||
|
app.menu_tab = "activity"
|
||||||
|
app.agent_menu_data = {
|
||||||
|
"tab": "activity",
|
||||||
|
"data": {
|
||||||
|
"items": [{"title": "Session started", "detail": "Active", "time": "12m ago"}]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
app.modal = "agent_menu"
|
||||||
|
# Render modal
|
||||||
|
app._render_modal(40, 120)
|
||||||
|
self.assertTrue(self.mock_stdscr.addstr.called)
|
||||||
|
|
||||||
|
# Tab key cycles menu tabs
|
||||||
|
with patch.object(app, "_async_load_menu") as mock_load:
|
||||||
|
app._handle_key(ord('\t'))
|
||||||
|
mock_load.assert_called_with("pip", "upcoming")
|
||||||
|
|
||||||
|
def test_transcript_sub_header_render_and_mouse_click(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="muse")
|
||||||
|
app.data.active_node = "dev"
|
||||||
|
app.focus_pane = "transcript"
|
||||||
|
app._render_transcript(0, 0, 30, 100)
|
||||||
|
self.assertIsNotNone(app._btn_case_actions_bounds)
|
||||||
|
btn_y, b_start, b_end = app._btn_case_actions_bounds
|
||||||
|
|
||||||
|
# Click [w:Case] (offset +2)
|
||||||
|
with patch.object(app, "_open_agent_menu") as mock_menu:
|
||||||
|
handled = app._handle_mouse(b_start + 2, btn_y, curses.BUTTON1_CLICKED)
|
||||||
|
self.assertTrue(handled)
|
||||||
|
self.assertEqual(app.modal, "work_dispatch")
|
||||||
|
app.modal = None
|
||||||
|
|
||||||
|
# Click [f:Refocus] (offset +12)
|
||||||
|
with patch.object(app, "_async_refocus_main") as mock_refocus:
|
||||||
|
handled = app._handle_mouse(b_start + 12, btn_y, curses.BUTTON1_CLICKED)
|
||||||
|
self.assertTrue(handled)
|
||||||
|
mock_refocus.assert_called_with("dev")
|
||||||
|
|
||||||
|
# Click [t:Thoughts] (offset +25)
|
||||||
|
with patch.object(app, "_open_agent_thoughts") as mock_thoughts:
|
||||||
|
handled = app._handle_mouse(b_start + 25, btn_y, curses.BUTTON1_CLICKED)
|
||||||
|
self.assertTrue(handled)
|
||||||
|
mock_thoughts.assert_called_with("dev")
|
||||||
|
|
||||||
|
# Click [p:Menu] (offset +38)
|
||||||
|
with patch.object(app, "_open_agent_menu") as mock_menu:
|
||||||
|
handled = app._handle_mouse(b_start + 38, btn_y, curses.BUTTON1_CLICKED)
|
||||||
|
self.assertTrue(handled)
|
||||||
|
mock_menu.assert_called_with("dev")
|
||||||
|
|
||||||
|
# Click [h:Heal] (offset +46)
|
||||||
|
with patch.object(app, "_async_heal_agent") as mock_heal:
|
||||||
|
handled = app._handle_mouse(b_start + 46, btn_y, curses.BUTTON1_CLICKED)
|
||||||
|
self.assertTrue(handled)
|
||||||
|
mock_heal.assert_called_with("dev")
|
||||||
|
|
||||||
|
def test_agent_context_twelve_actions_and_hotkeys(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="muse")
|
||||||
|
app.data.active_node = "646"
|
||||||
|
app.context_agent = {"node": "646", "status": {}, "index": 0}
|
||||||
|
app.modal = "agent_context"
|
||||||
|
app._render_modal(40, 120)
|
||||||
|
|
||||||
|
self.assertEqual(len(app._agent_context_actions), 12)
|
||||||
|
|
||||||
|
# Hotkey 2 / w: Dispatch
|
||||||
|
app.modal = "agent_context"
|
||||||
|
app._handle_key(ord('2'))
|
||||||
|
self.assertEqual(app.modal, "work_dispatch")
|
||||||
|
self.assertEqual(app.dispatch_target_node, "646")
|
||||||
|
|
||||||
|
# Hotkey 3 / t: Thoughts
|
||||||
|
app.modal = "agent_context"
|
||||||
|
with patch.object(app, "_open_agent_thoughts") as mock_th:
|
||||||
|
app._handle_key(ord('3'))
|
||||||
|
mock_th.assert_called_with("646")
|
||||||
|
|
||||||
|
# Hotkey 4 / p: Menu
|
||||||
|
app.modal = "agent_context"
|
||||||
|
with patch.object(app, "_open_agent_menu") as mock_menu:
|
||||||
|
app._handle_key(ord('4'))
|
||||||
|
mock_menu.assert_called_with("646")
|
||||||
|
|
||||||
|
# Hotkey 5 / f: Refocus
|
||||||
|
app.modal = "agent_context"
|
||||||
|
with patch.object(app, "_async_refocus_main") as mock_ref:
|
||||||
|
app._handle_key(ord('5'))
|
||||||
|
mock_ref.assert_called_with("646")
|
||||||
|
|
||||||
|
# Hotkey 7 / h: Heal
|
||||||
|
app.modal = "agent_context"
|
||||||
|
with patch.object(app, "_async_heal_agent") as mock_heal:
|
||||||
|
app._handle_key(ord('7'))
|
||||||
|
mock_heal.assert_called_with("646")
|
||||||
|
|
||||||
|
# Hotkey 8 / l: Loopback
|
||||||
|
app.modal = "agent_context"
|
||||||
|
with patch.object(app, "_async_run_loopback") as mock_loop:
|
||||||
|
app._handle_key(ord('8'))
|
||||||
|
mock_loop.assert_called_once()
|
||||||
|
|
||||||
|
def test_work_dispatch_presets_f1_to_f4(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="muse")
|
||||||
|
app.modal = "work_dispatch"
|
||||||
|
app.modal_input_buf = ""
|
||||||
|
app.modal_input_cursor = 0
|
||||||
|
app.dispatch_target_node = "dev"
|
||||||
|
|
||||||
|
app._handle_key(curses.KEY_F1)
|
||||||
|
self.assertEqual(app.modal_input_buf, "Build: Implement feature specification and tests")
|
||||||
|
|
||||||
|
app._handle_key(curses.KEY_F2)
|
||||||
|
self.assertEqual(app.modal_input_buf, "Bugfix: Investigate and resolve error trace")
|
||||||
|
|
||||||
|
app._handle_key(curses.KEY_F3)
|
||||||
|
self.assertEqual(app.modal_input_buf, "Review: Code review and verify pull request changes")
|
||||||
|
|
||||||
|
app._handle_key(curses.KEY_F4)
|
||||||
|
self.assertEqual(app.modal_input_buf, "Audit: Pre-flight health and security audit")
|
||||||
|
|
||||||
|
# Enter triggers dispatch
|
||||||
|
with patch("threading.Thread") as mock_thread:
|
||||||
|
app._handle_key(10)
|
||||||
|
mock_thread.assert_called()
|
||||||
|
self.assertIsNone(app.modal)
|
||||||
|
|
||||||
|
def test_main_panel_normal_mode_orchestration_hotkeys(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="muse")
|
||||||
|
app.data.active_node = "opm"
|
||||||
|
app.focus_pane = "transcript"
|
||||||
|
app.editor_mode = "NORMAL"
|
||||||
|
|
||||||
|
# 'w' -> work_dispatch
|
||||||
|
app._handle_key(ord('w'))
|
||||||
|
self.assertEqual(app.modal, "work_dispatch")
|
||||||
|
self.assertEqual(app.dispatch_target_node, "opm")
|
||||||
|
app.modal = None
|
||||||
|
|
||||||
|
# 'f' -> refocus
|
||||||
|
with patch.object(app, "_async_refocus_main") as mock_ref:
|
||||||
|
app._handle_key(ord('f'))
|
||||||
|
mock_ref.assert_called_with("opm")
|
||||||
|
|
||||||
|
# 't' -> thoughts
|
||||||
|
with patch.object(app, "_open_agent_thoughts") as mock_th:
|
||||||
|
app._handle_key(ord('t'))
|
||||||
|
mock_th.assert_called_with("opm")
|
||||||
|
|
||||||
|
# 'p' -> menu
|
||||||
|
with patch.object(app, "_open_agent_menu") as mock_menu:
|
||||||
|
app._handle_key(ord('p'))
|
||||||
|
mock_menu.assert_called_with("opm")
|
||||||
|
|
||||||
|
# 'h' / 'H' -> heal
|
||||||
|
with patch.object(app, "_async_heal_agent") as mock_heal:
|
||||||
|
app._handle_key(ord('h'))
|
||||||
|
mock_heal.assert_called_with("opm")
|
||||||
|
|
||||||
|
def test_work_pipeline_assign_and_merge_hotkeys(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="work")
|
||||||
|
app.box_tab = 7
|
||||||
|
app.work_sel_idx = 1 # 646 in all_workers
|
||||||
|
|
||||||
|
with patch.object(app, "_async_merge_pr") as mock_merge:
|
||||||
|
app._handle_key(ord('m'))
|
||||||
|
mock_merge.assert_called_once()
|
||||||
|
|
||||||
|
with patch.object(app, "_async_assign_ticket") as mock_assign:
|
||||||
|
app._handle_key(ord('a'))
|
||||||
|
mock_assign.assert_called_with("646")
|
||||||
|
|
||||||
|
def test_muse_tui_all_tabs_fuzz_dimensions_and_corrupt_data(self):
|
||||||
|
dimensions = [(8, 20), (12, 40), (24, 80), (50, 160)]
|
||||||
|
|
||||||
|
corrupt_dataset = {
|
||||||
|
"cognitive": {
|
||||||
|
"muse": None,
|
||||||
|
"pip": {"status": None, "cognitive_lock": None, "screen": None},
|
||||||
|
"dev": {"status": "THINKING", "cognitive_lock": True, "screen": {"url": None, "title": None}}
|
||||||
|
},
|
||||||
|
"issues": [
|
||||||
|
{"number": 1, "state": None, "title": None, "assignee": None, "labels": None},
|
||||||
|
{"number": 2, "state": "open", "title": "Test", "assignee": {"username": None}, "labels": [None, {"name": None}]}
|
||||||
|
],
|
||||||
|
"prs": [
|
||||||
|
{"number": 1, "state": None, "merged": False, "title": None, "head": None},
|
||||||
|
{"number": 2, "state": "open", "merged": True, "title": "PR test", "head": {"ref": None}}
|
||||||
|
],
|
||||||
|
"approvals": [
|
||||||
|
{"node": None, "status": None, "target": None},
|
||||||
|
{"node": "pip", "status": "INPUT_WAIT", "input_waits": [None, {"task": None}]}
|
||||||
|
],
|
||||||
|
"jobs": [
|
||||||
|
{"name": None, "agent": None, "schedule": None, "timeout": None, "description": None}
|
||||||
|
],
|
||||||
|
"tmux": ["", "invalid:session:colon"],
|
||||||
|
"dm_logs": [
|
||||||
|
{"type": None, "agent": None, "to": None, "target": None, "ts": None, "id": None}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
for h, w in dimensions:
|
||||||
|
stdscr = MagicMock()
|
||||||
|
stdscr.getmaxyx.return_value = (h, w)
|
||||||
|
stdscr.getch.return_value = -1
|
||||||
|
|
||||||
|
app = muse_tui.MuseTUI(stdscr, initial_mode="box", initial_tab="chat")
|
||||||
|
app._term_dims = (h, w)
|
||||||
|
|
||||||
|
# Test with empty caches
|
||||||
|
app.data.cognitive_cache = {}
|
||||||
|
app.data.work_issues_cache = []
|
||||||
|
app.data.work_prs_cache = []
|
||||||
|
app.data.approvals_cache = []
|
||||||
|
app.data.jobs_cache = []
|
||||||
|
app.data.dm_logs_cache = []
|
||||||
|
|
||||||
|
content_h = max(1, h - 4)
|
||||||
|
for tab_idx in range(8):
|
||||||
|
app.box_tab = tab_idx
|
||||||
|
if tab_idx == 0:
|
||||||
|
app._render_muse_view(2, 0, content_h, w)
|
||||||
|
elif tab_idx == 1:
|
||||||
|
app._render_fleet_table(2, 0, content_h, w)
|
||||||
|
elif tab_idx == 2:
|
||||||
|
app._render_approvals_view(2, 0, content_h, w)
|
||||||
|
elif tab_idx == 3:
|
||||||
|
app._render_jobs_view(2, 0, content_h, w)
|
||||||
|
elif tab_idx == 4:
|
||||||
|
with patch("subprocess.run") as mock_sub:
|
||||||
|
mock_sub.return_value = MagicMock(returncode=0, stdout="s1: 1 windows\n")
|
||||||
|
app._render_tmux_view(2, 0, content_h, w)
|
||||||
|
elif tab_idx == 5:
|
||||||
|
app._render_dm_logs_view(2, 0, content_h, w)
|
||||||
|
elif tab_idx == 6:
|
||||||
|
app._render_ssh_view(2, 0, content_h, w)
|
||||||
|
elif tab_idx == 7:
|
||||||
|
app._render_work_view(2, 0, content_h, w)
|
||||||
|
|
||||||
|
# Test with corrupt / null caches
|
||||||
|
app.data.cognitive_cache = corrupt_dataset["cognitive"]
|
||||||
|
app.data.work_issues_cache = corrupt_dataset["issues"]
|
||||||
|
app.data.work_prs_cache = corrupt_dataset["prs"]
|
||||||
|
app.data.approvals_cache = corrupt_dataset["approvals"]
|
||||||
|
app.data.jobs_cache = corrupt_dataset["jobs"]
|
||||||
|
app.data.dm_logs_cache = corrupt_dataset["dm_logs"]
|
||||||
|
|
||||||
|
for tab_idx in range(8):
|
||||||
|
app.box_tab = tab_idx
|
||||||
|
if tab_idx == 0:
|
||||||
|
app._render_muse_view(2, 0, content_h, w)
|
||||||
|
elif tab_idx == 1:
|
||||||
|
app._render_fleet_table(2, 0, content_h, w)
|
||||||
|
elif tab_idx == 2:
|
||||||
|
app._render_approvals_view(2, 0, content_h, w)
|
||||||
|
elif tab_idx == 3:
|
||||||
|
app._render_jobs_view(2, 0, content_h, w)
|
||||||
|
elif tab_idx == 4:
|
||||||
|
with patch("subprocess.run") as mock_sub:
|
||||||
|
mock_sub.return_value = MagicMock(returncode=0, stdout="s1: 1 windows\n")
|
||||||
|
app._render_tmux_view(2, 0, content_h, w)
|
||||||
|
elif tab_idx == 5:
|
||||||
|
app._render_dm_logs_view(2, 0, content_h, w)
|
||||||
|
elif tab_idx == 6:
|
||||||
|
app._render_ssh_view(2, 0, content_h, w)
|
||||||
|
elif tab_idx == 7:
|
||||||
|
app._render_work_view(2, 0, content_h, w)
|
||||||
|
|
||||||
|
def test_box_fleet_tui_all_tabs_fuzz_dimensions_and_corrupt_data(self):
|
||||||
|
dimensions = [(8, 20), (12, 40), (24, 80), (50, 160)]
|
||||||
|
for h, w in dimensions:
|
||||||
|
stdscr = MagicMock()
|
||||||
|
stdscr.getmaxyx.return_value = (h, w)
|
||||||
|
stdscr.getch.return_value = -1
|
||||||
|
|
||||||
|
fleet_app = box_fleet_tui.BoxFleetTUI(stdscr)
|
||||||
|
|
||||||
|
# 1. Empty snapshot
|
||||||
|
fleet_app.snapshot = {}
|
||||||
|
renderers = [fleet_app._render_timers, fleet_app._render_harvest,
|
||||||
|
fleet_app._render_followups, fleet_app._render_approvals,
|
||||||
|
fleet_app._render_activity, fleet_app._render_runtimes,
|
||||||
|
fleet_app._render_work]
|
||||||
|
|
||||||
|
for tab_idx, renderer in enumerate(renderers):
|
||||||
|
fleet_app.current_tab = tab_idx
|
||||||
|
renderer(h, w)
|
||||||
|
|
||||||
|
# 2. Corrupted / null snapshot
|
||||||
|
fleet_app.snapshot = {
|
||||||
|
"timers": {"rows": [{"timer": None, "next": None, "next_rel": None, "last": None, "last_rel": None, "activates": None}]},
|
||||||
|
"harvest": {"rows": [{"freshness": None, "agent": None, "thread": None, "watermark": None, "last": None, "ago": None}]},
|
||||||
|
"followups": {"counts": {"pending": 2}, "by_recipient": {"pip": 1}, "oldest_pending": None},
|
||||||
|
"approvals": {"total_pending": 1, "checked": 1, "unreachable": [], "pending": [{"node": None, "status": None, "target": None, "title": None}]},
|
||||||
|
"activity": {"rows": [{"freshness": None, "agent": None, "last": None, "last_rel": None, "source": None, "detail": None}]},
|
||||||
|
"runtimes": {"agents": [{"live": True, "briefed": None, "enabled": True, "session": None, "hat": None, "pane": None, "state": None, "mode": None, "watcher": None}], "queue": {}, "plan": {}},
|
||||||
|
"work": {
|
||||||
|
"cognitive": {"dev": None},
|
||||||
|
"issues": [{"number": None, "state": None, "assignee": None, "created_at": None, "title": None}],
|
||||||
|
"loopback_active": True
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for tab_idx, renderer in enumerate(renderers):
|
||||||
|
fleet_app.current_tab = tab_idx
|
||||||
|
renderer(h, w)
|
||||||
|
|
||||||
|
fleet_app._render_header(w)
|
||||||
|
fleet_app._render_footer(h, w)
|
||||||
|
fleet_app._render_help(h, w)
|
||||||
|
|
||||||
|
def test_work_pipeline_multi_component_focus_cycling_and_navigation(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="work")
|
||||||
|
app.box_tab = 7
|
||||||
|
|
||||||
|
# 1. Section cycling with Tab, Shift-Tab, l, h
|
||||||
|
self.assertEqual(app.work_focus_component, "tickets")
|
||||||
|
app._handle_key(ord('\t'))
|
||||||
|
self.assertEqual(app.work_focus_component, "prs")
|
||||||
|
app._handle_key(ord('\t'))
|
||||||
|
self.assertEqual(app.work_focus_component, "workers")
|
||||||
|
app._handle_key(ord('\t'))
|
||||||
|
self.assertEqual(app.work_focus_component, "tickets")
|
||||||
|
|
||||||
|
app._handle_key(curses.KEY_BTAB)
|
||||||
|
self.assertEqual(app.work_focus_component, "workers")
|
||||||
|
app._handle_key(curses.KEY_BTAB)
|
||||||
|
self.assertEqual(app.work_focus_component, "prs")
|
||||||
|
|
||||||
|
app._handle_key(ord('l'))
|
||||||
|
self.assertEqual(app.work_focus_component, "workers")
|
||||||
|
app._handle_key(ord('h'))
|
||||||
|
# If on workers, 'h' heals unless not on workers or cycled
|
||||||
|
app.work_focus_component = "prs"
|
||||||
|
app._handle_key(ord('h'))
|
||||||
|
self.assertEqual(app.work_focus_component, "tickets")
|
||||||
|
|
||||||
|
# 2. Navigation with j/k, PageUp/Down, Home/End
|
||||||
|
app.data.work_issues_cache = [{"number": i, "title": f"Issue {i}", "state": "open"} for i in range(10)]
|
||||||
|
app.data.work_prs_cache = [{"number": i, "title": f"PR {i}", "state": "open", "merged": False} for i in range(8)]
|
||||||
|
|
||||||
|
# On tickets
|
||||||
|
app.work_focus_component = "tickets"
|
||||||
|
app.work_tickets_sel_idx = 0
|
||||||
|
app._handle_key(ord('j'))
|
||||||
|
self.assertEqual(app.work_tickets_sel_idx, 1)
|
||||||
|
app._handle_key(curses.KEY_DOWN)
|
||||||
|
self.assertEqual(app.work_tickets_sel_idx, 2)
|
||||||
|
app._handle_key(ord('k'))
|
||||||
|
self.assertEqual(app.work_tickets_sel_idx, 1)
|
||||||
|
app._handle_key(curses.KEY_UP)
|
||||||
|
self.assertEqual(app.work_tickets_sel_idx, 0)
|
||||||
|
app._handle_key(curses.KEY_NPAGE) # PageDown (+5)
|
||||||
|
self.assertEqual(app.work_tickets_sel_idx, 5)
|
||||||
|
app._handle_key(curses.KEY_PPAGE) # PageUp (-5)
|
||||||
|
self.assertEqual(app.work_tickets_sel_idx, 0)
|
||||||
|
app._handle_key(ord('G')) # End
|
||||||
|
self.assertEqual(app.work_tickets_sel_idx, 9)
|
||||||
|
app._handle_key(ord('g')) # Home
|
||||||
|
self.assertEqual(app.work_tickets_sel_idx, 0)
|
||||||
|
|
||||||
|
# On PRs
|
||||||
|
app.work_focus_component = "prs"
|
||||||
|
app.work_prs_sel_idx = 0
|
||||||
|
app._handle_key(ord('j'))
|
||||||
|
self.assertEqual(app.work_prs_sel_idx, 1)
|
||||||
|
app._handle_key(ord('k'))
|
||||||
|
self.assertEqual(app.work_prs_sel_idx, 0)
|
||||||
|
app._handle_key(ord('G'))
|
||||||
|
self.assertEqual(app.work_prs_sel_idx, 7)
|
||||||
|
app._handle_key(ord('g'))
|
||||||
|
self.assertEqual(app.work_prs_sel_idx, 0)
|
||||||
|
|
||||||
|
# On workers
|
||||||
|
app.work_focus_component = "workers"
|
||||||
|
app.work_sel_idx = 0
|
||||||
|
app._handle_key(ord('j'))
|
||||||
|
self.assertEqual(app.work_sel_idx, 1)
|
||||||
|
app._handle_key(ord('k'))
|
||||||
|
self.assertEqual(app.work_sel_idx, 0)
|
||||||
|
app._handle_key(ord('G'))
|
||||||
|
self.assertEqual(app.work_sel_idx, 6)
|
||||||
|
app._handle_key(ord('g'))
|
||||||
|
self.assertEqual(app.work_sel_idx, 0)
|
||||||
|
|
||||||
|
def test_work_pipeline_mouse_bounds_and_section_selection(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="work")
|
||||||
|
app.box_tab = 7
|
||||||
|
app.data.work_issues_cache = [{"number": i, "title": f"Ticket {i}", "state": "open"} for i in range(12)]
|
||||||
|
app.data.work_prs_cache = [{"number": i, "title": f"PR {i}", "state": "open", "merged": False} for i in range(8)]
|
||||||
|
|
||||||
|
# Render to calculate bounds
|
||||||
|
app._render_work_view(2, 0, 30, 100)
|
||||||
|
self.assertTrue(hasattr(app, "_work_workers_bounds"))
|
||||||
|
self.assertTrue(hasattr(app, "_work_tickets_bounds"))
|
||||||
|
self.assertTrue(hasattr(app, "_work_prs_bounds"))
|
||||||
|
|
||||||
|
w_start, w_end = app._work_workers_bounds
|
||||||
|
t_start, t_end = app._work_tickets_bounds
|
||||||
|
p_start, p_end = app._work_prs_bounds
|
||||||
|
|
||||||
|
# Click inside tickets section
|
||||||
|
click_bstate = curses.BUTTON1_CLICKED
|
||||||
|
app._handle_mouse(10, t_start + 1, click_bstate)
|
||||||
|
self.assertEqual(app.work_focus_component, "tickets")
|
||||||
|
self.assertEqual(app.work_tickets_sel_idx, 1)
|
||||||
|
|
||||||
|
# Click inside PRs section
|
||||||
|
app._handle_mouse(10, p_start + 2, click_bstate)
|
||||||
|
self.assertEqual(app.work_focus_component, "prs")
|
||||||
|
self.assertEqual(app.work_prs_sel_idx, 2)
|
||||||
|
|
||||||
|
# Click inside workers section
|
||||||
|
app._handle_mouse(10, w_start, click_bstate)
|
||||||
|
self.assertEqual(app.work_focus_component, "workers")
|
||||||
|
self.assertEqual(app.work_sel_idx, 0)
|
||||||
|
|
||||||
|
# Wheel scroll down inside tickets
|
||||||
|
wheel_down = getattr(curses, "BUTTON5_PRESSED", 0x200000)
|
||||||
|
app._handle_mouse(10, t_start + 1, wheel_down)
|
||||||
|
self.assertEqual(app.work_focus_component, "tickets")
|
||||||
|
self.assertEqual(app.work_tickets_sel_idx, 2)
|
||||||
|
|
||||||
|
def test_box_tui_scrollable_tabs_approvals_tmux_dm_logs(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="approvals")
|
||||||
|
|
||||||
|
# Tab 2: Approvals scroll
|
||||||
|
app.box_tab = 2
|
||||||
|
app.data.approvals_cache = [{"node": f"agent{i}", "tool": f"tool{i}", "status": "PENDING"} for i in range(15)]
|
||||||
|
app.approvals_sel_idx = 10
|
||||||
|
app._render_approvals_view(2, 0, 10, 80)
|
||||||
|
self.assertGreater(app.approvals_scroll_start, 0)
|
||||||
|
|
||||||
|
# Tab 4: Tmux scroll
|
||||||
|
app.box_tab = 4
|
||||||
|
app.data.tmux_cache = [f"session_{i}: 1 windows" for i in range(12)]
|
||||||
|
app.tmux_sel_idx = 8
|
||||||
|
app._render_tmux_view(2, 0, 10, 80)
|
||||||
|
self.assertGreater(app.tmux_scroll_start, 0)
|
||||||
|
app._handle_key(ord('k'))
|
||||||
|
self.assertEqual(app.tmux_sel_idx, 7)
|
||||||
|
app._handle_key(ord('j'))
|
||||||
|
self.assertEqual(app.tmux_sel_idx, 8)
|
||||||
|
|
||||||
|
# Tab 5: DM Logs scroll
|
||||||
|
app.box_tab = 5
|
||||||
|
app.data.dm_logs_cache = [{"type": "sent", "agent": "opm", "to": "646", "target": "box", "ts": "2026-10-10T12:00:00", "id": f"m{i}"} for i in range(15)]
|
||||||
|
app.dm_logs_sel_idx = 12
|
||||||
|
app._render_dm_logs_view(2, 0, 10, 80)
|
||||||
|
self.assertGreater(app.dm_logs_scroll_start, 0)
|
||||||
|
app._handle_key(ord('k'))
|
||||||
|
self.assertEqual(app.dm_logs_sel_idx, 11)
|
||||||
|
app._handle_key(ord('j'))
|
||||||
|
self.assertEqual(app.dm_logs_sel_idx, 12)
|
||||||
|
app._handle_key(ord('g'))
|
||||||
|
self.assertEqual(app.dm_logs_sel_idx, 0)
|
||||||
|
app._handle_key(ord('G'))
|
||||||
|
self.assertEqual(app.dm_logs_sel_idx, 14)
|
||||||
|
|
||||||
|
def test_box_ssh_view_render_scrollbar_and_bounds(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="ssh")
|
||||||
|
self.assertEqual(app.box_tab, 6)
|
||||||
|
|
||||||
|
# Populate multiple nodes to ensure scrolling
|
||||||
|
app.data.nodes = [f"node_{i}" for i in range(15)]
|
||||||
|
app.ssh_sel_idx = 8
|
||||||
|
app._render_ssh_view(2, 0, 10, 100)
|
||||||
|
|
||||||
|
# Check bounds recorded
|
||||||
|
self.assertTrue(hasattr(app, "_ssh_view_bounds"))
|
||||||
|
self.assertEqual(app._ssh_view_bounds[0], 6) # y (2) + 4 = 6
|
||||||
|
self.assertGreater(app._ssh_view_bounds[1], app._ssh_view_bounds[0])
|
||||||
|
|
||||||
|
# Verify scrollbar and tags in screen output
|
||||||
|
rendered_texts = [call[0][2] for call in self.mock_stdscr.addstr.call_args_list if len(call[0]) >= 3 and isinstance(call[0][2], str)]
|
||||||
|
self.assertTrue(any("█" in t or "│" in t for t in rendered_texts))
|
||||||
|
self.assertTrue(any("[Diag]" in t for t in rendered_texts))
|
||||||
|
self.assertTrue(any("[SSH]" in t for t in rendered_texts))
|
||||||
|
|
||||||
|
def test_box_diagnostics_modal_and_hotkeys(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="ssh")
|
||||||
|
self.assertEqual(app.box_tab, 6)
|
||||||
|
app.data.nodes = ["muse-main", "muse", "646"]
|
||||||
|
app.ssh_sel_idx = 0
|
||||||
|
|
||||||
|
# 'd' opens box_diagnostics modal
|
||||||
|
handled = app._handle_key(ord('d'))
|
||||||
|
self.assertTrue(handled)
|
||||||
|
self.assertEqual(app.modal, "box_diagnostics")
|
||||||
|
|
||||||
|
# Render modal
|
||||||
|
app._render_modal(30, 100)
|
||||||
|
rendered_texts = [call[0][2] for call in self.mock_stdscr.addstr.call_args_list if len(call[0]) >= 3 and isinstance(call[0][2], str)]
|
||||||
|
self.assertTrue(any("BOX DIAGNOSTICS & TROUBLESHOOTING" in t for t in rendered_texts))
|
||||||
|
|
||||||
|
# Hotkey '2' / 'c' copies dial command
|
||||||
|
with patch.object(muse_tui, "copy_to_clipboard", return_value=True) as mock_cp:
|
||||||
|
app._handle_key(ord('c'))
|
||||||
|
self.assertTrue(mock_cp.called)
|
||||||
|
|
||||||
|
# Hotkey '4' / 'k' triggers key check
|
||||||
|
with patch.object(app, "_async_ssh_key_check") as mock_kc:
|
||||||
|
app._handle_key(ord('k'))
|
||||||
|
self.assertTrue(mock_kc.called)
|
||||||
|
|
||||||
|
# Hotkey '5' / 'h' triggers recovery probe
|
||||||
|
with patch.object(app, "_async_ssh_recovery_probe") as mock_rp:
|
||||||
|
app._handle_key(ord('h'))
|
||||||
|
self.assertTrue(mock_rp.called)
|
||||||
|
|
||||||
|
# Hotkey 'q' closes modal
|
||||||
|
app._handle_key(ord('q'))
|
||||||
|
self.assertIsNone(app.modal)
|
||||||
|
|
||||||
|
# Enter also opens diagnostics modal
|
||||||
|
app._handle_key(ord('\n'))
|
||||||
|
self.assertEqual(app.modal, "box_diagnostics")
|
||||||
|
app._handle_key(27) # Esc closes
|
||||||
|
self.assertIsNone(app.modal)
|
||||||
|
|
||||||
|
def test_box_ssh_mouse_interactions(self):
|
||||||
|
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="ssh")
|
||||||
|
app.data.nodes = ["muse-main", "muse", "646"]
|
||||||
|
app.ssh_sel_idx = 0
|
||||||
|
h, w = self.mock_stdscr.getmaxyx()
|
||||||
|
app._render_ssh_view(2, 0, 10, w)
|
||||||
|
|
||||||
|
bounds = app._ssh_view_bounds
|
||||||
|
# Click on row 1 (my = bounds[0] + 1)
|
||||||
|
btn1 = getattr(curses, "BUTTON1_CLICKED", 0x4)
|
||||||
|
app._handle_mouse(10, bounds[0] + 1, btn1)
|
||||||
|
self.assertEqual(app.ssh_sel_idx, 1)
|
||||||
|
|
||||||
|
# Click on [Diag] button (mx = w - 12)
|
||||||
|
app._handle_mouse(w - 12, bounds[0] + 1, btn1)
|
||||||
|
self.assertEqual(app.modal, "box_diagnostics")
|
||||||
|
app.modal = None
|
||||||
|
|
||||||
|
# Click on [SSH] button (mx = w - 4)
|
||||||
|
with patch.object(app, "_ssh_popout_selected") as mock_pop:
|
||||||
|
app._handle_mouse(w - 4, bounds[0] + 1, btn1)
|
||||||
|
self.assertTrue(mock_pop.called)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import unittest
|
||||||
|
import tempfile
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
sys.path.insert(0, str(REPO_ROOT / "bin"))
|
||||||
|
|
||||||
|
import box_work
|
||||||
|
|
||||||
|
class TestBoxWork(unittest.TestCase):
|
||||||
|
def test_workers_topology(self):
|
||||||
|
worker_names = [w["name"] for w in box_work.WORKERS]
|
||||||
|
self.assertIn("opm", worker_names)
|
||||||
|
self.assertIn("646", worker_names)
|
||||||
|
self.assertIn("dev", worker_names)
|
||||||
|
self.assertIn("pip", worker_names)
|
||||||
|
self.assertIn("def", worker_names)
|
||||||
|
self.assertIn("muse", worker_names)
|
||||||
|
self.assertIn("muse-main", worker_names)
|
||||||
|
|
||||||
|
def test_gitea_config_resolution(self):
|
||||||
|
api_base, token = box_work.get_gitea_config()
|
||||||
|
self.assertTrue(api_base.startswith("http"))
|
||||||
|
self.assertTrue(len(token) > 10)
|
||||||
|
|
||||||
|
def test_color_helpers(self):
|
||||||
|
self.assertTrue(len(box_work.c_bold("test")) >= 4)
|
||||||
|
self.assertTrue(len(box_work.c_green("test")) >= 4)
|
||||||
|
self.assertTrue(len(box_work.c_red("test")) >= 4)
|
||||||
|
|
||||||
|
def test_find_repo_root(self):
|
||||||
|
root = box_work.find_repo_root()
|
||||||
|
self.assertTrue(root.exists())
|
||||||
|
|
||||||
|
def test_claimed_tasks_empty_or_dict(self):
|
||||||
|
res = box_work.get_claimed_tasks()
|
||||||
|
self.assertIsInstance(res, dict)
|
||||||
|
|
||||||
|
def test_recent_done_tasks_list(self):
|
||||||
|
res = box_work.get_recent_done_tasks(limit=5)
|
||||||
|
self.assertIsInstance(res, list)
|
||||||
|
|
||||||
|
def test_check_agent_preflight_structure(self):
|
||||||
|
res = box_work.check_agent_preflight("opm")
|
||||||
|
self.assertIn("hatch", res)
|
||||||
|
self.assertIn("restore", res)
|
||||||
|
self.assertIn("git", res)
|
||||||
|
self.assertIn("status", res["hatch"])
|
||||||
|
self.assertIn("status", res["restore"])
|
||||||
|
self.assertIn("status", res["git"])
|
||||||
|
self.assertIn("ready", res)
|
||||||
|
|
||||||
|
def test_check_agent_preflight_unknown_fails(self):
|
||||||
|
res = box_work.check_agent_preflight("nonexistent_agent_xyz")
|
||||||
|
self.assertFalse(res["ready"])
|
||||||
|
self.assertEqual(res["overall"], "FAIL")
|
||||||
|
|
||||||
|
def test_cli_alone_prints_usage_reference(self):
|
||||||
|
import subprocess
|
||||||
|
proc = subprocess.run([sys.executable, str(REPO_ROOT / "bin" / "super-cli.py")], capture_output=True, text=True)
|
||||||
|
self.assertEqual(proc.returncode, 0)
|
||||||
|
self.assertIn("BOX ORCHESTRATOR: INPUT PARAMETERS & USAGE REFERENCE", proc.stdout)
|
||||||
|
self.assertIn("PRIMARY DOMAINS & INPUT PARAMETERS:", proc.stdout)
|
||||||
|
self.assertIn("box work", proc.stdout)
|
||||||
|
self.assertIn("box tasks", proc.stdout)
|
||||||
|
self.assertIn("box fleet", proc.stdout)
|
||||||
|
|
||||||
|
def test_cli_help_prints_master_manual(self):
|
||||||
|
import subprocess
|
||||||
|
proc = subprocess.run([sys.executable, str(REPO_ROOT / "bin" / "super-cli.py"), "help"], capture_output=True, text=True)
|
||||||
|
self.assertEqual(proc.returncode, 0)
|
||||||
|
self.assertIn("BOX ORCHESTRATOR COMPREHENSIVE CLI & RUNTIME MANUAL", proc.stdout)
|
||||||
|
self.assertIn("DOMAINS & ACTION SPECIFICATIONS:", proc.stdout)
|
||||||
|
|
||||||
|
def test_cli_domain_help_prints_subcommands_and_examples(self):
|
||||||
|
import subprocess
|
||||||
|
proc = subprocess.run([sys.executable, str(REPO_ROOT / "bin" / "super-cli.py"), "work", "help"], capture_output=True, text=True)
|
||||||
|
self.assertEqual(proc.returncode, 0)
|
||||||
|
self.assertIn("SHORTHAND EXAMPLES:", proc.stdout)
|
||||||
|
self.assertIn("OPERATIONAL GUIDELINES:", proc.stdout)
|
||||||
|
self.assertIn("box work start", proc.stdout)
|
||||||
|
|
||||||
|
def test_cli_missing_args_prints_error_and_shorthand_helper(self):
|
||||||
|
import subprocess
|
||||||
|
proc = subprocess.run([sys.executable, str(REPO_ROOT / "bin" / "super-cli.py"), "work", "start"], capture_output=True, text=True)
|
||||||
|
self.assertEqual(proc.returncode, 2)
|
||||||
|
err = proc.stderr
|
||||||
|
self.assertIn("CLI ERROR:", err)
|
||||||
|
self.assertIn("SHORTHAND USAGE HELPER:", err)
|
||||||
|
self.assertIn("title", err)
|
||||||
|
self.assertIn("--to", err)
|
||||||
|
self.assertIn("Quick Examples:", err)
|
||||||
|
|
||||||
|
def test_cli_invalid_subcommand_prints_shorthand_helper(self):
|
||||||
|
import subprocess
|
||||||
|
proc = subprocess.run([sys.executable, str(REPO_ROOT / "bin" / "super-cli.py"), "work", "invalid_action_xyz"], capture_output=True, text=True)
|
||||||
|
self.assertEqual(proc.returncode, 2)
|
||||||
|
err = proc.stderr
|
||||||
|
self.assertIn("CLI ERROR:", err)
|
||||||
|
self.assertIn("SHORTHAND USAGE HELPER:", err)
|
||||||
|
self.assertIn("Available Subcommands:", err)
|
||||||
|
self.assertIn("status", err)
|
||||||
|
self.assertIn("start", err)
|
||||||
|
|
||||||
|
def test_cli_invalid_domain_prints_shorthand_helper(self):
|
||||||
|
import subprocess
|
||||||
|
proc = subprocess.run([sys.executable, str(REPO_ROOT / "bin" / "super-cli.py"), "invalid_domain_xyz"], capture_output=True, text=True)
|
||||||
|
self.assertEqual(proc.returncode, 2)
|
||||||
|
err = proc.stderr
|
||||||
|
self.assertIn("CLI ERROR:", err)
|
||||||
|
self.assertIn("SHORTHAND USAGE HELPER:", err)
|
||||||
|
self.assertIn("Primary Domains & Commands:", err)
|
||||||
|
self.assertIn("work", err)
|
||||||
|
self.assertIn("fleet", err)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
# NetVM Watchers
|
||||||
|
|
||||||
|
Dedicated directory for background autonomous health, resource, and stability watchers on NetVM host `bl`.
|
||||||
|
|
||||||
|
## Components
|
||||||
|
|
||||||
|
- **`box-stability-watcher.py`**: Host resource supervisor and load shedder. Proactively monitors:
|
||||||
|
- System 1m/5m/15m load averages against core counts.
|
||||||
|
- Host RAM and Swap pressure percentages.
|
||||||
|
- Per-process memory leaks (critical RSS thresholds for `muse-bin`, headless Chromium renderers, Python workers).
|
||||||
|
- Rogue/leaked CPU hogs starving SSH/Tailscale.
|
||||||
|
- Failing systemd user services trapped in tight restart loops.
|
||||||
|
- Socket isolation violations (automated workers running on `/tmp/tmux-1000/default` instead of `/tmp/tmux-muse.sock`).
|
||||||
|
- **`box-stability.json`**: Tunable operational thresholds, notifications, and protected process whitelist.
|
||||||
|
- **`systemd/box-stability-watcher.service`**: Systemd user daemon running the watcher continuously with 10s evaluation ticks.
|
||||||
|
|
||||||
|
## Operational Tiers & Mitigations
|
||||||
|
|
||||||
|
| Tier | Status | Trigger Condition | Automated Action |
|
||||||
|
| :--- | :--- | :--- | :--- |
|
||||||
|
| **GREEN** | Normal | Load < 20, RAM < 80%, Swap < 75% | Silent monitoring. |
|
||||||
|
| **YELLOW** | Warning | Load >= 20, RAM >= 80%, or process RSS >= 2000MB | Renice CPU hogs (+15) to preserve interactive SSH responsiveness; log warning. |
|
||||||
|
| **ORANGE** | Critical | Load >= 35, RAM >= 90%, or process RSS >= 3000MB | **Pause (SIGSTOP)** runaway worker, record in `.state/stability-paused.json`, post alert to `646 tasks` sidechat, and allow 60s operator inspection before SIGTERM. |
|
||||||
|
| **RED** | Emergency | Load >= 60, RAM >= 95%, or Swap >= 92% | Emergency load shedding of non-protected heavy consumers (>1500MB). |
|
||||||
|
|
||||||
|
## Paused Process Lifecycle (60s Grace Window)
|
||||||
|
|
||||||
|
When a process is paused:
|
||||||
|
1. Sent `SIGSTOP` immediately.
|
||||||
|
2. Recorded in `.state/stability-paused.json` with timestamp and command info.
|
||||||
|
3. Alert posted to `646 tasks` sidechat.
|
||||||
|
4. An operator can inspect the runtime or resume it via:
|
||||||
|
```bash
|
||||||
|
box stability resume <PID>
|
||||||
|
```
|
||||||
|
5. If unresumed after 60 seconds, the watcher automatically culls the process via `SIGTERM`.
|
||||||
|
|
||||||
|
## Protected Whitelist
|
||||||
|
|
||||||
|
The watcher will **never** terminate or renice core system services or interactive terminal sessions:
|
||||||
|
`sshd`, `tailscaled`, `tailscale`, `systemd`, `dbus-broker`, `pipewire`, `wireplumber`, `tmux` (main server), `ghostty`, `alacritty`.
|
||||||
|
|
||||||
|
Interactive shells (`bash`, `zsh`, `sh`) are protected while operating within normal memory bounds (<2048MB RSS). Runaway scripts or test jobs executing under `bash`/`zsh` that exceed 2048MB RSS automatically lose whitelist immunity and are subjected to the standard ORANGE pause/cull lifecycle to protect the host against OOM crashes.
|
||||||
|
|
||||||
|
## Unified Box CLI Integration
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Host stability status & active socket warnings
|
||||||
|
box stability status
|
||||||
|
|
||||||
|
# Machine-readable JSON output
|
||||||
|
box stability json
|
||||||
|
|
||||||
|
# Single evaluation check
|
||||||
|
box stability check [--dry-run]
|
||||||
|
|
||||||
|
# Resume a paused process
|
||||||
|
box stability resume <PID>
|
||||||
|
|
||||||
|
# Top-line host health indicator
|
||||||
|
box fleet status
|
||||||
|
```
|
||||||
Binary file not shown.
Executable
+697
@@ -0,0 +1,697 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""box-stability-watcher.py — Proactive host and fleet stability guardrail for NetVM & Box.
|
||||||
|
|
||||||
|
Features:
|
||||||
|
1. System Load & Memory Monitoring (1m/5m/15m load, RAM%, Swap%)
|
||||||
|
2. Multi-tier Mitigation:
|
||||||
|
- GREEN: Normal.
|
||||||
|
- YELLOW: Renice runaway CPU hogs to +15.
|
||||||
|
- ORANGE: Pause runaway workers via SIGSTOP, record in .state/stability-paused.json,
|
||||||
|
post alert to sidechat (646 tasks), and give 60s grace period before SIGTERM.
|
||||||
|
- RED: Emergency shedder for processes >1500MB.
|
||||||
|
3. Tmux Socket Origin & Isolation:
|
||||||
|
- Allows user-launched agents on interactive desktop socket (/tmp/tmux-1000/default).
|
||||||
|
- Detects and logs automated workloads launched through Box on the desktop socket,
|
||||||
|
recommending migration to /tmp/tmux-muse.sock.
|
||||||
|
4. Systemd Loop Detection: Identifies crashing services trapped in tight restart loops.
|
||||||
|
5. State Management: Supports explicit "freeze" and "resume" commands.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
try:
|
||||||
|
import psutil
|
||||||
|
except ImportError:
|
||||||
|
psutil = None
|
||||||
|
|
||||||
|
WATCHERS_DIR = Path(__file__).resolve().parent
|
||||||
|
REPO_ROOT = WATCHERS_DIR.parent
|
||||||
|
DEFAULT_CONFIG = WATCHERS_DIR / "box-stability.json"
|
||||||
|
DEFAULT_LOG = REPO_ROOT / "logs" / "box-stability.jsonl"
|
||||||
|
PAUSED_STATE_FILE = REPO_ROOT / ".state" / "stability-paused.json"
|
||||||
|
BOX_LAUNCHED_SESSIONS_FILE = REPO_ROOT / ".state" / "box-launched-sessions.json"
|
||||||
|
PAUSE_GRACE_SECONDS = 60
|
||||||
|
|
||||||
|
|
||||||
|
def now_iso() -> str:
|
||||||
|
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
|
||||||
|
|
||||||
|
def now_epoch() -> float:
|
||||||
|
return time.time()
|
||||||
|
|
||||||
|
|
||||||
|
def load_config(config_path: Optional[Path] = None) -> Dict[str, Any]:
|
||||||
|
path = config_path or DEFAULT_CONFIG
|
||||||
|
if path.exists():
|
||||||
|
try:
|
||||||
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
return json.load(f)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {
|
||||||
|
"thresholds": {
|
||||||
|
"load_warning": 20.0,
|
||||||
|
"load_critical": 35.0,
|
||||||
|
"load_emergency": 60.0,
|
||||||
|
"ram_warning_pct": 80.0,
|
||||||
|
"ram_critical_pct": 90.0,
|
||||||
|
"swap_warning_pct": 75.0,
|
||||||
|
"swap_critical_pct": 85.0,
|
||||||
|
"process_rss_warning_mb": 2000,
|
||||||
|
"process_rss_critical_mb": 3000,
|
||||||
|
},
|
||||||
|
"protected_commands": [
|
||||||
|
"sshd", "tailscaled", "tailscale", "systemd", "dbus-broker",
|
||||||
|
"pipewire", "wireplumber", "tmux", "bash", "zsh", "ghostty", "alacritty"
|
||||||
|
],
|
||||||
|
"monitored_targets": [
|
||||||
|
"muse-bin", "chromium", "python3", "parec"
|
||||||
|
],
|
||||||
|
"actions": {
|
||||||
|
"enable_renice": True,
|
||||||
|
"enable_cull_runaway": True,
|
||||||
|
"enable_service_freeze": True,
|
||||||
|
"notify_sidechat": True,
|
||||||
|
},
|
||||||
|
"notification": {
|
||||||
|
"agent": "646",
|
||||||
|
"target": "646 tasks",
|
||||||
|
},
|
||||||
|
"interval_sec": 10,
|
||||||
|
"log_file": "logs/box-stability.jsonl",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def rotate_log_if_needed(log_path: Path, max_bytes: int = 10485760):
|
||||||
|
try:
|
||||||
|
if log_path.exists() and log_path.stat().st_size > max_bytes:
|
||||||
|
rotated = log_path.with_name(f"{log_path.name}.1")
|
||||||
|
os.replace(log_path, rotated)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def append_stability_event(event: Dict[str, Any], log_path: Optional[Path] = None):
|
||||||
|
target = log_path or DEFAULT_LOG
|
||||||
|
try:
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
rotate_log_if_needed(target)
|
||||||
|
with open(target, "a", encoding="utf-8") as f:
|
||||||
|
f.write(json.dumps(event) + "\n")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def read_paused_state(state_file: Optional[Path] = None) -> Dict[str, Any]:
|
||||||
|
target = state_file or PAUSED_STATE_FILE
|
||||||
|
if target.exists():
|
||||||
|
try:
|
||||||
|
with open(target, "r", encoding="utf-8") as f:
|
||||||
|
return json.load(f)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def write_paused_state(state: Dict[str, Any], state_file: Optional[Path] = None):
|
||||||
|
target = state_file or PAUSED_STATE_FILE
|
||||||
|
try:
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
tmp = target.with_suffix(".tmp")
|
||||||
|
with open(tmp, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(state, f, indent=2)
|
||||||
|
os.replace(tmp, target)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def get_box_launched_sessions(sessions_file: Optional[Path] = None) -> Dict[str, Any]:
|
||||||
|
target = sessions_file or BOX_LAUNCHED_SESSIONS_FILE
|
||||||
|
if target.exists():
|
||||||
|
try:
|
||||||
|
with open(target, "r", encoding="utf-8") as f:
|
||||||
|
return json.load(f)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def send_sidechat_alert(message: str, config: Dict[str, Any], dry_run: bool = False):
|
||||||
|
if dry_run or not config.get("actions", {}).get("notify_sidechat", True):
|
||||||
|
return
|
||||||
|
notif = config.get("notification", {})
|
||||||
|
agent = notif.get("agent", "646")
|
||||||
|
target = notif.get("target", "646 tasks")
|
||||||
|
box_cli = REPO_ROOT / "bin" / "super-cli.py"
|
||||||
|
if box_cli.exists():
|
||||||
|
cmd = [
|
||||||
|
sys.executable, str(box_cli), "dm", "send",
|
||||||
|
"--agent", agent,
|
||||||
|
"--to", agent,
|
||||||
|
"--target", target,
|
||||||
|
f"[STABILITY ALERT] {message}"
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
subprocess.run(cmd, capture_output=True, timeout=10)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def get_system_metrics() -> Dict[str, Any]:
|
||||||
|
load1, load5, load15 = os.getloadavg()
|
||||||
|
cpu_count = os.cpu_count() or 1
|
||||||
|
|
||||||
|
ram_total_mb = 0
|
||||||
|
ram_avail_mb = 0
|
||||||
|
ram_used_pct = 0.0
|
||||||
|
swap_total_mb = 0
|
||||||
|
swap_used_mb = 0
|
||||||
|
swap_used_pct = 0.0
|
||||||
|
|
||||||
|
if psutil:
|
||||||
|
vm = psutil.virtual_memory()
|
||||||
|
ram_total_mb = int(vm.total / (1024 * 1024))
|
||||||
|
ram_avail_mb = int(vm.available / (1024 * 1024))
|
||||||
|
ram_used_pct = round(vm.percent, 1)
|
||||||
|
|
||||||
|
sm = psutil.swap_memory()
|
||||||
|
swap_total_mb = int(sm.total / (1024 * 1024))
|
||||||
|
swap_used_mb = int(sm.used / (1024 * 1024))
|
||||||
|
swap_used_pct = round(sm.percent, 1)
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
meminfo = {}
|
||||||
|
with open("/proc/meminfo", "r") as f:
|
||||||
|
for line in f:
|
||||||
|
parts = line.split(":")
|
||||||
|
if len(parts) == 2:
|
||||||
|
key = parts[0].strip()
|
||||||
|
val = parts[1].strip().split()[0]
|
||||||
|
meminfo[key] = int(val)
|
||||||
|
total_kb = meminfo.get("MemTotal", 1)
|
||||||
|
avail_kb = meminfo.get("MemAvailable", meminfo.get("MemFree", 0))
|
||||||
|
ram_total_mb = total_kb // 1024
|
||||||
|
ram_avail_mb = avail_kb // 1024
|
||||||
|
ram_used_pct = round((1.0 - (avail_kb / total_kb)) * 100, 1)
|
||||||
|
|
||||||
|
swap_tot_kb = meminfo.get("SwapTotal", 0)
|
||||||
|
swap_free_kb = meminfo.get("SwapFree", 0)
|
||||||
|
if swap_tot_kb > 0:
|
||||||
|
swap_total_mb = swap_tot_kb // 1024
|
||||||
|
swap_used_mb = (swap_tot_kb - swap_free_kb) // 1024
|
||||||
|
swap_used_pct = round(((swap_tot_kb - swap_free_kb) / swap_tot_kb) * 100, 1)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return {
|
||||||
|
"load_1m": round(load1, 2),
|
||||||
|
"load_5m": round(load5, 2),
|
||||||
|
"load_15m": round(load15, 2),
|
||||||
|
"cpu_count": cpu_count,
|
||||||
|
"ram_total_mb": ram_total_mb,
|
||||||
|
"ram_avail_mb": ram_avail_mb,
|
||||||
|
"ram_used_pct": ram_used_pct,
|
||||||
|
"swap_total_mb": swap_total_mb,
|
||||||
|
"swap_used_mb": swap_used_mb,
|
||||||
|
"swap_used_pct": swap_used_pct,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def is_protected(pid: int, name: str, cmdline: str, config: Dict[str, Any], rss_mb: int = 0) -> bool:
|
||||||
|
if pid in (os.getpid(), os.getppid(), 1):
|
||||||
|
return True
|
||||||
|
low_name = name.lower()
|
||||||
|
low_cmd = cmdline.lower()
|
||||||
|
# Shells (bash/zsh) are only protected while of reasonable memory size.
|
||||||
|
# A shell consuming >= 2048 MB RSS is a runaway script/test or memory leak, not an interactive shell.
|
||||||
|
if low_name in ("bash", "zsh", "sh") and rss_mb >= 2048:
|
||||||
|
return False
|
||||||
|
for prot in config.get("protected_commands", []):
|
||||||
|
p_low = prot.lower()
|
||||||
|
if p_low == low_name or p_low in low_cmd.split():
|
||||||
|
return True
|
||||||
|
if "tmux new-session" in low_cmd or (low_name == "tmux" and "main" in low_cmd):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def get_tmux_socket_for_pid(pid: int) -> str:
|
||||||
|
try:
|
||||||
|
with open(f"/proc/{pid}/environ", "rb") as f:
|
||||||
|
env_data = f.read().split(b"\0")
|
||||||
|
for item in env_data:
|
||||||
|
if item.startswith(b"TMUX="):
|
||||||
|
val = item.decode("utf-8", errors="ignore")
|
||||||
|
parts = val.split(",")
|
||||||
|
if parts:
|
||||||
|
return parts[0].replace("TMUX=", "")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def inspect_processes(config: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||||
|
results = []
|
||||||
|
if not psutil:
|
||||||
|
return results
|
||||||
|
|
||||||
|
targets = [t.lower() for t in config.get("monitored_targets", [])]
|
||||||
|
|
||||||
|
for p in psutil.process_iter(["pid", "name", "cmdline", "cpu_percent", "memory_info", "nice"]):
|
||||||
|
try:
|
||||||
|
info = p.info
|
||||||
|
pid = info["pid"]
|
||||||
|
name = info["name"] or ""
|
||||||
|
cmdline_list = info["cmdline"] or []
|
||||||
|
cmdline = " ".join(cmdline_list)
|
||||||
|
mem_info = info["memory_info"]
|
||||||
|
rss_mb = int(mem_info.rss / (1024 * 1024)) if mem_info else 0
|
||||||
|
cpu_pct = info["cpu_percent"] or 0.0
|
||||||
|
nice = info["nice"] or 0
|
||||||
|
|
||||||
|
low_cmd = cmdline.lower()
|
||||||
|
low_name = name.lower()
|
||||||
|
matches_target = any(t in low_name or t in low_cmd for t in targets)
|
||||||
|
tmux_sock = get_tmux_socket_for_pid(pid) if matches_target else ""
|
||||||
|
|
||||||
|
results.append({
|
||||||
|
"pid": pid,
|
||||||
|
"name": name,
|
||||||
|
"cmdline": cmdline[:200],
|
||||||
|
"rss_mb": rss_mb,
|
||||||
|
"cpu_pct": cpu_pct,
|
||||||
|
"nice": nice,
|
||||||
|
"matches_target": matches_target,
|
||||||
|
"tmux_sock": tmux_sock,
|
||||||
|
"is_protected": is_protected(pid, name, cmdline, config, rss_mb=rss_mb),
|
||||||
|
})
|
||||||
|
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||||
|
continue
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def check_socket_isolation_violations(processes: List[Dict[str, Any]], box_sessions: Optional[Dict[str, Any]] = None) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
||||||
|
"""Distinguish user-launched agents (allowed) from Box/agent-launched workloads on the desktop socket.
|
||||||
|
|
||||||
|
Returns (violations, user_allowed).
|
||||||
|
"""
|
||||||
|
violations = []
|
||||||
|
user_allowed = []
|
||||||
|
tracked_box = box_sessions if box_sessions is not None else get_box_launched_sessions()
|
||||||
|
|
||||||
|
# Read subagent sessions as well
|
||||||
|
subagent_file = REPO_ROOT / "subagent-sessions.json"
|
||||||
|
subagent_ids = set()
|
||||||
|
if subagent_file.exists():
|
||||||
|
try:
|
||||||
|
with open(subagent_file, "r") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
if isinstance(data, dict):
|
||||||
|
subagent_ids = set(data.keys())
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
for p in processes:
|
||||||
|
if p.get("is_protected", False):
|
||||||
|
continue
|
||||||
|
sock = p.get("tmux_sock", "")
|
||||||
|
cmd = p.get("cmdline", "").lower()
|
||||||
|
|
||||||
|
if "default" in sock and ("muse-bin" in cmd or "auto-work" in cmd):
|
||||||
|
# Check if this workload originated from Box / automated scheduling
|
||||||
|
is_box_spawned = False
|
||||||
|
origin_reason = ""
|
||||||
|
|
||||||
|
# 1. Matches an automated session explicitly launched by Box CLI
|
||||||
|
for s_name in tracked_box:
|
||||||
|
if s_name.lower() in cmd:
|
||||||
|
is_box_spawned = True
|
||||||
|
origin_reason = f"tracked Box session '{s_name}'"
|
||||||
|
break
|
||||||
|
|
||||||
|
# 2. Matches subagent tracker UUID
|
||||||
|
if not is_box_spawned:
|
||||||
|
for sub_id in subagent_ids:
|
||||||
|
if sub_id.lower() in cmd:
|
||||||
|
is_box_spawned = True
|
||||||
|
origin_reason = f"tracked subagent '{sub_id[:8]}'"
|
||||||
|
break
|
||||||
|
|
||||||
|
# 3. Matches automated batch / flow naming conventions
|
||||||
|
if not is_box_spawned:
|
||||||
|
for pattern in ["auto-work", "flow-", "muse--runtime--"]:
|
||||||
|
if pattern in cmd:
|
||||||
|
is_box_spawned = True
|
||||||
|
origin_reason = f"automated job pattern '{pattern}'"
|
||||||
|
break
|
||||||
|
|
||||||
|
if is_box_spawned:
|
||||||
|
violations.append({
|
||||||
|
"pid": p["pid"],
|
||||||
|
"cmd": p["cmdline"][:60],
|
||||||
|
"socket": sock,
|
||||||
|
"origin": origin_reason,
|
||||||
|
"issue": f"Automated worker ({origin_reason}) running on desktop socket (/tmp/tmux-1000/default) instead of /tmp/tmux-muse.sock"
|
||||||
|
})
|
||||||
|
else:
|
||||||
|
# User-launched agent in interactive session
|
||||||
|
user_allowed.append({
|
||||||
|
"pid": p["pid"],
|
||||||
|
"cmd": p["cmdline"][:60],
|
||||||
|
"socket": sock,
|
||||||
|
"status": "user-launched (allowed in desktop socket)"
|
||||||
|
})
|
||||||
|
|
||||||
|
return violations, user_allowed
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_stability(metrics: Dict[str, Any], processes: List[Dict[str, Any]], config: Dict[str, Any]) -> Tuple[str, List[str], List[Dict[str, Any]]]:
|
||||||
|
th = config.get("thresholds", {})
|
||||||
|
tier = "GREEN"
|
||||||
|
reasons = []
|
||||||
|
actions_planned = []
|
||||||
|
|
||||||
|
load1 = metrics.get("load_1m", 0.0)
|
||||||
|
ram_pct = metrics.get("ram_used_pct", 0.0)
|
||||||
|
swap_pct = metrics.get("swap_used_pct", 0.0)
|
||||||
|
|
||||||
|
if load1 >= th.get("load_emergency", 60.0) or ram_pct >= 95.0 or swap_pct >= 92.0:
|
||||||
|
tier = "RED"
|
||||||
|
reasons.append(f"Emergency host pressure: load={load1}, RAM={ram_pct}%, Swap={swap_pct}%")
|
||||||
|
elif load1 >= th.get("load_critical", 35.0) or ram_pct >= th.get("ram_critical_pct", 90.0) or swap_pct >= th.get("swap_critical_pct", 85.0):
|
||||||
|
tier = "ORANGE"
|
||||||
|
reasons.append(f"Critical load/memory: load={load1}, RAM={ram_pct}%, Swap={swap_pct}%")
|
||||||
|
elif load1 >= th.get("load_warning", 20.0) or ram_pct >= th.get("ram_warning_pct", 80.0) or swap_pct >= th.get("swap_warning_pct", 75.0):
|
||||||
|
tier = "YELLOW"
|
||||||
|
reasons.append(f"Elevated load/memory: load={load1}, RAM={ram_pct}%, Swap={swap_pct}%")
|
||||||
|
|
||||||
|
rss_crit_mb = th.get("process_rss_critical_mb", 3000)
|
||||||
|
rss_warn_mb = th.get("process_rss_warning_mb", 2000)
|
||||||
|
|
||||||
|
for p in processes:
|
||||||
|
if p.get("is_protected", False):
|
||||||
|
continue
|
||||||
|
|
||||||
|
pid = p["pid"]
|
||||||
|
rss_mb = p.get("rss_mb", 0)
|
||||||
|
cpu_pct = p.get("cpu_pct", 0.0)
|
||||||
|
cmd_short = p.get("cmdline", "")[:60]
|
||||||
|
|
||||||
|
if rss_mb >= rss_crit_mb:
|
||||||
|
if tier in ("GREEN", "YELLOW"):
|
||||||
|
tier = "ORANGE"
|
||||||
|
reasons.append(f"Process PID {pid} exceeded critical RSS {rss_mb}MB >= {rss_crit_mb}MB: {cmd_short}")
|
||||||
|
actions_planned.append({
|
||||||
|
"action": "pause",
|
||||||
|
"pid": pid,
|
||||||
|
"reason": f"RSS {rss_mb}MB >= {rss_crit_mb}MB",
|
||||||
|
"cmd": cmd_short,
|
||||||
|
})
|
||||||
|
elif rss_mb >= rss_warn_mb:
|
||||||
|
if tier == "GREEN":
|
||||||
|
tier = "YELLOW"
|
||||||
|
reasons.append(f"Process PID {pid} elevated RSS {rss_mb}MB >= {rss_warn_mb}MB: {cmd_short}")
|
||||||
|
|
||||||
|
if tier in ("YELLOW", "ORANGE", "RED") and cpu_pct > 80.0 and p.get("nice", 0) < 10:
|
||||||
|
actions_planned.append({
|
||||||
|
"action": "renice",
|
||||||
|
"pid": pid,
|
||||||
|
"reason": f"CPU hog {cpu_pct}% under elevated load",
|
||||||
|
"nice_value": 15,
|
||||||
|
"cmd": cmd_short,
|
||||||
|
})
|
||||||
|
|
||||||
|
if tier == "RED":
|
||||||
|
for p in processes:
|
||||||
|
if not p.get("is_protected", False) and p.get("rss_mb", 0) > 1500:
|
||||||
|
actions_planned.append({
|
||||||
|
"action": "pause",
|
||||||
|
"pid": p["pid"],
|
||||||
|
"reason": f"Emergency RED shedder: RSS {p['rss_mb']}MB",
|
||||||
|
"cmd": p.get("cmdline", "")[:60],
|
||||||
|
})
|
||||||
|
|
||||||
|
return tier, reasons, actions_planned
|
||||||
|
|
||||||
|
|
||||||
|
def reconcile_paused_processes(state_file: Optional[Path] = None, dry_run: bool = False) -> List[Dict[str, Any]]:
|
||||||
|
actions = []
|
||||||
|
state = read_paused_state(state_file)
|
||||||
|
if not state:
|
||||||
|
return actions
|
||||||
|
|
||||||
|
now = now_epoch()
|
||||||
|
new_state = {}
|
||||||
|
|
||||||
|
for s_pid, info in state.items():
|
||||||
|
try:
|
||||||
|
pid = int(s_pid)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
paused_at = info.get("paused_at_epoch", now)
|
||||||
|
elapsed = now - paused_at
|
||||||
|
cmd = info.get("cmd", "")
|
||||||
|
|
||||||
|
if elapsed >= PAUSE_GRACE_SECONDS:
|
||||||
|
entry = {
|
||||||
|
"action": "cull_expired",
|
||||||
|
"pid": pid,
|
||||||
|
"cmd": cmd,
|
||||||
|
"reason": f"Grace period of {PAUSE_GRACE_SECONDS}s expired without resume",
|
||||||
|
"dry_run": dry_run,
|
||||||
|
"success": False,
|
||||||
|
}
|
||||||
|
if not dry_run:
|
||||||
|
try:
|
||||||
|
os.kill(pid, signal.SIGTERM)
|
||||||
|
entry["status"] = "SIGTERM sent"
|
||||||
|
entry["success"] = True
|
||||||
|
except ProcessLookupError:
|
||||||
|
entry["status"] = "process already dead"
|
||||||
|
entry["success"] = True
|
||||||
|
except Exception as e:
|
||||||
|
entry["status"] = f"error: {e}"
|
||||||
|
else:
|
||||||
|
entry["status"] = "skipped (dry-run)"
|
||||||
|
actions.append(entry)
|
||||||
|
else:
|
||||||
|
new_state[s_pid] = info
|
||||||
|
|
||||||
|
if not dry_run:
|
||||||
|
write_paused_state(new_state, state_file)
|
||||||
|
|
||||||
|
return actions
|
||||||
|
|
||||||
|
|
||||||
|
def execute_actions(actions: List[Dict[str, Any]], state_file: Optional[Path] = None, dry_run: bool = False) -> List[Dict[str, Any]]:
|
||||||
|
executed = []
|
||||||
|
paused_state = read_paused_state(state_file) if not dry_run else {}
|
||||||
|
|
||||||
|
for act in actions:
|
||||||
|
kind = act.get("action")
|
||||||
|
pid = act.get("pid")
|
||||||
|
entry = dict(act)
|
||||||
|
entry["dry_run"] = dry_run
|
||||||
|
entry["success"] = False
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
entry["status"] = "skipped (dry-run)"
|
||||||
|
executed.append(entry)
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
if kind == "renice":
|
||||||
|
nice_val = act.get("nice_value", 15)
|
||||||
|
os.setpriority(os.PRIO_PROCESS, pid, nice_val)
|
||||||
|
entry["status"] = f"reniced to {nice_val}"
|
||||||
|
entry["success"] = True
|
||||||
|
elif kind == "pause":
|
||||||
|
os.kill(pid, signal.SIGSTOP)
|
||||||
|
entry["status"] = "SIGSTOP sent (paused for 60s inspection)"
|
||||||
|
entry["success"] = True
|
||||||
|
paused_state[str(pid)] = {
|
||||||
|
"pid": pid,
|
||||||
|
"cmd": act.get("cmd", ""),
|
||||||
|
"reason": act.get("reason", ""),
|
||||||
|
"paused_at": now_iso(),
|
||||||
|
"paused_at_epoch": now_epoch(),
|
||||||
|
}
|
||||||
|
elif kind == "cull":
|
||||||
|
os.kill(pid, signal.SIGTERM)
|
||||||
|
entry["status"] = "SIGTERM sent"
|
||||||
|
entry["success"] = True
|
||||||
|
except ProcessLookupError:
|
||||||
|
entry["status"] = "process already gone"
|
||||||
|
entry["success"] = True
|
||||||
|
except PermissionError:
|
||||||
|
entry["status"] = "permission denied"
|
||||||
|
except Exception as e:
|
||||||
|
entry["status"] = f"error: {e}"
|
||||||
|
|
||||||
|
executed.append(entry)
|
||||||
|
|
||||||
|
if not dry_run and paused_state:
|
||||||
|
write_paused_state(paused_state, state_file)
|
||||||
|
|
||||||
|
return executed
|
||||||
|
|
||||||
|
|
||||||
|
def resume_process(pid: int, state_file: Optional[Path] = None) -> Dict[str, Any]:
|
||||||
|
state = read_paused_state(state_file)
|
||||||
|
res = {"pid": pid, "action": "resume", "success": False}
|
||||||
|
try:
|
||||||
|
os.kill(pid, signal.SIGCONT)
|
||||||
|
res["success"] = True
|
||||||
|
res["status"] = "SIGCONT sent (resumed)"
|
||||||
|
except ProcessLookupError:
|
||||||
|
res["status"] = "process does not exist"
|
||||||
|
except Exception as e:
|
||||||
|
res["status"] = f"error: {e}"
|
||||||
|
|
||||||
|
if str(pid) in state:
|
||||||
|
del state[str(pid)]
|
||||||
|
write_paused_state(state, state_file)
|
||||||
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
def run_cycle(config: Dict[str, Any], dry_run: bool = False) -> Dict[str, Any]:
|
||||||
|
metrics = get_system_metrics()
|
||||||
|
processes = inspect_processes(config)
|
||||||
|
socket_violations, user_allowed = check_socket_isolation_violations(processes)
|
||||||
|
|
||||||
|
tier, reasons, actions_planned = evaluate_stability(metrics, processes, config)
|
||||||
|
actions_taken = execute_actions(actions_planned, dry_run=dry_run)
|
||||||
|
expired_actions = reconcile_paused_processes(dry_run=dry_run)
|
||||||
|
actions_taken.extend(expired_actions)
|
||||||
|
|
||||||
|
if socket_violations:
|
||||||
|
for sv in socket_violations:
|
||||||
|
reasons.append(f"Automated workload on desktop socket: PID {sv['pid']} ({sv.get('origin', 'box')})")
|
||||||
|
|
||||||
|
event = {
|
||||||
|
"timestamp": now_iso(),
|
||||||
|
"tier": tier,
|
||||||
|
"metrics": metrics,
|
||||||
|
"reasons": reasons,
|
||||||
|
"socket_violations": socket_violations,
|
||||||
|
"user_allowed": user_allowed,
|
||||||
|
"actions_taken": actions_taken,
|
||||||
|
"dry_run": dry_run,
|
||||||
|
}
|
||||||
|
|
||||||
|
if tier in ("ORANGE", "RED") or any(a.get("action") == "pause" for a in actions_taken):
|
||||||
|
alert_msg = f"Tier: {tier}. Reasons: {reasons}. Actions: {actions_taken}"
|
||||||
|
send_sidechat_alert(alert_msg, config, dry_run=dry_run)
|
||||||
|
|
||||||
|
if tier != "GREEN" or actions_taken or socket_violations:
|
||||||
|
log_path = Path(config.get("log_file", DEFAULT_LOG))
|
||||||
|
if not log_path.is_absolute():
|
||||||
|
log_path = REPO_ROOT / log_path
|
||||||
|
append_stability_event(event, log_path)
|
||||||
|
|
||||||
|
return event
|
||||||
|
|
||||||
|
|
||||||
|
def print_status(event: Dict[str, Any]):
|
||||||
|
m = event["metrics"]
|
||||||
|
tier = event["tier"]
|
||||||
|
color_code = {
|
||||||
|
"GREEN": "\033[92m● GREEN\033[0m",
|
||||||
|
"YELLOW": "\033[93m▲ YELLOW\033[0m",
|
||||||
|
"ORANGE": "\033[91m■ ORANGE\033[0m",
|
||||||
|
"RED": "\033[1;41m✖ RED (EMERGENCY)\033[0m",
|
||||||
|
}.get(tier, tier)
|
||||||
|
|
||||||
|
print(f"\n=== BOX STABILITY STATUS: {color_code} ===")
|
||||||
|
print(f" Load Average: {m['load_1m']} (1m) | {m['load_5m']} (5m) | {m['load_15m']} (15m) [Cores: {m['cpu_count']}]")
|
||||||
|
print(f" RAM Usage: {m['ram_used_pct']}% ({m['ram_total_mb'] - m['ram_avail_mb']}MB used / {m['ram_total_mb']}MB total)")
|
||||||
|
print(f" Swap Usage: {m['swap_used_pct']}% ({m['swap_used_mb']}MB used / {m['swap_total_mb']}MB total)")
|
||||||
|
|
||||||
|
paused = read_paused_state()
|
||||||
|
if paused:
|
||||||
|
print("\n Paused Processes (60s Grace Window):")
|
||||||
|
for s_pid, info in paused.items():
|
||||||
|
print(f" - PID {s_pid}: {info.get('cmd')} (paused at {info.get('paused_at')})")
|
||||||
|
|
||||||
|
if event.get("socket_violations"):
|
||||||
|
print("\n Automated Workload Warnings (Detected on Desktop Socket):")
|
||||||
|
for v in event["socket_violations"]:
|
||||||
|
print(f" - PID {v['pid']} ({v.get('origin', 'box')}): {v['cmd']}")
|
||||||
|
|
||||||
|
if event.get("user_allowed"):
|
||||||
|
print(f"\n User-Launched Agents on Desktop Socket: {len(event['user_allowed'])} active (allowed)")
|
||||||
|
|
||||||
|
if event.get("reasons"):
|
||||||
|
print("\n Active Issues:")
|
||||||
|
for r in event["reasons"]:
|
||||||
|
print(f" - {r}")
|
||||||
|
|
||||||
|
if event.get("actions_taken"):
|
||||||
|
print("\n Mitigations:")
|
||||||
|
for a in event["actions_taken"]:
|
||||||
|
print(f" - [{a['action']}] PID {a['pid']} ({a['cmd']}): {a.get('status')}")
|
||||||
|
print("")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Box & Host Stability Watcher")
|
||||||
|
parser.add_argument("--config", type=Path, default=None, help="Path to box-stability.json")
|
||||||
|
parser.add_argument("--dry-run", action="store_true", help="Evaluate without executing kills or renices")
|
||||||
|
parser.add_argument("--check", "--once", dest="once", action="store_true", help="Run a single evaluation cycle and exit")
|
||||||
|
parser.add_argument("--status", action="store_true", help="Print human-readable status overview")
|
||||||
|
parser.add_argument("--json", action="store_true", help="Output JSON result")
|
||||||
|
parser.add_argument("--resume", type=int, help="Resume a paused PID with SIGCONT and remove from pause state")
|
||||||
|
parser.add_argument("--daemon", action="store_true", help="Run continuously in background daemon loop")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
config = load_config(args.config)
|
||||||
|
|
||||||
|
if args.resume:
|
||||||
|
res = resume_process(args.resume)
|
||||||
|
print(json.dumps(res, indent=2))
|
||||||
|
return 0 if res["success"] else 1
|
||||||
|
|
||||||
|
if args.status or args.once:
|
||||||
|
event = run_cycle(config, dry_run=args.dry_run or args.status)
|
||||||
|
if args.json:
|
||||||
|
print(json.dumps(event, indent=2))
|
||||||
|
else:
|
||||||
|
print_status(event)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if args.daemon:
|
||||||
|
interval = config.get("interval_sec", 10)
|
||||||
|
print(f"[{now_iso()}] Starting Box Stability Watcher daemon (interval: {interval}s)...")
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
run_cycle(config, dry_run=args.dry_run)
|
||||||
|
time.sleep(interval)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
print(f"[{now_iso()}] Watcher stopped by user.")
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[{now_iso()}] Watcher cycle error: {e}", file=sys.stderr)
|
||||||
|
time.sleep(interval)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
event = run_cycle(config, dry_run=args.dry_run)
|
||||||
|
if args.json:
|
||||||
|
print(json.dumps(event, indent=2))
|
||||||
|
else:
|
||||||
|
print_status(event)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
Reference in New Issue
Block a user