super/box
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

chore(fleet): sync operator memory, hatch menu dialogs, and watchdog alerts

Browse Source
This commit is contained in:
operator
2026-10-07 00:25:51 +00:00
parent 0065d11e97
commit 34b0ef9fe2
38 changed files with 2205 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/approvals.py
+136 -20
View File
@@ -13,10 +13,12 @@ Supports both:
4. Continuous watch & background integration into fleet status and loop health.
"""
import itertools
import json
import os
import re
import sys
import threading
import time
import urllib.request
from datetime import datetime, timezone
@@ -48,15 +50,28 @@ def load_responded_waits() -> dict:
return {}
def save_responded_waits(data: dict) -> None:
"""Persist the responded-waits map (best effort)."""
def _atomic_write_json(path: Path, data: dict) -> None:
"""Write JSON atomically via tmp + replace (best effort).
Plain write_text from concurrent writers (timers, box-ctl, TUI
threads) can interleave and corrupt the file; readers then fall
back to {} and silently drop state. Tmp names carry pid + thread
ident so concurrent writers never share a temp file.
"""
try:
RESPONDED_WAITS_FILE.parent.mkdir(parents=True, exist_ok=True)
RESPONDED_WAITS_FILE.write_text(json.dumps(data, indent=1))
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(f"{path.name}.tmp.{os.getpid()}.{threading.get_ident()}")
tmp.write_text(json.dumps(data, indent=1))
os.replace(tmp, path)
except Exception:
pass
def save_responded_waits(data: dict) -> None:
"""Persist the responded-waits map (best effort)."""
_atomic_write_json(RESPONDED_WAITS_FILE, data)
def load_first_seen_waits() -> dict:
"""Load map of when input waits were first observed: {node: {task: iso_timestamp}}."""
try:
@@ -69,11 +84,7 @@ def load_first_seen_waits() -> dict:
def save_first_seen_waits(data: dict) -> None:
"""Persist first-seen input waits map."""
try:
FIRST_SEEN_WAITS_FILE.parent.mkdir(parents=True, exist_ok=True)
FIRST_SEEN_WAITS_FILE.write_text(json.dumps(data, indent=1))
except Exception:
pass
_atomic_write_json(FIRST_SEEN_WAITS_FILE, data)
def is_wait_responded(node: str, task: str) -> bool:
@@ -466,9 +477,15 @@ def get_cdp_ws(node: str, page_idx: int = 0, timeout: float = 3.0):
return ws, target_page
_cdp_req_ids = itertools.count(1)
def cdp_evaluate(ws, js_expr: str, await_promise: bool = False, timeout: float = 3.0):
"""Evaluate a JavaScript expression via CDP Runtime.evaluate and return the result value."""
req_id = int(time.time() * 1000) % 100000
# Monotonic ids: millisecond-clock ids collide for rapid successive
# evaluates, letting a stale buffered response be misattributed to
# the wrong call (e.g. verify-after-click reading the click result).
req_id = next(_cdp_req_ids)
msg = {
"id": req_id,
"method": "Runtime.evaluate",
@@ -575,15 +592,33 @@ JS_INSPECT_APPROVALS = """(() => {
}
}
// Background queued approvals surface (e.g. "2 tasks need review", "Review")
const bgSurface = document.querySelector('[data-hatch-background-approval-surface="true"]');
let bgTasksCount = 0;
let bgText = '';
if (bgSurface) {
bgText = (bgSurface.innerText || '').trim();
const m = bgText.match(/(\\d+)\\s+tasks?\\s+need\\s+review/i);
if (m) {
bgTasksCount = parseInt(m[1], 10);
} else if (/a\\s+task\\s+needs\\s+review/i.test(bgText) || /tasks?\\s+need\\s+review/i.test(bgText)) {
bgTasksCount = 1;
}
}
const hasPendingApproval = (!!activeCard && (hasAllowOnce || hasDeny)) || (bgTasksCount > 0);
return JSON.stringify({
has_pending: !!activeCard && (hasAllowOnce || hasDeny),
card_text: cardText.slice(0, 1000),
has_pending: hasPendingApproval,
card_text: cardText.slice(0, 1000) || bgText,
buttons: buttons,
has_allow_once: hasAllowOnce,
has_allow_once: hasAllowOnce || (bgTasksCount > 0),
has_always_allow: hasAlwaysAllow,
has_deny: hasDeny,
history: historyBadges.slice(0, 5),
input_waits: inputWaits.slice(0, 10)
input_waits: inputWaits.slice(0, 10),
bg_tasks_count: bgTasksCount,
bg_text: bgText
});
})()"""
@@ -635,30 +670,68 @@ def inspect_node_approvals(node: str) -> dict:
"error": str(e),
"has_pending": False,
"host_cdp_ok": host_ok,
"title": "Node unreachable",
"purpose": "",
"ip": None,
"target": "-",
"is_trusted": False,
"buttons": [],
"has_allow_once": False,
"has_always_allow": False,
"has_deny": False,
"raw_text": "",
"history": [],
"input_waits": [],
"page_title": "",
"page_url": "",
"ws_url": "",
}
all_input_waits = []
first_page = pages[0]
last_err = None
inspected_ok = False
for page in pages:
ws_url = page.get("webSocketDebuggerUrl")
if not ws_url:
if last_err is None:
last_err = Exception("page has no webSocketDebuggerUrl")
continue
ws = None
try:
ws = websocket.create_connection(ws_url, timeout=2.0)
val_str = cdp_evaluate(ws, JS_INSPECT_APPROVALS, timeout=2.5)
if val_str and isinstance(val_str, str):
data = json.loads(val_str)
# If a background review banner is present and active card wasn't mounted, click review to reveal card
if data.get("bg_tasks_count", 0) > 0 and (not data.get("buttons") or "task" in (data.get("card_text") or "").lower()):
js_expand = """(() => {
const bgBtn = document.querySelector('[data-pel-click="chat_background_approval_review"]') ||
document.querySelector('[data-hatch-background-approval-surface="true"] button');
if (bgBtn) { bgBtn.click(); return 'CLICKED'; }
return 'NO_BTN';
})()"""
exp_res = cdp_evaluate(ws, js_expand, timeout=1.5)
if exp_res == "CLICKED":
time.sleep(0.35)
val_str2 = cdp_evaluate(ws, JS_INSPECT_APPROVALS, timeout=2.5)
if val_str2 and isinstance(val_str2, str):
val_str = val_str2
ws.close()
ws = None
if not val_str or not isinstance(val_str, str):
if last_err is None:
last_err = Exception("empty or invalid CDP evaluate result")
continue
data = json.loads(val_str)
inspected_ok = True
if data.get("input_waits"):
all_input_waits.extend(data["input_waits"])
if data.get("has_pending"):
card_text = data.get("card_text", "")
bg_tasks_count = data.get("bg_tasks_count", 0)
ip = None
target = None
m_t = re.search(
@@ -678,8 +751,14 @@ def inspect_node_approvals(node: str) -> dict:
target = m_domain.group(0)
lines = [line.strip() for line in card_text.split("\n") if line.strip()]
title = redact_sensitive(lines[0] if lines else "Permission request")
purpose = redact_sensitive(lines[1] if len(lines) > 1 else "")
if not lines and bg_tasks_count:
title = f"{bg_tasks_count} task(s) need review"
purpose = "Background tasks held up on review surface. Click 'Review' or allow to inspect."
else:
title = redact_sensitive(lines[0] if lines else "Permission request")
purpose = redact_sensitive(lines[1] if len(lines) > 1 else "")
if bg_tasks_count > 0 and "need review" not in purpose.lower() and "need review" not in title.lower():
purpose = f"{purpose} [{bg_tasks_count} queued task(s) awaiting review]".strip()
is_trusted = is_trusted_target(target or ip, card_text)
@@ -696,6 +775,7 @@ def inspect_node_approvals(node: str) -> dict:
"has_allow_once": data.get("has_allow_once", False),
"has_always_allow": data.get("has_always_allow", False),
"has_deny": data.get("has_deny", False),
"bg_tasks_count": bg_tasks_count,
"raw_text": redact_sensitive(card_text),
"history": data.get("history", []),
"input_waits": all_input_waits,
@@ -789,12 +869,24 @@ def inspect_node_approvals(node: str) -> dict:
"key_request": key_req,
}
status = "INPUT_WAIT" if unique_waits else ("ERROR" if last_err and not first_page else "CLEAR")
# A node whose pages all failed inspection must report ERROR, never a
# false CLEAR that hides pending approvals. (The old `last_err and not
# first_page` guard was dead: first_page is always truthy here.)
if unique_waits:
status = "INPUT_WAIT"
title = "No pending approvals"
elif not inspected_ok:
status = "ERROR"
title = "Approval inspection failed"
else:
status = "CLEAR"
title = "No pending approvals"
return {
"node": node,
"status": status,
"error": str(last_err) if status == "ERROR" and last_err else "",
"has_pending": False,
"title": "No pending approvals",
"title": title,
"purpose": "",
"ip": None,
"target": "-",
@@ -936,23 +1028,47 @@ def allow_node_approval(node: str, always: bool = False, force: bool = False, ca
btn.click();
return 'CLICKED_ALLOW';
}
const bgBtn = document.querySelector('[data-pel-click="chat_background_approval_review"]') ||
document.querySelector('[data-hatch-background-approval-surface="true"] button');
if (bgBtn) {
bgBtn.click();
return 'CLICKED_REVIEW_SURFACE';
}
return 'NOT_FOUND';
})()"""
click_res = cdp_evaluate(ws, js_click, timeout=3.0)
if click_res == "CLICKED_REVIEW_SURFACE":
time.sleep(0.6)
click_res2 = cdp_evaluate(ws, """(() => {
const primary = document.querySelector('button[data-hatch-approval-primary-action="true"]');
if (primary) { primary.click(); return 'CLICKED_PRIMARY'; }
const btns = Array.from(document.querySelectorAll('button'));
const btn = btns.find(b => {
const t = (b.innerText||'').trim().toLowerCase();
return t === 'allow once' || t === 'allow';
});
if (btn) { btn.click(); return 'CLICKED_ALLOW'; }
return 'NOT_FOUND';
})()""", timeout=2.0)
if click_res2 != "NOT_FOUND":
click_res = click_res2
# Verify dismissal
time.sleep(0.8)
js_verify = """(() => {
const primary = document.querySelector('button[data-hatch-approval-primary-action="true"]');
if (primary) return 'STILL_PRESENT';
const headers = document.querySelectorAll('[data-testid="approval-panel-header"]');
return headers.length === 0 ? 'DISMISSED' : 'STILL_PRESENT';
if (headers.length > 0) return 'STILL_PRESENT';
const bgSurface = document.querySelector('[data-hatch-background-approval-surface="true"]');
return bgSurface ? 'QUEUED_PRESENT' : 'DISMISSED';
})()"""
verify_res = cdp_evaluate(ws, js_verify, timeout=2.0)
ws.close()
dismissed = verify_res == "DISMISSED"
dismissed = verify_res in ("DISMISSED", "QUEUED_PRESENT")
mode = "always" if always else "allow_once"
log_box_ctl(
"approval-allow",