approvals: stale-wait cleanup, key-decision notify, TTLs, key/browser isolation
- bin/approvals.py: responded-wait filtering + auto-mark, key-decision sidechat-first notify (notified flag, --message, --allow-main-chat), TTL defaults (input 30m / browser 30m / key 2h), cross-type guard (browser actions cannot resolve key requests), sweep_expired_key_requests; restores check_node_key_request fallback in inspect_node_approvals - bin/box-ctl.py + bin/super-cli.py (approvals hunks only): --message/--allow-main-chat passthrough on allow/deny, clear/clear-all actions, def sidechat routing; restores sys.exit(1) on dismiss failure - bin/fleet-alert-check.sh: TTL-aware state_machine (EXPIRED action), auto-deny expired browser approvals (fail closed), auto-dismiss expired input waits, targeted per-agent DM for input waits - bin/job-dispatch.py + bin/gravity.py: KEY_APPROVAL excluded from auto-approval Reviewed by 5 independent reviewers (all APPROVE/APPROVE WITH NOTES); integration gate GO (17/17 tests). TUI hunks in super-cli.py intentionally excluded.
This commit is contained in:
operator
+337
-12
@@ -31,6 +31,98 @@ except ImportError:
|
||||
REPO_ROOT = Path("/home/super/Projects/NetVM")
|
||||
BIN_DIR = REPO_ROOT / "bin"
|
||||
CTL_LOG = REPO_ROOT / "box-ctl.jsonl"
|
||||
RESPONDED_WAITS_FILE = REPO_ROOT / ".state" / "approvals-responded.json"
|
||||
FIRST_SEEN_WAITS_FILE = REPO_ROOT / ".state" / "approvals-first-seen.json"
|
||||
|
||||
|
||||
def load_responded_waits() -> dict:
|
||||
"""Load the set of (node, task) input waits already responded to.
|
||||
|
||||
Returns {node: {task: iso_timestamp}}. Missing file -> {}.
|
||||
"""
|
||||
try:
|
||||
if RESPONDED_WAITS_FILE.exists():
|
||||
return json.loads(RESPONDED_WAITS_FILE.read_text())
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def save_responded_waits(data: dict) -> None:
|
||||
"""Persist the responded-waits map (best effort)."""
|
||||
try:
|
||||
RESPONDED_WAITS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
RESPONDED_WAITS_FILE.write_text(json.dumps(data, indent=1))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def load_first_seen_waits() -> dict:
|
||||
"""Load map of when input waits were first observed: {node: {task: iso_timestamp}}."""
|
||||
try:
|
||||
if FIRST_SEEN_WAITS_FILE.exists():
|
||||
return json.loads(FIRST_SEEN_WAITS_FILE.read_text())
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def save_first_seen_waits(data: dict) -> None:
|
||||
"""Persist first-seen input waits map."""
|
||||
try:
|
||||
FIRST_SEEN_WAITS_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
FIRST_SEEN_WAITS_FILE.write_text(json.dumps(data, indent=1))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def is_wait_responded(node: str, task: str) -> bool:
|
||||
"""True if this (node, task) wait was already answered."""
|
||||
return task in load_responded_waits().get(node, {})
|
||||
|
||||
|
||||
def mark_wait_responded(node: str, task: str, caller: str = "approvals") -> None:
|
||||
"""Record that (node, task) has been responded to, so future
|
||||
inspections filter it out of the live input-wait list."""
|
||||
data = load_responded_waits()
|
||||
node_map = data.setdefault(node, {})
|
||||
if task not in node_map:
|
||||
node_map[task] = datetime.now(timezone.utc).isoformat()
|
||||
save_responded_waits(data)
|
||||
log_box_ctl("approval-wait-responded", name=node, caller=caller,
|
||||
extra={"task": task})
|
||||
|
||||
|
||||
def clear_node_waits(node: str = None, caller: str = "box-approvals") -> dict:
|
||||
"""Clear and dismiss all pending input waits for a specific node or all nodes."""
|
||||
target_nodes = [node] if node else VALID_NODES
|
||||
total_cleared = 0
|
||||
cleared_per_node = {}
|
||||
data = load_responded_waits()
|
||||
now_iso = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
for n in target_nodes:
|
||||
node_map = data.setdefault(n, {})
|
||||
n_cleared = 0
|
||||
try:
|
||||
info = inspect_node_approvals(n)
|
||||
for w in info.get("input_waits", []) or []:
|
||||
t = w.get("task")
|
||||
if t and t not in node_map:
|
||||
node_map[t] = now_iso
|
||||
n_cleared += 1
|
||||
if n_cleared:
|
||||
log_box_ctl("approval-wait-cleared", name=n, caller=caller,
|
||||
extra={"cleared_count": n_cleared})
|
||||
except Exception:
|
||||
pass
|
||||
cleared_per_node[n] = n_cleared
|
||||
total_cleared += n_cleared
|
||||
|
||||
if total_cleared:
|
||||
save_responded_waits(data)
|
||||
|
||||
return {"ok": True, "total_cleared": total_cleared, "cleared_per_node": cleared_per_node}
|
||||
|
||||
# Add BIN_DIR to sys.path
|
||||
if str(BIN_DIR) not in sys.path:
|
||||
@@ -43,6 +135,14 @@ except ImportError:
|
||||
|
||||
VALID_NODES = ["muse", "pip", "646", "opm", "def", "dev"]
|
||||
|
||||
# Approval timeout defaults (seconds).
|
||||
# Key requests are sensitive operations -- give the operator 2h to decide.
|
||||
# Input waits and browser approvals block agent work -- expire after 30m so
|
||||
# agents unblock instead of sitting indefinitely (e.g. def waited 4h+).
|
||||
KEY_REQUEST_TTL_SECONDS = 2 * 3600
|
||||
INPUT_WAIT_TTL_SECONDS = 30 * 60
|
||||
BROWSER_APPROVAL_TTL_SECONDS = 30 * 60
|
||||
|
||||
# Trusted infrastructure IPs safe for automated approval
|
||||
TRUSTED_IPS = {
|
||||
"34.139.37.135", # VM (gateway)
|
||||
@@ -95,12 +195,30 @@ def redact_sensitive(text: str) -> str:
|
||||
return out
|
||||
|
||||
|
||||
def _approval_type(action: str) -> str:
|
||||
"""Classify an approval action into its request type.
|
||||
|
||||
Types: "key" (passkey/key requests), "browser" (browser dialog
|
||||
clicks), "input" (task input replies), "other". Used so that a
|
||||
resolution only clears requests of the matching type -- a browser
|
||||
approval-allow must never resolve a pending key request.
|
||||
"""
|
||||
if action.startswith("key-approval-"):
|
||||
return "key"
|
||||
if action == "approval-reply":
|
||||
return "input"
|
||||
if action.startswith("approval-"):
|
||||
return "browser"
|
||||
return "other"
|
||||
|
||||
|
||||
def log_box_ctl(action: str, name: str = None, caller: str = "box-approvals", extra: dict = None):
|
||||
"""Log an audit event to box-ctl.jsonl with secret redaction."""
|
||||
try:
|
||||
rec = {
|
||||
"ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"action": action,
|
||||
"type": _approval_type(action),
|
||||
"name": name,
|
||||
"caller": caller,
|
||||
}
|
||||
@@ -118,14 +236,23 @@ def log_box_ctl(action: str, name: str = None, caller: str = "box-approvals", ex
|
||||
pass
|
||||
|
||||
|
||||
def request_key_approval(node: str, reason: str = "", caller: str = "agent") -> dict:
|
||||
"""Register a key/passkey approval request for a node in box-ctl.jsonl."""
|
||||
def request_key_approval(node: str, reason: str = "", caller: str = "agent",
|
||||
ttl_seconds: int = None) -> dict:
|
||||
"""Register a key/passkey approval request for a node in box-ctl.jsonl.
|
||||
|
||||
ttl_seconds: how long the request stays valid (default KEY_REQUEST_TTL_SECONDS).
|
||||
After expiry the request is treated as denied; see check_node_key_request().
|
||||
"""
|
||||
reason = reason or "Operator passkey access requested"
|
||||
ttl = ttl_seconds if ttl_seconds is not None else KEY_REQUEST_TTL_SECONDS
|
||||
expires_iso = datetime.fromtimestamp(
|
||||
datetime.now(timezone.utc).timestamp() + ttl, tz=timezone.utc
|
||||
).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
log_box_ctl(
|
||||
"key-approval-request",
|
||||
name=node,
|
||||
caller=caller,
|
||||
extra={"reason": reason},
|
||||
extra={"reason": reason, "ttl_seconds": ttl, "expires_at": expires_iso},
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
@@ -133,16 +260,55 @@ def request_key_approval(node: str, reason: str = "", caller: str = "agent") ->
|
||||
"status": "KEY_APPROVAL_REQUESTED",
|
||||
"reason": reason,
|
||||
"caller": caller,
|
||||
"note": "Request recorded in audit log. Operator can approve via 'box approvals allow <node>'."
|
||||
"ttl_seconds": ttl,
|
||||
"expires_at": expires_iso,
|
||||
"note": "Request recorded in audit log. Operator can approve via 'box approvals allow <node>'.",
|
||||
}
|
||||
|
||||
|
||||
def _parse_ts(ts_str: str):
|
||||
"""Parse a box-ctl.jsonl ts ('%Y-%m-%dT%H:%M:%SZ') to epoch seconds. None on failure."""
|
||||
if not ts_str:
|
||||
return None
|
||||
try:
|
||||
dt = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
|
||||
return dt.timestamp()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def expire_key_request(node: str, caller: str = "approval-sweeper") -> dict:
|
||||
"""Mark a node's pending key request as expired (auto-deny on TTL)."""
|
||||
log_box_ctl("key-approval-expired", name=node, caller=caller,
|
||||
extra={"note": "TTL elapsed without operator decision; treated as denied"})
|
||||
return {"ok": True, "node": node, "status": "KEY_APPROVAL_EXPIRED"}
|
||||
|
||||
|
||||
def _rec_approval_type(rec: dict) -> str:
|
||||
"""Return the approval type of a log record.
|
||||
|
||||
Prefers the explicit "type" field (present on records written after the
|
||||
type-field fix); falls back to deriving from the action name for older
|
||||
records so history keeps working.
|
||||
"""
|
||||
t = rec.get("type")
|
||||
if t:
|
||||
return t
|
||||
return _approval_type(rec.get("action", ""))
|
||||
|
||||
|
||||
def check_node_key_request(node: str) -> dict:
|
||||
"""Check if node has an active unfulfilled key approval request in box-ctl.jsonl."""
|
||||
"""Check if node has an active unfulfilled key approval request in box-ctl.jsonl.
|
||||
|
||||
Requests expire after their TTL (default KEY_REQUEST_TTL_SECONDS). An expired
|
||||
request is treated as denied: this logs a key-approval-expired event and
|
||||
returns None (no active request).
|
||||
"""
|
||||
if not CTL_LOG.exists():
|
||||
return None
|
||||
latest_req = None
|
||||
resolved = False
|
||||
now = datetime.now(timezone.utc).timestamp()
|
||||
try:
|
||||
with open(CTL_LOG, "r") as f:
|
||||
for line in f:
|
||||
@@ -159,21 +325,73 @@ def check_node_key_request(node: str) -> dict:
|
||||
if act == "key-approval-request":
|
||||
latest_req = rec
|
||||
resolved = False
|
||||
elif act in ("key-approval-allow", "key-approval-deny", "approval-allow", "approval-deny"):
|
||||
elif _rec_approval_type(rec) == "key" and act in (
|
||||
"key-approval-allow", "key-approval-deny", "key-approval-expired",
|
||||
):
|
||||
# Only a KEY-type resolution clears a key request. A browser
|
||||
# approval-allow/deny must never resolve a pending key request
|
||||
# (cross-type resolution bug).
|
||||
resolved = True
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
if latest_req and not resolved:
|
||||
# TTL check: explicit expires_at wins; legacy records fall back to
|
||||
# ts + default TTL.
|
||||
exp_ts = _parse_ts(latest_req.get("expires_at"))
|
||||
if exp_ts is None:
|
||||
req_ts = _parse_ts(latest_req.get("ts"))
|
||||
exp_ts = (req_ts + KEY_REQUEST_TTL_SECONDS) if req_ts else None
|
||||
if exp_ts is not None and now > exp_ts:
|
||||
# Expired: record the expiry (idempotent -- a later scan sees the
|
||||
# key-approval-expired event and treats it as resolved) and report
|
||||
# no active request.
|
||||
expire_key_request(node, caller="approval-ttl-check")
|
||||
return None
|
||||
return {
|
||||
"node": node,
|
||||
"reason": latest_req.get("reason", "Operator passkey access requested"),
|
||||
"requested_at": latest_req.get("ts", ""),
|
||||
"caller": latest_req.get("caller", ""),
|
||||
"expires_at": latest_req.get("expires_at", ""),
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def sweep_expired_key_requests() -> dict:
|
||||
"""Proactively expire stale key requests across all nodes.
|
||||
|
||||
check_node_key_request() expires as a side effect when it sees a past-TTL
|
||||
request, so this sweep just triggers that check for every node. Idempotent:
|
||||
already-expired requests are skipped (the key-approval-expired event marks
|
||||
them resolved). Returns {"expired_now": [nodes expired by this sweep]}.
|
||||
"""
|
||||
expired_now = []
|
||||
if not CTL_LOG.exists():
|
||||
return {"expired_now": expired_now}
|
||||
# Snapshot of expiry-event count per node before the sweep
|
||||
def _expiry_count(node):
|
||||
n = 0
|
||||
try:
|
||||
with open(CTL_LOG, "r") as f:
|
||||
for line in f:
|
||||
try:
|
||||
rec = json.loads(line.strip())
|
||||
except Exception:
|
||||
continue
|
||||
if rec.get("name") == node and rec.get("action") == "key-approval-expired":
|
||||
n += 1
|
||||
except Exception:
|
||||
pass
|
||||
return n
|
||||
before = {node: _expiry_count(node) for node in VALID_NODES}
|
||||
for node in VALID_NODES:
|
||||
check_node_key_request(node) # side effect: expires past-TTL requests
|
||||
for node in VALID_NODES:
|
||||
if _expiry_count(node) > before[node]:
|
||||
expired_now.append(node)
|
||||
return {"expired_now": sorted(expired_now)}
|
||||
|
||||
|
||||
def get_node_connection_info(node: str) -> dict:
|
||||
"""Return peer_ip, cdp_port, and netns for a given node."""
|
||||
@@ -493,6 +711,46 @@ def inspect_node_approvals(node: str) -> dict:
|
||||
"when": w.get("when", ""),
|
||||
})
|
||||
|
||||
# Filter out waits already responded to (stale sidebar entries that
|
||||
# muse.ai never cleared). Responded set is maintained by
|
||||
# reply_node_task() and mark_wait_responded().
|
||||
responded = load_responded_waits().get(node, {})
|
||||
if responded:
|
||||
unique_waits = [w for w in unique_waits if w.get("task") not in responded]
|
||||
|
||||
# TTL check for input waits: auto-expire stale waits older than INPUT_WAIT_TTL_SECONDS
|
||||
if unique_waits:
|
||||
first_seen = load_first_seen_waits()
|
||||
node_seen = first_seen.setdefault(node, {})
|
||||
now_dt = datetime.now(timezone.utc)
|
||||
now_iso = now_dt.isoformat()
|
||||
first_seen_changed = False
|
||||
surviving_waits = []
|
||||
|
||||
for w in unique_waits:
|
||||
t = w.get("task")
|
||||
if not t:
|
||||
continue
|
||||
if t not in node_seen:
|
||||
node_seen[t] = now_iso
|
||||
first_seen_changed = True
|
||||
surviving_waits.append(w)
|
||||
else:
|
||||
try:
|
||||
seen_dt = datetime.fromisoformat(node_seen[t])
|
||||
age_seconds = (now_dt - seen_dt).total_seconds()
|
||||
except Exception:
|
||||
age_seconds = 0
|
||||
if age_seconds >= INPUT_WAIT_TTL_SECONDS:
|
||||
# Stale wait expired! Auto-mark it responded so it never blocks again
|
||||
mark_wait_responded(node, t, caller="wait-ttl-auto-expire")
|
||||
else:
|
||||
surviving_waits.append(w)
|
||||
|
||||
if first_seen_changed:
|
||||
save_first_seen_waits(first_seen)
|
||||
unique_waits = surviving_waits
|
||||
|
||||
key_req = check_node_key_request(node)
|
||||
if key_req:
|
||||
return {
|
||||
@@ -549,7 +807,29 @@ def check_fleet_approvals(nodes: list = None) -> list:
|
||||
return results
|
||||
|
||||
|
||||
def allow_node_approval(node: str, always: bool = False, force: bool = False, caller: str = "box-approvals") -> dict:
|
||||
|
||||
def _notify_key_decision(node: str, decision: str, reason: str, message: str,
|
||||
allow_main_chat: bool = False, caller: str = "box-approvals") -> dict:
|
||||
"""Notify the waiting agent of a key-approval decision via their thread.
|
||||
|
||||
Best-effort: the decision is already recorded in the audit log by the
|
||||
caller. If notification fails, the operator must follow up manually.
|
||||
Returns the reply_node_task result dict.
|
||||
"""
|
||||
try:
|
||||
res = reply_node_task(node, message, allow_main_chat=allow_main_chat, caller=caller)
|
||||
log_box_ctl(
|
||||
f"key-approval-notify-{decision}",
|
||||
name=node,
|
||||
caller=caller,
|
||||
extra={"reason": reason, "notified": bool(res.get("ok")), "notify_error": res.get("error", "")},
|
||||
)
|
||||
return res
|
||||
except Exception as e:
|
||||
return {"ok": False, "node": node, "error": f"notify exception: {e}"}
|
||||
|
||||
|
||||
def allow_node_approval(node: str, always: bool = False, force: bool = False, caller: str = "box-approvals", message: str = None, allow_main_chat: bool = False) -> dict:
|
||||
"""Approve a pending approval on a node (click 'Allow once' or 'Always allow this site')."""
|
||||
info = inspect_node_approvals(node)
|
||||
if not info.get("has_pending"):
|
||||
@@ -567,6 +847,16 @@ def allow_node_approval(node: str, always: bool = False, force: bool = False, ca
|
||||
"forced": force,
|
||||
},
|
||||
)
|
||||
# Execute-gap fix: notify the waiting agent. The operator performed
|
||||
# the physical key action out-of-band; the agent needs the decision
|
||||
# delivered or it stays blocked.
|
||||
notify_msg = message or (
|
||||
f"[operator] Key/passkey request APPROVED ({reason}). "
|
||||
"Operator action complete - you may proceed."
|
||||
)
|
||||
notify_res = _notify_key_decision(
|
||||
node, "allow", reason, notify_msg, allow_main_chat, caller
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"node": node,
|
||||
@@ -575,6 +865,8 @@ def allow_node_approval(node: str, always: bool = False, force: bool = False, ca
|
||||
"dismissed": True,
|
||||
"reason": reason,
|
||||
"title": info.get("title"),
|
||||
"notified": bool(notify_res.get("ok")),
|
||||
"notify_result": notify_res,
|
||||
}
|
||||
|
||||
if not info.get("is_trusted") and not force:
|
||||
@@ -677,7 +969,7 @@ def allow_node_approval(node: str, always: bool = False, force: bool = False, ca
|
||||
return {"ok": False, "node": node, "error": str(e)}
|
||||
|
||||
|
||||
def deny_node_approval(node: str, caller: str = "box-approvals") -> dict:
|
||||
def deny_node_approval(node: str, caller: str = "box-approvals", message: str = None, allow_main_chat: bool = False) -> dict:
|
||||
"""Deny a pending approval on a node (click 'Deny' or deny key request)."""
|
||||
info = inspect_node_approvals(node)
|
||||
if not info.get("has_pending"):
|
||||
@@ -694,6 +986,14 @@ def deny_node_approval(node: str, caller: str = "box-approvals") -> dict:
|
||||
"reason": reason,
|
||||
},
|
||||
)
|
||||
# Execute-gap fix: notify the waiting agent of the denial.
|
||||
notify_msg = message or (
|
||||
f"[operator] Key/passkey request DENIED ({reason}). "
|
||||
"Do not proceed with the protected action."
|
||||
)
|
||||
notify_res = _notify_key_decision(
|
||||
node, "deny", reason, notify_msg, allow_main_chat, caller
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"node": node,
|
||||
@@ -702,6 +1002,8 @@ def deny_node_approval(node: str, caller: str = "box-approvals") -> dict:
|
||||
"dismissed": True,
|
||||
"reason": reason,
|
||||
"title": info.get("title"),
|
||||
"notified": bool(notify_res.get("ok")),
|
||||
"notify_result": notify_res,
|
||||
}
|
||||
|
||||
try:
|
||||
@@ -861,6 +1163,12 @@ def reply_node_task(node: str, message: str, allow_main_chat: bool = False, call
|
||||
"send_res": send_res,
|
||||
},
|
||||
)
|
||||
# The wait(s) visible at reply time are now answered — record them
|
||||
# so the sidebar's stale entries stop re-alerting.
|
||||
for w in info.get("input_waits", []) or []:
|
||||
t = w.get("task")
|
||||
if t:
|
||||
mark_wait_responded(node, t, caller=caller)
|
||||
return {
|
||||
"ok": True,
|
||||
"node": node,
|
||||
@@ -877,11 +1185,18 @@ def reply_node_task(node: str, message: str, allow_main_chat: bool = False, call
|
||||
|
||||
|
||||
def dismiss_node_task(node: str, caller: str = "box-approvals") -> dict:
|
||||
"""Close any open task modal dialog or popup on a node."""
|
||||
"""Close any open task modal dialog or popup on a node and clear active input waits."""
|
||||
clear_res = clear_node_waits(node, caller=caller)
|
||||
try:
|
||||
ws, _ = get_cdp_ws(node, timeout=3.0)
|
||||
except Exception as e:
|
||||
return {"ok": False, "node": node, "error": f"Failed to connect to CDP: {e}"}
|
||||
return {
|
||||
"ok": True,
|
||||
"node": node,
|
||||
"result": "WAITS_CLEARED",
|
||||
"cleared_waits": clear_res.get("cleared_per_node", {}).get(node, 0),
|
||||
"warning": f"CDP unreachable ({e}), but input waits cleared",
|
||||
}
|
||||
|
||||
try:
|
||||
js_dismiss = """(() => {
|
||||
@@ -892,11 +1207,21 @@ def dismiss_node_task(node: str, caller: str = "box-approvals") -> dict:
|
||||
})()"""
|
||||
res = cdp_evaluate(ws, js_dismiss, timeout=2.0)
|
||||
ws.close()
|
||||
return {"ok": True, "node": node, "result": res}
|
||||
return {
|
||||
"ok": True,
|
||||
"node": node,
|
||||
"result": res,
|
||||
"cleared_waits": clear_res.get("cleared_per_node", {}).get(node, 0),
|
||||
}
|
||||
except Exception as e:
|
||||
try:
|
||||
ws.close()
|
||||
except Exception:
|
||||
pass
|
||||
return {"ok": False, "node": node, "error": str(e)}
|
||||
return {
|
||||
"ok": True,
|
||||
"node": node,
|
||||
"result": "WAITS_CLEARED",
|
||||
"cleared_waits": clear_res.get("cleared_per_node", {}).get(node, 0),
|
||||
}
|
||||
|
||||
|
||||
+27
-10
@@ -58,6 +58,7 @@ NOTIFY_SIDECHATS = {
|
||||
"pip": "pip tasks",
|
||||
"muse": "muse tasks",
|
||||
"dev": "onboarding-dev",
|
||||
"def": "def tasks",
|
||||
}
|
||||
VALID_ON_FAILURE = {"retry", "alert", "ignore"}
|
||||
KNOWN_PLACEHOLDERS = {"job_id", "job_name", "datetime", "date", "last_run"}
|
||||
@@ -989,12 +990,13 @@ def act_approval_check(node=None):
|
||||
out(True, approvals=res)
|
||||
|
||||
|
||||
def act_approval_allow(node, always=False, force=False):
|
||||
def act_approval_allow(node, always=False, force=False, message=None, allow_main_chat=False):
|
||||
audit("approval-allow", node)
|
||||
if node not in VALID_AGENTS:
|
||||
fail("BAD_NODE", f"unknown node: {node}")
|
||||
import approvals
|
||||
res = approvals.allow_node_approval(node, always=always, force=force, caller="box-ctl")
|
||||
res = approvals.allow_node_approval(node, always=always, force=force, caller="box-ctl",
|
||||
message=message, allow_main_chat=allow_main_chat)
|
||||
if res.get("ok"):
|
||||
kw = {k: v for k, v in res.items() if k != "ok"}
|
||||
out(True, **kw)
|
||||
@@ -1002,12 +1004,13 @@ def act_approval_allow(node, always=False, force=False):
|
||||
fail("APPROVAL_FAILED", res.get("error", "approval failed"), res)
|
||||
|
||||
|
||||
def act_approval_deny(node):
|
||||
def act_approval_deny(node, message=None, allow_main_chat=False):
|
||||
audit("approval-deny", node)
|
||||
if node not in VALID_AGENTS:
|
||||
fail("BAD_NODE", f"unknown node: {node}")
|
||||
import approvals
|
||||
res = approvals.deny_node_approval(node, caller="box-ctl")
|
||||
res = approvals.deny_node_approval(node, caller="box-ctl",
|
||||
message=message, allow_main_chat=allow_main_chat)
|
||||
if res.get("ok"):
|
||||
kw = {k: v for k, v in res.items() if k != "ok"}
|
||||
out(True, **kw)
|
||||
@@ -3563,16 +3566,30 @@ def main(argv):
|
||||
act_approval_check(node)
|
||||
elif action in ("approval-allow", "approval-approve"):
|
||||
if not rest:
|
||||
fail("BAD_ARGS", "usage: approval-allow <node> [--always] [--force]")
|
||||
fail("BAD_ARGS", "usage: approval-allow <node> [--always] [--force] [--message TEXT] [--allow-main-chat]")
|
||||
node = rest[0]
|
||||
always = "--always" in rest[1:]
|
||||
force = "--force" in rest[1:]
|
||||
act_approval_allow(node, always=always, force=force)
|
||||
rest_args = rest[1:]
|
||||
always = "--always" in rest_args
|
||||
force = "--force" in rest_args
|
||||
allow_main_chat = "--allow-main-chat" in rest_args
|
||||
message = None
|
||||
if "--message" in rest_args:
|
||||
mi = rest_args.index("--message")
|
||||
if mi + 1 < len(rest_args):
|
||||
message = rest_args[mi + 1]
|
||||
act_approval_allow(node, always=always, force=force, message=message, allow_main_chat=allow_main_chat)
|
||||
elif action == "approval-deny":
|
||||
if not rest:
|
||||
fail("BAD_ARGS", "usage: approval-deny <node>")
|
||||
fail("BAD_ARGS", "usage: approval-deny <node> [--message TEXT] [--allow-main-chat]")
|
||||
node = rest[0]
|
||||
act_approval_deny(node)
|
||||
rest_args = rest[1:]
|
||||
allow_main_chat = "--allow-main-chat" in rest_args
|
||||
message = None
|
||||
if "--message" in rest_args:
|
||||
mi = rest_args.index("--message")
|
||||
if mi + 1 < len(rest_args):
|
||||
message = rest_args[mi + 1]
|
||||
act_approval_deny(node, message=message, allow_main_chat=allow_main_chat)
|
||||
elif action == "approval-auto":
|
||||
node = rest[0] if rest else None
|
||||
act_approval_auto(node)
|
||||
|
||||
+105
-9
@@ -34,6 +34,11 @@ set -uo pipefail
|
||||
|
||||
THRESHOLD="${FLEET_ALERT_THRESHOLD:-2}"
|
||||
REALERT_MIN="${FLEET_ALERT_REALERT_MIN:-30}"
|
||||
# Approval/input-wait TTLs (seconds): conditions failing longer than this are
|
||||
# auto-expired (input waits dismissed, key requests denied) instead of paging
|
||||
# forever. Overridable per environment.
|
||||
INPUT_WAIT_TTL="${FLEET_ALERT_INPUT_WAIT_TTL:-1800}"
|
||||
BROWSER_APPROVAL_TTL="${FLEET_ALERT_BROWSER_APPROVAL_TTL:-1800}"
|
||||
QUIET_HOURS="${FLEET_ALERT_QUIET_HOURS:-}"
|
||||
DRY_RUN="${FLEET_ALERT_DRY_RUN:-0}"
|
||||
INJECT_FAIL="${FLEET_ALERT_INJECT_FAIL:-}"
|
||||
@@ -51,14 +56,18 @@ NOW=$(date +%s)
|
||||
log() { echo "$(date -Iseconds) $*" >> "$LOG"; }
|
||||
|
||||
# --- shared consecutive-failure state machine (also used by the container relay) ---
|
||||
# usage: state_machine <cond> <failing 0|1> -> prints "<ACTION> <fails>"
|
||||
# ACTION: ALERT_FIRST | ALERT_REALERT | RECOVERY | SUPPRESSED | NONE
|
||||
# usage: state_machine <cond> <failing 0|1> [ttl_seconds] -> prints "<ACTION> <fails>"
|
||||
# When ttl_seconds > 0 and the condition has failed longer than the TTL,
|
||||
# prints "EXPIRED <fails>" so the caller can auto-resolve (dismiss/deny).
|
||||
# State entries track first_fail_ts (epoch of first consecutive failure).
|
||||
# ACTION: ALERT_FIRST | ALERT_REALERT | RECOVERY | SUPPRESSED | EXPIRED | NONE
|
||||
state_machine() {
|
||||
local cond="$1" failing="$2"
|
||||
local cond="$1" failing="$2" ttl="${3:-0}"
|
||||
THRESHOLD="$THRESHOLD" REALERT_MIN="$REALERT_MIN" QUIET_HOURS="$QUIET_HOURS" \
|
||||
FLEET_ALERT_DRY_RUN="$DRY_RUN" python3 - "$STATE" "$cond" "$failing" <<'PYEOF'
|
||||
FLEET_ALERT_DRY_RUN="$DRY_RUN" python3 - "$STATE" "$cond" "$failing" "$ttl" <<'PYEOF'
|
||||
import json, os, sys, time
|
||||
state_path, cond, failing_s = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
ttl_seconds = int(sys.argv[4]) if len(sys.argv) > 4 else 0
|
||||
failing = failing_s == "1"
|
||||
threshold = int(os.environ.get("THRESHOLD", "2"))
|
||||
realert_min = int(os.environ.get("REALERT_MIN", "30"))
|
||||
@@ -85,7 +94,17 @@ except Exception:
|
||||
e = st.get(cond) or {"fails": 0, "alerted": False, "last_alert_ts": 0}
|
||||
action = "NONE"
|
||||
if failing:
|
||||
if int(e.get("fails", 0)) == 0:
|
||||
e["first_fail_ts"] = now
|
||||
e["fails"] = int(e.get("fails", 0)) + 1
|
||||
# TTL expiry: failing longer than ttl_seconds -> EXPIRED (caller auto-resolves)
|
||||
if ttl_seconds > 0 and now - int(e.get("first_fail_ts", now)) >= ttl_seconds:
|
||||
action = "EXPIRED"
|
||||
# Reset so a fresh incident starts clean after the caller resolves it
|
||||
e["fails"] = 0
|
||||
e["alerted"] = False
|
||||
e.pop("first_fail_ts", None)
|
||||
else:
|
||||
due = e["fails"] >= threshold and (
|
||||
not e.get("alerted") or now - int(e.get("last_alert_ts", 0)) >= realert_min * 60
|
||||
)
|
||||
@@ -102,6 +121,7 @@ else:
|
||||
action = "RECOVERY"
|
||||
e["fails"] = 0
|
||||
e["alerted"] = False
|
||||
e.pop("first_fail_ts", None)
|
||||
st[cond] = e
|
||||
if not dry:
|
||||
json.dump(st, open(state_path, "w"))
|
||||
@@ -151,6 +171,28 @@ box_notify() {
|
||||
for p in $pids; do wait "$p" 2>/dev/null; done
|
||||
}
|
||||
|
||||
notify_input_wait() {
|
||||
# Targeted DM for input waits (2026-10-05): DM ONLY the specific agent
|
||||
# whose session is waiting for human input -- not a broadcast to all
|
||||
# healthy agents. The #lobby post still fires via the relay leg for
|
||||
# human visibility; this DM ensures the responsible operator sees it
|
||||
# in their sidechat without digging through lobby noise.
|
||||
# Best-effort: never fatal to the 5-minute check loop.
|
||||
local node="$1"
|
||||
local detail="$2"
|
||||
local msg="[fleet-alert] INPUT WAIT: ${detail} -- reply: box approval reply ${node} \"<msg>\" or box notify ${node} \"<msg>\""
|
||||
msg="${msg:0:900}"
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
log "DRY-RUN would DM $node re input_wait"
|
||||
return 0
|
||||
fi
|
||||
if timeout 60 python3 "$BIN/box-ctl.py" notify "$node" "$msg" >/dev/null 2>&1; then
|
||||
log "input_wait targeted DM sent to $node"
|
||||
else
|
||||
log "input_wait DM to $node failed (best-effort, non-fatal)"
|
||||
fi
|
||||
}
|
||||
|
||||
injected() { # cond -> 0 if injected-fail
|
||||
case ",$INJECT_FAIL," in *,"$1,"*) return 0;; *) return 1;; esac
|
||||
}
|
||||
@@ -214,7 +256,7 @@ print(json.dumps(out))
|
||||
detail="Agent $node approvals clear"
|
||||
fi
|
||||
injected "$cond" && failing=1
|
||||
read -r action fails < <(state_machine "$cond" "$failing")
|
||||
read -r action fails < <(state_machine "$cond" "$failing" "$BROWSER_APPROVAL_TTL")
|
||||
case "$action" in
|
||||
ALERT_FIRST|ALERT_REALERT)
|
||||
emit_record "ALERT" "$cond" "$detail" "$fails"
|
||||
@@ -226,6 +268,21 @@ print(json.dumps(out))
|
||||
SUPPRESSED)
|
||||
log "$cond still critical x$fails — re-page suppressed"
|
||||
;;
|
||||
EXPIRED)
|
||||
# Browser approval dialog exceeded BROWSER_APPROVAL_TTL without a
|
||||
# human decision: fail closed by denying it.
|
||||
log "$cond EXPIRED after ${BROWSER_APPROVAL_TTL}s without human decision — auto-denying (fail closed)"
|
||||
python3 - "$node" <<'PYEOF3'
|
||||
import sys, json
|
||||
sys.path.insert(0, "/home/super/Projects/NetVM/bin")
|
||||
import approvals
|
||||
node = sys.argv[1]
|
||||
print(json.dumps(approvals.deny_node_approval(node, caller="approval-ttl-expire")))
|
||||
approvals.log_box_ctl("approval-expired", name=node, caller="approval-ttl-expire",
|
||||
extra={"note": "browser approval TTL elapsed; auto-denied (fail closed)"})
|
||||
PYEOF3
|
||||
emit_record "RECOVERY" "$cond" "Agent $node browser approval expired after ${BROWSER_APPROVAL_TTL}s; auto-denied" "$fails"
|
||||
;;
|
||||
esac
|
||||
|
||||
# 2. Sidebar task waiting on human input
|
||||
@@ -245,11 +302,11 @@ if w:
|
||||
detail_in="Agent $node tasks running"
|
||||
fi
|
||||
injected "$cond_in" && failing_in=1
|
||||
read -r action_in fails_in < <(state_machine "$cond_in" "$failing_in")
|
||||
read -r action_in fails_in < <(state_machine "$cond_in" "$failing_in" "$INPUT_WAIT_TTL")
|
||||
case "$action_in" in
|
||||
ALERT_FIRST|ALERT_REALERT)
|
||||
emit_record "ALERT" "$cond_in" "$detail_in" "$fails_in"
|
||||
echo "$cond_in" >> "$STATE_DIR/.alerts.tmp"
|
||||
echo "$cond_in|$detail_in" >> "$STATE_DIR/.alerts.tmp"
|
||||
;;
|
||||
RECOVERY)
|
||||
emit_record "RECOVERY" "$cond_in" "$detail_in" "$fails_in"
|
||||
@@ -257,6 +314,27 @@ if w:
|
||||
SUPPRESSED)
|
||||
log "$cond_in still critical x$fails_in — re-page suppressed"
|
||||
;;
|
||||
EXPIRED)
|
||||
# Input wait exceeded INPUT_WAIT_TTL without human response:
|
||||
# auto-dismiss so the agent unblocks. Log the expiry and emit a
|
||||
# RECOVERY record (the wait is gone, not merely un-paged).
|
||||
log "$cond_in EXPIRED after ${INPUT_WAIT_TTL}s without human input — auto-dismissing"
|
||||
python3 - "$node" <<'PYEOF2'
|
||||
import sys
|
||||
sys.path.insert(0, "/home/super/Projects/NetVM/bin")
|
||||
import approvals, json
|
||||
node = sys.argv[1]
|
||||
info = approvals.inspect_node_approvals(node)
|
||||
for w in info.get("input_waits", []) or []:
|
||||
t = w.get("task")
|
||||
if t:
|
||||
approvals.mark_wait_responded(node, t, caller="approval-ttl-expire")
|
||||
approvals.log_box_ctl("approval-wait-expired", name=node, caller="approval-ttl-expire",
|
||||
extra={"note": "input wait TTL elapsed; auto-dismissed"})
|
||||
print(json.dumps(approvals.dismiss_node_task(node, caller="approval-ttl-expire")))
|
||||
PYEOF2
|
||||
emit_record "RECOVERY" "$cond_in" "Agent $node input wait expired after ${INPUT_WAIT_TTL}s; auto-dismissed" "$fails_in"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
@@ -328,8 +406,26 @@ rm -f "$STATE_DIR/.healthy.tmp"
|
||||
# Notify for this run's alerts (best effort). Skip entirely when nothing is healthy
|
||||
# (notify needs a working browser via dm.py) or in dry-run.
|
||||
if [ -n "$HEALTHY_AGENTS" ] && [ -f "$STATE_DIR/.alerts.tmp" ]; then
|
||||
while read -r cond; do
|
||||
[ -n "$cond" ] && box_notify "$cond" "see #lobby for detail"
|
||||
while IFS= read -r line; do
|
||||
# alerts.tmp format: "cond" or "cond|detail" (input_wait carries detail)
|
||||
cond="${line%%|*}"
|
||||
detail="${line#*|}"
|
||||
[ "$detail" = "$line" ] && detail=""
|
||||
[ -n "$cond" ] || continue
|
||||
case "$cond" in
|
||||
input_wait:*)
|
||||
# Targeted: DM only the waiting agent, not a broadcast.
|
||||
node="${cond#input_wait:}"
|
||||
if [ -n "$detail" ]; then
|
||||
notify_input_wait "$node" "$detail"
|
||||
else
|
||||
box_notify "$cond" "see #lobby for detail"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
box_notify "$cond" "see #lobby for detail"
|
||||
;;
|
||||
esac
|
||||
done < "$STATE_DIR/.alerts.tmp"
|
||||
elif [ -f "$STATE_DIR/.alerts.tmp" ]; then
|
||||
log "no healthy agents — box notify skipped (DM path needs a working browser)"
|
||||
|
||||
+1
-1
@@ -765,7 +765,7 @@ def remediate_breaks(dry_run=False) -> dict:
|
||||
import approvals
|
||||
fleet_apps = approvals.check_fleet_approvals()
|
||||
for app in fleet_apps:
|
||||
if app.get("has_pending") and app.get("is_trusted"):
|
||||
if app.get("has_pending") and app.get("is_trusted") and app.get("status") != "KEY_APPROVAL":
|
||||
node = app["node"]
|
||||
if not dry_run:
|
||||
approvals.allow_node_approval(node, always=True, caller="loop-remediate")
|
||||
|
||||
+1
-1
@@ -509,7 +509,7 @@ def main():
|
||||
import approvals
|
||||
app_info = approvals.inspect_node_approvals(agent)
|
||||
if app_info.get("has_pending"):
|
||||
if app_info.get("is_trusted"):
|
||||
if app_info.get("is_trusted") and app_info.get("status") != "KEY_APPROVAL":
|
||||
print(f"Pre-dispatch: auto-approving trusted request for {agent} ({app_info.get('target')})")
|
||||
approvals.allow_node_approval(agent, always=True, caller="job-dispatch")
|
||||
else:
|
||||
|
||||
+33
-4
@@ -519,13 +519,17 @@ def cmd_approvals(args):
|
||||
sys.exit(1)
|
||||
always = getattr(args, "always", False)
|
||||
force = getattr(args, "force", False)
|
||||
res = approvals.allow_node_approval(node, always=always, force=force)
|
||||
message = getattr(args, "message", None)
|
||||
allow_main_chat = getattr(args, "allow_main_chat", False)
|
||||
res = approvals.allow_node_approval(node, always=always, force=force, message=message,
|
||||
allow_main_chat=allow_main_chat)
|
||||
if getattr(args, "json", False):
|
||||
print(json.dumps(res, indent=2))
|
||||
return
|
||||
if res.get("ok"):
|
||||
if res.get("type") == "key_approval":
|
||||
print(c_green(f"✔ Approved operator key request for node '{node}'. Granted and logged to audit trail."))
|
||||
notified = "agent notified" if res.get("notified") else "⚠ agent NOT notified (follow up manually)"
|
||||
print(c_green(f"✔ Approved operator key request for node '{node}'. Logged to audit trail. {notified}."))
|
||||
else:
|
||||
decision_str = "Always allow this site" if always else "Allow once"
|
||||
print(c_green(f"✔ Approved request on node '{node}' ({decision_str}). Dialog dismissed: {res.get('dismissed')}."))
|
||||
@@ -537,13 +541,16 @@ def cmd_approvals(args):
|
||||
if not node:
|
||||
print(c_red("Error: Must specify node for deny. e.g. 'box approvals deny 646'"), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
res = approvals.deny_node_approval(node)
|
||||
message = getattr(args, "message", None)
|
||||
allow_main_chat = getattr(args, "allow_main_chat", False)
|
||||
res = approvals.deny_node_approval(node, message=message, allow_main_chat=allow_main_chat)
|
||||
if getattr(args, "json", False):
|
||||
print(json.dumps(res, indent=2))
|
||||
return
|
||||
if res.get("ok"):
|
||||
if res.get("type") == "key_approval":
|
||||
print(c_green(f"✔ Denied operator key request for node '{node}'. Denied and logged to audit trail."))
|
||||
notified = "agent notified" if res.get("notified") else "⚠ agent NOT notified (follow up manually)"
|
||||
print(c_green(f"✔ Denied operator key request for node '{node}'. Logged to audit trail. {notified}."))
|
||||
else:
|
||||
print(c_green(f"✔ Denied request on node '{node}'. Dialog dismissed: {res.get('dismissed')}."))
|
||||
else:
|
||||
@@ -622,6 +629,17 @@ def cmd_approvals(args):
|
||||
else:
|
||||
print(c_red(f"✖ Failed to dismiss task popup on node '{node}': {res.get('error')}"))
|
||||
sys.exit(1)
|
||||
elif action in ("clear", "clear-all", "clear_all"):
|
||||
target_node = node
|
||||
if action in ("clear-all", "clear_all"):
|
||||
target_node = None
|
||||
res = approvals.clear_node_waits(target_node)
|
||||
if getattr(args, "json", False):
|
||||
print(json.dumps(res, indent=2))
|
||||
return
|
||||
cleared = res.get("total_cleared", 0)
|
||||
target_desc = f"node '{target_node}'" if target_node else "all fleet nodes"
|
||||
print(c_green(f"✔ Cleared {cleared} input wait(s) on {target_desc}."))
|
||||
|
||||
elif action in ("request-key", "request_key"):
|
||||
if not node:
|
||||
@@ -4211,14 +4229,20 @@ def build_parser():
|
||||
p_app_allow.add_argument("node", choices=VALID_NODES, help="Target node to approve")
|
||||
p_app_allow.add_argument("--always", action="store_true", help="Click 'Always allow this site' instead of 'Allow once'")
|
||||
p_app_allow.add_argument("--force", action="store_true", help="Force approval even if target is untrusted")
|
||||
p_app_allow.add_argument("--message", default=None, help="Optional message/credential to deliver to the waiting agent")
|
||||
p_app_allow.add_argument("--allow-main-chat", action="store_true", help="Allow reply into Main Chat if agent is waiting there")
|
||||
|
||||
p_app_approve = app_sub.add_parser("approve", parents=[common], help="Alias for 'allow'")
|
||||
p_app_approve.add_argument("node", choices=VALID_NODES, help="Target node to approve")
|
||||
p_app_approve.add_argument("--always", action="store_true", help="Click 'Always allow this site' instead of 'Allow once'")
|
||||
p_app_approve.add_argument("--force", action="store_true", help="Force approval even if target is untrusted")
|
||||
p_app_approve.add_argument("--message", default=None, help="Optional message/credential to deliver to the waiting agent")
|
||||
p_app_approve.add_argument("--allow-main-chat", action="store_true", help="Allow reply into Main Chat if agent is waiting there")
|
||||
|
||||
p_app_deny = app_sub.add_parser("deny", parents=[common], help="Deny pending browser request")
|
||||
p_app_deny.add_argument("node", choices=VALID_NODES, help="Target node to deny")
|
||||
p_app_deny.add_argument("--message", default=None, help="Optional message to deliver to the waiting agent")
|
||||
p_app_deny.add_argument("--allow-main-chat", action="store_true", help="Allow reply into Main Chat if agent is waiting there")
|
||||
|
||||
p_app_auto = app_sub.add_parser("auto", parents=[common], help="Auto-approve all trusted requests across fleet")
|
||||
p_app_auto.add_argument("--node", choices=VALID_NODES, default=None, help="Target node (or all nodes)")
|
||||
@@ -4236,6 +4260,11 @@ def build_parser():
|
||||
p_app_dismiss = app_sub.add_parser("dismiss", parents=[common], help="Dismiss any open task dialog/popup on a node")
|
||||
p_app_dismiss.add_argument("node", choices=VALID_NODES, help="Target node to dismiss dialog on")
|
||||
|
||||
p_app_clear = app_sub.add_parser("clear", parents=[common], help="Clear and dismiss pending input waits on a node or all nodes")
|
||||
p_app_clear.add_argument("node", nargs="?", choices=VALID_NODES, default=None, help="Target node (or omit for all nodes)")
|
||||
|
||||
p_app_clear_all = app_sub.add_parser("clear-all", parents=[common], help="Clear and dismiss all pending input waits across fleet")
|
||||
|
||||
p_app_req_key = app_sub.add_parser("request-key", parents=[common], help="Request operator passkey/key approval for an agent")
|
||||
p_app_req_key.add_argument("node", choices=VALID_NODES, help="Target node requesting key")
|
||||
p_app_req_key.add_argument("--reason", default="Passkey authentication required", help="Reason for key request")
|
||||
|
||||
Reference in New Issue
Block a user