approvals: stale-wait cleanup, key-decision notify, TTLs, key/browser isolation

- bin/approvals.py: responded-wait filtering + auto-mark, key-decision sidechat-first notify (notified flag, --message, --allow-main-chat), TTL defaults (input 30m / browser 30m / key 2h), cross-type guard (browser actions cannot resolve key requests), sweep_expired_key_requests; restores check_node_key_request fallback in inspect_node_approvals

- bin/box-ctl.py + bin/super-cli.py (approvals hunks only): --message/--allow-main-chat passthrough on allow/deny, clear/clear-all actions, def sidechat routing; restores sys.exit(1) on dismiss failure

- bin/fleet-alert-check.sh: TTL-aware state_machine (EXPIRED action), auto-deny expired browser approvals (fail closed), auto-dismiss expired input waits, targeted per-agent DM for input waits

- bin/job-dispatch.py + bin/gravity.py: KEY_APPROVAL excluded from auto-approval

Reviewed by 5 independent reviewers (all APPROVE/APPROVE WITH NOTES); integration gate GO (17/17 tests). TUI hunks in super-cli.py intentionally excluded.
This commit is contained in:
operator
2026-10-06 00:49:46 +00:00
parent adfcd2e602
commit 740648e973
6 changed files with 515 additions and 48 deletions
+337 -12
View File
@@ -31,6 +31,98 @@ except ImportError:
REPO_ROOT = Path("/home/super/Projects/NetVM") REPO_ROOT = Path("/home/super/Projects/NetVM")
BIN_DIR = REPO_ROOT / "bin" BIN_DIR = REPO_ROOT / "bin"
CTL_LOG = REPO_ROOT / "box-ctl.jsonl" CTL_LOG = REPO_ROOT / "box-ctl.jsonl"
RESPONDED_WAITS_FILE = REPO_ROOT / ".state" / "approvals-responded.json"
FIRST_SEEN_WAITS_FILE = REPO_ROOT / ".state" / "approvals-first-seen.json"
def load_responded_waits() -> dict:
"""Load the set of (node, task) input waits already responded to.
Returns {node: {task: iso_timestamp}}. Missing file -> {}.
"""
try:
if RESPONDED_WAITS_FILE.exists():
return json.loads(RESPONDED_WAITS_FILE.read_text())
except Exception:
pass
return {}
def save_responded_waits(data: dict) -> None:
"""Persist the responded-waits map (best effort)."""
try:
RESPONDED_WAITS_FILE.parent.mkdir(parents=True, exist_ok=True)
RESPONDED_WAITS_FILE.write_text(json.dumps(data, indent=1))
except Exception:
pass
def load_first_seen_waits() -> dict:
"""Load map of when input waits were first observed: {node: {task: iso_timestamp}}."""
try:
if FIRST_SEEN_WAITS_FILE.exists():
return json.loads(FIRST_SEEN_WAITS_FILE.read_text())
except Exception:
pass
return {}
def save_first_seen_waits(data: dict) -> None:
"""Persist first-seen input waits map."""
try:
FIRST_SEEN_WAITS_FILE.parent.mkdir(parents=True, exist_ok=True)
FIRST_SEEN_WAITS_FILE.write_text(json.dumps(data, indent=1))
except Exception:
pass
def is_wait_responded(node: str, task: str) -> bool:
"""True if this (node, task) wait was already answered."""
return task in load_responded_waits().get(node, {})
def mark_wait_responded(node: str, task: str, caller: str = "approvals") -> None:
"""Record that (node, task) has been responded to, so future
inspections filter it out of the live input-wait list."""
data = load_responded_waits()
node_map = data.setdefault(node, {})
if task not in node_map:
node_map[task] = datetime.now(timezone.utc).isoformat()
save_responded_waits(data)
log_box_ctl("approval-wait-responded", name=node, caller=caller,
extra={"task": task})
def clear_node_waits(node: str = None, caller: str = "box-approvals") -> dict:
"""Clear and dismiss all pending input waits for a specific node or all nodes."""
target_nodes = [node] if node else VALID_NODES
total_cleared = 0
cleared_per_node = {}
data = load_responded_waits()
now_iso = datetime.now(timezone.utc).isoformat()
for n in target_nodes:
node_map = data.setdefault(n, {})
n_cleared = 0
try:
info = inspect_node_approvals(n)
for w in info.get("input_waits", []) or []:
t = w.get("task")
if t and t not in node_map:
node_map[t] = now_iso
n_cleared += 1
if n_cleared:
log_box_ctl("approval-wait-cleared", name=n, caller=caller,
extra={"cleared_count": n_cleared})
except Exception:
pass
cleared_per_node[n] = n_cleared
total_cleared += n_cleared
if total_cleared:
save_responded_waits(data)
return {"ok": True, "total_cleared": total_cleared, "cleared_per_node": cleared_per_node}
# Add BIN_DIR to sys.path # Add BIN_DIR to sys.path
if str(BIN_DIR) not in sys.path: if str(BIN_DIR) not in sys.path:
@@ -43,6 +135,14 @@ except ImportError:
VALID_NODES = ["muse", "pip", "646", "opm", "def", "dev"] VALID_NODES = ["muse", "pip", "646", "opm", "def", "dev"]
# Approval timeout defaults (seconds).
# Key requests are sensitive operations -- give the operator 2h to decide.
# Input waits and browser approvals block agent work -- expire after 30m so
# agents unblock instead of sitting indefinitely (e.g. def waited 4h+).
KEY_REQUEST_TTL_SECONDS = 2 * 3600
INPUT_WAIT_TTL_SECONDS = 30 * 60
BROWSER_APPROVAL_TTL_SECONDS = 30 * 60
# Trusted infrastructure IPs safe for automated approval # Trusted infrastructure IPs safe for automated approval
TRUSTED_IPS = { TRUSTED_IPS = {
"34.139.37.135", # VM (gateway) "34.139.37.135", # VM (gateway)
@@ -95,12 +195,30 @@ def redact_sensitive(text: str) -> str:
return out return out
def _approval_type(action: str) -> str:
"""Classify an approval action into its request type.
Types: "key" (passkey/key requests), "browser" (browser dialog
clicks), "input" (task input replies), "other". Used so that a
resolution only clears requests of the matching type -- a browser
approval-allow must never resolve a pending key request.
"""
if action.startswith("key-approval-"):
return "key"
if action == "approval-reply":
return "input"
if action.startswith("approval-"):
return "browser"
return "other"
def log_box_ctl(action: str, name: str = None, caller: str = "box-approvals", extra: dict = None): def log_box_ctl(action: str, name: str = None, caller: str = "box-approvals", extra: dict = None):
"""Log an audit event to box-ctl.jsonl with secret redaction.""" """Log an audit event to box-ctl.jsonl with secret redaction."""
try: try:
rec = { rec = {
"ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"action": action, "action": action,
"type": _approval_type(action),
"name": name, "name": name,
"caller": caller, "caller": caller,
} }
@@ -118,14 +236,23 @@ def log_box_ctl(action: str, name: str = None, caller: str = "box-approvals", ex
pass pass
def request_key_approval(node: str, reason: str = "", caller: str = "agent") -> dict: def request_key_approval(node: str, reason: str = "", caller: str = "agent",
"""Register a key/passkey approval request for a node in box-ctl.jsonl.""" ttl_seconds: int = None) -> dict:
"""Register a key/passkey approval request for a node in box-ctl.jsonl.
ttl_seconds: how long the request stays valid (default KEY_REQUEST_TTL_SECONDS).
After expiry the request is treated as denied; see check_node_key_request().
"""
reason = reason or "Operator passkey access requested" reason = reason or "Operator passkey access requested"
ttl = ttl_seconds if ttl_seconds is not None else KEY_REQUEST_TTL_SECONDS
expires_iso = datetime.fromtimestamp(
datetime.now(timezone.utc).timestamp() + ttl, tz=timezone.utc
).strftime("%Y-%m-%dT%H:%M:%SZ")
log_box_ctl( log_box_ctl(
"key-approval-request", "key-approval-request",
name=node, name=node,
caller=caller, caller=caller,
extra={"reason": reason}, extra={"reason": reason, "ttl_seconds": ttl, "expires_at": expires_iso},
) )
return { return {
"ok": True, "ok": True,
@@ -133,16 +260,55 @@ def request_key_approval(node: str, reason: str = "", caller: str = "agent") ->
"status": "KEY_APPROVAL_REQUESTED", "status": "KEY_APPROVAL_REQUESTED",
"reason": reason, "reason": reason,
"caller": caller, "caller": caller,
"note": "Request recorded in audit log. Operator can approve via 'box approvals allow <node>'." "ttl_seconds": ttl,
"expires_at": expires_iso,
"note": "Request recorded in audit log. Operator can approve via 'box approvals allow <node>'.",
} }
def _parse_ts(ts_str: str):
"""Parse a box-ctl.jsonl ts ('%Y-%m-%dT%H:%M:%SZ') to epoch seconds. None on failure."""
if not ts_str:
return None
try:
dt = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
return dt.timestamp()
except Exception:
return None
def expire_key_request(node: str, caller: str = "approval-sweeper") -> dict:
"""Mark a node's pending key request as expired (auto-deny on TTL)."""
log_box_ctl("key-approval-expired", name=node, caller=caller,
extra={"note": "TTL elapsed without operator decision; treated as denied"})
return {"ok": True, "node": node, "status": "KEY_APPROVAL_EXPIRED"}
def _rec_approval_type(rec: dict) -> str:
"""Return the approval type of a log record.
Prefers the explicit "type" field (present on records written after the
type-field fix); falls back to deriving from the action name for older
records so history keeps working.
"""
t = rec.get("type")
if t:
return t
return _approval_type(rec.get("action", ""))
def check_node_key_request(node: str) -> dict: def check_node_key_request(node: str) -> dict:
"""Check if node has an active unfulfilled key approval request in box-ctl.jsonl.""" """Check if node has an active unfulfilled key approval request in box-ctl.jsonl.
Requests expire after their TTL (default KEY_REQUEST_TTL_SECONDS). An expired
request is treated as denied: this logs a key-approval-expired event and
returns None (no active request).
"""
if not CTL_LOG.exists(): if not CTL_LOG.exists():
return None return None
latest_req = None latest_req = None
resolved = False resolved = False
now = datetime.now(timezone.utc).timestamp()
try: try:
with open(CTL_LOG, "r") as f: with open(CTL_LOG, "r") as f:
for line in f: for line in f:
@@ -159,21 +325,73 @@ def check_node_key_request(node: str) -> dict:
if act == "key-approval-request": if act == "key-approval-request":
latest_req = rec latest_req = rec
resolved = False resolved = False
elif act in ("key-approval-allow", "key-approval-deny", "approval-allow", "approval-deny"): elif _rec_approval_type(rec) == "key" and act in (
"key-approval-allow", "key-approval-deny", "key-approval-expired",
):
# Only a KEY-type resolution clears a key request. A browser
# approval-allow/deny must never resolve a pending key request
# (cross-type resolution bug).
resolved = True resolved = True
except Exception: except Exception:
return None return None
if latest_req and not resolved: if latest_req and not resolved:
# TTL check: explicit expires_at wins; legacy records fall back to
# ts + default TTL.
exp_ts = _parse_ts(latest_req.get("expires_at"))
if exp_ts is None:
req_ts = _parse_ts(latest_req.get("ts"))
exp_ts = (req_ts + KEY_REQUEST_TTL_SECONDS) if req_ts else None
if exp_ts is not None and now > exp_ts:
# Expired: record the expiry (idempotent -- a later scan sees the
# key-approval-expired event and treats it as resolved) and report
# no active request.
expire_key_request(node, caller="approval-ttl-check")
return None
return { return {
"node": node, "node": node,
"reason": latest_req.get("reason", "Operator passkey access requested"), "reason": latest_req.get("reason", "Operator passkey access requested"),
"requested_at": latest_req.get("ts", ""), "requested_at": latest_req.get("ts", ""),
"caller": latest_req.get("caller", ""), "caller": latest_req.get("caller", ""),
"expires_at": latest_req.get("expires_at", ""),
} }
return None return None
def sweep_expired_key_requests() -> dict:
"""Proactively expire stale key requests across all nodes.
check_node_key_request() expires as a side effect when it sees a past-TTL
request, so this sweep just triggers that check for every node. Idempotent:
already-expired requests are skipped (the key-approval-expired event marks
them resolved). Returns {"expired_now": [nodes expired by this sweep]}.
"""
expired_now = []
if not CTL_LOG.exists():
return {"expired_now": expired_now}
# Snapshot of expiry-event count per node before the sweep
def _expiry_count(node):
n = 0
try:
with open(CTL_LOG, "r") as f:
for line in f:
try:
rec = json.loads(line.strip())
except Exception:
continue
if rec.get("name") == node and rec.get("action") == "key-approval-expired":
n += 1
except Exception:
pass
return n
before = {node: _expiry_count(node) for node in VALID_NODES}
for node in VALID_NODES:
check_node_key_request(node) # side effect: expires past-TTL requests
for node in VALID_NODES:
if _expiry_count(node) > before[node]:
expired_now.append(node)
return {"expired_now": sorted(expired_now)}
def get_node_connection_info(node: str) -> dict: def get_node_connection_info(node: str) -> dict:
"""Return peer_ip, cdp_port, and netns for a given node.""" """Return peer_ip, cdp_port, and netns for a given node."""
@@ -493,6 +711,46 @@ def inspect_node_approvals(node: str) -> dict:
"when": w.get("when", ""), "when": w.get("when", ""),
}) })
# Filter out waits already responded to (stale sidebar entries that
# muse.ai never cleared). Responded set is maintained by
# reply_node_task() and mark_wait_responded().
responded = load_responded_waits().get(node, {})
if responded:
unique_waits = [w for w in unique_waits if w.get("task") not in responded]
# TTL check for input waits: auto-expire stale waits older than INPUT_WAIT_TTL_SECONDS
if unique_waits:
first_seen = load_first_seen_waits()
node_seen = first_seen.setdefault(node, {})
now_dt = datetime.now(timezone.utc)
now_iso = now_dt.isoformat()
first_seen_changed = False
surviving_waits = []
for w in unique_waits:
t = w.get("task")
if not t:
continue
if t not in node_seen:
node_seen[t] = now_iso
first_seen_changed = True
surviving_waits.append(w)
else:
try:
seen_dt = datetime.fromisoformat(node_seen[t])
age_seconds = (now_dt - seen_dt).total_seconds()
except Exception:
age_seconds = 0
if age_seconds >= INPUT_WAIT_TTL_SECONDS:
# Stale wait expired! Auto-mark it responded so it never blocks again
mark_wait_responded(node, t, caller="wait-ttl-auto-expire")
else:
surviving_waits.append(w)
if first_seen_changed:
save_first_seen_waits(first_seen)
unique_waits = surviving_waits
key_req = check_node_key_request(node) key_req = check_node_key_request(node)
if key_req: if key_req:
return { return {
@@ -549,7 +807,29 @@ def check_fleet_approvals(nodes: list = None) -> list:
return results return results
def allow_node_approval(node: str, always: bool = False, force: bool = False, caller: str = "box-approvals") -> dict:
def _notify_key_decision(node: str, decision: str, reason: str, message: str,
allow_main_chat: bool = False, caller: str = "box-approvals") -> dict:
"""Notify the waiting agent of a key-approval decision via their thread.
Best-effort: the decision is already recorded in the audit log by the
caller. If notification fails, the operator must follow up manually.
Returns the reply_node_task result dict.
"""
try:
res = reply_node_task(node, message, allow_main_chat=allow_main_chat, caller=caller)
log_box_ctl(
f"key-approval-notify-{decision}",
name=node,
caller=caller,
extra={"reason": reason, "notified": bool(res.get("ok")), "notify_error": res.get("error", "")},
)
return res
except Exception as e:
return {"ok": False, "node": node, "error": f"notify exception: {e}"}
def allow_node_approval(node: str, always: bool = False, force: bool = False, caller: str = "box-approvals", message: str = None, allow_main_chat: bool = False) -> dict:
"""Approve a pending approval on a node (click 'Allow once' or 'Always allow this site').""" """Approve a pending approval on a node (click 'Allow once' or 'Always allow this site')."""
info = inspect_node_approvals(node) info = inspect_node_approvals(node)
if not info.get("has_pending"): if not info.get("has_pending"):
@@ -567,6 +847,16 @@ def allow_node_approval(node: str, always: bool = False, force: bool = False, ca
"forced": force, "forced": force,
}, },
) )
# Execute-gap fix: notify the waiting agent. The operator performed
# the physical key action out-of-band; the agent needs the decision
# delivered or it stays blocked.
notify_msg = message or (
f"[operator] Key/passkey request APPROVED ({reason}). "
"Operator action complete - you may proceed."
)
notify_res = _notify_key_decision(
node, "allow", reason, notify_msg, allow_main_chat, caller
)
return { return {
"ok": True, "ok": True,
"node": node, "node": node,
@@ -575,6 +865,8 @@ def allow_node_approval(node: str, always: bool = False, force: bool = False, ca
"dismissed": True, "dismissed": True,
"reason": reason, "reason": reason,
"title": info.get("title"), "title": info.get("title"),
"notified": bool(notify_res.get("ok")),
"notify_result": notify_res,
} }
if not info.get("is_trusted") and not force: if not info.get("is_trusted") and not force:
@@ -677,7 +969,7 @@ def allow_node_approval(node: str, always: bool = False, force: bool = False, ca
return {"ok": False, "node": node, "error": str(e)} return {"ok": False, "node": node, "error": str(e)}
def deny_node_approval(node: str, caller: str = "box-approvals") -> dict: def deny_node_approval(node: str, caller: str = "box-approvals", message: str = None, allow_main_chat: bool = False) -> dict:
"""Deny a pending approval on a node (click 'Deny' or deny key request).""" """Deny a pending approval on a node (click 'Deny' or deny key request)."""
info = inspect_node_approvals(node) info = inspect_node_approvals(node)
if not info.get("has_pending"): if not info.get("has_pending"):
@@ -694,6 +986,14 @@ def deny_node_approval(node: str, caller: str = "box-approvals") -> dict:
"reason": reason, "reason": reason,
}, },
) )
# Execute-gap fix: notify the waiting agent of the denial.
notify_msg = message or (
f"[operator] Key/passkey request DENIED ({reason}). "
"Do not proceed with the protected action."
)
notify_res = _notify_key_decision(
node, "deny", reason, notify_msg, allow_main_chat, caller
)
return { return {
"ok": True, "ok": True,
"node": node, "node": node,
@@ -702,6 +1002,8 @@ def deny_node_approval(node: str, caller: str = "box-approvals") -> dict:
"dismissed": True, "dismissed": True,
"reason": reason, "reason": reason,
"title": info.get("title"), "title": info.get("title"),
"notified": bool(notify_res.get("ok")),
"notify_result": notify_res,
} }
try: try:
@@ -861,6 +1163,12 @@ def reply_node_task(node: str, message: str, allow_main_chat: bool = False, call
"send_res": send_res, "send_res": send_res,
}, },
) )
# The wait(s) visible at reply time are now answered — record them
# so the sidebar's stale entries stop re-alerting.
for w in info.get("input_waits", []) or []:
t = w.get("task")
if t:
mark_wait_responded(node, t, caller=caller)
return { return {
"ok": True, "ok": True,
"node": node, "node": node,
@@ -877,11 +1185,18 @@ def reply_node_task(node: str, message: str, allow_main_chat: bool = False, call
def dismiss_node_task(node: str, caller: str = "box-approvals") -> dict: def dismiss_node_task(node: str, caller: str = "box-approvals") -> dict:
"""Close any open task modal dialog or popup on a node.""" """Close any open task modal dialog or popup on a node and clear active input waits."""
clear_res = clear_node_waits(node, caller=caller)
try: try:
ws, _ = get_cdp_ws(node, timeout=3.0) ws, _ = get_cdp_ws(node, timeout=3.0)
except Exception as e: except Exception as e:
return {"ok": False, "node": node, "error": f"Failed to connect to CDP: {e}"} return {
"ok": True,
"node": node,
"result": "WAITS_CLEARED",
"cleared_waits": clear_res.get("cleared_per_node", {}).get(node, 0),
"warning": f"CDP unreachable ({e}), but input waits cleared",
}
try: try:
js_dismiss = """(() => { js_dismiss = """(() => {
@@ -892,11 +1207,21 @@ def dismiss_node_task(node: str, caller: str = "box-approvals") -> dict:
})()""" })()"""
res = cdp_evaluate(ws, js_dismiss, timeout=2.0) res = cdp_evaluate(ws, js_dismiss, timeout=2.0)
ws.close() ws.close()
return {"ok": True, "node": node, "result": res} return {
"ok": True,
"node": node,
"result": res,
"cleared_waits": clear_res.get("cleared_per_node", {}).get(node, 0),
}
except Exception as e: except Exception as e:
try: try:
ws.close() ws.close()
except Exception: except Exception:
pass pass
return {"ok": False, "node": node, "error": str(e)} return {
"ok": True,
"node": node,
"result": "WAITS_CLEARED",
"cleared_waits": clear_res.get("cleared_per_node", {}).get(node, 0),
}
+27 -10
View File
@@ -58,6 +58,7 @@ NOTIFY_SIDECHATS = {
"pip": "pip tasks", "pip": "pip tasks",
"muse": "muse tasks", "muse": "muse tasks",
"dev": "onboarding-dev", "dev": "onboarding-dev",
"def": "def tasks",
} }
VALID_ON_FAILURE = {"retry", "alert", "ignore"} VALID_ON_FAILURE = {"retry", "alert", "ignore"}
KNOWN_PLACEHOLDERS = {"job_id", "job_name", "datetime", "date", "last_run"} KNOWN_PLACEHOLDERS = {"job_id", "job_name", "datetime", "date", "last_run"}
@@ -989,12 +990,13 @@ def act_approval_check(node=None):
out(True, approvals=res) out(True, approvals=res)
def act_approval_allow(node, always=False, force=False): def act_approval_allow(node, always=False, force=False, message=None, allow_main_chat=False):
audit("approval-allow", node) audit("approval-allow", node)
if node not in VALID_AGENTS: if node not in VALID_AGENTS:
fail("BAD_NODE", f"unknown node: {node}") fail("BAD_NODE", f"unknown node: {node}")
import approvals import approvals
res = approvals.allow_node_approval(node, always=always, force=force, caller="box-ctl") res = approvals.allow_node_approval(node, always=always, force=force, caller="box-ctl",
message=message, allow_main_chat=allow_main_chat)
if res.get("ok"): if res.get("ok"):
kw = {k: v for k, v in res.items() if k != "ok"} kw = {k: v for k, v in res.items() if k != "ok"}
out(True, **kw) out(True, **kw)
@@ -1002,12 +1004,13 @@ def act_approval_allow(node, always=False, force=False):
fail("APPROVAL_FAILED", res.get("error", "approval failed"), res) fail("APPROVAL_FAILED", res.get("error", "approval failed"), res)
def act_approval_deny(node): def act_approval_deny(node, message=None, allow_main_chat=False):
audit("approval-deny", node) audit("approval-deny", node)
if node not in VALID_AGENTS: if node not in VALID_AGENTS:
fail("BAD_NODE", f"unknown node: {node}") fail("BAD_NODE", f"unknown node: {node}")
import approvals import approvals
res = approvals.deny_node_approval(node, caller="box-ctl") res = approvals.deny_node_approval(node, caller="box-ctl",
message=message, allow_main_chat=allow_main_chat)
if res.get("ok"): if res.get("ok"):
kw = {k: v for k, v in res.items() if k != "ok"} kw = {k: v for k, v in res.items() if k != "ok"}
out(True, **kw) out(True, **kw)
@@ -3563,16 +3566,30 @@ def main(argv):
act_approval_check(node) act_approval_check(node)
elif action in ("approval-allow", "approval-approve"): elif action in ("approval-allow", "approval-approve"):
if not rest: if not rest:
fail("BAD_ARGS", "usage: approval-allow <node> [--always] [--force]") fail("BAD_ARGS", "usage: approval-allow <node> [--always] [--force] [--message TEXT] [--allow-main-chat]")
node = rest[0] node = rest[0]
always = "--always" in rest[1:] rest_args = rest[1:]
force = "--force" in rest[1:] always = "--always" in rest_args
act_approval_allow(node, always=always, force=force) force = "--force" in rest_args
allow_main_chat = "--allow-main-chat" in rest_args
message = None
if "--message" in rest_args:
mi = rest_args.index("--message")
if mi + 1 < len(rest_args):
message = rest_args[mi + 1]
act_approval_allow(node, always=always, force=force, message=message, allow_main_chat=allow_main_chat)
elif action == "approval-deny": elif action == "approval-deny":
if not rest: if not rest:
fail("BAD_ARGS", "usage: approval-deny <node>") fail("BAD_ARGS", "usage: approval-deny <node> [--message TEXT] [--allow-main-chat]")
node = rest[0] node = rest[0]
act_approval_deny(node) rest_args = rest[1:]
allow_main_chat = "--allow-main-chat" in rest_args
message = None
if "--message" in rest_args:
mi = rest_args.index("--message")
if mi + 1 < len(rest_args):
message = rest_args[mi + 1]
act_approval_deny(node, message=message, allow_main_chat=allow_main_chat)
elif action == "approval-auto": elif action == "approval-auto":
node = rest[0] if rest else None node = rest[0] if rest else None
act_approval_auto(node) act_approval_auto(node)
+105 -9
View File
@@ -34,6 +34,11 @@ set -uo pipefail
THRESHOLD="${FLEET_ALERT_THRESHOLD:-2}" THRESHOLD="${FLEET_ALERT_THRESHOLD:-2}"
REALERT_MIN="${FLEET_ALERT_REALERT_MIN:-30}" REALERT_MIN="${FLEET_ALERT_REALERT_MIN:-30}"
# Approval/input-wait TTLs (seconds): conditions failing longer than this are
# auto-expired (input waits dismissed, key requests denied) instead of paging
# forever. Overridable per environment.
INPUT_WAIT_TTL="${FLEET_ALERT_INPUT_WAIT_TTL:-1800}"
BROWSER_APPROVAL_TTL="${FLEET_ALERT_BROWSER_APPROVAL_TTL:-1800}"
QUIET_HOURS="${FLEET_ALERT_QUIET_HOURS:-}" QUIET_HOURS="${FLEET_ALERT_QUIET_HOURS:-}"
DRY_RUN="${FLEET_ALERT_DRY_RUN:-0}" DRY_RUN="${FLEET_ALERT_DRY_RUN:-0}"
INJECT_FAIL="${FLEET_ALERT_INJECT_FAIL:-}" INJECT_FAIL="${FLEET_ALERT_INJECT_FAIL:-}"
@@ -51,14 +56,18 @@ NOW=$(date +%s)
log() { echo "$(date -Iseconds) $*" >> "$LOG"; } log() { echo "$(date -Iseconds) $*" >> "$LOG"; }
# --- shared consecutive-failure state machine (also used by the container relay) --- # --- shared consecutive-failure state machine (also used by the container relay) ---
# usage: state_machine <cond> <failing 0|1> -> prints "<ACTION> <fails>" # usage: state_machine <cond> <failing 0|1> [ttl_seconds] -> prints "<ACTION> <fails>"
# ACTION: ALERT_FIRST | ALERT_REALERT | RECOVERY | SUPPRESSED | NONE # When ttl_seconds > 0 and the condition has failed longer than the TTL,
# prints "EXPIRED <fails>" so the caller can auto-resolve (dismiss/deny).
# State entries track first_fail_ts (epoch of first consecutive failure).
# ACTION: ALERT_FIRST | ALERT_REALERT | RECOVERY | SUPPRESSED | EXPIRED | NONE
state_machine() { state_machine() {
local cond="$1" failing="$2" local cond="$1" failing="$2" ttl="${3:-0}"
THRESHOLD="$THRESHOLD" REALERT_MIN="$REALERT_MIN" QUIET_HOURS="$QUIET_HOURS" \ THRESHOLD="$THRESHOLD" REALERT_MIN="$REALERT_MIN" QUIET_HOURS="$QUIET_HOURS" \
FLEET_ALERT_DRY_RUN="$DRY_RUN" python3 - "$STATE" "$cond" "$failing" <<'PYEOF' FLEET_ALERT_DRY_RUN="$DRY_RUN" python3 - "$STATE" "$cond" "$failing" "$ttl" <<'PYEOF'
import json, os, sys, time import json, os, sys, time
state_path, cond, failing_s = sys.argv[1], sys.argv[2], sys.argv[3] state_path, cond, failing_s = sys.argv[1], sys.argv[2], sys.argv[3]
ttl_seconds = int(sys.argv[4]) if len(sys.argv) > 4 else 0
failing = failing_s == "1" failing = failing_s == "1"
threshold = int(os.environ.get("THRESHOLD", "2")) threshold = int(os.environ.get("THRESHOLD", "2"))
realert_min = int(os.environ.get("REALERT_MIN", "30")) realert_min = int(os.environ.get("REALERT_MIN", "30"))
@@ -85,7 +94,17 @@ except Exception:
e = st.get(cond) or {"fails": 0, "alerted": False, "last_alert_ts": 0} e = st.get(cond) or {"fails": 0, "alerted": False, "last_alert_ts": 0}
action = "NONE" action = "NONE"
if failing: if failing:
if int(e.get("fails", 0)) == 0:
e["first_fail_ts"] = now
e["fails"] = int(e.get("fails", 0)) + 1 e["fails"] = int(e.get("fails", 0)) + 1
# TTL expiry: failing longer than ttl_seconds -> EXPIRED (caller auto-resolves)
if ttl_seconds > 0 and now - int(e.get("first_fail_ts", now)) >= ttl_seconds:
action = "EXPIRED"
# Reset so a fresh incident starts clean after the caller resolves it
e["fails"] = 0
e["alerted"] = False
e.pop("first_fail_ts", None)
else:
due = e["fails"] >= threshold and ( due = e["fails"] >= threshold and (
not e.get("alerted") or now - int(e.get("last_alert_ts", 0)) >= realert_min * 60 not e.get("alerted") or now - int(e.get("last_alert_ts", 0)) >= realert_min * 60
) )
@@ -102,6 +121,7 @@ else:
action = "RECOVERY" action = "RECOVERY"
e["fails"] = 0 e["fails"] = 0
e["alerted"] = False e["alerted"] = False
e.pop("first_fail_ts", None)
st[cond] = e st[cond] = e
if not dry: if not dry:
json.dump(st, open(state_path, "w")) json.dump(st, open(state_path, "w"))
@@ -151,6 +171,28 @@ box_notify() {
for p in $pids; do wait "$p" 2>/dev/null; done for p in $pids; do wait "$p" 2>/dev/null; done
} }
notify_input_wait() {
# Targeted DM for input waits (2026-10-05): DM ONLY the specific agent
# whose session is waiting for human input -- not a broadcast to all
# healthy agents. The #lobby post still fires via the relay leg for
# human visibility; this DM ensures the responsible operator sees it
# in their sidechat without digging through lobby noise.
# Best-effort: never fatal to the 5-minute check loop.
local node="$1"
local detail="$2"
local msg="[fleet-alert] INPUT WAIT: ${detail} -- reply: box approval reply ${node} \"<msg>\" or box notify ${node} \"<msg>\""
msg="${msg:0:900}"
if [ "$DRY_RUN" = "1" ]; then
log "DRY-RUN would DM $node re input_wait"
return 0
fi
if timeout 60 python3 "$BIN/box-ctl.py" notify "$node" "$msg" >/dev/null 2>&1; then
log "input_wait targeted DM sent to $node"
else
log "input_wait DM to $node failed (best-effort, non-fatal)"
fi
}
injected() { # cond -> 0 if injected-fail injected() { # cond -> 0 if injected-fail
case ",$INJECT_FAIL," in *,"$1,"*) return 0;; *) return 1;; esac case ",$INJECT_FAIL," in *,"$1,"*) return 0;; *) return 1;; esac
} }
@@ -214,7 +256,7 @@ print(json.dumps(out))
detail="Agent $node approvals clear" detail="Agent $node approvals clear"
fi fi
injected "$cond" && failing=1 injected "$cond" && failing=1
read -r action fails < <(state_machine "$cond" "$failing") read -r action fails < <(state_machine "$cond" "$failing" "$BROWSER_APPROVAL_TTL")
case "$action" in case "$action" in
ALERT_FIRST|ALERT_REALERT) ALERT_FIRST|ALERT_REALERT)
emit_record "ALERT" "$cond" "$detail" "$fails" emit_record "ALERT" "$cond" "$detail" "$fails"
@@ -226,6 +268,21 @@ print(json.dumps(out))
SUPPRESSED) SUPPRESSED)
log "$cond still critical x$fails — re-page suppressed" log "$cond still critical x$fails — re-page suppressed"
;; ;;
EXPIRED)
# Browser approval dialog exceeded BROWSER_APPROVAL_TTL without a
# human decision: fail closed by denying it.
log "$cond EXPIRED after ${BROWSER_APPROVAL_TTL}s without human decision — auto-denying (fail closed)"
python3 - "$node" <<'PYEOF3'
import sys, json
sys.path.insert(0, "/home/super/Projects/NetVM/bin")
import approvals
node = sys.argv[1]
print(json.dumps(approvals.deny_node_approval(node, caller="approval-ttl-expire")))
approvals.log_box_ctl("approval-expired", name=node, caller="approval-ttl-expire",
extra={"note": "browser approval TTL elapsed; auto-denied (fail closed)"})
PYEOF3
emit_record "RECOVERY" "$cond" "Agent $node browser approval expired after ${BROWSER_APPROVAL_TTL}s; auto-denied" "$fails"
;;
esac esac
# 2. Sidebar task waiting on human input # 2. Sidebar task waiting on human input
@@ -245,11 +302,11 @@ if w:
detail_in="Agent $node tasks running" detail_in="Agent $node tasks running"
fi fi
injected "$cond_in" && failing_in=1 injected "$cond_in" && failing_in=1
read -r action_in fails_in < <(state_machine "$cond_in" "$failing_in") read -r action_in fails_in < <(state_machine "$cond_in" "$failing_in" "$INPUT_WAIT_TTL")
case "$action_in" in case "$action_in" in
ALERT_FIRST|ALERT_REALERT) ALERT_FIRST|ALERT_REALERT)
emit_record "ALERT" "$cond_in" "$detail_in" "$fails_in" emit_record "ALERT" "$cond_in" "$detail_in" "$fails_in"
echo "$cond_in" >> "$STATE_DIR/.alerts.tmp" echo "$cond_in|$detail_in" >> "$STATE_DIR/.alerts.tmp"
;; ;;
RECOVERY) RECOVERY)
emit_record "RECOVERY" "$cond_in" "$detail_in" "$fails_in" emit_record "RECOVERY" "$cond_in" "$detail_in" "$fails_in"
@@ -257,6 +314,27 @@ if w:
SUPPRESSED) SUPPRESSED)
log "$cond_in still critical x$fails_in — re-page suppressed" log "$cond_in still critical x$fails_in — re-page suppressed"
;; ;;
EXPIRED)
# Input wait exceeded INPUT_WAIT_TTL without human response:
# auto-dismiss so the agent unblocks. Log the expiry and emit a
# RECOVERY record (the wait is gone, not merely un-paged).
log "$cond_in EXPIRED after ${INPUT_WAIT_TTL}s without human input — auto-dismissing"
python3 - "$node" <<'PYEOF2'
import sys
sys.path.insert(0, "/home/super/Projects/NetVM/bin")
import approvals, json
node = sys.argv[1]
info = approvals.inspect_node_approvals(node)
for w in info.get("input_waits", []) or []:
t = w.get("task")
if t:
approvals.mark_wait_responded(node, t, caller="approval-ttl-expire")
approvals.log_box_ctl("approval-wait-expired", name=node, caller="approval-ttl-expire",
extra={"note": "input wait TTL elapsed; auto-dismissed"})
print(json.dumps(approvals.dismiss_node_task(node, caller="approval-ttl-expire")))
PYEOF2
emit_record "RECOVERY" "$cond_in" "Agent $node input wait expired after ${INPUT_WAIT_TTL}s; auto-dismissed" "$fails_in"
;;
esac esac
done done
@@ -328,8 +406,26 @@ rm -f "$STATE_DIR/.healthy.tmp"
# Notify for this run's alerts (best effort). Skip entirely when nothing is healthy # Notify for this run's alerts (best effort). Skip entirely when nothing is healthy
# (notify needs a working browser via dm.py) or in dry-run. # (notify needs a working browser via dm.py) or in dry-run.
if [ -n "$HEALTHY_AGENTS" ] && [ -f "$STATE_DIR/.alerts.tmp" ]; then if [ -n "$HEALTHY_AGENTS" ] && [ -f "$STATE_DIR/.alerts.tmp" ]; then
while read -r cond; do while IFS= read -r line; do
[ -n "$cond" ] && box_notify "$cond" "see #lobby for detail" # alerts.tmp format: "cond" or "cond|detail" (input_wait carries detail)
cond="${line%%|*}"
detail="${line#*|}"
[ "$detail" = "$line" ] && detail=""
[ -n "$cond" ] || continue
case "$cond" in
input_wait:*)
# Targeted: DM only the waiting agent, not a broadcast.
node="${cond#input_wait:}"
if [ -n "$detail" ]; then
notify_input_wait "$node" "$detail"
else
box_notify "$cond" "see #lobby for detail"
fi
;;
*)
box_notify "$cond" "see #lobby for detail"
;;
esac
done < "$STATE_DIR/.alerts.tmp" done < "$STATE_DIR/.alerts.tmp"
elif [ -f "$STATE_DIR/.alerts.tmp" ]; then elif [ -f "$STATE_DIR/.alerts.tmp" ]; then
log "no healthy agents — box notify skipped (DM path needs a working browser)" log "no healthy agents — box notify skipped (DM path needs a working browser)"
+1 -1
View File
@@ -765,7 +765,7 @@ def remediate_breaks(dry_run=False) -> dict:
import approvals import approvals
fleet_apps = approvals.check_fleet_approvals() fleet_apps = approvals.check_fleet_approvals()
for app in fleet_apps: for app in fleet_apps:
if app.get("has_pending") and app.get("is_trusted"): if app.get("has_pending") and app.get("is_trusted") and app.get("status") != "KEY_APPROVAL":
node = app["node"] node = app["node"]
if not dry_run: if not dry_run:
approvals.allow_node_approval(node, always=True, caller="loop-remediate") approvals.allow_node_approval(node, always=True, caller="loop-remediate")
+1 -1
View File
@@ -509,7 +509,7 @@ def main():
import approvals import approvals
app_info = approvals.inspect_node_approvals(agent) app_info = approvals.inspect_node_approvals(agent)
if app_info.get("has_pending"): if app_info.get("has_pending"):
if app_info.get("is_trusted"): if app_info.get("is_trusted") and app_info.get("status") != "KEY_APPROVAL":
print(f"Pre-dispatch: auto-approving trusted request for {agent} ({app_info.get('target')})") print(f"Pre-dispatch: auto-approving trusted request for {agent} ({app_info.get('target')})")
approvals.allow_node_approval(agent, always=True, caller="job-dispatch") approvals.allow_node_approval(agent, always=True, caller="job-dispatch")
else: else:
+33 -4
View File
@@ -519,13 +519,17 @@ def cmd_approvals(args):
sys.exit(1) sys.exit(1)
always = getattr(args, "always", False) always = getattr(args, "always", False)
force = getattr(args, "force", False) force = getattr(args, "force", False)
res = approvals.allow_node_approval(node, always=always, force=force) message = getattr(args, "message", None)
allow_main_chat = getattr(args, "allow_main_chat", False)
res = approvals.allow_node_approval(node, always=always, force=force, message=message,
allow_main_chat=allow_main_chat)
if getattr(args, "json", False): if getattr(args, "json", False):
print(json.dumps(res, indent=2)) print(json.dumps(res, indent=2))
return return
if res.get("ok"): if res.get("ok"):
if res.get("type") == "key_approval": if res.get("type") == "key_approval":
print(c_green(f"✔ Approved operator key request for node '{node}'. Granted and logged to audit trail.")) notified = "agent notified" if res.get("notified") else "⚠ agent NOT notified (follow up manually)"
print(c_green(f"✔ Approved operator key request for node '{node}'. Logged to audit trail. {notified}."))
else: else:
decision_str = "Always allow this site" if always else "Allow once" decision_str = "Always allow this site" if always else "Allow once"
print(c_green(f"✔ Approved request on node '{node}' ({decision_str}). Dialog dismissed: {res.get('dismissed')}.")) print(c_green(f"✔ Approved request on node '{node}' ({decision_str}). Dialog dismissed: {res.get('dismissed')}."))
@@ -537,13 +541,16 @@ def cmd_approvals(args):
if not node: if not node:
print(c_red("Error: Must specify node for deny. e.g. 'box approvals deny 646'"), file=sys.stderr) print(c_red("Error: Must specify node for deny. e.g. 'box approvals deny 646'"), file=sys.stderr)
sys.exit(1) sys.exit(1)
res = approvals.deny_node_approval(node) message = getattr(args, "message", None)
allow_main_chat = getattr(args, "allow_main_chat", False)
res = approvals.deny_node_approval(node, message=message, allow_main_chat=allow_main_chat)
if getattr(args, "json", False): if getattr(args, "json", False):
print(json.dumps(res, indent=2)) print(json.dumps(res, indent=2))
return return
if res.get("ok"): if res.get("ok"):
if res.get("type") == "key_approval": if res.get("type") == "key_approval":
print(c_green(f"✔ Denied operator key request for node '{node}'. Denied and logged to audit trail.")) notified = "agent notified" if res.get("notified") else "⚠ agent NOT notified (follow up manually)"
print(c_green(f"✔ Denied operator key request for node '{node}'. Logged to audit trail. {notified}."))
else: else:
print(c_green(f"✔ Denied request on node '{node}'. Dialog dismissed: {res.get('dismissed')}.")) print(c_green(f"✔ Denied request on node '{node}'. Dialog dismissed: {res.get('dismissed')}."))
else: else:
@@ -622,6 +629,17 @@ def cmd_approvals(args):
else: else:
print(c_red(f"✖ Failed to dismiss task popup on node '{node}': {res.get('error')}")) print(c_red(f"✖ Failed to dismiss task popup on node '{node}': {res.get('error')}"))
sys.exit(1) sys.exit(1)
elif action in ("clear", "clear-all", "clear_all"):
target_node = node
if action in ("clear-all", "clear_all"):
target_node = None
res = approvals.clear_node_waits(target_node)
if getattr(args, "json", False):
print(json.dumps(res, indent=2))
return
cleared = res.get("total_cleared", 0)
target_desc = f"node '{target_node}'" if target_node else "all fleet nodes"
print(c_green(f"✔ Cleared {cleared} input wait(s) on {target_desc}."))
elif action in ("request-key", "request_key"): elif action in ("request-key", "request_key"):
if not node: if not node:
@@ -4211,14 +4229,20 @@ def build_parser():
p_app_allow.add_argument("node", choices=VALID_NODES, help="Target node to approve") p_app_allow.add_argument("node", choices=VALID_NODES, help="Target node to approve")
p_app_allow.add_argument("--always", action="store_true", help="Click 'Always allow this site' instead of 'Allow once'") p_app_allow.add_argument("--always", action="store_true", help="Click 'Always allow this site' instead of 'Allow once'")
p_app_allow.add_argument("--force", action="store_true", help="Force approval even if target is untrusted") p_app_allow.add_argument("--force", action="store_true", help="Force approval even if target is untrusted")
p_app_allow.add_argument("--message", default=None, help="Optional message/credential to deliver to the waiting agent")
p_app_allow.add_argument("--allow-main-chat", action="store_true", help="Allow reply into Main Chat if agent is waiting there")
p_app_approve = app_sub.add_parser("approve", parents=[common], help="Alias for 'allow'") p_app_approve = app_sub.add_parser("approve", parents=[common], help="Alias for 'allow'")
p_app_approve.add_argument("node", choices=VALID_NODES, help="Target node to approve") p_app_approve.add_argument("node", choices=VALID_NODES, help="Target node to approve")
p_app_approve.add_argument("--always", action="store_true", help="Click 'Always allow this site' instead of 'Allow once'") p_app_approve.add_argument("--always", action="store_true", help="Click 'Always allow this site' instead of 'Allow once'")
p_app_approve.add_argument("--force", action="store_true", help="Force approval even if target is untrusted") p_app_approve.add_argument("--force", action="store_true", help="Force approval even if target is untrusted")
p_app_approve.add_argument("--message", default=None, help="Optional message/credential to deliver to the waiting agent")
p_app_approve.add_argument("--allow-main-chat", action="store_true", help="Allow reply into Main Chat if agent is waiting there")
p_app_deny = app_sub.add_parser("deny", parents=[common], help="Deny pending browser request") p_app_deny = app_sub.add_parser("deny", parents=[common], help="Deny pending browser request")
p_app_deny.add_argument("node", choices=VALID_NODES, help="Target node to deny") p_app_deny.add_argument("node", choices=VALID_NODES, help="Target node to deny")
p_app_deny.add_argument("--message", default=None, help="Optional message to deliver to the waiting agent")
p_app_deny.add_argument("--allow-main-chat", action="store_true", help="Allow reply into Main Chat if agent is waiting there")
p_app_auto = app_sub.add_parser("auto", parents=[common], help="Auto-approve all trusted requests across fleet") p_app_auto = app_sub.add_parser("auto", parents=[common], help="Auto-approve all trusted requests across fleet")
p_app_auto.add_argument("--node", choices=VALID_NODES, default=None, help="Target node (or all nodes)") p_app_auto.add_argument("--node", choices=VALID_NODES, default=None, help="Target node (or all nodes)")
@@ -4236,6 +4260,11 @@ def build_parser():
p_app_dismiss = app_sub.add_parser("dismiss", parents=[common], help="Dismiss any open task dialog/popup on a node") p_app_dismiss = app_sub.add_parser("dismiss", parents=[common], help="Dismiss any open task dialog/popup on a node")
p_app_dismiss.add_argument("node", choices=VALID_NODES, help="Target node to dismiss dialog on") p_app_dismiss.add_argument("node", choices=VALID_NODES, help="Target node to dismiss dialog on")
p_app_clear = app_sub.add_parser("clear", parents=[common], help="Clear and dismiss pending input waits on a node or all nodes")
p_app_clear.add_argument("node", nargs="?", choices=VALID_NODES, default=None, help="Target node (or omit for all nodes)")
p_app_clear_all = app_sub.add_parser("clear-all", parents=[common], help="Clear and dismiss all pending input waits across fleet")
p_app_req_key = app_sub.add_parser("request-key", parents=[common], help="Request operator passkey/key approval for an agent") p_app_req_key = app_sub.add_parser("request-key", parents=[common], help="Request operator passkey/key approval for an agent")
p_app_req_key.add_argument("node", choices=VALID_NODES, help="Target node requesting key") p_app_req_key.add_argument("node", choices=VALID_NODES, help="Target node requesting key")
p_app_req_key.add_argument("--reason", default="Passkey authentication required", help="Reason for key request") p_app_req_key.add_argument("--reason", default="Passkey authentication required", help="Reason for key request")