diff --git a/bin/approvals.py b/bin/approvals.py index 10004f8..855131b 100755 --- a/bin/approvals.py +++ b/bin/approvals.py @@ -31,6 +31,98 @@ except ImportError: REPO_ROOT = Path("/home/super/Projects/NetVM") BIN_DIR = REPO_ROOT / "bin" CTL_LOG = REPO_ROOT / "box-ctl.jsonl" +RESPONDED_WAITS_FILE = REPO_ROOT / ".state" / "approvals-responded.json" +FIRST_SEEN_WAITS_FILE = REPO_ROOT / ".state" / "approvals-first-seen.json" + + +def load_responded_waits() -> dict: + """Load the set of (node, task) input waits already responded to. + + Returns {node: {task: iso_timestamp}}. Missing file -> {}. + """ + try: + if RESPONDED_WAITS_FILE.exists(): + return json.loads(RESPONDED_WAITS_FILE.read_text()) + except Exception: + pass + return {} + + +def save_responded_waits(data: dict) -> None: + """Persist the responded-waits map (best effort).""" + try: + RESPONDED_WAITS_FILE.parent.mkdir(parents=True, exist_ok=True) + RESPONDED_WAITS_FILE.write_text(json.dumps(data, indent=1)) + except Exception: + pass + + +def load_first_seen_waits() -> dict: + """Load map of when input waits were first observed: {node: {task: iso_timestamp}}.""" + try: + if FIRST_SEEN_WAITS_FILE.exists(): + return json.loads(FIRST_SEEN_WAITS_FILE.read_text()) + except Exception: + pass + return {} + + +def save_first_seen_waits(data: dict) -> None: + """Persist first-seen input waits map.""" + try: + FIRST_SEEN_WAITS_FILE.parent.mkdir(parents=True, exist_ok=True) + FIRST_SEEN_WAITS_FILE.write_text(json.dumps(data, indent=1)) + except Exception: + pass + + +def is_wait_responded(node: str, task: str) -> bool: + """True if this (node, task) wait was already answered.""" + return task in load_responded_waits().get(node, {}) + + +def mark_wait_responded(node: str, task: str, caller: str = "approvals") -> None: + """Record that (node, task) has been responded to, so future + inspections filter it out of the live input-wait list.""" + data = load_responded_waits() + node_map = data.setdefault(node, {}) + if task not in node_map: + node_map[task] = datetime.now(timezone.utc).isoformat() + save_responded_waits(data) + log_box_ctl("approval-wait-responded", name=node, caller=caller, + extra={"task": task}) + + +def clear_node_waits(node: str = None, caller: str = "box-approvals") -> dict: + """Clear and dismiss all pending input waits for a specific node or all nodes.""" + target_nodes = [node] if node else VALID_NODES + total_cleared = 0 + cleared_per_node = {} + data = load_responded_waits() + now_iso = datetime.now(timezone.utc).isoformat() + + for n in target_nodes: + node_map = data.setdefault(n, {}) + n_cleared = 0 + try: + info = inspect_node_approvals(n) + for w in info.get("input_waits", []) or []: + t = w.get("task") + if t and t not in node_map: + node_map[t] = now_iso + n_cleared += 1 + if n_cleared: + log_box_ctl("approval-wait-cleared", name=n, caller=caller, + extra={"cleared_count": n_cleared}) + except Exception: + pass + cleared_per_node[n] = n_cleared + total_cleared += n_cleared + + if total_cleared: + save_responded_waits(data) + + return {"ok": True, "total_cleared": total_cleared, "cleared_per_node": cleared_per_node} # Add BIN_DIR to sys.path if str(BIN_DIR) not in sys.path: @@ -43,6 +135,14 @@ except ImportError: VALID_NODES = ["muse", "pip", "646", "opm", "def", "dev"] +# Approval timeout defaults (seconds). +# Key requests are sensitive operations -- give the operator 2h to decide. +# Input waits and browser approvals block agent work -- expire after 30m so +# agents unblock instead of sitting indefinitely (e.g. def waited 4h+). +KEY_REQUEST_TTL_SECONDS = 2 * 3600 +INPUT_WAIT_TTL_SECONDS = 30 * 60 +BROWSER_APPROVAL_TTL_SECONDS = 30 * 60 + # Trusted infrastructure IPs safe for automated approval TRUSTED_IPS = { "34.139.37.135", # VM (gateway) @@ -95,12 +195,30 @@ def redact_sensitive(text: str) -> str: return out +def _approval_type(action: str) -> str: + """Classify an approval action into its request type. + + Types: "key" (passkey/key requests), "browser" (browser dialog + clicks), "input" (task input replies), "other". Used so that a + resolution only clears requests of the matching type -- a browser + approval-allow must never resolve a pending key request. + """ + if action.startswith("key-approval-"): + return "key" + if action == "approval-reply": + return "input" + if action.startswith("approval-"): + return "browser" + return "other" + + def log_box_ctl(action: str, name: str = None, caller: str = "box-approvals", extra: dict = None): """Log an audit event to box-ctl.jsonl with secret redaction.""" try: rec = { "ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "action": action, + "type": _approval_type(action), "name": name, "caller": caller, } @@ -118,14 +236,23 @@ def log_box_ctl(action: str, name: str = None, caller: str = "box-approvals", ex pass -def request_key_approval(node: str, reason: str = "", caller: str = "agent") -> dict: - """Register a key/passkey approval request for a node in box-ctl.jsonl.""" +def request_key_approval(node: str, reason: str = "", caller: str = "agent", + ttl_seconds: int = None) -> dict: + """Register a key/passkey approval request for a node in box-ctl.jsonl. + + ttl_seconds: how long the request stays valid (default KEY_REQUEST_TTL_SECONDS). + After expiry the request is treated as denied; see check_node_key_request(). + """ reason = reason or "Operator passkey access requested" + ttl = ttl_seconds if ttl_seconds is not None else KEY_REQUEST_TTL_SECONDS + expires_iso = datetime.fromtimestamp( + datetime.now(timezone.utc).timestamp() + ttl, tz=timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") log_box_ctl( "key-approval-request", name=node, caller=caller, - extra={"reason": reason}, + extra={"reason": reason, "ttl_seconds": ttl, "expires_at": expires_iso}, ) return { "ok": True, @@ -133,16 +260,55 @@ def request_key_approval(node: str, reason: str = "", caller: str = "agent") -> "status": "KEY_APPROVAL_REQUESTED", "reason": reason, "caller": caller, - "note": "Request recorded in audit log. Operator can approve via 'box approvals allow '." + "ttl_seconds": ttl, + "expires_at": expires_iso, + "note": "Request recorded in audit log. Operator can approve via 'box approvals allow '.", } +def _parse_ts(ts_str: str): + """Parse a box-ctl.jsonl ts ('%Y-%m-%dT%H:%M:%SZ') to epoch seconds. None on failure.""" + if not ts_str: + return None + try: + dt = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + return dt.timestamp() + except Exception: + return None + + +def expire_key_request(node: str, caller: str = "approval-sweeper") -> dict: + """Mark a node's pending key request as expired (auto-deny on TTL).""" + log_box_ctl("key-approval-expired", name=node, caller=caller, + extra={"note": "TTL elapsed without operator decision; treated as denied"}) + return {"ok": True, "node": node, "status": "KEY_APPROVAL_EXPIRED"} + + +def _rec_approval_type(rec: dict) -> str: + """Return the approval type of a log record. + + Prefers the explicit "type" field (present on records written after the + type-field fix); falls back to deriving from the action name for older + records so history keeps working. + """ + t = rec.get("type") + if t: + return t + return _approval_type(rec.get("action", "")) + + def check_node_key_request(node: str) -> dict: - """Check if node has an active unfulfilled key approval request in box-ctl.jsonl.""" + """Check if node has an active unfulfilled key approval request in box-ctl.jsonl. + + Requests expire after their TTL (default KEY_REQUEST_TTL_SECONDS). An expired + request is treated as denied: this logs a key-approval-expired event and + returns None (no active request). + """ if not CTL_LOG.exists(): return None latest_req = None resolved = False + now = datetime.now(timezone.utc).timestamp() try: with open(CTL_LOG, "r") as f: for line in f: @@ -159,21 +325,73 @@ def check_node_key_request(node: str) -> dict: if act == "key-approval-request": latest_req = rec resolved = False - elif act in ("key-approval-allow", "key-approval-deny", "approval-allow", "approval-deny"): + elif _rec_approval_type(rec) == "key" and act in ( + "key-approval-allow", "key-approval-deny", "key-approval-expired", + ): + # Only a KEY-type resolution clears a key request. A browser + # approval-allow/deny must never resolve a pending key request + # (cross-type resolution bug). resolved = True except Exception: return None if latest_req and not resolved: + # TTL check: explicit expires_at wins; legacy records fall back to + # ts + default TTL. + exp_ts = _parse_ts(latest_req.get("expires_at")) + if exp_ts is None: + req_ts = _parse_ts(latest_req.get("ts")) + exp_ts = (req_ts + KEY_REQUEST_TTL_SECONDS) if req_ts else None + if exp_ts is not None and now > exp_ts: + # Expired: record the expiry (idempotent -- a later scan sees the + # key-approval-expired event and treats it as resolved) and report + # no active request. + expire_key_request(node, caller="approval-ttl-check") + return None return { "node": node, "reason": latest_req.get("reason", "Operator passkey access requested"), "requested_at": latest_req.get("ts", ""), "caller": latest_req.get("caller", ""), + "expires_at": latest_req.get("expires_at", ""), } return None +def sweep_expired_key_requests() -> dict: + """Proactively expire stale key requests across all nodes. + + check_node_key_request() expires as a side effect when it sees a past-TTL + request, so this sweep just triggers that check for every node. Idempotent: + already-expired requests are skipped (the key-approval-expired event marks + them resolved). Returns {"expired_now": [nodes expired by this sweep]}. + """ + expired_now = [] + if not CTL_LOG.exists(): + return {"expired_now": expired_now} + # Snapshot of expiry-event count per node before the sweep + def _expiry_count(node): + n = 0 + try: + with open(CTL_LOG, "r") as f: + for line in f: + try: + rec = json.loads(line.strip()) + except Exception: + continue + if rec.get("name") == node and rec.get("action") == "key-approval-expired": + n += 1 + except Exception: + pass + return n + before = {node: _expiry_count(node) for node in VALID_NODES} + for node in VALID_NODES: + check_node_key_request(node) # side effect: expires past-TTL requests + for node in VALID_NODES: + if _expiry_count(node) > before[node]: + expired_now.append(node) + return {"expired_now": sorted(expired_now)} + def get_node_connection_info(node: str) -> dict: """Return peer_ip, cdp_port, and netns for a given node.""" @@ -493,6 +711,46 @@ def inspect_node_approvals(node: str) -> dict: "when": w.get("when", ""), }) + # Filter out waits already responded to (stale sidebar entries that + # muse.ai never cleared). Responded set is maintained by + # reply_node_task() and mark_wait_responded(). + responded = load_responded_waits().get(node, {}) + if responded: + unique_waits = [w for w in unique_waits if w.get("task") not in responded] + + # TTL check for input waits: auto-expire stale waits older than INPUT_WAIT_TTL_SECONDS + if unique_waits: + first_seen = load_first_seen_waits() + node_seen = first_seen.setdefault(node, {}) + now_dt = datetime.now(timezone.utc) + now_iso = now_dt.isoformat() + first_seen_changed = False + surviving_waits = [] + + for w in unique_waits: + t = w.get("task") + if not t: + continue + if t not in node_seen: + node_seen[t] = now_iso + first_seen_changed = True + surviving_waits.append(w) + else: + try: + seen_dt = datetime.fromisoformat(node_seen[t]) + age_seconds = (now_dt - seen_dt).total_seconds() + except Exception: + age_seconds = 0 + if age_seconds >= INPUT_WAIT_TTL_SECONDS: + # Stale wait expired! Auto-mark it responded so it never blocks again + mark_wait_responded(node, t, caller="wait-ttl-auto-expire") + else: + surviving_waits.append(w) + + if first_seen_changed: + save_first_seen_waits(first_seen) + unique_waits = surviving_waits + key_req = check_node_key_request(node) if key_req: return { @@ -549,7 +807,29 @@ def check_fleet_approvals(nodes: list = None) -> list: return results -def allow_node_approval(node: str, always: bool = False, force: bool = False, caller: str = "box-approvals") -> dict: + +def _notify_key_decision(node: str, decision: str, reason: str, message: str, + allow_main_chat: bool = False, caller: str = "box-approvals") -> dict: + """Notify the waiting agent of a key-approval decision via their thread. + + Best-effort: the decision is already recorded in the audit log by the + caller. If notification fails, the operator must follow up manually. + Returns the reply_node_task result dict. + """ + try: + res = reply_node_task(node, message, allow_main_chat=allow_main_chat, caller=caller) + log_box_ctl( + f"key-approval-notify-{decision}", + name=node, + caller=caller, + extra={"reason": reason, "notified": bool(res.get("ok")), "notify_error": res.get("error", "")}, + ) + return res + except Exception as e: + return {"ok": False, "node": node, "error": f"notify exception: {e}"} + + +def allow_node_approval(node: str, always: bool = False, force: bool = False, caller: str = "box-approvals", message: str = None, allow_main_chat: bool = False) -> dict: """Approve a pending approval on a node (click 'Allow once' or 'Always allow this site').""" info = inspect_node_approvals(node) if not info.get("has_pending"): @@ -567,6 +847,16 @@ def allow_node_approval(node: str, always: bool = False, force: bool = False, ca "forced": force, }, ) + # Execute-gap fix: notify the waiting agent. The operator performed + # the physical key action out-of-band; the agent needs the decision + # delivered or it stays blocked. + notify_msg = message or ( + f"[operator] Key/passkey request APPROVED ({reason}). " + "Operator action complete - you may proceed." + ) + notify_res = _notify_key_decision( + node, "allow", reason, notify_msg, allow_main_chat, caller + ) return { "ok": True, "node": node, @@ -575,6 +865,8 @@ def allow_node_approval(node: str, always: bool = False, force: bool = False, ca "dismissed": True, "reason": reason, "title": info.get("title"), + "notified": bool(notify_res.get("ok")), + "notify_result": notify_res, } if not info.get("is_trusted") and not force: @@ -677,7 +969,7 @@ def allow_node_approval(node: str, always: bool = False, force: bool = False, ca return {"ok": False, "node": node, "error": str(e)} -def deny_node_approval(node: str, caller: str = "box-approvals") -> dict: +def deny_node_approval(node: str, caller: str = "box-approvals", message: str = None, allow_main_chat: bool = False) -> dict: """Deny a pending approval on a node (click 'Deny' or deny key request).""" info = inspect_node_approvals(node) if not info.get("has_pending"): @@ -694,6 +986,14 @@ def deny_node_approval(node: str, caller: str = "box-approvals") -> dict: "reason": reason, }, ) + # Execute-gap fix: notify the waiting agent of the denial. + notify_msg = message or ( + f"[operator] Key/passkey request DENIED ({reason}). " + "Do not proceed with the protected action." + ) + notify_res = _notify_key_decision( + node, "deny", reason, notify_msg, allow_main_chat, caller + ) return { "ok": True, "node": node, @@ -702,6 +1002,8 @@ def deny_node_approval(node: str, caller: str = "box-approvals") -> dict: "dismissed": True, "reason": reason, "title": info.get("title"), + "notified": bool(notify_res.get("ok")), + "notify_result": notify_res, } try: @@ -861,6 +1163,12 @@ def reply_node_task(node: str, message: str, allow_main_chat: bool = False, call "send_res": send_res, }, ) + # The wait(s) visible at reply time are now answered — record them + # so the sidebar's stale entries stop re-alerting. + for w in info.get("input_waits", []) or []: + t = w.get("task") + if t: + mark_wait_responded(node, t, caller=caller) return { "ok": True, "node": node, @@ -877,11 +1185,18 @@ def reply_node_task(node: str, message: str, allow_main_chat: bool = False, call def dismiss_node_task(node: str, caller: str = "box-approvals") -> dict: - """Close any open task modal dialog or popup on a node.""" + """Close any open task modal dialog or popup on a node and clear active input waits.""" + clear_res = clear_node_waits(node, caller=caller) try: ws, _ = get_cdp_ws(node, timeout=3.0) except Exception as e: - return {"ok": False, "node": node, "error": f"Failed to connect to CDP: {e}"} + return { + "ok": True, + "node": node, + "result": "WAITS_CLEARED", + "cleared_waits": clear_res.get("cleared_per_node", {}).get(node, 0), + "warning": f"CDP unreachable ({e}), but input waits cleared", + } try: js_dismiss = """(() => { @@ -892,11 +1207,21 @@ def dismiss_node_task(node: str, caller: str = "box-approvals") -> dict: })()""" res = cdp_evaluate(ws, js_dismiss, timeout=2.0) ws.close() - return {"ok": True, "node": node, "result": res} + return { + "ok": True, + "node": node, + "result": res, + "cleared_waits": clear_res.get("cleared_per_node", {}).get(node, 0), + } except Exception as e: try: ws.close() except Exception: pass - return {"ok": False, "node": node, "error": str(e)} + return { + "ok": True, + "node": node, + "result": "WAITS_CLEARED", + "cleared_waits": clear_res.get("cleared_per_node", {}).get(node, 0), + } diff --git a/bin/box-ctl.py b/bin/box-ctl.py index 9a437ff..6c0ac9f 100755 --- a/bin/box-ctl.py +++ b/bin/box-ctl.py @@ -58,6 +58,7 @@ NOTIFY_SIDECHATS = { "pip": "pip tasks", "muse": "muse tasks", "dev": "onboarding-dev", + "def": "def tasks", } VALID_ON_FAILURE = {"retry", "alert", "ignore"} KNOWN_PLACEHOLDERS = {"job_id", "job_name", "datetime", "date", "last_run"} @@ -989,12 +990,13 @@ def act_approval_check(node=None): out(True, approvals=res) -def act_approval_allow(node, always=False, force=False): +def act_approval_allow(node, always=False, force=False, message=None, allow_main_chat=False): audit("approval-allow", node) if node not in VALID_AGENTS: fail("BAD_NODE", f"unknown node: {node}") import approvals - res = approvals.allow_node_approval(node, always=always, force=force, caller="box-ctl") + res = approvals.allow_node_approval(node, always=always, force=force, caller="box-ctl", + message=message, allow_main_chat=allow_main_chat) if res.get("ok"): kw = {k: v for k, v in res.items() if k != "ok"} out(True, **kw) @@ -1002,12 +1004,13 @@ def act_approval_allow(node, always=False, force=False): fail("APPROVAL_FAILED", res.get("error", "approval failed"), res) -def act_approval_deny(node): +def act_approval_deny(node, message=None, allow_main_chat=False): audit("approval-deny", node) if node not in VALID_AGENTS: fail("BAD_NODE", f"unknown node: {node}") import approvals - res = approvals.deny_node_approval(node, caller="box-ctl") + res = approvals.deny_node_approval(node, caller="box-ctl", + message=message, allow_main_chat=allow_main_chat) if res.get("ok"): kw = {k: v for k, v in res.items() if k != "ok"} out(True, **kw) @@ -3563,16 +3566,30 @@ def main(argv): act_approval_check(node) elif action in ("approval-allow", "approval-approve"): if not rest: - fail("BAD_ARGS", "usage: approval-allow [--always] [--force]") + fail("BAD_ARGS", "usage: approval-allow [--always] [--force] [--message TEXT] [--allow-main-chat]") node = rest[0] - always = "--always" in rest[1:] - force = "--force" in rest[1:] - act_approval_allow(node, always=always, force=force) + rest_args = rest[1:] + always = "--always" in rest_args + force = "--force" in rest_args + allow_main_chat = "--allow-main-chat" in rest_args + message = None + if "--message" in rest_args: + mi = rest_args.index("--message") + if mi + 1 < len(rest_args): + message = rest_args[mi + 1] + act_approval_allow(node, always=always, force=force, message=message, allow_main_chat=allow_main_chat) elif action == "approval-deny": if not rest: - fail("BAD_ARGS", "usage: approval-deny ") + fail("BAD_ARGS", "usage: approval-deny [--message TEXT] [--allow-main-chat]") node = rest[0] - act_approval_deny(node) + rest_args = rest[1:] + allow_main_chat = "--allow-main-chat" in rest_args + message = None + if "--message" in rest_args: + mi = rest_args.index("--message") + if mi + 1 < len(rest_args): + message = rest_args[mi + 1] + act_approval_deny(node, message=message, allow_main_chat=allow_main_chat) elif action == "approval-auto": node = rest[0] if rest else None act_approval_auto(node) diff --git a/bin/fleet-alert-check.sh b/bin/fleet-alert-check.sh index 3917161..5eeb084 100755 --- a/bin/fleet-alert-check.sh +++ b/bin/fleet-alert-check.sh @@ -34,6 +34,11 @@ set -uo pipefail THRESHOLD="${FLEET_ALERT_THRESHOLD:-2}" REALERT_MIN="${FLEET_ALERT_REALERT_MIN:-30}" +# Approval/input-wait TTLs (seconds): conditions failing longer than this are +# auto-expired (input waits dismissed, key requests denied) instead of paging +# forever. Overridable per environment. +INPUT_WAIT_TTL="${FLEET_ALERT_INPUT_WAIT_TTL:-1800}" +BROWSER_APPROVAL_TTL="${FLEET_ALERT_BROWSER_APPROVAL_TTL:-1800}" QUIET_HOURS="${FLEET_ALERT_QUIET_HOURS:-}" DRY_RUN="${FLEET_ALERT_DRY_RUN:-0}" INJECT_FAIL="${FLEET_ALERT_INJECT_FAIL:-}" @@ -51,14 +56,18 @@ NOW=$(date +%s) log() { echo "$(date -Iseconds) $*" >> "$LOG"; } # --- shared consecutive-failure state machine (also used by the container relay) --- -# usage: state_machine -> prints " " -# ACTION: ALERT_FIRST | ALERT_REALERT | RECOVERY | SUPPRESSED | NONE +# usage: state_machine [ttl_seconds] -> prints " " +# When ttl_seconds > 0 and the condition has failed longer than the TTL, +# prints "EXPIRED " so the caller can auto-resolve (dismiss/deny). +# State entries track first_fail_ts (epoch of first consecutive failure). +# ACTION: ALERT_FIRST | ALERT_REALERT | RECOVERY | SUPPRESSED | EXPIRED | NONE state_machine() { - local cond="$1" failing="$2" + local cond="$1" failing="$2" ttl="${3:-0}" THRESHOLD="$THRESHOLD" REALERT_MIN="$REALERT_MIN" QUIET_HOURS="$QUIET_HOURS" \ - FLEET_ALERT_DRY_RUN="$DRY_RUN" python3 - "$STATE" "$cond" "$failing" <<'PYEOF' + FLEET_ALERT_DRY_RUN="$DRY_RUN" python3 - "$STATE" "$cond" "$failing" "$ttl" <<'PYEOF' import json, os, sys, time state_path, cond, failing_s = sys.argv[1], sys.argv[2], sys.argv[3] +ttl_seconds = int(sys.argv[4]) if len(sys.argv) > 4 else 0 failing = failing_s == "1" threshold = int(os.environ.get("THRESHOLD", "2")) realert_min = int(os.environ.get("REALERT_MIN", "30")) @@ -85,23 +94,34 @@ except Exception: e = st.get(cond) or {"fails": 0, "alerted": False, "last_alert_ts": 0} action = "NONE" if failing: + if int(e.get("fails", 0)) == 0: + e["first_fail_ts"] = now e["fails"] = int(e.get("fails", 0)) + 1 - due = e["fails"] >= threshold and ( - not e.get("alerted") or now - int(e.get("last_alert_ts", 0)) >= realert_min * 60 - ) - if due: - first = not e.get("alerted") - if not first and in_quiet(qh): - action = "SUPPRESSED" - else: - action = "ALERT_FIRST" if first else "ALERT_REALERT" - e["alerted"] = True - e["last_alert_ts"] = now + # TTL expiry: failing longer than ttl_seconds -> EXPIRED (caller auto-resolves) + if ttl_seconds > 0 and now - int(e.get("first_fail_ts", now)) >= ttl_seconds: + action = "EXPIRED" + # Reset so a fresh incident starts clean after the caller resolves it + e["fails"] = 0 + e["alerted"] = False + e.pop("first_fail_ts", None) + else: + due = e["fails"] >= threshold and ( + not e.get("alerted") or now - int(e.get("last_alert_ts", 0)) >= realert_min * 60 + ) + if due: + first = not e.get("alerted") + if not first and in_quiet(qh): + action = "SUPPRESSED" + else: + action = "ALERT_FIRST" if first else "ALERT_REALERT" + e["alerted"] = True + e["last_alert_ts"] = now else: if e.get("alerted"): action = "RECOVERY" e["fails"] = 0 e["alerted"] = False + e.pop("first_fail_ts", None) st[cond] = e if not dry: json.dump(st, open(state_path, "w")) @@ -151,6 +171,28 @@ box_notify() { for p in $pids; do wait "$p" 2>/dev/null; done } +notify_input_wait() { + # Targeted DM for input waits (2026-10-05): DM ONLY the specific agent + # whose session is waiting for human input -- not a broadcast to all + # healthy agents. The #lobby post still fires via the relay leg for + # human visibility; this DM ensures the responsible operator sees it + # in their sidechat without digging through lobby noise. + # Best-effort: never fatal to the 5-minute check loop. + local node="$1" + local detail="$2" + local msg="[fleet-alert] INPUT WAIT: ${detail} -- reply: box approval reply ${node} \"\" or box notify ${node} \"\"" + msg="${msg:0:900}" + if [ "$DRY_RUN" = "1" ]; then + log "DRY-RUN would DM $node re input_wait" + return 0 + fi + if timeout 60 python3 "$BIN/box-ctl.py" notify "$node" "$msg" >/dev/null 2>&1; then + log "input_wait targeted DM sent to $node" + else + log "input_wait DM to $node failed (best-effort, non-fatal)" + fi +} + injected() { # cond -> 0 if injected-fail case ",$INJECT_FAIL," in *,"$1,"*) return 0;; *) return 1;; esac } @@ -214,7 +256,7 @@ print(json.dumps(out)) detail="Agent $node approvals clear" fi injected "$cond" && failing=1 - read -r action fails < <(state_machine "$cond" "$failing") + read -r action fails < <(state_machine "$cond" "$failing" "$BROWSER_APPROVAL_TTL") case "$action" in ALERT_FIRST|ALERT_REALERT) emit_record "ALERT" "$cond" "$detail" "$fails" @@ -226,6 +268,21 @@ print(json.dumps(out)) SUPPRESSED) log "$cond still critical x$fails — re-page suppressed" ;; + EXPIRED) + # Browser approval dialog exceeded BROWSER_APPROVAL_TTL without a + # human decision: fail closed by denying it. + log "$cond EXPIRED after ${BROWSER_APPROVAL_TTL}s without human decision — auto-denying (fail closed)" + python3 - "$node" <<'PYEOF3' +import sys, json +sys.path.insert(0, "/home/super/Projects/NetVM/bin") +import approvals +node = sys.argv[1] +print(json.dumps(approvals.deny_node_approval(node, caller="approval-ttl-expire"))) +approvals.log_box_ctl("approval-expired", name=node, caller="approval-ttl-expire", + extra={"note": "browser approval TTL elapsed; auto-denied (fail closed)"}) +PYEOF3 + emit_record "RECOVERY" "$cond" "Agent $node browser approval expired after ${BROWSER_APPROVAL_TTL}s; auto-denied" "$fails" + ;; esac # 2. Sidebar task waiting on human input @@ -245,11 +302,11 @@ if w: detail_in="Agent $node tasks running" fi injected "$cond_in" && failing_in=1 - read -r action_in fails_in < <(state_machine "$cond_in" "$failing_in") + read -r action_in fails_in < <(state_machine "$cond_in" "$failing_in" "$INPUT_WAIT_TTL") case "$action_in" in ALERT_FIRST|ALERT_REALERT) emit_record "ALERT" "$cond_in" "$detail_in" "$fails_in" - echo "$cond_in" >> "$STATE_DIR/.alerts.tmp" + echo "$cond_in|$detail_in" >> "$STATE_DIR/.alerts.tmp" ;; RECOVERY) emit_record "RECOVERY" "$cond_in" "$detail_in" "$fails_in" @@ -257,6 +314,27 @@ if w: SUPPRESSED) log "$cond_in still critical x$fails_in — re-page suppressed" ;; + EXPIRED) + # Input wait exceeded INPUT_WAIT_TTL without human response: + # auto-dismiss so the agent unblocks. Log the expiry and emit a + # RECOVERY record (the wait is gone, not merely un-paged). + log "$cond_in EXPIRED after ${INPUT_WAIT_TTL}s without human input — auto-dismissing" + python3 - "$node" <<'PYEOF2' +import sys +sys.path.insert(0, "/home/super/Projects/NetVM/bin") +import approvals, json +node = sys.argv[1] +info = approvals.inspect_node_approvals(node) +for w in info.get("input_waits", []) or []: + t = w.get("task") + if t: + approvals.mark_wait_responded(node, t, caller="approval-ttl-expire") +approvals.log_box_ctl("approval-wait-expired", name=node, caller="approval-ttl-expire", + extra={"note": "input wait TTL elapsed; auto-dismissed"}) +print(json.dumps(approvals.dismiss_node_task(node, caller="approval-ttl-expire"))) +PYEOF2 + emit_record "RECOVERY" "$cond_in" "Agent $node input wait expired after ${INPUT_WAIT_TTL}s; auto-dismissed" "$fails_in" + ;; esac done @@ -328,8 +406,26 @@ rm -f "$STATE_DIR/.healthy.tmp" # Notify for this run's alerts (best effort). Skip entirely when nothing is healthy # (notify needs a working browser via dm.py) or in dry-run. if [ -n "$HEALTHY_AGENTS" ] && [ -f "$STATE_DIR/.alerts.tmp" ]; then - while read -r cond; do - [ -n "$cond" ] && box_notify "$cond" "see #lobby for detail" + while IFS= read -r line; do + # alerts.tmp format: "cond" or "cond|detail" (input_wait carries detail) + cond="${line%%|*}" + detail="${line#*|}" + [ "$detail" = "$line" ] && detail="" + [ -n "$cond" ] || continue + case "$cond" in + input_wait:*) + # Targeted: DM only the waiting agent, not a broadcast. + node="${cond#input_wait:}" + if [ -n "$detail" ]; then + notify_input_wait "$node" "$detail" + else + box_notify "$cond" "see #lobby for detail" + fi + ;; + *) + box_notify "$cond" "see #lobby for detail" + ;; + esac done < "$STATE_DIR/.alerts.tmp" elif [ -f "$STATE_DIR/.alerts.tmp" ]; then log "no healthy agents — box notify skipped (DM path needs a working browser)" diff --git a/bin/gravity.py b/bin/gravity.py index 2b06fa3..397fa68 100644 --- a/bin/gravity.py +++ b/bin/gravity.py @@ -765,7 +765,7 @@ def remediate_breaks(dry_run=False) -> dict: import approvals fleet_apps = approvals.check_fleet_approvals() for app in fleet_apps: - if app.get("has_pending") and app.get("is_trusted"): + if app.get("has_pending") and app.get("is_trusted") and app.get("status") != "KEY_APPROVAL": node = app["node"] if not dry_run: approvals.allow_node_approval(node, always=True, caller="loop-remediate") diff --git a/bin/job-dispatch.py b/bin/job-dispatch.py index aab02a2..2e5e8ba 100755 --- a/bin/job-dispatch.py +++ b/bin/job-dispatch.py @@ -509,7 +509,7 @@ def main(): import approvals app_info = approvals.inspect_node_approvals(agent) if app_info.get("has_pending"): - if app_info.get("is_trusted"): + if app_info.get("is_trusted") and app_info.get("status") != "KEY_APPROVAL": print(f"Pre-dispatch: auto-approving trusted request for {agent} ({app_info.get('target')})") approvals.allow_node_approval(agent, always=True, caller="job-dispatch") else: diff --git a/bin/super-cli.py b/bin/super-cli.py index 3f27a7e..9492ae3 100755 --- a/bin/super-cli.py +++ b/bin/super-cli.py @@ -519,13 +519,17 @@ def cmd_approvals(args): sys.exit(1) always = getattr(args, "always", False) force = getattr(args, "force", False) - res = approvals.allow_node_approval(node, always=always, force=force) + message = getattr(args, "message", None) + allow_main_chat = getattr(args, "allow_main_chat", False) + res = approvals.allow_node_approval(node, always=always, force=force, message=message, + allow_main_chat=allow_main_chat) if getattr(args, "json", False): print(json.dumps(res, indent=2)) return if res.get("ok"): if res.get("type") == "key_approval": - print(c_green(f"✔ Approved operator key request for node '{node}'. Granted and logged to audit trail.")) + notified = "agent notified" if res.get("notified") else "⚠ agent NOT notified (follow up manually)" + print(c_green(f"✔ Approved operator key request for node '{node}'. Logged to audit trail. {notified}.")) else: decision_str = "Always allow this site" if always else "Allow once" print(c_green(f"✔ Approved request on node '{node}' ({decision_str}). Dialog dismissed: {res.get('dismissed')}.")) @@ -537,13 +541,16 @@ def cmd_approvals(args): if not node: print(c_red("Error: Must specify node for deny. e.g. 'box approvals deny 646'"), file=sys.stderr) sys.exit(1) - res = approvals.deny_node_approval(node) + message = getattr(args, "message", None) + allow_main_chat = getattr(args, "allow_main_chat", False) + res = approvals.deny_node_approval(node, message=message, allow_main_chat=allow_main_chat) if getattr(args, "json", False): print(json.dumps(res, indent=2)) return if res.get("ok"): if res.get("type") == "key_approval": - print(c_green(f"✔ Denied operator key request for node '{node}'. Denied and logged to audit trail.")) + notified = "agent notified" if res.get("notified") else "⚠ agent NOT notified (follow up manually)" + print(c_green(f"✔ Denied operator key request for node '{node}'. Logged to audit trail. {notified}.")) else: print(c_green(f"✔ Denied request on node '{node}'. Dialog dismissed: {res.get('dismissed')}.")) else: @@ -622,6 +629,17 @@ def cmd_approvals(args): else: print(c_red(f"✖ Failed to dismiss task popup on node '{node}': {res.get('error')}")) sys.exit(1) + elif action in ("clear", "clear-all", "clear_all"): + target_node = node + if action in ("clear-all", "clear_all"): + target_node = None + res = approvals.clear_node_waits(target_node) + if getattr(args, "json", False): + print(json.dumps(res, indent=2)) + return + cleared = res.get("total_cleared", 0) + target_desc = f"node '{target_node}'" if target_node else "all fleet nodes" + print(c_green(f"✔ Cleared {cleared} input wait(s) on {target_desc}.")) elif action in ("request-key", "request_key"): if not node: @@ -4211,14 +4229,20 @@ def build_parser(): p_app_allow.add_argument("node", choices=VALID_NODES, help="Target node to approve") p_app_allow.add_argument("--always", action="store_true", help="Click 'Always allow this site' instead of 'Allow once'") p_app_allow.add_argument("--force", action="store_true", help="Force approval even if target is untrusted") + p_app_allow.add_argument("--message", default=None, help="Optional message/credential to deliver to the waiting agent") + p_app_allow.add_argument("--allow-main-chat", action="store_true", help="Allow reply into Main Chat if agent is waiting there") p_app_approve = app_sub.add_parser("approve", parents=[common], help="Alias for 'allow'") p_app_approve.add_argument("node", choices=VALID_NODES, help="Target node to approve") p_app_approve.add_argument("--always", action="store_true", help="Click 'Always allow this site' instead of 'Allow once'") p_app_approve.add_argument("--force", action="store_true", help="Force approval even if target is untrusted") + p_app_approve.add_argument("--message", default=None, help="Optional message/credential to deliver to the waiting agent") + p_app_approve.add_argument("--allow-main-chat", action="store_true", help="Allow reply into Main Chat if agent is waiting there") p_app_deny = app_sub.add_parser("deny", parents=[common], help="Deny pending browser request") p_app_deny.add_argument("node", choices=VALID_NODES, help="Target node to deny") + p_app_deny.add_argument("--message", default=None, help="Optional message to deliver to the waiting agent") + p_app_deny.add_argument("--allow-main-chat", action="store_true", help="Allow reply into Main Chat if agent is waiting there") p_app_auto = app_sub.add_parser("auto", parents=[common], help="Auto-approve all trusted requests across fleet") p_app_auto.add_argument("--node", choices=VALID_NODES, default=None, help="Target node (or all nodes)") @@ -4236,6 +4260,11 @@ def build_parser(): p_app_dismiss = app_sub.add_parser("dismiss", parents=[common], help="Dismiss any open task dialog/popup on a node") p_app_dismiss.add_argument("node", choices=VALID_NODES, help="Target node to dismiss dialog on") + p_app_clear = app_sub.add_parser("clear", parents=[common], help="Clear and dismiss pending input waits on a node or all nodes") + p_app_clear.add_argument("node", nargs="?", choices=VALID_NODES, default=None, help="Target node (or omit for all nodes)") + + p_app_clear_all = app_sub.add_parser("clear-all", parents=[common], help="Clear and dismiss all pending input waits across fleet") + p_app_req_key = app_sub.add_parser("request-key", parents=[common], help="Request operator passkey/key approval for an agent") p_app_req_key.add_argument("node", choices=VALID_NODES, help="Target node requesting key") p_app_req_key.add_argument("--reason", default="Passkey authentication required", help="Reason for key request")