feat(kpi): expand KPI runtime monitoring, prompt advisory envelopes, and missing field resiliency

2026-10-06 20:02:48 +00:00
parent b824f6d405
commit ae4df2f29c
13 changed files with 1702 additions and 85 deletions
@@ -55,11 +55,49 @@ class RedemptionResult:
    reason: Optional[str]
    detail: Optional[str]
    method_used: str
+    field_missing: bool = False
+    loopback_notified: bool = False
+    loopback_detail: Optional[str] = None

    def to_dict(self) -> Dict[str, Any]:
        return asdict(self)


+def send_loopback_notice(
+    recipient: str,
+    target: str,
+    message: str,
+    sender: str = "super",
+    timeout: float = 15.0,
+) -> bool:
+    """Send a loopback notification DM via bin/dm.py to a sidechat without blocking or failing."""
+    try:
+        from pathlib import Path
+        bin_dir = Path(__file__).resolve().parent
+        dm_script = bin_dir / "dm.py"
+        if not dm_script.exists():
+            return False
+        import subprocess
+        cmd = [
+            sys.executable,
+            str(dm_script),
+            "send",
+            "--agent",
+            sender,
+            "--to",
+            recipient,
+            "--target",
+            target,
+            message,
+        ]
+        if target == "main":
+            cmd.insert(-1, "--allow-main-chat")
+        res = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
+        return res.returncode == 0
+    except Exception:
+        return False
+
+
 class InviteHandler:
    """Handles invite codes discovery, inspection, and redemption for NetVM nodes."""

@@ -68,6 +106,15 @@ class InviteHandler:
        self.timeout = timeout
        self.ws = None

+    def _dispatch_loopback(self, res: RedemptionResult, agent: str, target: str) -> bool:
+        """Post a loopback notification DM via box dm."""
+        msg = f"[BOX-INVITE-LOOPBACK] Node @{self.node} redeem status: {res.status}. Reason: {res.reason or 'unknown'}. Detail: {res.detail or '-'}"
+        ok = send_loopback_notice(recipient=agent, target=target, message=msg)
+        res.loopback_notified = ok
+        res.loopback_detail = f"Notified @{agent}/{target}" if ok else f"Failed to notify @{agent}/{target}"
+        return ok
+
+
    def connect(self) -> InviteHandler:
        if self.ws is None:
            self.ws, _ = get_cdp_ws(self.node, timeout=self.timeout)
@@ -262,87 +309,169 @@ class InviteHandler:
            method_used="api",
        )

-    def redeem_code_dom(self, code: str) -> RedemptionResult:
-        """Redeem invite code using Settings Menu RPA -> Settings -> Redeem Invite Code dialog."""
+    def redeem_code_dom(
+        self,
+        code: str,
+        notify_target: Optional[str] = None,
+        notify_agent: Optional[str] = None,
+    ) -> RedemptionResult:
+        """Redeem invite code using Settings Menu RPA -> Settings -> Redeem Invite Code dialog.
+        Gracefully handles missing entrypoint rows, missing input fields, and async loading races.
+        Falls back to in-page API automatically if DOM fields are absent, and dispatches DM loopback if requested."""
        self.connect()
        clean_code = code.strip().upper()
        with SettingsRPA(self.node, timeout=self.timeout) as rpa:
            # 1. Open Settings dialog
            opened = rpa.open_settings_dialog()
            if not opened:
-                return RedemptionResult(
+                # If dialog failed to open, try API fallback directly
+                api_res = self.redeem_code_api(clean_code)
+                if api_res.success:
+                    api_res.detail = f"Settings dialog could not open; redemption completed via API fallback ({api_res.detail or ''})".strip()
+                    return api_res
+
+                res = RedemptionResult(
                    target_node=self.node,
                    code=clean_code,
                    success=False,
                    status="error",
                    reason="settings_open_failed",
-                    detail="Could not open Settings dialog via RPA",
+                    detail=f"Could not open Settings dialog via RPA on @{self.node}. API fallback: {api_res.detail or api_res.reason or 'failed'}",
                    method_used="dom",
+                    field_missing=True,
                )
+                if notify_target and notify_agent:
+                    self._dispatch_loopback(res, notify_agent, notify_target)
+                return res

            rpa.select_tab("General")
            time.sleep(0.3)

-            # 2. Check for "Redeem invite code" entry
+            # 2. Check for "Redeem invite code" entry (poll up to 2.5s for React rendering)
            js_find_and_click = """(() => {
                const dialog = document.querySelector('[role="dialog"]');
-                if (!dialog) return {found: false};
+                if (!dialog) return {found: false, dialog_present: false};
                const items = Array.from(dialog.querySelectorAll('button, div, span'));
                const redeemItem = items.find(el => (el.innerText || '').trim() === 'Redeem invite code');
                if (redeemItem) {
                    redeemItem.click();
-                    return {found: true};
+                    return {found: true, dialog_present: true};
                }
-                return {found: false, text: dialog.innerText};
+                return {
+                    found: false,
+                    dialog_present: true,
+                    text: dialog.innerText || '',
+                    has_additional: (dialog.innerText || '').includes('Additional tokens')
+                };
            })()"""
-            click_res = cdp_evaluate(self.ws, js_find_and_click)
+            click_res = None
+            deadline = time.time() + 2.5
+            while time.time() < deadline:
+                click_res = cdp_evaluate(self.ws, js_find_and_click)
+                if click_res and click_res.get("found"):
+                    break
+                time.sleep(0.3)
+
            if not click_res or not click_res.get("found"):
+                # "Redeem invite code" field is missing in General settings!
                rpa.close_settings_dialog()
-                # If entrypoint missing, likely already redeemed
-                api_check = self.find_code_api()
-                if api_check.has_redeemed:
-                    return RedemptionResult(
-                        target_node=self.node,
-                        code=clean_code,
-                        success=False,
-                        status="already_redeemed",
-                        reason="already_redeemed",
-                        detail="Node has already redeemed an invite code (entrypoint hidden)",
-                        method_used="dom",
-                    )
-                return RedemptionResult(
+
+                # Attempt automatic API fallback first
+                api_res = self.redeem_code_api(clean_code)
+                if api_res.success:
+                    api_res.detail = f"Redeem field was missing in Settings DOM; redeemed successfully via API fallback! ({api_res.detail or ''})".strip()
+                    return api_res
+
+                # Diagnose why field is missing
+                diag_text = click_res.get("text", "") if click_res else ""
+                has_extra = click_res.get("has_additional", False) if click_res else False
+
+                is_already = has_extra or "Additional tokens" in diag_text
+                if not is_already:
+                    try:
+                        api_check = self.find_code_api()
+                        is_already = bool(api_check.has_redeemed)
+                    except Exception:
+                        pass
+
+                if is_already:
+                    status = "already_redeemed"
+                    reason = "already_redeemed"
+                    detail = f"Node @{self.node} has already redeemed an invite code (entrypoint hidden by active Additional tokens ticker)."
+                elif api_res.reason in ["window_expired", "already_redeemed", "invalid", "used_up"]:
+                    status = api_res.status
+                    reason = api_res.reason
+                    detail = f"Redeem invite code field not present in Settings on @{self.node}: server reports {api_res.reason} ({api_res.detail or ''})."
+                else:
+                    status = "entrypoint_not_found"
+                    reason = "field_missing"
+                    detail = f"Redeem invite code field missing in General settings on @{self.node} (account may be past 48h onboarding window or already redeemed)."
+
+                res = RedemptionResult(
                    target_node=self.node,
                    code=clean_code,
                    success=False,
-                    status="entrypoint_not_found",
-                    reason="not_eligible",
-                    detail="Redeem invite code item not present in General settings",
+                    status=status,
+                    reason=reason,
+                    detail=detail,
                    method_used="dom",
+                    field_missing=True,
                )
+                if notify_target and notify_agent:
+                    self._dispatch_loopback(res, notify_agent, notify_target)
+                return res

-            time.sleep(0.8)
-
-            # 3. Handle HatchInviteRedemptionDialog
+            # 3. Handle HatchInviteRedemptionDialog (sub-dialog opened by clicking Redeem)
+            # Poll up to 2.5s for input box to mount
            js_input_code = f"""(() => {{
-                // Look for dialog titled "Redeem a code"
+                // Look for dialog titled "Redeem a code" or input with label
                const input = document.querySelector('input[aria-label="Invite code"]') ||
                              document.querySelector('[role="dialog"] input[type="text"]');
                if (!input) return {{found_input: false}};

-                // Type code into input
                input.focus();
                input.value = {json.dumps(clean_code)};
                input.dispatchEvent(new Event('input', {{bubbles: true}}));
                input.dispatchEvent(new Event('change', {{bubbles: true}}));

-                // Submit via Enter key
                input.dispatchEvent(new KeyboardEvent('keydown', {{key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true}}));
                return {{found_input: true}};
            }})()"""
-            input_res = cdp_evaluate(self.ws, js_input_code)
-            time.sleep(1.5)
+            input_res = None
+            deadline = time.time() + 2.5
+            while time.time() < deadline:
+                input_res = cdp_evaluate(self.ws, js_input_code)
+                if input_res and input_res.get("found_input"):
+                    break
+                time.sleep(0.3)

-            # 4. Check outcome in dialog
+            if not input_res or not input_res.get("found_input"):
+                # Subdialog opened or clicked, but input box is absent!
+                cdp_send_escape(self.ws)
+                rpa.close_settings_dialog()
+
+                # Automatic API fallback
+                api_res = self.redeem_code_api(clean_code)
+                if api_res.success:
+                    api_res.detail = f"Redemption input box was missing in dialog; redeemed successfully via API fallback! ({api_res.detail or ''})".strip()
+                    return api_res
+
+                res = RedemptionResult(
+                    target_node=self.node,
+                    code=clean_code,
+                    success=False,
+                    status=api_res.status or "dom_input_missing",
+                    reason=api_res.reason or "input_field_missing",
+                    detail=f"Invite code input field was not found in redemption dialog on @{self.node}. API fallback: {api_res.detail or api_res.reason or 'failed'}.",
+                    method_used="dom",
+                    field_missing=True,
+                )
+                if notify_target and notify_agent:
+                    self._dispatch_loopback(res, notify_agent, notify_target)
+                return res
+
+            # Input submitted; poll for outcome text
+            time.sleep(1.2)
            js_check_outcome = """(() => {
                const dialog = document.querySelector('[role="dialog"]');
                if (!dialog) return {found: false};
@@ -365,7 +494,14 @@ class InviteHandler:
                }
                return {status: 'unknown', detail: text};
            })()"""
-            outcome = cdp_evaluate(self.ws, js_check_outcome)
+            outcome = None
+            deadline = time.time() + 2.5
+            while time.time() < deadline:
+                outcome = cdp_evaluate(self.ws, js_check_outcome)
+                if outcome and outcome.get("status") != "unknown":
+                    break
+                time.sleep(0.3)
+
            rpa.close_settings_dialog()

            if outcome and outcome.get("success"):
@@ -379,7 +515,7 @@ class InviteHandler:
                    method_used="dom",
                )
            elif outcome and outcome.get("status") in ["already_redeemed", "invalid", "used_up", "window_expired"]:
-                return RedemptionResult(
+                res = RedemptionResult(
                    target_node=self.node,
                    code=clean_code,
                    success=False,
@@ -388,19 +524,40 @@ class InviteHandler:
                    detail=outcome.get("detail"),
                    method_used="dom",
                )
+                if notify_target and notify_agent:
+                    self._dispatch_loopback(res, notify_agent, notify_target)
+                return res

            # Fallback to API check if DOM did not confirm status
-            return self.redeem_code_api(clean_code)
+            api_res = self.redeem_code_api(clean_code)
+            if not api_res.success and notify_target and notify_agent:
+                self._dispatch_loopback(api_res, notify_agent, notify_target)
+            return api_res

-    def redeem_code(self, code: str, method: str = "auto") -> RedemptionResult:
+    def redeem_code(
+        self,
+        code: str,
+        method: str = "auto",
+        notify_target: Optional[str] = None,
+        notify_agent: Optional[str] = None,
+    ) -> RedemptionResult:
        """Redeem invite code using specified method ('auto', 'dom', or 'api')."""
        if method == "dom":
-            return self.redeem_code_dom(code)
+            return self.redeem_code_dom(code, notify_target=notify_target, notify_agent=notify_agent)
        elif method == "api":
-            return self.redeem_code_api(code)
+            res = self.redeem_code_api(code)
+            if not res.success and notify_target and notify_agent:
+                self._dispatch_loopback(res, notify_agent, notify_target)
+            return res
        else:
-            # Auto: Validate and attempt via API for reliability
-            return self.redeem_code_api(code)
+            # Auto: Validate and attempt via API for reliability. Fallback to DOM on transport error.
+            res = self.redeem_code_api(code)
+            if not res.success and res.reason == "transport_failure":
+                res = self.redeem_code_dom(code, notify_target=notify_target, notify_agent=notify_agent)
+            elif not res.success and notify_target and notify_agent:
+                self._dispatch_loopback(res, notify_agent, notify_target)
+            return res
+


 def scan_fleet_invites(nodes: Optional[List[str]] = None) -> List[Dict[str, Any]]:
@@ -454,7 +611,12 @@ def scan_fleet_usage(nodes: Optional[List[str]] = None) -> List[Dict[str, Any]]:
    return results


-def salvage_blocked_node(blocked_node: str = "646", helper_node: Optional[str] = None) -> Dict[str, Any]:
+def salvage_blocked_node(
+    blocked_node: str = "646",
+    helper_node: Optional[str] = None,
+    notify_target: Optional[str] = None,
+    notify_agent: Optional[str] = None,
+) -> Dict[str, Any]:
    """Salvage an out-of-tokens node by identifying its code and redeeming it on an eligible peer."""
    # 1. Fetch blocked node invite code
    with InviteHandler(blocked_node) as h_blocked:
@@ -483,18 +645,28 @@ def salvage_blocked_node(blocked_node: str = "646", helper_node: Optional[str] =
            continue

    if not eligible_peer:
-        return {
+        res = {
            "success": False,
            "blocked_node": blocked_node,
            "invite_code": code_to_redeem,
            "error": "No existing fleet peer is currently eligible (all active peers have already redeemed an invite code). An onboarding agent or fresh client profile must redeem this code.",
            "code_to_redeem": code_to_redeem,
-            "share_instruction": f"Redeem code '{code_to_redeem}' on a newly provisioned agent to credit 1 billion tokens to {blocked_node}."
+            "share_instruction": f"Redeem code '{code_to_redeem}' on a newly provisioned agent to credit 1 billion tokens to {blocked_node}.",
+            "field_missing": True,
+            "loopback_notified": False,
        }
+        if notify_target and notify_agent:
+            msg = f"[SALVAGE-NOTICE] Node @{blocked_node} is blocked (code: {code_to_redeem}), but no eligible peer is available. Fresh onboarding required."
+            res["loopback_notified"] = send_loopback_notice(notify_agent, notify_target, msg)
+        return res

    # 3. Redeem on eligible peer
    with InviteHandler(eligible_peer) as h_peer:
-        redemption = h_peer.redeem_code(code_to_redeem)
+        redemption = h_peer.redeem_code(
+            code_to_redeem,
+            notify_target=notify_target,
+            notify_agent=notify_agent,
+        )

    return {
        "success": redemption.success,
@@ -502,6 +674,8 @@ def salvage_blocked_node(blocked_node: str = "646", helper_node: Optional[str] =
        "helper_node": eligible_peer,
        "code_redeemed": code_to_redeem,
        "redemption_result": redemption.to_dict(),
+        "field_missing": redemption.field_missing,
+        "loopback_notified": redemption.loopback_notified,
    }


@@ -518,6 +692,8 @@ def main() -> None:
    p_redeem.add_argument("node", help="Target node to redeem the code on")
    p_redeem.add_argument("code", help="6-character invite code")
    p_redeem.add_argument("--method", choices=["auto", "api", "dom"], default="auto")
+    p_redeem.add_argument("--notify-target", default=None, help="Sidechat to notify on loopback")
+    p_redeem.add_argument("--notify-agent", default=None, help="Agent to notify on loopback")
    p_redeem.add_argument("--json", action="store_true")

    p_list = subparsers.add_parser("list", help="List invite codes across fleet")
@@ -526,6 +702,8 @@ def main() -> None:
    p_salvage = subparsers.add_parser("salvage", help="Salvage a blocked agent (e.g. 646)")
    p_salvage.add_argument("node", default="646", nargs="?", help="Blocked node (default: 646)")
    p_salvage.add_argument("--helper", help="Specific helper node to redeem code")
+    p_salvage.add_argument("--notify-target", default=None, help="Sidechat to notify on loopback")
+    p_salvage.add_argument("--notify-agent", default=None, help="Agent to notify on loopback")
    p_salvage.add_argument("--json", action="store_true")

    args = parser.parse_args()
@@ -546,7 +724,12 @@ def main() -> None:

    elif args.command == "redeem":
        with InviteHandler(args.node) as h:
-            res = h.redeem_code(args.code, method=args.method)
+            res = h.redeem_code(
+                args.code,
+                method=args.method,
+                notify_target=getattr(args, "notify_target", None),
+                notify_agent=getattr(args, "notify_agent", None),
+            )
            if args.json:
                print(json.dumps(res.to_dict(), indent=2))
            else:
@@ -556,6 +739,8 @@ def main() -> None:
                print(f"  Status:  {res.status}")
                if res.detail:
                    print(f"  Detail:  {res.detail}")
+                if res.loopback_notified:
+                    print(f"  Loopback:{res.loopback_detail}")

    elif args.command == "list":
        fleet = scan_fleet_invites()
@@ -576,7 +761,12 @@ def main() -> None:
            print()

    elif args.command == "salvage":
-        salvage_res = salvage_blocked_node(args.node, helper_node=args.helper)
+        salvage_res = salvage_blocked_node(
+            args.node,
+            helper_node=args.helper,
+            notify_target=getattr(args, "notify_target", None),
+            notify_agent=getattr(args, "notify_agent", None),
+        )
        if args.json:
            print(json.dumps(salvage_res, indent=2))
        else:
@@ -588,7 +778,10 @@ def main() -> None:
                print(f"  Status: {salvage_res.get('error')}")
                if salvage_res.get("share_instruction"):
                    print(f"  Next Step: {salvage_res.get('share_instruction')}")
+            if salvage_res.get("loopback_notified"):
+                print("  Loopback: Notice sent to requesting target.")
            print()
+
    else:
        parser.print_help()