feat(kpi): expand KPI runtime monitoring, prompt advisory envelopes, and missing field resiliency

2026-10-06 20:02:48 +00:00
parent b824f6d405
commit ae4df2f29c
13 changed files with 1702 additions and 85 deletions
@@ -12,6 +12,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(REPO_ROOT / "bin"))

 import invite
+import invite_handler
 from invite_handler import InviteCodeInfo, InviteHandler, RedemptionResult, salvage_blocked_node
 from settings_rpa import NodeUsage, SettingsRPA

@@ -176,7 +177,136 @@ class TestInviteHandlerMocked(unittest.TestCase):
            self.assertFalse(res["success"])
            self.assertEqual(res["invite_code"], "REDCJ7")
            self.assertIn("An onboarding agent or fresh client profile must redeem this code", res["error"])
+            self.assertTrue(res.get("field_missing"))
+
+    def test_salvage_blocked_node_with_loopback_dm(self):
+        with patch.object(InviteHandler, "find_code") as mock_find:
+            mock_find.return_value = InviteCodeInfo(
+                node="646",
+                code="REDCJ7",
+                uses_remaining=30,
+                use_count=0,
+                max_uses=30,
+                has_redeemed=True,
+                invite_state="ELIGIBLE",
+                reward=None,
+                method_used="api",
+            )
+            with patch("invite_handler.send_loopback_notice", return_value=True) as mock_notice:
+                res = salvage_blocked_node("646", notify_target="heartbeat-opm", notify_agent="opm")
+                self.assertFalse(res["success"])
+                self.assertTrue(res.get("field_missing"))
+                self.assertTrue(res.get("loopback_notified"))
+                mock_notice.assert_called_once()
+                call_args = mock_notice.call_args[0]
+                self.assertEqual(call_args[0], "opm")
+                self.assertEqual(call_args[1], "heartbeat-opm")
+                self.assertIn("REDCJ7", call_args[2])
+
+    @patch("invite_handler.SettingsRPA")
+    def test_redeem_code_dom_missing_row_api_fallback_success(self, mock_rpa_cls):
+        mock_rpa = MagicMock()
+        mock_rpa.open_settings_dialog.return_value = True
+        mock_rpa.select_tab.return_value = True
+        mock_rpa_cls.return_value.__enter__.return_value = mock_rpa
+
+        h = InviteHandler("dev")
+        h.ws = MagicMock()
+        with patch.object(h, "connect"), patch("time.sleep", return_value=None):
+            with patch("invite_handler.cdp_evaluate", return_value={"found": False, "text": "General"}):
+                with patch.object(h, "redeem_code_api") as mock_api:
+                    mock_api.return_value = RedemptionResult(
+                        target_node="dev",
+                        code="REDCJ7",
+                        success=True,
+                        status="redeemed",
+                        reason=None,
+                        detail="API success",
+                        method_used="api",
+                    )
+                    res = h.redeem_code_dom("REDCJ7")
+                    self.assertTrue(res.success)
+                    self.assertEqual(res.status, "redeemed")
+                    self.assertIn("Redeem field was missing in Settings DOM; redeemed successfully via API fallback!", res.detail)
+                    mock_rpa.close_settings_dialog.assert_called_once()
+
+    @patch("invite_handler.SettingsRPA")
+    def test_redeem_code_dom_missing_row_already_redeemed(self, mock_rpa_cls):
+        mock_rpa = MagicMock()
+        mock_rpa.open_settings_dialog.return_value = True
+        mock_rpa.select_tab.return_value = True
+        mock_rpa_cls.return_value.__enter__.return_value = mock_rpa
+
+        h = InviteHandler("646")
+        h.ws = MagicMock()
+        with patch.object(h, "connect"), patch("time.sleep", return_value=None):
+            with patch("invite_handler.cdp_evaluate", return_value={"found": False, "has_additional": True, "text": "Additional tokens"}):
+                with patch.object(h, "redeem_code_api") as mock_api:
+                    mock_api.return_value = RedemptionResult(
+                        target_node="646",
+                        code="REDCJ7",
+                        success=False,
+                        status="already_redeemed",
+                        reason="already_redeemed",
+                        detail="Already redeemed",
+                        method_used="api",
+                    )
+                    res = h.redeem_code_dom("REDCJ7")
+                    self.assertFalse(res.success)
+                    self.assertEqual(res.status, "already_redeemed")
+                    self.assertTrue(res.field_missing)
+                    self.assertIn("already redeemed an invite code", res.detail)
+
+    @patch("invite_handler.SettingsRPA")
+    def test_redeem_code_dom_missing_input_box_with_loopback(self, mock_rpa_cls):
+        mock_rpa = MagicMock()
+        mock_rpa.open_settings_dialog.return_value = True
+        mock_rpa.select_tab.return_value = True
+        mock_rpa_cls.return_value.__enter__.return_value = mock_rpa
+
+        h = InviteHandler("pip")
+        h.ws = MagicMock()
+
+        def mock_eval(ws, js, *args, **kwargs):
+            if "Redeem invite code" in js:
+                return {"found": True}
+            if "found_input" in js:
+                return {"found_input": False}
+            return True
+
+        with patch.object(h, "connect"), patch("time.sleep", return_value=None):
+            with patch("invite_handler.cdp_evaluate", side_effect=mock_eval), patch("invite_handler.cdp_send_escape"):
+                with patch.object(h, "redeem_code_api") as mock_api:
+                    mock_api.return_value = RedemptionResult(
+                        target_node="pip",
+                        code="TEST01",
+                        success=False,
+                        status="dom_input_missing",
+                        reason="input_field_missing",
+                        detail="API fallback also failed",
+                        method_used="api",
+                    )
+                    with patch("invite_handler.send_loopback_notice", return_value=True) as mock_notice:
+                        res = h.redeem_code_dom("TEST01", notify_target="pip tasks", notify_agent="pip")
+                        self.assertFalse(res.success)
+                        self.assertTrue(res.field_missing)
+                        self.assertEqual(res.reason, "input_field_missing")
+                        self.assertTrue(res.loopback_notified)
+                        mock_notice.assert_called_once()
+                        self.assertIn("pip tasks", res.loopback_detail)
+
+    @patch("subprocess.run")
+    def test_send_loopback_notice(self, mock_subproc):
+        mock_subproc.return_value = MagicMock(returncode=0)
+        ok = invite_handler.send_loopback_notice("opm", "646 tasks", "Test alert")
+        self.assertTrue(ok)
+        mock_subproc.assert_called_once()
+        cmd = mock_subproc.call_args[0][0]
+        self.assertIn("send", cmd)
+        self.assertIn("646 tasks", cmd)
+        self.assertIn("Test alert", cmd)


 if __name__ == "__main__":
    unittest.main()
+
@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""test_kpi.py — Unit tests for NetVM Fleet KPI, spend monitor and runtime preservation engine."""
+
+import json
+import sys
+import unittest
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "bin"))
+
+import kpi
+from kpi import (
+    calculate_efficiency,
+    check_node_routes,
+    generate_preservation_advisory,
+    get_agent_dm_metrics,
+    get_agent_kpi,
+    get_live_advisory_block,
+    spawn_tmux_worker,
+)
+
+
+class TestKPIMetrics(unittest.TestCase):
+    def test_calculate_efficiency_high(self):
+        # High work units relative to burn
+        idx, rating = calculate_efficiency(
+            jobs_done=5,
+            subagents_active=2,
+            tmux_workers=2,
+            calls_verified=20,
+            weekly_used_pct=30,
+            uptime_hours=10.0,
+        )
+        self.assertGreaterEqual(idx, 3.0)
+        self.assertEqual(rating, "HIGH_EFFICIENCY")
+
+    def test_calculate_efficiency_vanity_idle(self):
+        # Long uptime but zero workers and zero jobs
+        idx, rating = calculate_efficiency(
+            jobs_done=0,
+            subagents_active=0,
+            tmux_workers=0,
+            calls_verified=0,
+            weekly_used_pct=10,
+            uptime_hours=5.0,
+        )
+        self.assertEqual(rating, "VANITY_IDLE")
+
+    def test_generate_preservation_advisory_critical_quota(self):
+        adv = generate_preservation_advisory(
+            node="646",
+            weekly_used_pct=100,
+            extra_tokens_remaining="0 tokens left",
+            subagents_active=0,
+            tmux_workers=0,
+            rating="VANITY_IDLE",
+        )
+        self.assertIn("CRITICAL: Quota exhausted", adv)
+        self.assertIn("box onboard start", adv)
+
+    def test_generate_preservation_advisory_high_quota_idle(self):
+        adv = generate_preservation_advisory(
+            node="pip",
+            weekly_used_pct=75,
+            extra_tokens_remaining="500M tokens left",
+            subagents_active=0,
+            tmux_workers=0,
+            rating="VANITY_IDLE",
+        )
+        self.assertIn("Cease prose chatter", adv)
+        self.assertIn("Spawn subagents", adv)
+
+    @patch("kpi.Path.exists")
+    @patch("approvals.get_node_pages")
+    def test_check_node_routes_online(self, mock_pages, mock_exists):
+        mock_exists.return_value = True
+        mock_pages.return_value = [{"title": "Chat"}]
+        st = check_node_routes("646")
+        self.assertEqual(st, "ONLINE")
+
+    @patch("kpi.Path.exists")
+    def test_check_node_routes_no_netns(self, mock_exists):
+        mock_exists.return_value = False
+        st = check_node_routes("646")
+        self.assertEqual(st, "NO_NETNS")
+
+    @patch("kpi.get_agent_dm_metrics")
+    @patch("kpi.get_agent_job_metrics")
+    @patch("kpi.get_agent_subagent_count")
+    @patch("kpi.get_agent_tmux_workers")
+    @patch("kpi.get_agent_uptime_hours")
+    @patch("kpi.check_node_routes")
+    def test_get_agent_kpi(
+        self,
+        mock_routes,
+        mock_uptime,
+        mock_tmux,
+        mock_sub,
+        mock_jobs,
+        mock_dm,
+    ):
+        mock_dm.return_value = {"sent": 50, "verified": 40, "total_events": 100}
+        mock_jobs.return_value = {"assigned": 10, "completed": 4}
+        mock_sub.return_value = 2
+        mock_tmux.return_value = ["worker-1"]
+        mock_uptime.return_value = 8.5
+        mock_routes.return_value = "ONLINE"
+
+        usage_cache = {
+            "weekly_used_pct": 25,
+            "extra_tokens_remaining": "900M tokens left",
+            "is_blocked": False,
+        }
+
+        res = get_agent_kpi("dev", usage_cache=usage_cache)
+        self.assertEqual(res.node, "dev")
+        self.assertEqual(res.weekly_used_pct, 25)
+        self.assertEqual(res.calls_sent, 50)
+        self.assertEqual(res.calls_verified, 40)
+        self.assertEqual(res.jobs_completed, 4)
+        self.assertEqual(res.subagents_active, 2)
+        self.assertEqual(res.tmux_workers_active, 1)
+        self.assertEqual(res.route_status, "ONLINE")
+
+    @patch("kpi.get_agent_kpi")
+    def test_get_live_advisory_block(self, mock_kpi):
+        mock_kpi.return_value = MagicMock(
+            node="646",
+            weekly_used_pct=30,
+            extra_tokens_remaining="1B tokens left",
+            uptime_hours=12.0,
+            jobs_completed=5,
+            subagents_active=2,
+            tmux_workers_active=1,
+            efficiency_rating="HIGH_EFFICIENCY",
+            efficiency_index=4.5,
+            route_status="ONLINE",
+            preservation_advisory="Runtime healthy.",
+        )
+        block = get_live_advisory_block("646")
+        self.assertIn("BOX PERFORMANCE & RUNTIME ADVISORY", block)
+        self.assertIn("AGENT: @646", block)
+        self.assertIn("30% weekly limit used", block)
+        self.assertIn("HIGH_EFFICIENCY", block)
+        self.assertIn("Runtime healthy.", block)
+
+    @patch("subprocess.run")
+    @patch("subagent_tracker.register_session")
+    def test_spawn_tmux_worker(self, mock_register, mock_subproc):
+        mock_subproc.return_value = MagicMock(returncode=0)
+        res = spawn_tmux_worker("dev", "audit-sub", "python3 audit.py")
+        self.assertTrue(res["ok"])
+        self.assertEqual(res["node"], "dev")
+        self.assertEqual(res["session"], "dev-audit-sub")
+        mock_register.assert_called_once()
+        call_kwargs = mock_register.call_args[1]
+        self.assertEqual(call_kwargs["parent"], "dev")
+        self.assertIn("dev-audit-sub", call_kwargs["session_id"])
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -83,6 +83,40 @@ class TestOnboardPipeline(unittest.TestCase):
        self.assertEqual(res["beneficiary_node"], "646")
        mock_redeem.assert_called_once_with("testnode", "REDCJ7", timeout=20.0)

+    @patch("invite.redeem_invite")
+    @patch("subprocess.run")
+    @patch("invite_handler.send_loopback_notice")
+    def test_finish_onboarding_redemption_failure_with_loopback(self, mock_notice, mock_subproc, mock_redeem):
+        mock_redeem.return_value = {"ok": False, "reason": "already_redeemed", "detail": "Already redeemed"}
+        mock_notice.return_value = True
+        state = OnboardState(
+            node="testnode",
+            email="client@test.com",
+            beneficiary_node="646",
+            invite_code="REDCJ7",
+            stage="auth_active",
+        )
+
+        res = finish_onboarding_redemption(state)
+        self.assertTrue(res["ok"])
+        self.assertEqual(res["status"], "auth_active_redemption_warning")
+        self.assertIn("Redemption failed: already_redeemed", state.detail)
+        mock_notice.assert_called_once()
+        kwargs = mock_notice.call_args.kwargs
+        self.assertEqual(kwargs.get("recipient"), "646")
+        self.assertEqual(kwargs.get("target"), "646 tasks")
+        self.assertIn("already_redeemed", kwargs.get("message", ""))
+
+
+    def test_get_all_connects_fast(self):
+        from onboard_pipeline import get_all_connects
+        connects = get_all_connects(fast=True)
+        self.assertIsInstance(connects, list)
+        self.assertGreaterEqual(len(connects), 6)
+        nodes = {c["node"] for c in connects}
+        for expected in ("muse", "pip", "646", "opm", "dev", "def"):
+            self.assertIn(expected, nodes)
+

 if __name__ == "__main__":
    unittest.main()
@@ -104,7 +104,32 @@ class TestSettingsRPAPrimitives(unittest.TestCase):
            self.assertEqual(usage.extra_percent_used, 0)
            self.assertFalse(usage.is_blocked)
            self.assertFalse(usage.has_redeemed)
+            self.assertTrue(usage.stats_loaded)
+
+    @patch("settings_rpa.get_cdp_ws")
+    @patch("settings_rpa.cdp_evaluate")
+    def test_read_usage_stats_did_not_render(self, mock_eval, mock_ws):
+        mock_ws.return_value = (MagicMock(), MagicMock())
+
+        def eval_side_effect(ws, js, *args, **kwargs):
+            if "role=\"dialog\"" in js and "Weekly limit" in js:
+                return None
+            return True
+
+        mock_eval.side_effect = eval_side_effect
+
+        with SettingsRPA("646", timeout=0.5) as rpa:
+            # Shorten deadline by patching time.time or passing small timeout
+            with patch("time.sleep", return_value=None):
+                usage = rpa.read_usage(keep_dialog_open=True)
+                self.assertEqual(usage.node, "646")
+                self.assertFalse(usage.stats_loaded)
+                self.assertFalse(usage.is_blocked)
+                self.assertIn("Unavailable", usage.weekly_reset_text)
+                self.assertEqual(usage.extra_tokens_remaining, "Unavailable")
+


 if __name__ == "__main__":
    unittest.main()
+