box/tests/test_loop_health_remediation.py

#!/usr/bin/env python3
"""
test_loop_health_remediation.py — Unit and integration tests for:
1. Reconstructing intrinsic loops from followups.json and dm-log.jsonl
2. Fleet loop health metrics computation and threshold evaluation
3. Progressive remediation:
   - Soft break healing (auto-resolving answered follow-ups)
   - Deadline re-arming for pending expired nudges
4. Hard break diagnostic escalations (silent agent, auth rot, systemd failure)
5. Audit event logging in job-log.jsonl
"""
import unittest
import subprocess
import json
import os
import sys
import tempfile
import shutil
from pathlib import Path
from datetime import datetime, timezone
from unittest.mock import patch

REPO_ROOT = Path("/home/super/Projects/NetVM")
BIN_DIR = REPO_ROOT / "bin"
BOX_CTL = BIN_DIR / "box-ctl.py"

sys.path.insert(0, str(BIN_DIR))
import gravity


def _load(name, relpath):
    import importlib.util
    spec = importlib.util.spec_from_file_location(name, REPO_ROOT / relpath)
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod


box_ctl = _load("box_ctl_loophealthtest", "bin/box-ctl.py")


class _InProcResult:
    """Minimal CompletedProcess stand-in (returncode/stdout only)."""

    def __init__(self, returncode, stdout):
        self.returncode = returncode
        self.stdout = stdout


def _box_ctl_inproc(*args):
    """In-process box-ctl call (proven pattern from test_box_loop_https)."""
    import io
    from contextlib import redirect_stdout
    buf = io.StringIO()
    returncode = 0
    with redirect_stdout(buf):
        try:
            box_ctl.main(["box-ctl.py", *args])
        except SystemExit as e:
            returncode = e.code if isinstance(e.code, int) else 1
    return _InProcResult(returncode, buf.getvalue())


class TestLoopDiagnosticsAndRemediation(unittest.TestCase):
    """Test gravity.py loop health and progressive remediation."""

    def test_fleet_loop_health_calculation(self):
        health = gravity.get_fleet_loop_health(threshold=0.5)
        self.assertIn("agents", health)
        self.assertIn("summary", health)
        self.assertIn("healthy", health["summary"])
        self.assertIn("overall_health", health["summary"])

    def test_reconstruct_loops(self):
        loops = gravity.reconstruct_loops(limit=10)
        self.assertIsInstance(loops, list)
        if loops:
            l = loops[0]
            self.assertIn("loop_id", l)
            self.assertIn("agent", l)
            self.assertIn("state", l)

    def test_diagnose_breaks_structure(self):
        breaks = gravity.diagnose_breaks()
        self.assertIsInstance(breaks, list)
        for b in breaks:
            self.assertIn("type", b)
            self.assertIn("severity", b)
            self.assertIn("detail", b)

    def test_remediate_breaks_dry_run(self):
        res = gravity.remediate_breaks(dry_run=True)
        self.assertTrue(res.get("ok"))
        self.assertTrue(res.get("dry_run"))
        self.assertIsInstance(res.get("remediated"), list)
        self.assertIsInstance(res.get("escalated"), list)

    def test_remediate_single_approval_scan(self):
        import approvals
        real = approvals.check_fleet_approvals
        with patch.object(approvals, "check_fleet_approvals", wraps=real) as spy:
            res = gravity.remediate_breaks(dry_run=True)
        self.assertTrue(res.get("ok"))
        self.assertEqual(spy.call_count, 1)

    def test_remediate_single_loop_parse(self):
        real = gravity._load_loop_candidates
        with patch.object(gravity, "_load_loop_candidates", wraps=real) as spy:
            res = gravity.remediate_breaks(dry_run=True)
        self.assertTrue(res.get("ok"))
        self.assertEqual(spy.call_count, 1)


class TestLoopRpc(unittest.TestCase):
    """Test box-ctl.py allowlisted RPC actions for loops."""

    def test_box_ctl_loop_health(self):
        res = _box_ctl_inproc("loop-health")
        self.assertEqual(res.returncode, 0)
        data = json.loads(res.stdout)
        self.assertTrue(data.get("ok"))
        self.assertIn("agents", data)
        self.assertIn("summary", data)


    def test_box_ctl_loop_status(self):
        res = _box_ctl_inproc("loop-status", "--limit", "5")
        self.assertEqual(res.returncode, 0)
        data = json.loads(res.stdout)
        self.assertTrue(data.get("ok"))
        self.assertIn("loops", data)

    def test_box_ctl_loop_remediate_dry_run(self):
        res = _box_ctl_inproc("loop-remediate", "--dry-run")
        self.assertEqual(res.returncode, 0)
        data = json.loads(res.stdout)
        self.assertTrue(data.get("ok"))
        self.assertTrue(data.get("dry_run"))


if __name__ == "__main__":
    unittest.main()