operator
371 lines
14 KiB
Python
371 lines
14 KiB
Python
"""Tests for the completion-enforcement loop: on_no_result fallbacks,
|
|
proof-of-result followups, the NACK acted-variant, and the auditor funnel."""
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
import types
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
|
|
def _load(mod_name, rel_path):
|
|
spec = importlib.util.spec_from_file_location(mod_name, REPO_ROOT / rel_path)
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
return mod
|
|
|
|
|
|
sw = _load("sweeper_completion", "bin/followup-sweeper.py")
|
|
harv = _load("harvester_completion", "bin/response-harvester.py")
|
|
aud = _load("completion_audit_mod", "bin/completion-audit.py")
|
|
grav = _load("gravity_completion", "bin/gravity.py")
|
|
|
|
|
|
class DeriveJobName(unittest.TestCase):
|
|
def test_valid(self):
|
|
self.assertEqual(
|
|
sw.derive_job_name("autonomy-pulse-646-20261006-060000-ed9a26af"),
|
|
"autonomy-pulse-646",
|
|
)
|
|
|
|
def test_invalid_shapes(self):
|
|
for bad in (None, "", "nonsense", "job-2026-1-abc",
|
|
"nonexistent-job-20261006-060000-ed9a26af"):
|
|
self.assertIsNone(sw.derive_job_name(bad), bad)
|
|
|
|
|
|
class LoadJobFallback(unittest.TestCase):
|
|
def test_seeded_pulse_jobs(self):
|
|
for name in ("autonomy-pulse-646", "autonomy-pulse-pip",
|
|
"autonomy-pulse-opm"):
|
|
spec, err = sw.load_job_fallback(name)
|
|
self.assertIsNone(err, name)
|
|
self.assertIsNotNone(spec, name)
|
|
self.assertEqual(spec["op"], "swarm.spawn")
|
|
self.assertEqual(spec["args"]["count"], 2)
|
|
|
|
def test_absent_is_none(self):
|
|
spec, err = sw.load_job_fallback("646-daily-checkin")
|
|
self.assertIsNone(spec)
|
|
self.assertIsNone(err)
|
|
|
|
|
|
class RunNoResultFallback(unittest.TestCase):
|
|
def test_unresolvable_rec(self):
|
|
out = sw.run_no_result_fallback({"dm_id": "x"})
|
|
self.assertFalse(out["ran"])
|
|
self.assertFalse(out["configured"])
|
|
|
|
def test_job_without_spec(self):
|
|
out = sw.run_no_result_fallback(
|
|
{"dm_id": "x", "job_id": "646-daily-checkin-20261006-090000-abcdef12",
|
|
"recipient": "646"})
|
|
self.assertFalse(out["ran"])
|
|
self.assertFalse(out["configured"])
|
|
|
|
def test_dry_run_marks_configured(self):
|
|
out = sw.run_no_result_fallback(
|
|
{"dm_id": "x",
|
|
"job_id": "autonomy-pulse-646-20261006-060000-ed9a26af",
|
|
"recipient": "646"},
|
|
dry_run=True)
|
|
self.assertTrue(out["configured"])
|
|
self.assertFalse(out["ran"])
|
|
|
|
def test_op_path_uses_validated_build(self):
|
|
calls = {}
|
|
|
|
class FakeOpError(Exception):
|
|
pass
|
|
|
|
def fake_validate(args):
|
|
calls["validated"] = dict(args)
|
|
return {"echo": "yes"}
|
|
|
|
def fake_build(clean):
|
|
calls["built"] = clean
|
|
return ["/bin/echo", "fallback-ok"]
|
|
|
|
fake_mod = types.SimpleNamespace(
|
|
OPS={"probe.op": {"validate": fake_validate, "build": fake_build,
|
|
"timeout": 10}})
|
|
orig_exec, orig_load, orig_derive = (
|
|
sw._load_exec_ops, sw.load_job_fallback, sw.derive_job_name)
|
|
sw._load_exec_ops = lambda: fake_mod
|
|
sw.load_job_fallback = lambda name: (
|
|
({"op": "probe.op", "args": {"a": 1}}, None))
|
|
sw.derive_job_name = lambda jid: "autonomy-pulse-646"
|
|
try:
|
|
out = sw.run_no_result_fallback(
|
|
{"dm_id": "x", "job_id": "whatever", "recipient": "646"})
|
|
finally:
|
|
sw._load_exec_ops, sw.load_job_fallback, sw.derive_job_name = (
|
|
orig_exec, orig_load, orig_derive)
|
|
self.assertTrue(out["ran"], out)
|
|
self.assertEqual(out["mode"], "op")
|
|
self.assertEqual(calls["validated"], {"a": 1})
|
|
self.assertEqual(calls["built"], {"echo": "yes"})
|
|
self.assertIn("fallback-ok", out["detail"])
|
|
|
|
def test_unknown_op_is_outcome_not_raise(self):
|
|
orig_exec, orig_load, orig_derive = (
|
|
sw._load_exec_ops, sw.load_job_fallback, sw.derive_job_name)
|
|
sw._load_exec_ops = lambda: types.SimpleNamespace(OPS={})
|
|
sw.load_job_fallback = lambda name: ({"op": "nope.nope"}, None)
|
|
sw.derive_job_name = lambda jid: "autonomy-pulse-646"
|
|
try:
|
|
out = sw.run_no_result_fallback(
|
|
{"dm_id": "x", "job_id": "whatever", "recipient": "646"})
|
|
finally:
|
|
sw._load_exec_ops, sw.load_job_fallback, sw.derive_job_name = (
|
|
orig_exec, orig_load, orig_derive)
|
|
self.assertTrue(out["configured"])
|
|
self.assertFalse(out["ran"])
|
|
self.assertIn("unknown op", out["detail"])
|
|
|
|
|
|
class FallbackDue(unittest.TestCase):
|
|
def test_fresh_record_due(self):
|
|
self.assertTrue(sw.fallback_due({"dm_id": "x"}))
|
|
|
|
def test_ran_never_due(self):
|
|
self.assertFalse(sw.fallback_due(
|
|
{"fallback": {"ran": True, "ts": "2026-10-06T00:00:00+00:00"}}))
|
|
|
|
def test_failed_recent_not_due(self):
|
|
from datetime import datetime, timezone, timedelta
|
|
ts = (datetime.now(timezone.utc) - timedelta(minutes=5)).isoformat()
|
|
self.assertFalse(sw.fallback_due(
|
|
{"fallback": {"ran": False, "ts": ts}}))
|
|
|
|
def test_failed_old_due(self):
|
|
self.assertTrue(sw.fallback_due(
|
|
{"fallback": {"ran": False, "ts": "2026-10-05T00:00:00+00:00"}}))
|
|
|
|
|
|
class GravityFallback(unittest.TestCase):
|
|
def test_dry_run_configured(self):
|
|
rec = {"dm_id": "x",
|
|
"job_id": "autonomy-pulse-646-20261006-060000-ed9a26af",
|
|
"recipient": "646"}
|
|
out = grav.maybe_run_terminal_fallback(
|
|
rec, "2026-10-06T08:00:00+00:00", dry_run=True)
|
|
self.assertIsNotNone(out)
|
|
self.assertFalse(out["ran"])
|
|
self.assertNotIn("fallback", rec)
|
|
|
|
def test_no_spec_returns_none(self):
|
|
rec = {"dm_id": "x",
|
|
"job_id": "646-daily-checkin-20261006-090000-abcdef12",
|
|
"recipient": "646"}
|
|
self.assertIsNone(grav.maybe_run_terminal_fallback(
|
|
rec, "2026-10-06T08:00:00+00:00", dry_run=True))
|
|
|
|
def test_already_ran_returns_none(self):
|
|
rec = {"dm_id": "x",
|
|
"job_id": "autonomy-pulse-646-20261006-060000-ed9a26af",
|
|
"recipient": "646",
|
|
"fallback": {"ran": True, "ts": "2026-10-06T07:00:00+00:00"}}
|
|
self.assertIsNone(grav.maybe_run_terminal_fallback(
|
|
rec, "2026-10-06T08:00:00+00:00", dry_run=True))
|
|
|
|
|
|
class GravityEntry(unittest.TestCase):
|
|
def test_no_args_prints_help(self):
|
|
import io
|
|
from contextlib import redirect_stdout
|
|
buf = io.StringIO()
|
|
with redirect_stdout(buf):
|
|
rc = grav.main([])
|
|
self.assertEqual(rc, 2)
|
|
self.assertIn("remediate", buf.getvalue())
|
|
|
|
def test_remediate_dry_run_returns_json(self):
|
|
import io
|
|
from contextlib import redirect_stdout
|
|
buf = io.StringIO()
|
|
with redirect_stdout(buf):
|
|
rc = grav.main(["--remediate", "--dry-run"])
|
|
self.assertEqual(rc, 0)
|
|
data = json.loads(buf.getvalue())
|
|
self.assertTrue(data["ok"])
|
|
self.assertTrue(data["dry_run"])
|
|
|
|
|
|
class ResultEvidence(unittest.TestCase):
|
|
def test_positives(self):
|
|
for text in (
|
|
"done, swarm sw-20261006-060000-ab12 reported 2/2",
|
|
"wrote /tmp/out.json with 40 rows",
|
|
"timer id: pulse-15m restarted",
|
|
"thread 1dfb3199-2f99-446c-83e3-848ae2da0a12 swept",
|
|
"3/3 slots complete",
|
|
):
|
|
self.assertTrue(harv.result_has_evidence(text), text)
|
|
|
|
def test_negatives(self):
|
|
for text in ("OK all good", "done, nothing to report", "", None):
|
|
self.assertFalse(harv.result_has_evidence(text), repr(text))
|
|
|
|
|
|
class ProofRequest(unittest.TestCase):
|
|
def _patch(self, tmp_path):
|
|
orig = (harv.NUDGE_TRACKER_FILE, harv.JOB_LOG, harv.execute_agent_tool,
|
|
getattr(harv, "PROOF_REQUESTS_ENABLED", False))
|
|
harv.NUDGE_TRACKER_FILE = tmp_path / "tracker.json"
|
|
harv.JOB_LOG = tmp_path / "job-log.jsonl"
|
|
harv.PROOF_REQUESTS_ENABLED = True
|
|
calls = []
|
|
harv.execute_agent_tool = lambda a, op, args: (
|
|
calls.append((a, op, args)), (True, "scheduled"))[1]
|
|
return orig, calls
|
|
|
|
def test_bare_result_schedules_proof(self):
|
|
import tempfile
|
|
with tempfile.TemporaryDirectory() as td:
|
|
orig, calls = self._patch(Path(td))
|
|
try:
|
|
ok = harv.maybe_request_proof(
|
|
"646", "1dfb3199-2f99-446c-83e3-848ae2da0a12",
|
|
"job-1", "OK all good")
|
|
finally:
|
|
(harv.NUDGE_TRACKER_FILE, harv.JOB_LOG,
|
|
harv.execute_agent_tool,
|
|
harv.PROOF_REQUESTS_ENABLED) = orig
|
|
self.assertTrue(ok)
|
|
self.assertEqual(calls[0][1], "followup.create")
|
|
self.assertEqual(calls[0][2]["in_m"], 30)
|
|
self.assertIn("PROOF", calls[0][2]["prompt"])
|
|
|
|
def test_evidence_skips(self):
|
|
ok = harv.maybe_request_proof(
|
|
"646", "1dfb3199-2f99-446c-83e3-848ae2da0a12",
|
|
"job-1", "done, swarm sw-20261006-060000-ab12")
|
|
self.assertFalse(ok)
|
|
|
|
def test_bad_thread_and_dry_run_skip(self):
|
|
self.assertFalse(harv.maybe_request_proof(
|
|
"646", "main", "job-1", "OK"))
|
|
self.assertFalse(harv.maybe_request_proof(
|
|
"646", "1dfb3199-2f99-446c-83e3-848ae2da0a12",
|
|
"job-1", "OK", dry_run=True))
|
|
|
|
def test_one_shot_per_job(self):
|
|
import tempfile
|
|
with tempfile.TemporaryDirectory() as td:
|
|
orig, calls = self._patch(Path(td))
|
|
try:
|
|
kw = dict(agent="646",
|
|
thread_id="1dfb3199-2f99-446c-83e3-848ae2da0a12",
|
|
job_id="job-9", result_text="OK")
|
|
self.assertTrue(harv.maybe_request_proof(**kw))
|
|
self.assertFalse(harv.maybe_request_proof(**kw))
|
|
finally:
|
|
(harv.NUDGE_TRACKER_FILE, harv.JOB_LOG,
|
|
harv.execute_agent_tool,
|
|
harv.PROOF_REQUESTS_ENABLED) = orig
|
|
self.assertEqual(len(calls), 1)
|
|
|
|
|
|
class NackActedVariant(unittest.TestCase):
|
|
def test_acted_variant_text(self):
|
|
import tempfile
|
|
sent = []
|
|
fake = types.ModuleType("muse_hybrid")
|
|
fake.send_message = lambda a, m, thread_id=None, wait=0: sent.append(m)
|
|
with tempfile.TemporaryDirectory() as td:
|
|
tp = Path(td)
|
|
(tp / "followups.json").write_text(json.dumps({
|
|
"m1": {"status": "pending",
|
|
"thread_uuid": "1dfb3199-2f99-446c-83e3-848ae2da0a12",
|
|
"job_id": "job-7", "recipient": "646"}}))
|
|
orig = (harv.NUDGE_TRACKER_FILE, harv.FOLLOWUPS_FILE,
|
|
sys.modules.get("muse_hybrid"))
|
|
harv.NUDGE_TRACKER_FILE = tp / "tracker.json"
|
|
harv.FOLLOWUPS_FILE = tp / "followups.json"
|
|
sys.modules["muse_hybrid"] = fake
|
|
try:
|
|
harv.maybe_nudge_untagged_sidechat(
|
|
"646", "1dfb3199-2f99-446c-83e3-848ae2da0a12",
|
|
"646 tasks", "mid-1", "some prose", acted=True)
|
|
finally:
|
|
(harv.NUDGE_TRACKER_FILE, harv.FOLLOWUPS_FILE,
|
|
old_mod) = orig
|
|
if old_mod is None:
|
|
sys.modules.pop("muse_hybrid", None)
|
|
else:
|
|
sys.modules["muse_hybrid"] = old_mod
|
|
self.assertEqual(len(sent), 1)
|
|
self.assertIn("Action received", sent[0])
|
|
self.assertIn("[RESULT job-7]", sent[0])
|
|
self.assertNotIn("STRICT ENFORCEMENT", sent[0])
|
|
|
|
|
|
class AuditorFunnel(unittest.TestCase):
|
|
def _events(self):
|
|
base = "2026-10-06T07:00:00+00:00"
|
|
return [
|
|
{"ts": base, "type": "job_sent",
|
|
"job_id": "work-finder-20261006-070000-aaaaaaaa"},
|
|
{"ts": base, "type": "job_dispatched",
|
|
"job_id": "work-finder-20261006-070000-aaaaaaaa"},
|
|
{"ts": base, "type": "tool_exec", "op": "swarm.spawn",
|
|
"success": True},
|
|
{"ts": base, "type": "job_result",
|
|
"job_id": "work-finder-20261006-070000-aaaaaaaa",
|
|
"success": True},
|
|
{"ts": base, "type": "job_sent",
|
|
"job_id": "pulse-20261006-070000-bbbbbbbb"},
|
|
{"ts": base, "type": "job_failed",
|
|
"job_id": "pulse-20261006-070000-bbbbbbbb"},
|
|
]
|
|
|
|
def test_funnel_counts(self):
|
|
from datetime import datetime, timezone
|
|
fam, tools = aud.compute_funnel(
|
|
self._events(), datetime(2026, 10, 6, 6, 0, tzinfo=timezone.utc))
|
|
self.assertEqual(fam["work-finder"]["sent"], 1)
|
|
self.assertEqual(fam["work-finder"]["results"], 1)
|
|
self.assertEqual(fam["pulse"]["failed"], 1)
|
|
self.assertEqual(tools["tools"]["total"], 1)
|
|
self.assertEqual(tools["tools"]["ok"], 1)
|
|
|
|
def test_family_of(self):
|
|
self.assertEqual(aud.family_of("a-b-20261006-070000-aaaaaaaa"), "a-b")
|
|
self.assertEqual(aud.family_of("weird"), "weird")
|
|
|
|
def test_digest_verdicts(self):
|
|
healthy = {"ts": "2026-10-06T07:00:00+00:00", "window_h": 6,
|
|
"totals": {"sent": 4, "dispatched": 4, "results": 4,
|
|
"ok": 4},
|
|
"tools": {"tools": {"total": 3, "ok": 3}, "tool_errs": {}},
|
|
"families": {}, "silent_families": [], "swarms": {},
|
|
"stale_running": [], "followups": {"pending": 1},
|
|
"degraded": False, "reasons": []}
|
|
out = aud.render_digest(healthy)
|
|
self.assertIn("HEALTHY", out)
|
|
self.assertIn("4 sent", out)
|
|
bad = dict(healthy, degraded=True,
|
|
reasons=["1 silent families: x"],
|
|
silent_families=["x"])
|
|
self.assertIn("DEGRADED", aud.render_digest(bad))
|
|
|
|
def test_should_post_policy(self):
|
|
import tempfile
|
|
with tempfile.TemporaryDirectory() as td:
|
|
orig = aud.STATE_FILE
|
|
aud.STATE_FILE = Path(td) / "state.json"
|
|
try:
|
|
self.assertTrue(aud.should_post({"degraded": True})[0])
|
|
ok, why = aud.should_post({"degraded": False})
|
|
self.assertTrue(ok)
|
|
self.assertEqual(why, "heartbeat")
|
|
finally:
|
|
aud.STATE_FILE = orig
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|