feat: completion-enforcement loop (fallback, proof, emit-model, auditor)
Close the loop so dispatched work actually completes on bl: - on_no_result fallback in followup-sweeper (op + job forms via exec-constrained registry / job-dispatch), seeded on the three autonomy-pulse jobs; fallback_due() dedupes the gravity path - gravity.py: add __main__ entry (loop-remediator.timer was a no-op), 300s re-arm budget, fallback firing + stamp/skip logic - harvester: proof-of-result followups (result_has_evidence), acted-variant NACK, emit-model tool-hint wording - envelope: RESPONSE RULE states the emit model (agents EMIT directives verbatim; runtime executes; works from bare containers) - completion-audit.py + systemd 15-min timer: per-family funnel, swarm drain, followup backlog; digest DM when degraded, 6h heartbeat - tests/test_completion.py (29 tests), JOB-SPEC.md docs Tests: 67/67 focused green (completion + tool_calls).
This commit is contained in:
Muse Sidechat
@@ -0,0 +1,366 @@
|
||||
"""Tests for the completion-enforcement loop: on_no_result fallbacks,
|
||||
proof-of-result followups, the NACK acted-variant, and the auditor funnel."""
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def _load(mod_name, rel_path):
|
||||
spec = importlib.util.spec_from_file_location(mod_name, REPO_ROOT / rel_path)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
sw = _load("sweeper_completion", "bin/followup-sweeper.py")
|
||||
harv = _load("harvester_completion", "bin/response-harvester.py")
|
||||
aud = _load("completion_audit_mod", "bin/completion-audit.py")
|
||||
grav = _load("gravity_completion", "bin/gravity.py")
|
||||
|
||||
|
||||
class DeriveJobName(unittest.TestCase):
|
||||
def test_valid(self):
|
||||
self.assertEqual(
|
||||
sw.derive_job_name("autonomy-pulse-646-20261006-060000-ed9a26af"),
|
||||
"autonomy-pulse-646",
|
||||
)
|
||||
|
||||
def test_invalid_shapes(self):
|
||||
for bad in (None, "", "nonsense", "job-2026-1-abc",
|
||||
"nonexistent-job-20261006-060000-ed9a26af"):
|
||||
self.assertIsNone(sw.derive_job_name(bad), bad)
|
||||
|
||||
|
||||
class LoadJobFallback(unittest.TestCase):
|
||||
def test_seeded_pulse_jobs(self):
|
||||
for name in ("autonomy-pulse-646", "autonomy-pulse-pip",
|
||||
"autonomy-pulse-opm"):
|
||||
spec, err = sw.load_job_fallback(name)
|
||||
self.assertIsNone(err, name)
|
||||
self.assertIsNotNone(spec, name)
|
||||
self.assertEqual(spec["op"], "swarm.spawn")
|
||||
self.assertEqual(spec["args"]["count"], 2)
|
||||
|
||||
def test_absent_is_none(self):
|
||||
spec, err = sw.load_job_fallback("646-daily-checkin")
|
||||
self.assertIsNone(spec)
|
||||
self.assertIsNone(err)
|
||||
|
||||
|
||||
class RunNoResultFallback(unittest.TestCase):
|
||||
def test_unresolvable_rec(self):
|
||||
out = sw.run_no_result_fallback({"dm_id": "x"})
|
||||
self.assertFalse(out["ran"])
|
||||
self.assertFalse(out["configured"])
|
||||
|
||||
def test_job_without_spec(self):
|
||||
out = sw.run_no_result_fallback(
|
||||
{"dm_id": "x", "job_id": "646-daily-checkin-20261006-090000-abcdef12",
|
||||
"recipient": "646"})
|
||||
self.assertFalse(out["ran"])
|
||||
self.assertFalse(out["configured"])
|
||||
|
||||
def test_dry_run_marks_configured(self):
|
||||
out = sw.run_no_result_fallback(
|
||||
{"dm_id": "x",
|
||||
"job_id": "autonomy-pulse-646-20261006-060000-ed9a26af",
|
||||
"recipient": "646"},
|
||||
dry_run=True)
|
||||
self.assertTrue(out["configured"])
|
||||
self.assertFalse(out["ran"])
|
||||
|
||||
def test_op_path_uses_validated_build(self):
|
||||
calls = {}
|
||||
|
||||
class FakeOpError(Exception):
|
||||
pass
|
||||
|
||||
def fake_validate(args):
|
||||
calls["validated"] = dict(args)
|
||||
return {"echo": "yes"}
|
||||
|
||||
def fake_build(clean):
|
||||
calls["built"] = clean
|
||||
return ["/bin/echo", "fallback-ok"]
|
||||
|
||||
fake_mod = types.SimpleNamespace(
|
||||
OPS={"probe.op": {"validate": fake_validate, "build": fake_build,
|
||||
"timeout": 10}})
|
||||
orig_exec, orig_load, orig_derive = (
|
||||
sw._load_exec_ops, sw.load_job_fallback, sw.derive_job_name)
|
||||
sw._load_exec_ops = lambda: fake_mod
|
||||
sw.load_job_fallback = lambda name: (
|
||||
({"op": "probe.op", "args": {"a": 1}}, None))
|
||||
sw.derive_job_name = lambda jid: "autonomy-pulse-646"
|
||||
try:
|
||||
out = sw.run_no_result_fallback(
|
||||
{"dm_id": "x", "job_id": "whatever", "recipient": "646"})
|
||||
finally:
|
||||
sw._load_exec_ops, sw.load_job_fallback, sw.derive_job_name = (
|
||||
orig_exec, orig_load, orig_derive)
|
||||
self.assertTrue(out["ran"], out)
|
||||
self.assertEqual(out["mode"], "op")
|
||||
self.assertEqual(calls["validated"], {"a": 1})
|
||||
self.assertEqual(calls["built"], {"echo": "yes"})
|
||||
self.assertIn("fallback-ok", out["detail"])
|
||||
|
||||
def test_unknown_op_is_outcome_not_raise(self):
|
||||
orig_exec, orig_load, orig_derive = (
|
||||
sw._load_exec_ops, sw.load_job_fallback, sw.derive_job_name)
|
||||
sw._load_exec_ops = lambda: types.SimpleNamespace(OPS={})
|
||||
sw.load_job_fallback = lambda name: ({"op": "nope.nope"}, None)
|
||||
sw.derive_job_name = lambda jid: "autonomy-pulse-646"
|
||||
try:
|
||||
out = sw.run_no_result_fallback(
|
||||
{"dm_id": "x", "job_id": "whatever", "recipient": "646"})
|
||||
finally:
|
||||
sw._load_exec_ops, sw.load_job_fallback, sw.derive_job_name = (
|
||||
orig_exec, orig_load, orig_derive)
|
||||
self.assertTrue(out["configured"])
|
||||
self.assertFalse(out["ran"])
|
||||
self.assertIn("unknown op", out["detail"])
|
||||
|
||||
|
||||
class FallbackDue(unittest.TestCase):
|
||||
def test_fresh_record_due(self):
|
||||
self.assertTrue(sw.fallback_due({"dm_id": "x"}))
|
||||
|
||||
def test_ran_never_due(self):
|
||||
self.assertFalse(sw.fallback_due(
|
||||
{"fallback": {"ran": True, "ts": "2026-10-06T00:00:00+00:00"}}))
|
||||
|
||||
def test_failed_recent_not_due(self):
|
||||
from datetime import datetime, timezone, timedelta
|
||||
ts = (datetime.now(timezone.utc) - timedelta(minutes=5)).isoformat()
|
||||
self.assertFalse(sw.fallback_due(
|
||||
{"fallback": {"ran": False, "ts": ts}}))
|
||||
|
||||
def test_failed_old_due(self):
|
||||
self.assertTrue(sw.fallback_due(
|
||||
{"fallback": {"ran": False, "ts": "2026-10-05T00:00:00+00:00"}}))
|
||||
|
||||
|
||||
class GravityFallback(unittest.TestCase):
|
||||
def test_dry_run_configured(self):
|
||||
rec = {"dm_id": "x",
|
||||
"job_id": "autonomy-pulse-646-20261006-060000-ed9a26af",
|
||||
"recipient": "646"}
|
||||
out = grav.maybe_run_terminal_fallback(
|
||||
rec, "2026-10-06T08:00:00+00:00", dry_run=True)
|
||||
self.assertIsNotNone(out)
|
||||
self.assertFalse(out["ran"])
|
||||
self.assertNotIn("fallback", rec)
|
||||
|
||||
def test_no_spec_returns_none(self):
|
||||
rec = {"dm_id": "x",
|
||||
"job_id": "646-daily-checkin-20261006-090000-abcdef12",
|
||||
"recipient": "646"}
|
||||
self.assertIsNone(grav.maybe_run_terminal_fallback(
|
||||
rec, "2026-10-06T08:00:00+00:00", dry_run=True))
|
||||
|
||||
def test_already_ran_returns_none(self):
|
||||
rec = {"dm_id": "x",
|
||||
"job_id": "autonomy-pulse-646-20261006-060000-ed9a26af",
|
||||
"recipient": "646",
|
||||
"fallback": {"ran": True, "ts": "2026-10-06T07:00:00+00:00"}}
|
||||
self.assertIsNone(grav.maybe_run_terminal_fallback(
|
||||
rec, "2026-10-06T08:00:00+00:00", dry_run=True))
|
||||
|
||||
|
||||
class GravityEntry(unittest.TestCase):
|
||||
def test_no_args_prints_help(self):
|
||||
import io
|
||||
from contextlib import redirect_stdout
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
rc = grav.main([])
|
||||
self.assertEqual(rc, 2)
|
||||
self.assertIn("remediate", buf.getvalue())
|
||||
|
||||
def test_remediate_dry_run_returns_json(self):
|
||||
import io
|
||||
from contextlib import redirect_stdout
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
rc = grav.main(["--remediate", "--dry-run"])
|
||||
self.assertEqual(rc, 0)
|
||||
data = json.loads(buf.getvalue())
|
||||
self.assertTrue(data["ok"])
|
||||
self.assertTrue(data["dry_run"])
|
||||
|
||||
|
||||
class ResultEvidence(unittest.TestCase):
|
||||
def test_positives(self):
|
||||
for text in (
|
||||
"done, swarm sw-20261006-060000-ab12 reported 2/2",
|
||||
"wrote /tmp/out.json with 40 rows",
|
||||
"timer id: pulse-15m restarted",
|
||||
"thread 1dfb3199-2f99-446c-83e3-848ae2da0a12 swept",
|
||||
"3/3 slots complete",
|
||||
):
|
||||
self.assertTrue(harv.result_has_evidence(text), text)
|
||||
|
||||
def test_negatives(self):
|
||||
for text in ("OK all good", "done, nothing to report", "", None):
|
||||
self.assertFalse(harv.result_has_evidence(text), repr(text))
|
||||
|
||||
|
||||
class ProofRequest(unittest.TestCase):
|
||||
def _patch(self, tmp_path):
|
||||
orig = (harv.NUDGE_TRACKER_FILE, harv.JOB_LOG, harv.execute_agent_tool)
|
||||
harv.NUDGE_TRACKER_FILE = tmp_path / "tracker.json"
|
||||
harv.JOB_LOG = tmp_path / "job-log.jsonl"
|
||||
calls = []
|
||||
harv.execute_agent_tool = lambda a, op, args: (
|
||||
calls.append((a, op, args)), (True, "scheduled"))[1]
|
||||
return orig, calls
|
||||
|
||||
def test_bare_result_schedules_proof(self):
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
orig, calls = self._patch(Path(td))
|
||||
try:
|
||||
ok = harv.maybe_request_proof(
|
||||
"646", "1dfb3199-2f99-446c-83e3-848ae2da0a12",
|
||||
"job-1", "OK all good")
|
||||
finally:
|
||||
(harv.NUDGE_TRACKER_FILE, harv.JOB_LOG,
|
||||
harv.execute_agent_tool) = orig
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual(calls[0][1], "followup.create")
|
||||
self.assertEqual(calls[0][2]["in_m"], 30)
|
||||
self.assertIn("PROOF", calls[0][2]["prompt"])
|
||||
|
||||
def test_evidence_skips(self):
|
||||
ok = harv.maybe_request_proof(
|
||||
"646", "1dfb3199-2f99-446c-83e3-848ae2da0a12",
|
||||
"job-1", "done, swarm sw-20261006-060000-ab12")
|
||||
self.assertFalse(ok)
|
||||
|
||||
def test_bad_thread_and_dry_run_skip(self):
|
||||
self.assertFalse(harv.maybe_request_proof(
|
||||
"646", "main", "job-1", "OK"))
|
||||
self.assertFalse(harv.maybe_request_proof(
|
||||
"646", "1dfb3199-2f99-446c-83e3-848ae2da0a12",
|
||||
"job-1", "OK", dry_run=True))
|
||||
|
||||
def test_one_shot_per_job(self):
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
orig, calls = self._patch(Path(td))
|
||||
try:
|
||||
kw = dict(agent="646",
|
||||
thread_id="1dfb3199-2f99-446c-83e3-848ae2da0a12",
|
||||
job_id="job-9", result_text="OK")
|
||||
self.assertTrue(harv.maybe_request_proof(**kw))
|
||||
self.assertFalse(harv.maybe_request_proof(**kw))
|
||||
finally:
|
||||
(harv.NUDGE_TRACKER_FILE, harv.JOB_LOG,
|
||||
harv.execute_agent_tool) = orig
|
||||
self.assertEqual(len(calls), 1)
|
||||
|
||||
|
||||
class NackActedVariant(unittest.TestCase):
|
||||
def test_acted_variant_text(self):
|
||||
import tempfile
|
||||
sent = []
|
||||
fake = types.ModuleType("muse_hybrid")
|
||||
fake.send_message = lambda a, m, thread_id=None, wait=0: sent.append(m)
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
tp = Path(td)
|
||||
(tp / "followups.json").write_text(json.dumps({
|
||||
"m1": {"status": "pending",
|
||||
"thread_uuid": "1dfb3199-2f99-446c-83e3-848ae2da0a12",
|
||||
"job_id": "job-7", "recipient": "646"}}))
|
||||
orig = (harv.NUDGE_TRACKER_FILE, harv.FOLLOWUPS_FILE,
|
||||
sys.modules.get("muse_hybrid"))
|
||||
harv.NUDGE_TRACKER_FILE = tp / "tracker.json"
|
||||
harv.FOLLOWUPS_FILE = tp / "followups.json"
|
||||
sys.modules["muse_hybrid"] = fake
|
||||
try:
|
||||
harv.maybe_nudge_untagged_sidechat(
|
||||
"646", "1dfb3199-2f99-446c-83e3-848ae2da0a12",
|
||||
"646 tasks", "mid-1", "some prose", acted=True)
|
||||
finally:
|
||||
(harv.NUDGE_TRACKER_FILE, harv.FOLLOWUPS_FILE,
|
||||
old_mod) = orig
|
||||
if old_mod is None:
|
||||
sys.modules.pop("muse_hybrid", None)
|
||||
else:
|
||||
sys.modules["muse_hybrid"] = old_mod
|
||||
self.assertEqual(len(sent), 1)
|
||||
self.assertIn("Action received", sent[0])
|
||||
self.assertIn("[RESULT job-7]", sent[0])
|
||||
self.assertNotIn("STRICT ENFORCEMENT", sent[0])
|
||||
|
||||
|
||||
class AuditorFunnel(unittest.TestCase):
|
||||
def _events(self):
|
||||
base = "2026-10-06T07:00:00+00:00"
|
||||
return [
|
||||
{"ts": base, "type": "job_sent",
|
||||
"job_id": "work-finder-20261006-070000-aaaaaaaa"},
|
||||
{"ts": base, "type": "job_dispatched",
|
||||
"job_id": "work-finder-20261006-070000-aaaaaaaa"},
|
||||
{"ts": base, "type": "tool_exec", "op": "swarm.spawn",
|
||||
"success": True},
|
||||
{"ts": base, "type": "job_result",
|
||||
"job_id": "work-finder-20261006-070000-aaaaaaaa",
|
||||
"success": True},
|
||||
{"ts": base, "type": "job_sent",
|
||||
"job_id": "pulse-20261006-070000-bbbbbbbb"},
|
||||
{"ts": base, "type": "job_failed",
|
||||
"job_id": "pulse-20261006-070000-bbbbbbbb"},
|
||||
]
|
||||
|
||||
def test_funnel_counts(self):
|
||||
from datetime import datetime, timezone
|
||||
fam, tools = aud.compute_funnel(
|
||||
self._events(), datetime(2026, 10, 6, 6, 0, tzinfo=timezone.utc))
|
||||
self.assertEqual(fam["work-finder"]["sent"], 1)
|
||||
self.assertEqual(fam["work-finder"]["results"], 1)
|
||||
self.assertEqual(fam["pulse"]["failed"], 1)
|
||||
self.assertEqual(tools["tools"]["total"], 1)
|
||||
self.assertEqual(tools["tools"]["ok"], 1)
|
||||
|
||||
def test_family_of(self):
|
||||
self.assertEqual(aud.family_of("a-b-20261006-070000-aaaaaaaa"), "a-b")
|
||||
self.assertEqual(aud.family_of("weird"), "weird")
|
||||
|
||||
def test_digest_verdicts(self):
|
||||
healthy = {"ts": "2026-10-06T07:00:00+00:00", "window_h": 6,
|
||||
"totals": {"sent": 4, "dispatched": 4, "results": 4,
|
||||
"ok": 4},
|
||||
"tools": {"tools": {"total": 3, "ok": 3}, "tool_errs": {}},
|
||||
"families": {}, "silent_families": [], "swarms": {},
|
||||
"stale_running": [], "followups": {"pending": 1},
|
||||
"degraded": False, "reasons": []}
|
||||
out = aud.render_digest(healthy)
|
||||
self.assertIn("HEALTHY", out)
|
||||
self.assertIn("4 sent", out)
|
||||
bad = dict(healthy, degraded=True,
|
||||
reasons=["1 silent families: x"],
|
||||
silent_families=["x"])
|
||||
self.assertIn("DEGRADED", aud.render_digest(bad))
|
||||
|
||||
def test_should_post_policy(self):
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
orig = aud.STATE_FILE
|
||||
aud.STATE_FILE = Path(td) / "state.json"
|
||||
try:
|
||||
self.assertTrue(aud.should_post({"degraded": True})[0])
|
||||
ok, why = aud.should_post({"degraded": False})
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual(why, "heartbeat")
|
||||
finally:
|
||||
aud.STATE_FILE = orig
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user