feat: completion-enforcement loop (fallback, proof, emit-model, auditor)
Close the loop so dispatched work actually completes on bl: - on_no_result fallback in followup-sweeper (op + job forms via exec-constrained registry / job-dispatch), seeded on the three autonomy-pulse jobs; fallback_due() dedupes the gravity path - gravity.py: add __main__ entry (loop-remediator.timer was a no-op), 300s re-arm budget, fallback firing + stamp/skip logic - harvester: proof-of-result followups (result_has_evidence), acted-variant NACK, emit-model tool-hint wording - envelope: RESPONSE RULE states the emit model (agents EMIT directives verbatim; runtime executes; works from bare containers) - completion-audit.py + systemd 15-min timer: per-family funnel, swarm drain, followup backlog; digest DM when degraded, 6h heartbeat - tests/test_completion.py (29 tests), JOB-SPEC.md docs Tests: 67/67 focused green (completion + tool_calls).
This commit is contained in:
+94
-1
@@ -657,6 +657,62 @@ def diagnose_breaks() -> list:
|
||||
return breaks
|
||||
|
||||
|
||||
def _load_sweeper_module():
|
||||
import importlib.util
|
||||
mod_spec = importlib.util.spec_from_file_location(
|
||||
"followup_sweeper_gravity",
|
||||
str(Path(__file__).resolve().parent / "followup-sweeper.py"))
|
||||
mod = importlib.util.module_from_spec(mod_spec)
|
||||
mod_spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
def maybe_run_terminal_fallback(rec, now_iso, dry_run=False):
|
||||
"""Run a job's on_no_result fallback once at terminal followup expiry.
|
||||
|
||||
Returns an outcome dict, or None when the record has no resolvable
|
||||
fallback. Never raises. The sweeper terminal path shares the
|
||||
fallback_due() guard, so the two firing paths can't double-execute.
|
||||
"""
|
||||
try:
|
||||
sw = _load_sweeper_module()
|
||||
except Exception as e:
|
||||
return {"ran": False, "mode": None,
|
||||
"detail": f"sweeper import failed: {e}"}
|
||||
try:
|
||||
if not sw.fallback_due(rec):
|
||||
return None
|
||||
job_name = sw.derive_job_name(rec.get("job_id"))
|
||||
if not job_name:
|
||||
return None
|
||||
spec, err = sw.load_job_fallback(job_name)
|
||||
if err or spec is None:
|
||||
return None
|
||||
if dry_run:
|
||||
return {"ran": False, "mode": "dry_run",
|
||||
"detail": json.dumps(spec)[:200]}
|
||||
out = sw.run_no_result_fallback(rec)
|
||||
rec["fallback"] = {"ran": out["ran"], "mode": out["mode"],
|
||||
"detail": out["detail"][:200], "ts": now_iso}
|
||||
try:
|
||||
sw.append_job_log({
|
||||
"ts": now_iso,
|
||||
"type": ("fallback_executed" if out["ran"]
|
||||
else "fallback_failed"),
|
||||
"dm_id": rec.get("dm_id"),
|
||||
"recipient": rec.get("recipient"),
|
||||
"job_id": rec.get("job_id"),
|
||||
"mode": out["mode"],
|
||||
"detail": out["detail"][:300],
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
return out
|
||||
except Exception as e:
|
||||
return {"ran": False, "mode": None,
|
||||
"detail": f"fallback exception: {e}"}
|
||||
|
||||
|
||||
def remediate_breaks(dry_run=False) -> dict:
|
||||
"""Progressively auto-remediate soft loop breakages while escalating hard breakages.
|
||||
|
||||
@@ -746,6 +802,20 @@ def remediate_breaks(dry_run=False) -> dict:
|
||||
f_modified = True
|
||||
rearm_sweeper = True
|
||||
|
||||
# Terminal: nudges exhausted and still no reply. Run the
|
||||
# job's on_no_result fallback (server-side guarantee).
|
||||
if is_expired and nudges_sent >= nudges_allowed:
|
||||
fb = maybe_run_terminal_fallback(rec, now_iso, dry_run)
|
||||
if fb is not None:
|
||||
remediated.append({
|
||||
"action": "terminal_fallback",
|
||||
"loop_id": dm_id,
|
||||
"agent": rec.get("recipient"),
|
||||
"detail": f"on_no_result ran={fb.get('ran')}: {fb.get('detail', '')[:160]}",
|
||||
})
|
||||
if not dry_run and "fallback" in rec:
|
||||
f_modified = True
|
||||
|
||||
if f_modified and not dry_run:
|
||||
tmp = f"{f_path}.tmp.{os.getpid()}"
|
||||
with open(tmp, "w") as f:
|
||||
@@ -756,7 +826,10 @@ def remediate_breaks(dry_run=False) -> dict:
|
||||
sweeper_py = Path("/home/super/Projects/NetVM/bin/followup-sweeper.py")
|
||||
if sweeper_py.exists():
|
||||
try:
|
||||
subprocess.run([sys.executable, str(sweeper_py), "--once"], timeout=10)
|
||||
# A single nudge send takes ~10s median; give the sweep
|
||||
# room to finish or it dies mid-first-send every time.
|
||||
subprocess.run([sys.executable, str(sweeper_py), "--once"],
|
||||
timeout=300)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -819,4 +892,24 @@ def remediate_breaks(dry_run=False) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
import argparse
|
||||
ap = argparse.ArgumentParser(description="Loop gravity: reconcile and remediate followup loops")
|
||||
ap.add_argument("--remediate", action="store_true",
|
||||
help="Run remediate_breaks once (what loop-remediator.timer invokes)")
|
||||
ap.add_argument("--dry-run", action="store_true",
|
||||
help="Report actions without writing state or sending anything")
|
||||
args = ap.parse_args(argv)
|
||||
if not args.remediate:
|
||||
ap.print_help()
|
||||
return 2
|
||||
result = remediate_breaks(dry_run=args.dry_run)
|
||||
print(json.dumps(result, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user