feat(tmux): add server death watchdog daemon and multi-socket approver enhancements

bin/tmux_auto_approver.py

@@ -285,13 +285,44 @@ def run_tmux_cmd(socket_path: str, *args: str, timeout: float = 3.0) -> Tuple[in
 
 
 def capture_pane_text(socket_path: str, pane_id: str, lines: int = 30) -> str:
-    """Capture recent lines from a pane."""
-    rc, out, _ = run_tmux_cmd(socket_path, "capture-pane", "-p", "-t", pane_id, "-S", f"-{lines}")
+    """Capture recent lines from a pane, joining wrapped rows.
+
+    -J joins physical wrapped lines into logical lines so matching is
+    width-independent: narrow panes wrap the same dialog onto more
+    rows, which otherwise breaks cue/option regexes.
+    """
+    rc, out, _ = run_tmux_cmd(socket_path, "capture-pane", "-p", "-J",
+                              "-t", pane_id, "-S", f"-{lines}")
     if rc == 0:
         return out
     return ""
 
 
+MUSE_COMMAND_HINTS = ("muse-bin", "muse-code")
+
+
+def should_defer_to_muse_watcher(socket_path: str, pane_id: str,
+                                 current_command: str) -> bool:
+    """True when a per-pane muse watcher owns this pane.
+
+    Single-owner rule: muse_choice_watcher is authoritative for muse
+    panes (stability + re-verify + once-per-prompt + decided-block
+    guard). When its daemon is alive for this socket:pane, tmux must
+    skip the pane entirely, or both daemons answer the same prompt
+    within the same second ('11' + stray keys, observed live). Never
+    raises: import or liveness failures mean no owner, handle here.
+    """
+    try:
+        cmd = current_command or ""
+        if not any(h in cmd for h in MUSE_COMMAND_HINTS):
+            return False
+        import muse_choice_watcher as mcw
+        alive = getattr(mcw, "watcher_alive", mcw.is_running)
+        return alive(socket_path, pane_id) is not None
+    except Exception:
+        return False
+
+
 def gather_tmux_tally(state: Optional[AutoApproverState] = None) -> TmuxWorkerTally:
     """Scan all sockets and build a comprehensive tally of tmux workers."""
     if state is None:
@@ -497,6 +528,9 @@ class AutoApproverRunner:
         for p in tally.panes:
             if not p.auto_approve:
                 continue
+            if should_defer_to_muse_watcher(p.socket, p.pane_id,
+                                            p.current_command):
+                continue
 
             text = capture_pane_text(p.socket, p.pane_id, lines=30)
             if not text:
@@ -516,9 +550,12 @@ class AutoApproverRunner:
                 continue
 
             if verdict.matched and verdict.key:
-                # Deduplicate identical prompt to avoid infinite loop
+                # Deduplicate identical prompt to avoid infinite loop.
+                # Keyed by socket:pane: bare pane ids repeat on every
+                # tmux socket, so %1 on pip must not suppress %1 on opm.
                 sig = hashlib.sha1(f"{verdict.rule_id}:{verdict.excerpt}".encode()).hexdigest()
-                last_time, last_sig = self.recent_signatures.get(p.pane_id, (0, ""))
+                dedup_key = "%s:%s" % (p.socket, p.pane_id)
+                last_time, last_sig = self.recent_signatures.get(dedup_key, (0, ""))
                 if last_sig == sig and (time.time() - last_time) < 15.0:
                     continue  # already handled recently
 
@@ -544,7 +581,7 @@ class AutoApproverRunner:
                     success = True  # dry-run simulated
 
                 now = time.time()
-                self.recent_signatures[p.pane_id] = (now, sig)
+                self.recent_signatures[dedup_key] = (now, sig)
                 self.approval_counts.append(now)
 
                 event = {

bin/tmux_server_watchdog.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+"""tmux_server_watchdog.py — Death-capture for tmux servers.
+
+Runs on a 1-minute systemd timer. Remembers each known socket's server
+pid; when a server dies or its pid changes without a witnessed death,
+appends a forensics bundle (dmesg OOM/kill lines, memory, uptime,
+journal tail) to logs/tmux-server-deaths.jsonl so the next "tmux
+crashed" leaves evidence instead of a mystery.
+
+Read-only against tmux itself: one `display-message -p` probe per
+socket. Never raises; a watchdog must not need its own watchdog.
+"""
+
+import json
+import os
+import subprocess
+import sys
+from datetime import datetime, timezone
+
+BIN_DIR = os.path.dirname(os.path.abspath(__file__))
+REPO_ROOT = os.path.dirname(BIN_DIR)
+sys.path.insert(0, BIN_DIR)
+
+try:
+    from muse_choice_watcher import KNOWN_SOCKETS
+except Exception:
+    KNOWN_SOCKETS = ["/tmp/tmux-1000/default"]
+
+STATE_FILE = os.path.join(REPO_ROOT, ".state", "tmux-servers.json")
+DEATH_LOG = os.path.join(REPO_ROOT, "logs", "tmux-server-deaths.jsonl")
+
+
+def _now():
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _run(cmd, timeout=10):
+    try:
+        r = subprocess.run(cmd, capture_output=True, text=True,
+                           timeout=timeout)
+        return r.returncode, (r.stdout or "").strip()
+    except Exception as e:
+        return -1, "exec failed: %r" % (e,)
+
+
+def probe(socket_path):
+    """Server pid for a socket, or None when unreachable."""
+    rc, out = _run(["tmux", "-S", socket_path, "display-message",
+                    "-p", "#{pid}"], timeout=10)
+    if rc != 0:
+        return None
+    try:
+        return int(out.strip().split()[0])
+    except (ValueError, IndexError):
+        return None
+
+
+def collect_forensics(socket_path, last_pid):
+    """Best-effort death evidence. Dict of strings, never raises."""
+    ev = {"ts": _now(), "socket": socket_path, "last_pid": last_pid}
+    rc, dmesg = _run(["dmesg"], timeout=10)
+    if rc != 0:
+        ev["dmesg"] = "unavailable: %s" % dmesg[:200]
+    else:
+        hits = [ln for ln in dmesg.split("\n")
+                if any(k in ln.lower() for k in
+                       ("oom", "killed process", "segfault", "tmux"))]
+        ev["dmesg_hits"] = hits[-15:]
+    _, ev["memory"] = _run(["free", "-m"], timeout=10)
+    _, ev["uptime"] = _run(["uptime"], timeout=10)
+    rc, journal = _run(["journalctl", "--user", "-n", "50"], timeout=10)
+    if rc == 0:
+        ev["journal_tmux"] = [ln for ln in journal.split("\n")
+                              if "tmux" in ln.lower()][-10:]
+    else:
+        ev["journal_tmux"] = []
+    return ev
+
+
+def read_state(path=None):
+    try:
+        with open(path or STATE_FILE) as f:
+            data = json.load(f)
+        return data if isinstance(data, dict) else {}
+    except Exception:
+        return {}
+
+
+def write_state(state, path=None):
+    path = path or STATE_FILE
+    try:
+        parent = os.path.dirname(path)
+        if parent:
+            os.makedirs(parent, exist_ok=True)
+        tmp = "%s.tmp.%d" % (path, os.getpid())
+        with open(tmp, "w") as f:
+            json.dump(state, f, indent=1)
+        os.replace(tmp, path)
+    except Exception:
+        pass
+
+
+def append_death(ev, path=None):
+    path = path or DEATH_LOG
+    try:
+        parent = os.path.dirname(path)
+        if parent:
+            os.makedirs(parent, exist_ok=True)
+        with open(path, "a") as f:
+            f.write(json.dumps(ev) + "\n")
+    except Exception:
+        pass
+
+
+def evaluate(previous, probed):
+    """Pure transition logic: (prev_state, {sock: pid|None}) ->
+    (new_state, events). Events: death | restart | started."""
+    new_state, events = {}, []
+    for sock, pid in sorted(probed.items()):
+        prev = (previous.get(sock) or {})
+        prev_pid = prev.get("pid")
+        if pid is None:
+            new_state[sock] = {"pid": None, "died": _now(),
+                               "last_pid": prev_pid}
+            if prev_pid:
+                events.append({"type": "death", "socket": sock,
+                               "last_pid": prev_pid})
+        else:
+            new_state[sock] = {"pid": pid, "since": _now()}
+            if prev_pid and prev_pid != pid:
+                # Changed with no witnessed death: restart inside one
+                # tick gap (or pid recycled under us). Treat as a
+                # restart, still worth a forensics note.
+                events.append({"type": "restart", "socket": sock,
+                               "old_pid": prev_pid, "pid": pid})
+            elif not prev_pid and prev.get("died"):
+                events.append({"type": "started", "socket": sock,
+                               "pid": pid})
+            elif not prev_pid and not prev:
+                events.append({"type": "started", "socket": sock,
+                               "pid": pid})
+    return new_state, events
+
+
+def check(sockets=None, dry_run=False):
+    """Probe, transition state, log deaths. Returns summary dict."""
+    probed = {s: probe(s) for s in (sockets or KNOWN_SOCKETS)}
+    previous = read_state()
+    new_state, events = evaluate(previous, probed)
+    for ev in events:
+        if ev["type"] == "death":
+            bundle = collect_forensics(ev["socket"], ev["last_pid"])
+            bundle["event"] = "death"
+            if not dry_run:
+                append_death(bundle)
+            ev["forensics"] = bundle
+        elif ev["type"] == "restart":
+            bundle = collect_forensics(ev["socket"], ev["old_pid"])
+            bundle["event"] = "restart-gap-missed"
+            if not dry_run:
+                append_death(bundle)
+            ev["forensics"] = bundle
+    if not dry_run:
+        write_state(new_state)
+    return {"probed": probed, "events": events, "dry_run": dry_run}
+
+
+def main(argv=None):
+    import argparse
+    ap = argparse.ArgumentParser(description="tmux server death-capture")
+    ap.add_argument("--sockets", nargs="*", default=None)
+    ap.add_argument("--dry-run", action="store_true")
+    ap.add_argument("--json", action="store_true")
+    args = ap.parse_args(argv)
+    try:
+        res = check(sockets=args.sockets, dry_run=args.dry_run)
+    except Exception as e:
+        print("watchdog failed: %r" % (e,), file=sys.stderr)
+        return 1
+    if args.json or args.dry_run:
+        print(json.dumps(res, indent=1, default=str))
+    else:
+        for ev in res["events"]:
+            print("%s: %s" % (ev["type"], ev["socket"]))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())