super/box
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(work): import hashlib and wire heal subparser into main CLI

Browse Source
This commit is contained in:
operator
2026-10-09 23:13:43 +00:00
parent c3b4b1cd28
commit f75977ca6e
202 changed files with 4185 additions and 4142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/tmux_server_watchdog.py
+90 -17
View File
@@ -2,10 +2,11 @@
"""tmux_server_watchdog.py — Death-capture for tmux servers.
Runs on a 1-minute systemd timer. Remembers each known socket's server
pid; when a server dies or its pid changes without a witnessed death,
appends a forensics bundle (dmesg OOM/kill lines, memory, uptime,
journal tail) to logs/tmux-server-deaths.jsonl so the next "tmux
crashed" leaves evidence instead of a mystery.
identity (pid + /proc starttime + ppid + cmdline); when a server dies,
its pid changes, or its pid is recycled under us without a witnessed
death, appends a forensics bundle (dmesg OOM/kill lines, memory,
uptime, journal tail) to logs/tmux-server-deaths.jsonl so the next
"tmux crashed" leaves evidence instead of a mystery.
Read-only against tmux itself: one `display-message -p` probe per
socket. Never raises; a watchdog must not need its own watchdog.
@@ -55,9 +56,46 @@ def probe(socket_path):
return None
def collect_forensics(socket_path, last_pid):
def proc_identity(pid):
"""Identity dict for a pid: starttime defeats PID-reuse confusion.
Never raises; on any failure returns {"pid": pid} so callers can
still snapshot. starttime is the raw /proc starttime tick (field
22), stable for the life of the process."""
ident = {"pid": pid}
try:
with open("/proc/%d/stat" % pid) as f:
parts = f.read().rsplit(")", 1)[1].split()
# After "(comm)": state ppid pgrp session tty_nr ... starttime
# is field 22 overall, i.e. parts[19] after the split above.
ident["ppid"] = int(parts[1])
ident["starttime"] = int(parts[19])
except Exception:
pass
try:
with open("/proc/%d/cmdline" % pid, "rb") as f:
raw = f.read().replace(b"\0", b" ").decode(
"utf-8", "replace").strip()
if raw:
ident["cmd"] = raw[:200]
except Exception:
pass
return ident
def probe_identity(socket_path):
"""Enriched snapshot for a socket: identity dict or None."""
pid = probe(socket_path)
if pid is None:
return None
return proc_identity(pid)
def collect_forensics(socket_path, last_pid, last_identity=None):
"""Best-effort death evidence. Dict of strings, never raises."""
ev = {"ts": _now(), "socket": socket_path, "last_pid": last_pid}
if last_identity:
ev["last_identity"] = last_identity
rc, dmesg = _run(["dmesg"], timeout=10)
if rc != 0:
ev["dmesg"] = "unavailable: %s" % dmesg[:200]
@@ -112,27 +150,60 @@ def append_death(ev, path=None):
pass
def _as_identity(value):
"""Normalize a probed value to an identity dict (legacy int ok)."""
if value is None:
return None
if isinstance(value, dict):
return value
return {"pid": value}
def _prev_identity(prev):
ident = {"pid": prev.get("pid")}
for key in ("starttime", "ppid", "cmd"):
if prev.get(key) is not None:
ident[key] = prev[key]
return ident
def evaluate(previous, probed):
"""Pure transition logic: (prev_state, {sock: pid|None}) ->
(new_state, events). Events: death | restart | started."""
"""Pure transition logic: (prev_state, {sock: pid|identity|None}) ->
(new_state, events). Events: death | restart | started.
Probed values may be a bare pid (legacy) or an identity dict from
probe_identity(). Same pid with a different starttime is a restart
(pid recycled under us), not steady state."""
new_state, events = {}, []
for sock, pid in sorted(probed.items()):
for sock, raw in sorted(probed.items()):
ident = _as_identity(raw)
prev = (previous.get(sock) or {})
prev_pid = prev.get("pid")
if pid is None:
if ident is None:
new_state[sock] = {"pid": None, "died": _now(),
"last_pid": prev_pid}
if prev_pid:
events.append({"type": "death", "socket": sock,
"last_pid": prev_pid})
"last_pid": prev_pid,
"last_identity": _prev_identity(prev)})
else:
new_state[sock] = {"pid": pid, "since": _now()}
pid = ident.get("pid")
new_state[sock] = dict(ident, since=_now())
if prev_pid and prev_pid != pid:
# Changed with no witnessed death: restart inside one
# tick gap (or pid recycled under us). Treat as a
# restart, still worth a forensics note.
# tick gap. Worth a forensics note.
events.append({"type": "restart", "socket": sock,
"old_pid": prev_pid, "pid": pid})
"old_pid": prev_pid, "pid": pid,
"last_identity": _prev_identity(prev)})
elif (prev_pid and prev_pid == pid
and prev.get("starttime") is not None
and ident.get("starttime") is not None
and prev["starttime"] != ident["starttime"]):
# Same pid, different process: pid recycled under us.
events.append({"type": "restart", "socket": sock,
"old_pid": prev_pid, "pid": pid,
"pid_reused": True,
"last_identity": _prev_identity(prev)})
elif not prev_pid and prev.get("died"):
events.append({"type": "started", "socket": sock,
"pid": pid})
@@ -144,18 +215,20 @@ def evaluate(previous, probed):
def check(sockets=None, dry_run=False):
"""Probe, transition state, log deaths. Returns summary dict."""
probed = {s: probe(s) for s in (sockets or KNOWN_SOCKETS)}
probed = {s: probe_identity(s) for s in (sockets or KNOWN_SOCKETS)}
previous = read_state()
new_state, events = evaluate(previous, probed)
for ev in events:
if ev["type"] == "death":
bundle = collect_forensics(ev["socket"], ev["last_pid"])
bundle = collect_forensics(ev["socket"], ev["last_pid"],
ev.get("last_identity"))
bundle["event"] = "death"
if not dry_run:
append_death(bundle)
ev["forensics"] = bundle
elif ev["type"] == "restart":
bundle = collect_forensics(ev["socket"], ev["old_pid"])
bundle = collect_forensics(ev["socket"], ev["old_pid"],
ev.get("last_identity"))
bundle["event"] = "restart-gap-missed"
if not dry_run:
append_death(bundle)