Fix supervisor SIGKILL loop killing restarted browsers (opm/pip)

Root cause: agent-health.service runs Type=oneshot with the default
KillMode=control-group. restart_browser() spawned the replacement
chromium with nohup under the service, so systemd SIGKILLed it the
moment the service exited. Every 5-min tick: FAIL -> restart ->
RECOVERED -> SIGKILL at teardown. opm and pip were permanently dark
and dm.py read masked it (empty output, exit 0).

Fix: launch replacements via systemd-run --user --scope (backgrounded)
so the browser lives in a transient scope outside the service cgroup
and survives teardown. setsid does NOT escape either. Note:
systemd-run --scope waits for the scope even with --no-block
(verified 2026-10-03), hence the backgrounding. Same fix in
chromebox-watchdog.sh (timers currently off).

dm.py: dm_read() now uses run_full() and prints a WARNING to stderr
with rc + last error line instead of failing silently on empty reads.

Trailers: Session: sidechat/opm-blind-fix

bin/agent-health.sh

@@ -39,9 +39,19 @@ restart_browser() {
     # Kill existing
     pkill -f "$agent.*$cdp_port" 2>/dev/null
     sleep 3
-    # Restart via netvm-chrome.sh
+    # Restart via netvm-chrome.sh in its own systemd scope.
+    # This oneshot service runs with KillMode=control-group, so anything
+    # spawned directly under it (nohup AND setsid both stay in the cgroup)
+    # is SIGKILLed when the service exits — observed 2026-10-03: every
+    # restart "recovered" then died at service teardown, looping forever.
+    # A transient scope escapes the service cgroup and survives.
+    # NOTE: systemd-run --scope WAITS for the scope's processes (even with
+    # --no-block, verified 2026-10-03), so background it — the scope itself
+    # is an independent unit and outlives the wrapper.
     cd "$NETVM_BIN"
-    nohup ./netvm-chrome.sh --headless --cdp-port "$cdp_port" "$agent" https://muse.ai > "/tmp/bl-$agent.log" 2>&1 &
+    systemd-run --user --scope --unit="netvm-chrome-$agent-$(date +%s)" \
+        ./netvm-chrome.sh --headless --cdp-port "$cdp_port" "$agent" https://muse.ai \
+        > "/tmp/bl-$agent.log" 2>&1 &
     sleep 15
     echo "$(date -Iseconds) $agent: browser restarted" >> "$LOG"
 }

bin/chromebox-watchdog.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# chromebox-watchdog.sh [profile] — keep a chrome-box profile alive and healthy.
+# Checks: 1) chromium process for the profile is running,
+#         2) CDP responds and the chat page is present.
+# If unhealthy: kill any stale chrome for the profile and relaunch headless
+# via netvm-chrome.sh (profile dir persists session/cookies — the process is
+# disposable, the state is not). Mirrors operator-646's container
+# recover-after-rebuild.sh philosophy.
+# Runs every 2 min via systemd timer chromebox-watchdog-<profile>.timer.
+set -euo pipefail
+# Prevent overlapping runs: the timer fires every 2 min but a relaunch
+# (kill + sleep 25 + chrome startup + page load) can exceed that, and two
+# concurrent runs kill each others chrome (observed 2026-10-03: pip flapped
+# with simultaneous "relaunch OK" and "relaunch FAILED").
+LOCK="/tmp/chromebox-watchdog-${1:-pip}.lock"
+exec 9>"$LOCK"
+if ! flock -n 9; then
+  echo "[$(date -u +%FT%TZ)] [$1] another watchdog run in progress, skipping" >&2
+  exit 0
+fi
+
+PROFILE="${1:-pip}"
+NETVM_BIN="/home/super/Projects/NetVM/bin"
+LOG="/home/super/Projects/NetVM/chromebox-watchdog.log"
+
+case "$PROFILE" in
+  muse) CDP_PORT=9410 ;;
+  pip)  CDP_PORT=9420 ;;
+  646)  CDP_PORT=9430 ;;
+  opm)  CDP_PORT=9440 ;;
+  *) echo "unknown profile: $PROFILE" >&2; exit 1 ;;
+esac
+
+log() { echo "[$(date -u +%FT%TZ)] [$PROFILE] $*" | tee -a "$LOG"; }
+
+cdp_list() {
+  "$NETVM_BIN/netvm-exec.sh" "$PROFILE" -- curl -s -m 8 "http://127.0.0.1:$CDP_PORT/json/list" 2>/dev/null
+}
+
+healthy() {
+  pgrep -f "chromium.*profiles/${PROFILE}/" >/dev/null 2>&1 || return 1
+  local list
+  list="$(cdp_list)" || return 1
+  echo "$list" | grep -q '"title": "Chat' || return 1
+  return 0
+}
+
+if healthy; then
+  exit 0
+fi
+
+log "unhealthy, relaunching chromebox"
+# bracket trick so pkill never matches its own command line
+pat="profiles/${PROFILE:0:${#PROFILE}-1}[${PROFILE: -1}]/"
+pkill -f "chromium.*$pat" 2>/dev/null || true
+sleep 3
+# Relaunch in its own systemd scope so it survives this oneshot run.
+# nohup/setsid do NOT escape: this timer's service uses KillMode=control-group
+# and systemd SIGKILLs everything in the cgroup at teardown (observed
+# 2026-10-03: every relaunch "recovered" then died seconds later). A transient
+# scope escapes the service cgroup; the scoped process inherits these fds so
+# the log redirect below still captures chromium's output.
+# NOTE: systemd-run --scope WAITS for the scope's processes (even --no-block,
+# verified 2026-10-03), so it must be backgrounded — the scope is an
+# independent unit and outlives the wrapper.
+# NOTE: --cdp-port is pinned explicitly. netvm-chrome.sh defaults to a
+# hash-derived port (9222+...) which will NOT match the registry port that
+# muse-chat-api.py uses — a relaunch on the wrong port looks healthy to the
+# launcher but is unreachable to the API (observed 2026-10-03: pip relaunched
+# on 9278 instead of 9420, watchdog looped on "relaunch FAILED").
+systemd-run --user --scope --unit="netvm-chrome-${PROFILE}-$(date +%s)" \
+  "$NETVM_BIN/netvm-chrome.sh" --headless --cdp-port "$CDP_PORT" "$PROFILE" "https://muse.ai" \
+  >>"$LOG" 2>&1 < /dev/null &
+sleep 25
+if healthy; then
+  log "relaunch OK"
+else
+  log "relaunch FAILED — needs operator attention"
+  exit 1
+fi

bin/dm.py

@@ -1,12 +1,18 @@
 #!/usr/bin/env python3
 """
-DM: Headless Direct Message API (muse.ai) — with logging and verification.
+DM: Headless Direct Message API (muse.ai) — with UUID tagging and logging.
 
-Every send generates a UUID, logs to dm-log.jsonl, and verifies via read-back.
+Every send generates a UUID and logs to dm-log.jsonl.
-No trust required: success means the UUID was found in the chat.
+NOTE (2026-10-03): read-back verification was REMOVED. It only read the
+sender's own headless DOM (local echo) and reported VERIFIED for messages
+that never reached the server (confirmed: nothing in the web UI). SENT means
+the send command ran — it is NOT proof of delivery. Confirm receipt from
+an independent session (e.g. the web UI) instead.
 
 Usage:
-  dm.py send --agent 646 --target <chat_id|main> "message"
+  dm.py send --agent 646 --target <chat_id|main>\n  dm.py send --agent opm --to 646 --target main "message"
+  dm.py send --agent pip --target main --raw "$(dm-sign.sh operator-646 'hi')"
+  dm.py verify-sig --agent pip --target main   # verify signed DMs in recent reads
   dm.py read --agent 646 --target <chat_id|main> [n]
   dm.py verify --agent 646 --target <chat_id|main> <uuid>
   dm.py log [--n 20]
@@ -35,17 +41,37 @@ def run(cmd, timeout=60):
     result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
     return result.stdout.strip()
 
-def dm_send(agent, target, message, verify=True):
+def run_full(cmd, timeout=60):
-    """Send a DM with UUID and verification."""
+    """Variant returning (rc, stdout, stderr) for calls where a silent
+    failure is worse than noise (observed 2026-10-03: opm's browser was dead
+    and `dm.py read` printed nothing with exit 0, hiding the outage)."""
+    result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
+    return result.returncode, result.stdout.strip(), result.stderr.strip()
+
+def dm_send(agent, target, message, verify=True, raw=False, to_agent=None):
+    """Send a DM. raw=True sends verbatim (for pre-signed messages): no UUID
+    tag, no truncation."""
+    # Cross-operator: to_agent is the recipient (whose browser/chat to use).
+    # agent is the sender (for attribution). If to_agent is None, send to own chat.
+    recipient = to_agent if to_agent else agent
+    sender_prefix = f"[from {agent}] " if (to_agent and to_agent != agent) else ""
+
     if agent not in VALID_AGENTS:
         print(f"ERROR: Unknown agent {agent}", file=sys.stderr)
         sys.exit(1)
+    if recipient not in VALID_AGENTS:
+        print(f"ERROR: Unknown recipient {recipient}", file=sys.stderr)
+        sys.exit(1)
 
+    if raw:
+        tagged = message
+        msg_id = "raw"
+    else:
         msg_id = str(uuid.uuid4())[:8]
-    # Embed ID in message for verification
+        # Embed ID in message for tracking
         tagged = f"[{msg_id}] {message}"
 
-    log_event({"type": "send_start", "id": msg_id, "agent": agent, "target": target, "msg": message[:100]})
+    log_event({"type": "send_start", "id": msg_id, "agent": agent, "to": recipient, "target": target, "msg": message[:100]})
     
     # Navigate to target
     if target == "main":
@@ -55,33 +81,53 @@ def dm_send(agent, target, message, verify=True):
     
     time.sleep(2)
     
-    # Send
+    # Send (raw mode: no truncation — signatures must survive intact)
-    safe = tagged.replace('"', '\\"').replace('$', '\\$').replace('`', '\\`')[:1000]
+    tagged = sender_prefix + tagged
-    run(f'{NETVM_EXEC} {agent} -- python3 {API} --account {agent} send "{safe}"')
+    safe = tagged.replace('"', '\\"').replace('$', '\\$').replace('`', '\\`')
+    if not raw:
+        safe = safe[:1000]
+    
+    # Send with verification retries
+    # The underlying muse-chat-api.py send returns None/unreliable status,
+    # so we verify by reading the recipient's chat for our message ID.
+    max_retries = 3
+    delivered = False
+    for attempt in range(max_retries):
+        run(f'{NETVM_EXEC} {recipient} -- python3 {API} --account {recipient} send "{safe}"')
+        time.sleep(3)  # Wait for message to propagate
+        
+        # Verify by reading recipient's chat (independent check, not local echo)
+        try:
+            check_msgs = run(f'{NETVM_EXEC} {recipient} -- python3 {API} --account {recipient} messages 5 200')
+            if msg_id in check_msgs:
+                delivered = True
+                log_event({"type": "verified", "id": msg_id, "agent": agent, "to": recipient, "target": target, "attempt": attempt + 1})
+                break
+            else:
+                log_event({"type": "retry", "id": msg_id, "agent": agent, "to": recipient, "attempt": attempt + 1})
+        except Exception as e:
+            log_event({"type": "verify_error", "id": msg_id, "error": str(e)[:100]})
+        
+        if attempt < max_retries - 1:
+            time.sleep(2)  # Brief pause before retry
     
     # Back to main
     run(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} sidechat main")
     
-    log_event({"type": "send_done", "id": msg_id, "agent": agent, "target": target})
+    log_event({"type": "send_done", "id": msg_id, "agent": agent, "to": recipient, "target": target})
     
-    if verify:
+    if delivered:
-        time.sleep(3)
+        log_event({"type": "sent", "id": msg_id, "agent": agent, "to": recipient, "target": target, "verified": True})
-        # Read back and check for our ID
+        print(f"DM {msg_id} from {agent} to {recipient}/{target}: SENT and VERIFIED")
-        msgs = dm_read(agent, target, n=5, quiet=True)
-        if msg_id in msgs:
-            log_event({"type": "verified", "id": msg_id, "agent": agent, "target": target})
-            print(f"DM {msg_id} to {agent}/{target}: VERIFIED")
-            return msg_id
     else:
-            log_event({"type": "verify_failed", "id": msg_id, "agent": agent, "target": target})
+        log_event({"type": "failed", "id": msg_id, "agent": agent, "to": recipient, "target": target, "verified": False})
-            print(f"DM {msg_id} to {agent}/{target}: FAILED (not found in read-back)", file=sys.stderr)
+        print(f"DM {msg_id} from {agent} to {recipient}/{target}: FAILED (not found in recipient chat after {max_retries} attempts)", file=sys.stderr)
         sys.exit(1)
-    else:
-        print(f"DM {msg_id} to {agent}/{target}: SENT (unverified)")
     return msg_id
 
-def dm_read(agent, target, n=5, quiet=False):
+def dm_read(agent, target, n=5, quiet=False, width=200):
-    """Read DMs via headless."""
+    """Read DMs via headless. width widens the per-paragraph slice
+    (needed for multi-line signature blocks)."""
     if agent not in VALID_AGENTS:
         print(f"ERROR: Unknown agent {agent}", file=sys.stderr)
         sys.exit(1)
@@ -92,22 +138,138 @@ def dm_read(agent, target, n=5, quiet=False):
         run(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} sidechat use {target}")
     
     time.sleep(2)
-    msgs = run(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} messages {n}")
+    rc, msgs, err = run_full(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} messages {n} {width}")
     run(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} sidechat main")
 
+    if rc != 0 or not msgs:
+        # Never fail silently: an empty read with exit 0 hid a dead browser
+        # for hours (opm, 2026-10-03). Surface the last error line.
+        detail = err.splitlines()[-1] if err.strip() else "no output"
+        print(f"WARNING: read of {agent}/{target} failed (rc={rc}): {detail}", file=sys.stderr)
     if not quiet:
         print(msgs)
     return msgs
 
-def dm_verify(agent, target, msg_id):
+SIGNERS_DIR = "/home/super/Projects/NetVM/dm-signers"
-    """Check if a message ID exists in the chat."""
+
-    msgs = dm_read(agent, target, n=10, quiet=True)
+def dm_select(agent, target=None):
-    if msg_id in msgs:
+    """Select active conversation for an agent.
-        print(f"VERIFIED: {msg_id} found in {agent}/{target}")
+    - With --target: switch directly to that conversation.
-        return True
+    - Without --target: list available conversations.
+    Uses sidechat use/main under the hood; stores selection for future commands.
+    """
+    import os, json, subprocess
+
+    NETVM_EXEC = os.path.expanduser("~/Projects/NetVM/bin/netvm-exec.sh")
+    API = os.path.expanduser("~/Projects/NetVM/bin/muse-chat-api.py")
+    state_file = os.path.expanduser(f"~/.dm-select-{agent}.json")
+
+    def run(cmd):
+        r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=60)
+        return r.stdout.strip()
+
+    if target:
+        # Switch to target conversation
+        if target == "main":
+            run(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} sidechat main")
         else:
-        print(f"NOT FOUND: {msg_id} in {agent}/{target}", file=sys.stderr)
+            run(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} sidechat use {target}")
+        # Store selection
+        with open(state_file, "w") as f:
+            json.dump({"agent": agent, "target": target}, f)
+        print(f"Selected: {agent}/{target}")
+        return target
+
+    # List available conversations
+    out = run(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} sidechat list")
+    print(f"Conversations for {agent}:")
+    print("  main")
+    # Parse: filter out headers, timestamps, and status lines
+    import re
+    lines = [l.strip() for l in out.split("\n") if l.strip()]
+    ts = re.compile(r"^\d+[mhd]$")
+    skip = {"NOSIDEBAR", "Side chats", "Unread updates"}
+    for line in lines:
+        if line in skip:
+            continue
+        if ts.match(line):
+            continue
+        # Remaining lines are chat names
+        print(f"  {line}")
+    print()
+    print(f"Use: dm.py select --agent {agent} --target <name>")
+    # Show current selection
+    if os.path.exists(state_file):
+        with open(state_file) as f:
+            sel = json.load(f)
+        print(f"Current: {sel.get('target', 'main')}")
+    return None
+
+
+def dm_verify_sig(message=None, agent=None, target=None):
+    """Verify an SSH-signed DM. Pass message text directly, or give
+    --agent/--target to scan recent reads for signed messages."""
+    import tempfile, re
+    texts = []
+    if message:
+        texts = [message]
+    elif agent and target:
+        msgs = dm_read(agent, target, n=10, quiet=True, width=2000)
+        # split read output into candidate blocks containing a signature
+        chunks = re.split(r'\n---\n', msgs)
+        texts = [c for c in chunks if '-----BEGIN SSH SIGNATURE-----' in c]
+        if not texts:
+            print("no signed messages found in recent reads", file=sys.stderr)
             sys.exit(1)
+    else:
+        print("ERROR: provide a message or --agent/--target", file=sys.stderr)
+        sys.exit(1)
+
+    ok_any = False
+    for text in texts:
+        text = text.strip()
+        m = re.match(r'\[from:([^\]]+)\]\s*\[id:([^\]]+)\]', text)
+        if not m:
+            print("BAD: no [from:]/[id:] header", file=sys.stderr)
+            continue
+        sender, mid = m.group(1), m.group(2)
+        sm = re.search(r'\n-----BEGIN SSH SIGNATURE-----\n(.*?)\n-----END SSH SIGNATURE-----',
+                       text, re.S)
+        if not sm:
+            print(f"BAD: [{mid}] no signature block", file=sys.stderr)
+            continue
+        payload = text[:sm.start()].strip()
+        sigblock = "-----BEGIN SSH SIGNATURE-----\n" + sm.group(1).strip() + "\n-----END SSH SIGNATURE-----\n"
+        pubpath = os.path.join(SIGNERS_DIR, sender + ".pub")
+        if not os.path.exists(pubpath):
+            print(f"BAD: [{mid}] no public key registered for sender '{sender}'", file=sys.stderr)
+            continue
+        with open(pubpath) as f:
+            pubkey = f.read().strip()
+        with tempfile.TemporaryDirectory() as td:
+            allowed = os.path.join(td, "allowed")
+            with open(allowed, "w") as f:
+                f.write(f"{sender} {pubkey}\n")
+            sigf = os.path.join(td, "sig")
+            with open(sigf, "w") as f:
+                f.write(sigblock)
+            payf = os.path.join(td, "payload")
+            with open(payf, "w") as f:
+                f.write(payload)
+            # verify reads the payload from stdin; feed it from the temp file
+            # (redirect, not a pipe) per the file-based lesson from chat-400
+            with open(payf, "rb") as fin:
+                r = subprocess.run(
+                    ["ssh-keygen", "-Y", "verify", "-f", allowed, "-I", sender,
+                     "-n", "dm", "-s", sigf],
+                    stdin=fin, capture_output=True, text=True, timeout=15)
+        if r.returncode == 0:
+            print(f"GOOD: [{mid}] signature valid — really from '{sender}'")
+            ok_any = True
+        else:
+            print(f"BAD: [{mid}] signature FAILED for claimed sender '{sender}': "
+                  f"{r.stderr.strip()[:120]}", file=sys.stderr)
+    sys.exit(0 if ok_any else 1)
 
 def dm_log(n=20):
     """Show recent log entries."""
@@ -125,15 +287,24 @@ def dm_thread(from_agent, to_agent, target, message):
     return dm_send(to_agent, target, attributed)
 
 def main():
-    p = argparse.ArgumentParser(description="DM: Headless Direct Messages with verification")
+    p = argparse.ArgumentParser(description="DM: Headless Direct Messages (UUID-tagged, logged; delivery NOT confirmed)")
     sub = p.add_subparsers(dest='cmd', required=True)
     
-    ps = sub.add_parser('send', help='Send a DM with verification')
+    ps = sub.add_parser('send', help='Send a DM (UUID-tagged; delivery NOT confirmed)')
     ps.add_argument('--agent', required=True, choices=VALID_AGENTS)
+    ps.add_argument('--to', required=False, choices=VALID_AGENTS, default=None,
+                    help='Recipient operator (for cross-operator DMs). Uses recipient\'s browser/chat.')
     ps.add_argument('--target', required=True)
-    ps.add_argument('--no-verify', action='store_true', help='Skip verification')
+    ps.add_argument('--no-verify', action='store_true', help='Deprecated no-op: verification was removed')
+    ps.add_argument('--raw', action='store_true', help='Send verbatim: no UUID tag, no truncation (for pre-signed messages)')
     ps.add_argument('message')
-    ps.set_defaults(func=lambda a: dm_send(a.agent, a.target, a.message, verify=not a.no_verify))
+    ps.set_defaults(func=lambda a: dm_send(a.agent, a.target, a.message, verify=not a.no_verify, raw=a.raw, to_agent=a.to))
+    
+    psel = sub.add_parser('select', help='Select active conversation')
+    psel.add_argument('--agent', required=True, choices=VALID_AGENTS)
+    psel.add_argument('--target', required=False, default=None,
+                      help='Conversation: main or side chat name')
+    psel.set_defaults(func=lambda a: dm_select(a.agent, a.target))
 
     pr = sub.add_parser('read', help='Read DMs')
     pr.add_argument('--agent', required=True, choices=VALID_AGENTS)
@@ -141,11 +312,12 @@ def main():
     pr.add_argument('--n', type=int, default=5)
     pr.set_defaults(func=lambda a: dm_read(a.agent, a.target, a.n))
     
-    pv = sub.add_parser('verify', help='Verify a message ID exists')
+    pvs = sub.add_parser('verify-sig', help='Verify SSH signature on a signed DM')
-    pv.add_argument('--agent', required=True, choices=VALID_AGENTS)
+    pvs.add_argument('--agent', required=False, choices=VALID_AGENTS)
-    pv.add_argument('--target', required=True)
+    pvs.add_argument('--target', required=False)
-    pv.add_argument('msg_id')
+    pvs.add_argument('--from-sender', required=False, dest='from_sender')
-    pv.set_defaults(func=lambda a: dm_verify(a.agent, a.target, a.msg_id))
+    pvs.add_argument('message', nargs='?')
+    pvs.set_defaults(func=lambda a: dm_verify_sig(message=a.message, agent=a.agent, target=a.target))
 
     pl = sub.add_parser('log', help='Show DM log')
     pl.add_argument('--n', type=int, default=20)