diff --git a/bin/agent-health.sh b/bin/agent-health.sh index bc33ab6..8a1dfaa 100755 --- a/bin/agent-health.sh +++ b/bin/agent-health.sh @@ -39,9 +39,19 @@ restart_browser() { # Kill existing pkill -f "$agent.*$cdp_port" 2>/dev/null sleep 3 - # Restart via netvm-chrome.sh + # Restart via netvm-chrome.sh in its own systemd scope. + # This oneshot service runs with KillMode=control-group, so anything + # spawned directly under it (nohup AND setsid both stay in the cgroup) + # is SIGKILLed when the service exits — observed 2026-10-03: every + # restart "recovered" then died at service teardown, looping forever. + # A transient scope escapes the service cgroup and survives. + # NOTE: systemd-run --scope WAITS for the scope's processes (even with + # --no-block, verified 2026-10-03), so background it — the scope itself + # is an independent unit and outlives the wrapper. cd "$NETVM_BIN" - nohup ./netvm-chrome.sh --headless --cdp-port "$cdp_port" "$agent" https://muse.ai > "/tmp/bl-$agent.log" 2>&1 & + systemd-run --user --scope --unit="netvm-chrome-$agent-$(date +%s)" \ + ./netvm-chrome.sh --headless --cdp-port "$cdp_port" "$agent" https://muse.ai \ + > "/tmp/bl-$agent.log" 2>&1 & sleep 15 echo "$(date -Iseconds) $agent: browser restarted" >> "$LOG" } diff --git a/bin/chromebox-watchdog.sh b/bin/chromebox-watchdog.sh new file mode 100755 index 0000000..480cf40 --- /dev/null +++ b/bin/chromebox-watchdog.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# chromebox-watchdog.sh [profile] — keep a chrome-box profile alive and healthy. +# Checks: 1) chromium process for the profile is running, +# 2) CDP responds and the chat page is present. +# If unhealthy: kill any stale chrome for the profile and relaunch headless +# via netvm-chrome.sh (profile dir persists session/cookies — the process is +# disposable, the state is not). Mirrors operator-646's container +# recover-after-rebuild.sh philosophy. +# Runs every 2 min via systemd timer chromebox-watchdog-.timer. +set -euo pipefail +# Prevent overlapping runs: the timer fires every 2 min but a relaunch +# (kill + sleep 25 + chrome startup + page load) can exceed that, and two +# concurrent runs kill each others chrome (observed 2026-10-03: pip flapped +# with simultaneous "relaunch OK" and "relaunch FAILED"). +LOCK="/tmp/chromebox-watchdog-${1:-pip}.lock" +exec 9>"$LOCK" +if ! flock -n 9; then + echo "[$(date -u +%FT%TZ)] [$1] another watchdog run in progress, skipping" >&2 + exit 0 +fi + +PROFILE="${1:-pip}" +NETVM_BIN="/home/super/Projects/NetVM/bin" +LOG="/home/super/Projects/NetVM/chromebox-watchdog.log" + +case "$PROFILE" in + muse) CDP_PORT=9410 ;; + pip) CDP_PORT=9420 ;; + 646) CDP_PORT=9430 ;; + opm) CDP_PORT=9440 ;; + *) echo "unknown profile: $PROFILE" >&2; exit 1 ;; +esac + +log() { echo "[$(date -u +%FT%TZ)] [$PROFILE] $*" | tee -a "$LOG"; } + +cdp_list() { + "$NETVM_BIN/netvm-exec.sh" "$PROFILE" -- curl -s -m 8 "http://127.0.0.1:$CDP_PORT/json/list" 2>/dev/null +} + +healthy() { + pgrep -f "chromium.*profiles/${PROFILE}/" >/dev/null 2>&1 || return 1 + local list + list="$(cdp_list)" || return 1 + echo "$list" | grep -q '"title": "Chat' || return 1 + return 0 +} + +if healthy; then + exit 0 +fi + +log "unhealthy, relaunching chromebox" +# bracket trick so pkill never matches its own command line +pat="profiles/${PROFILE:0:${#PROFILE}-1}[${PROFILE: -1}]/" +pkill -f "chromium.*$pat" 2>/dev/null || true +sleep 3 +# Relaunch in its own systemd scope so it survives this oneshot run. +# nohup/setsid do NOT escape: this timer's service uses KillMode=control-group +# and systemd SIGKILLs everything in the cgroup at teardown (observed +# 2026-10-03: every relaunch "recovered" then died seconds later). A transient +# scope escapes the service cgroup; the scoped process inherits these fds so +# the log redirect below still captures chromium's output. +# NOTE: systemd-run --scope WAITS for the scope's processes (even --no-block, +# verified 2026-10-03), so it must be backgrounded — the scope is an +# independent unit and outlives the wrapper. +# NOTE: --cdp-port is pinned explicitly. netvm-chrome.sh defaults to a +# hash-derived port (9222+...) which will NOT match the registry port that +# muse-chat-api.py uses — a relaunch on the wrong port looks healthy to the +# launcher but is unreachable to the API (observed 2026-10-03: pip relaunched +# on 9278 instead of 9420, watchdog looped on "relaunch FAILED"). +systemd-run --user --scope --unit="netvm-chrome-${PROFILE}-$(date +%s)" \ + "$NETVM_BIN/netvm-chrome.sh" --headless --cdp-port "$CDP_PORT" "$PROFILE" "https://muse.ai" \ + >>"$LOG" 2>&1 < /dev/null & +sleep 25 +if healthy; then + log "relaunch OK" +else + log "relaunch FAILED — needs operator attention" + exit 1 +fi diff --git a/bin/dm.py b/bin/dm.py index 32e4f7f..dfb651b 100755 --- a/bin/dm.py +++ b/bin/dm.py @@ -1,12 +1,18 @@ #!/usr/bin/env python3 """ -DM: Headless Direct Message API (muse.ai) — with logging and verification. +DM: Headless Direct Message API (muse.ai) — with UUID tagging and logging. -Every send generates a UUID, logs to dm-log.jsonl, and verifies via read-back. -No trust required: success means the UUID was found in the chat. +Every send generates a UUID and logs to dm-log.jsonl. +NOTE (2026-10-03): read-back verification was REMOVED. It only read the +sender's own headless DOM (local echo) and reported VERIFIED for messages +that never reached the server (confirmed: nothing in the web UI). SENT means +the send command ran — it is NOT proof of delivery. Confirm receipt from +an independent session (e.g. the web UI) instead. Usage: - dm.py send --agent 646 --target "message" + dm.py send --agent 646 --target \n dm.py send --agent opm --to 646 --target main "message" + dm.py send --agent pip --target main --raw "$(dm-sign.sh operator-646 'hi')" + dm.py verify-sig --agent pip --target main # verify signed DMs in recent reads dm.py read --agent 646 --target [n] dm.py verify --agent 646 --target dm.py log [--n 20] @@ -35,17 +41,37 @@ def run(cmd, timeout=60): result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout) return result.stdout.strip() -def dm_send(agent, target, message, verify=True): - """Send a DM with UUID and verification.""" +def run_full(cmd, timeout=60): + """Variant returning (rc, stdout, stderr) for calls where a silent + failure is worse than noise (observed 2026-10-03: opm's browser was dead + and `dm.py read` printed nothing with exit 0, hiding the outage).""" + result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout) + return result.returncode, result.stdout.strip(), result.stderr.strip() + +def dm_send(agent, target, message, verify=True, raw=False, to_agent=None): + """Send a DM. raw=True sends verbatim (for pre-signed messages): no UUID + tag, no truncation.""" + # Cross-operator: to_agent is the recipient (whose browser/chat to use). + # agent is the sender (for attribution). If to_agent is None, send to own chat. + recipient = to_agent if to_agent else agent + sender_prefix = f"[from {agent}] " if (to_agent and to_agent != agent) else "" + if agent not in VALID_AGENTS: print(f"ERROR: Unknown agent {agent}", file=sys.stderr) sys.exit(1) - - msg_id = str(uuid.uuid4())[:8] - # Embed ID in message for verification - tagged = f"[{msg_id}] {message}" - - log_event({"type": "send_start", "id": msg_id, "agent": agent, "target": target, "msg": message[:100]}) + if recipient not in VALID_AGENTS: + print(f"ERROR: Unknown recipient {recipient}", file=sys.stderr) + sys.exit(1) + + if raw: + tagged = message + msg_id = "raw" + else: + msg_id = str(uuid.uuid4())[:8] + # Embed ID in message for tracking + tagged = f"[{msg_id}] {message}" + + log_event({"type": "send_start", "id": msg_id, "agent": agent, "to": recipient, "target": target, "msg": message[:100]}) # Navigate to target if target == "main": @@ -55,33 +81,53 @@ def dm_send(agent, target, message, verify=True): time.sleep(2) - # Send - safe = tagged.replace('"', '\\"').replace('$', '\\$').replace('`', '\\`')[:1000] - run(f'{NETVM_EXEC} {agent} -- python3 {API} --account {agent} send "{safe}"') + # Send (raw mode: no truncation — signatures must survive intact) + tagged = sender_prefix + tagged + safe = tagged.replace('"', '\\"').replace('$', '\\$').replace('`', '\\`') + if not raw: + safe = safe[:1000] + + # Send with verification retries + # The underlying muse-chat-api.py send returns None/unreliable status, + # so we verify by reading the recipient's chat for our message ID. + max_retries = 3 + delivered = False + for attempt in range(max_retries): + run(f'{NETVM_EXEC} {recipient} -- python3 {API} --account {recipient} send "{safe}"') + time.sleep(3) # Wait for message to propagate + + # Verify by reading recipient's chat (independent check, not local echo) + try: + check_msgs = run(f'{NETVM_EXEC} {recipient} -- python3 {API} --account {recipient} messages 5 200') + if msg_id in check_msgs: + delivered = True + log_event({"type": "verified", "id": msg_id, "agent": agent, "to": recipient, "target": target, "attempt": attempt + 1}) + break + else: + log_event({"type": "retry", "id": msg_id, "agent": agent, "to": recipient, "attempt": attempt + 1}) + except Exception as e: + log_event({"type": "verify_error", "id": msg_id, "error": str(e)[:100]}) + + if attempt < max_retries - 1: + time.sleep(2) # Brief pause before retry # Back to main run(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} sidechat main") - log_event({"type": "send_done", "id": msg_id, "agent": agent, "target": target}) + log_event({"type": "send_done", "id": msg_id, "agent": agent, "to": recipient, "target": target}) - if verify: - time.sleep(3) - # Read back and check for our ID - msgs = dm_read(agent, target, n=5, quiet=True) - if msg_id in msgs: - log_event({"type": "verified", "id": msg_id, "agent": agent, "target": target}) - print(f"DM {msg_id} to {agent}/{target}: VERIFIED") - return msg_id - else: - log_event({"type": "verify_failed", "id": msg_id, "agent": agent, "target": target}) - print(f"DM {msg_id} to {agent}/{target}: FAILED (not found in read-back)", file=sys.stderr) - sys.exit(1) + if delivered: + log_event({"type": "sent", "id": msg_id, "agent": agent, "to": recipient, "target": target, "verified": True}) + print(f"DM {msg_id} from {agent} to {recipient}/{target}: SENT and VERIFIED") else: - print(f"DM {msg_id} to {agent}/{target}: SENT (unverified)") - return msg_id + log_event({"type": "failed", "id": msg_id, "agent": agent, "to": recipient, "target": target, "verified": False}) + print(f"DM {msg_id} from {agent} to {recipient}/{target}: FAILED (not found in recipient chat after {max_retries} attempts)", file=sys.stderr) + sys.exit(1) + return msg_id -def dm_read(agent, target, n=5, quiet=False): - """Read DMs via headless.""" +def dm_read(agent, target, n=5, quiet=False, width=200): + """Read DMs via headless. width widens the per-paragraph slice + (needed for multi-line signature blocks).""" if agent not in VALID_AGENTS: print(f"ERROR: Unknown agent {agent}", file=sys.stderr) sys.exit(1) @@ -92,23 +138,139 @@ def dm_read(agent, target, n=5, quiet=False): run(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} sidechat use {target}") time.sleep(2) - msgs = run(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} messages {n}") + rc, msgs, err = run_full(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} messages {n} {width}") run(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} sidechat main") - + + if rc != 0 or not msgs: + # Never fail silently: an empty read with exit 0 hid a dead browser + # for hours (opm, 2026-10-03). Surface the last error line. + detail = err.splitlines()[-1] if err.strip() else "no output" + print(f"WARNING: read of {agent}/{target} failed (rc={rc}): {detail}", file=sys.stderr) if not quiet: print(msgs) return msgs -def dm_verify(agent, target, msg_id): - """Check if a message ID exists in the chat.""" - msgs = dm_read(agent, target, n=10, quiet=True) - if msg_id in msgs: - print(f"VERIFIED: {msg_id} found in {agent}/{target}") - return True +SIGNERS_DIR = "/home/super/Projects/NetVM/dm-signers" + +def dm_select(agent, target=None): + """Select active conversation for an agent. + - With --target: switch directly to that conversation. + - Without --target: list available conversations. + Uses sidechat use/main under the hood; stores selection for future commands. + """ + import os, json, subprocess + + NETVM_EXEC = os.path.expanduser("~/Projects/NetVM/bin/netvm-exec.sh") + API = os.path.expanduser("~/Projects/NetVM/bin/muse-chat-api.py") + state_file = os.path.expanduser(f"~/.dm-select-{agent}.json") + + def run(cmd): + r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=60) + return r.stdout.strip() + + if target: + # Switch to target conversation + if target == "main": + run(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} sidechat main") + else: + run(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} sidechat use {target}") + # Store selection + with open(state_file, "w") as f: + json.dump({"agent": agent, "target": target}, f) + print(f"Selected: {agent}/{target}") + return target + + # List available conversations + out = run(f"{NETVM_EXEC} {agent} -- python3 {API} --account {agent} sidechat list") + print(f"Conversations for {agent}:") + print(" main") + # Parse: filter out headers, timestamps, and status lines + import re + lines = [l.strip() for l in out.split("\n") if l.strip()] + ts = re.compile(r"^\d+[mhd]$") + skip = {"NOSIDEBAR", "Side chats", "Unread updates"} + for line in lines: + if line in skip: + continue + if ts.match(line): + continue + # Remaining lines are chat names + print(f" {line}") + print() + print(f"Use: dm.py select --agent {agent} --target ") + # Show current selection + if os.path.exists(state_file): + with open(state_file) as f: + sel = json.load(f) + print(f"Current: {sel.get('target', 'main')}") + return None + + +def dm_verify_sig(message=None, agent=None, target=None): + """Verify an SSH-signed DM. Pass message text directly, or give + --agent/--target to scan recent reads for signed messages.""" + import tempfile, re + texts = [] + if message: + texts = [message] + elif agent and target: + msgs = dm_read(agent, target, n=10, quiet=True, width=2000) + # split read output into candidate blocks containing a signature + chunks = re.split(r'\n---\n', msgs) + texts = [c for c in chunks if '-----BEGIN SSH SIGNATURE-----' in c] + if not texts: + print("no signed messages found in recent reads", file=sys.stderr) + sys.exit(1) else: - print(f"NOT FOUND: {msg_id} in {agent}/{target}", file=sys.stderr) + print("ERROR: provide a message or --agent/--target", file=sys.stderr) sys.exit(1) + ok_any = False + for text in texts: + text = text.strip() + m = re.match(r'\[from:([^\]]+)\]\s*\[id:([^\]]+)\]', text) + if not m: + print("BAD: no [from:]/[id:] header", file=sys.stderr) + continue + sender, mid = m.group(1), m.group(2) + sm = re.search(r'\n-----BEGIN SSH SIGNATURE-----\n(.*?)\n-----END SSH SIGNATURE-----', + text, re.S) + if not sm: + print(f"BAD: [{mid}] no signature block", file=sys.stderr) + continue + payload = text[:sm.start()].strip() + sigblock = "-----BEGIN SSH SIGNATURE-----\n" + sm.group(1).strip() + "\n-----END SSH SIGNATURE-----\n" + pubpath = os.path.join(SIGNERS_DIR, sender + ".pub") + if not os.path.exists(pubpath): + print(f"BAD: [{mid}] no public key registered for sender '{sender}'", file=sys.stderr) + continue + with open(pubpath) as f: + pubkey = f.read().strip() + with tempfile.TemporaryDirectory() as td: + allowed = os.path.join(td, "allowed") + with open(allowed, "w") as f: + f.write(f"{sender} {pubkey}\n") + sigf = os.path.join(td, "sig") + with open(sigf, "w") as f: + f.write(sigblock) + payf = os.path.join(td, "payload") + with open(payf, "w") as f: + f.write(payload) + # verify reads the payload from stdin; feed it from the temp file + # (redirect, not a pipe) per the file-based lesson from chat-400 + with open(payf, "rb") as fin: + r = subprocess.run( + ["ssh-keygen", "-Y", "verify", "-f", allowed, "-I", sender, + "-n", "dm", "-s", sigf], + stdin=fin, capture_output=True, text=True, timeout=15) + if r.returncode == 0: + print(f"GOOD: [{mid}] signature valid — really from '{sender}'") + ok_any = True + else: + print(f"BAD: [{mid}] signature FAILED for claimed sender '{sender}': " + f"{r.stderr.strip()[:120]}", file=sys.stderr) + sys.exit(0 if ok_any else 1) + def dm_log(n=20): """Show recent log entries.""" if not os.path.exists(LOG_FILE): @@ -125,28 +287,38 @@ def dm_thread(from_agent, to_agent, target, message): return dm_send(to_agent, target, attributed) def main(): - p = argparse.ArgumentParser(description="DM: Headless Direct Messages with verification") + p = argparse.ArgumentParser(description="DM: Headless Direct Messages (UUID-tagged, logged; delivery NOT confirmed)") sub = p.add_subparsers(dest='cmd', required=True) - ps = sub.add_parser('send', help='Send a DM with verification') + ps = sub.add_parser('send', help='Send a DM (UUID-tagged; delivery NOT confirmed)') ps.add_argument('--agent', required=True, choices=VALID_AGENTS) + ps.add_argument('--to', required=False, choices=VALID_AGENTS, default=None, + help='Recipient operator (for cross-operator DMs). Uses recipient\'s browser/chat.') ps.add_argument('--target', required=True) - ps.add_argument('--no-verify', action='store_true', help='Skip verification') + ps.add_argument('--no-verify', action='store_true', help='Deprecated no-op: verification was removed') + ps.add_argument('--raw', action='store_true', help='Send verbatim: no UUID tag, no truncation (for pre-signed messages)') ps.add_argument('message') - ps.set_defaults(func=lambda a: dm_send(a.agent, a.target, a.message, verify=not a.no_verify)) + ps.set_defaults(func=lambda a: dm_send(a.agent, a.target, a.message, verify=not a.no_verify, raw=a.raw, to_agent=a.to)) + psel = sub.add_parser('select', help='Select active conversation') + psel.add_argument('--agent', required=True, choices=VALID_AGENTS) + psel.add_argument('--target', required=False, default=None, + help='Conversation: main or side chat name') + psel.set_defaults(func=lambda a: dm_select(a.agent, a.target)) + pr = sub.add_parser('read', help='Read DMs') pr.add_argument('--agent', required=True, choices=VALID_AGENTS) pr.add_argument('--target', required=True) pr.add_argument('--n', type=int, default=5) pr.set_defaults(func=lambda a: dm_read(a.agent, a.target, a.n)) - pv = sub.add_parser('verify', help='Verify a message ID exists') - pv.add_argument('--agent', required=True, choices=VALID_AGENTS) - pv.add_argument('--target', required=True) - pv.add_argument('msg_id') - pv.set_defaults(func=lambda a: dm_verify(a.agent, a.target, a.msg_id)) - + pvs = sub.add_parser('verify-sig', help='Verify SSH signature on a signed DM') + pvs.add_argument('--agent', required=False, choices=VALID_AGENTS) + pvs.add_argument('--target', required=False) + pvs.add_argument('--from-sender', required=False, dest='from_sender') + pvs.add_argument('message', nargs='?') + pvs.set_defaults(func=lambda a: dm_verify_sig(message=a.message, agent=a.agent, target=a.target)) + pl = sub.add_parser('log', help='Show DM log') pl.add_argument('--n', type=int, default=20) pl.set_defaults(func=lambda a: dm_log(a.n))