box/bin/chromebox-watchdog.sh

#!/usr/bin/env bash
# chromebox-watchdog.sh [profile] — keep a chrome-box profile alive and healthy.
# Checks: 1) chromium process for the profile is running,
#         2) CDP responds and the chat page is present.
# If unhealthy: kill any stale chrome for the profile and relaunch headless
# via netvm-chrome.sh (profile dir persists session/cookies — the process is
# disposable, the state is not). Mirrors operator-646's container
# recover-after-rebuild.sh philosophy.
# Runs every 2 min via systemd timer chromebox-watchdog-<profile>.timer.
set -euo pipefail
# Prevent overlapping runs: the timer fires every 2 min but a relaunch
# (kill + sleep 25 + chrome startup + page load) can exceed that, and two
# concurrent runs kill each others chrome (observed 2026-10-03: pip flapped
# with simultaneous "relaunch OK" and "relaunch FAILED").
LOCK="/tmp/chromebox-watchdog-${1:-pip}.lock"
exec 9>"$LOCK"
if ! flock -n 9; then
  echo "[$(date -u +%FT%TZ)] [$1] another watchdog run in progress, skipping" >&2
  exit 0
fi

PROFILE="${1:-pip}"
NETVM_BIN="/home/super/Projects/NetVM/bin"
LOG="/home/super/Projects/NetVM/chromebox-watchdog.log"
CHROME_LOG="/home/super/Projects/NetVM/chromebox-${PROFILE}.log"

# Rotate a log file past 10MB (keep one generation)
rotate_log() {
  local f="$1"
  [ -f "$f" ] || return 0
  local sz
  sz=$(stat -c%s "$f" 2>/dev/null || echo 0)
  if [ "$sz" -gt 10485760 ]; then
    mv -f "$f" "$f.1"
    echo "[$(date -u +%FT%TZ)] [$PROFILE] log rotated" > "$f"
  fi
}
rotate_log "$LOG"
# CHROME_LOG rotation happens after PROFILE is set (see below)

case "$PROFILE" in
  muse) CDP_PORT=9410 ;;
  pip)  CDP_PORT=9420 ;;
  646)  CDP_PORT=9430 ;;
  opm)  CDP_PORT=9440 ;;
  *) echo "unknown profile: $PROFILE" >&2; exit 1 ;;
esac
rotate_log "$CHROME_LOG"

log() { echo "[$(date -u +%FT%TZ)] [$PROFILE] $*" | tee -a "$LOG"; }

cdp_list() {
  "$NETVM_BIN/netvm-exec.sh" "$PROFILE" -- curl -s -m 8 "http://127.0.0.1:$CDP_PORT/json/list" 2>/dev/null
}

# HEALTH_FAIL_REASON is set by healthy() on failure: which stage broke.
HEALTH_FAIL_REASON=""
healthy() {
  HEALTH_FAIL_REASON=""
  pgrep -f "chromium.*profiles/${PROFILE}/" >/dev/null 2>&1 \
    || { HEALTH_FAIL_REASON="no chromium process for profile"; return 1; }
  local list
  list="$(cdp_list)" \
    || { HEALTH_FAIL_REASON="CDP unreachable on :$CDP_PORT"; return 1; }
  echo "$list" | grep -q '"title": ".*Chat' \
    || { HEALTH_FAIL_REASON="CDP up but no Chat page in target list"; return 1; }
  return 0
}

if healthy; then
  exit 0
fi

log "unhealthy ($HEALTH_FAIL_REASON), relaunching chromebox"
# bracket trick so pkill never matches its own command line
pat="profiles/${PROFILE:0:${#PROFILE}-1}[${PROFILE: -1}]/"
pkill -f "chromium.*$pat" 2>/dev/null || true
sleep 3
# Relaunch in its own systemd scope so it survives this oneshot run.
# nohup/setsid do NOT escape: this timer's service uses KillMode=control-group
# and systemd SIGKILLs everything in the cgroup at teardown (observed
# 2026-10-03: every relaunch "recovered" then died seconds later). A transient
# scope escapes the service cgroup; the scoped process inherits these fds so
# the log redirect below still captures chromium's output.
# NOTE: systemd-run --scope WAITS for the scope's processes (even --no-block,
# verified 2026-10-03), so it must be backgrounded — the scope is an
# independent unit and outlives the wrapper.
# NOTE: --cdp-port is pinned explicitly. netvm-chrome.sh defaults to a
# hash-derived port (9222+...) which will NOT match the registry port that
# muse-chat-api.py uses — a relaunch on the wrong port looks healthy to the
# launcher but is unreachable to the API (observed 2026-10-03: pip relaunched
# on 9278 instead of 9420, watchdog looped on "relaunch FAILED").
systemd-run --user --scope --unit="netvm-chrome-${PROFILE}-$(date +%s)" \
  "$NETVM_BIN/netvm-chrome.sh" --headless --cdp-port "$CDP_PORT" "$PROFILE" "https://muse.ai" \
  >>"$CHROME_LOG" 2>&1 < /dev/null &
sleep 25
if healthy; then
  log "relaunch OK"
else
  log "relaunch FAILED ($HEALTH_FAIL_REASON) \u2014 needs operator attention"
  exit 1
fi