From f9f12b450e51f79d209e525563b9e6086e4259db Mon Sep 17 00:00:00 2001 From: operator Date: Sat, 10 Oct 2026 13:23:56 -0400 Subject: [PATCH] feat(tui): upgrade Tab 7 SSH/Boxes with diagnostics modal, key verification, and recovery watcher hardening --- bin/muse-tui.py | 251 ++++++++++++++++++--- cloud-uptime/recover-after-rebuild.sh | 299 ++++++++++++++++++++++++++ cloud-uptime/uptime-watcher.sh | 130 +++++++++++ tests/test_box_tui_render.py | 84 ++++++++ 4 files changed, 731 insertions(+), 33 deletions(-) create mode 100755 cloud-uptime/recover-after-rebuild.sh create mode 100755 cloud-uptime/uptime-watcher.sh diff --git a/bin/muse-tui.py b/bin/muse-tui.py index b8ece26..c5aee9d 100755 --- a/bin/muse-tui.py +++ b/bin/muse-tui.py @@ -1424,6 +1424,21 @@ class FleetDataManager: err_lines = (stderr or stdout or f"exit {rc}").strip().splitlines() return False, (err_lines[-1] if err_lines else f"exit {rc}")[:200] + def probe_container_keys(self, account: str) -> tuple[bool, str]: + """On-demand probe: check presence of critical SSH keys inside container.""" + if account not in SSH_TUNNEL_PORTS: + return False, f"No tunnel registered for '{account}'" + cmd = build_ssh_dial_command( + account, + ssh_options=["-o", "BatchMode=yes", "-o", "ConnectTimeout=12"], + remote_command="ls -1 ~/.ssh/ ~/workspace/.ssh-keys/ 2>/dev/null", + ) + rc, stdout, stderr = run_command_isolated(cmd, timeout=25.0) + if rc == 0 and (stdout or "").strip(): + return True, stdout.strip() + err_lines = (stderr or stdout or f"exit {rc}").strip().splitlines() + return False, (err_lines[-1] if err_lines else f"exit {rc}")[:200] + def _fetch_dm_logs(self): log_path = REPO_ROOT / "dm-log.jsonl" if not log_path.exists(): @@ -3521,7 +3536,21 @@ class MuseTUI: return ssh_row_order(nodes, extras) def _render_ssh_view(self, y: int, x: int, h: int, w: int): - self.safe_addstr(self.stdscr, y, x + 1, f"CONTAINER SSH TUNNEL HEALTH (jump: {SSH_OPERATOR_USER}@{SSH_JUMP_HOST})", self._attr("bold")) + rows = self.get_ssh_rows() + if self.ssh_sel_idx >= len(rows): + self.ssh_sel_idx = max(0, len(rows) - 1) + + visible_rows = max(3, h - 6) + scroll_start = getattr(self, "ssh_scroll_idx", 0) + if self.ssh_sel_idx < scroll_start: + scroll_start = self.ssh_sel_idx + elif self.ssh_sel_idx >= scroll_start + visible_rows: + scroll_start = self.ssh_sel_idx - visible_rows + 1 + scroll_start = max(0, min(scroll_start, max(0, len(rows) - visible_rows))) + self.ssh_scroll_idx = scroll_start + + pos_tag = f" [{scroll_start + 1}-{min(len(rows), scroll_start + visible_rows)}/{len(rows)}]" if rows else "" + self.safe_addstr(self.stdscr, y, x + 1, f"CONTAINER SSH TUNNEL HEALTH (jump: {SSH_OPERATOR_USER}@{SSH_JUMP_HOST}){pos_tag}", self._attr("bold")) with self.data.lock: jump = self.data.ssh_jump_reachable @@ -3542,18 +3571,8 @@ class MuseTUI: self.safe_addstr(self.stdscr, y + 2, x + 1, " ACCOUNT SSH PORT SSH STATE LAT SSH BANNER / HOSTKEY TERM PORT TERM STATE SINCE", self._attr("dim")) self.safe_addstr(self.stdscr, y + 3, x + 1, "─" * (w - 2), self._attr("dim")) - rows = self.get_ssh_rows() - if self.ssh_sel_idx >= len(rows): - self.ssh_sel_idx = max(0, len(rows) - 1) - - visible_rows = max(3, h - 6) - scroll_start = getattr(self, "ssh_scroll_idx", 0) - if self.ssh_sel_idx < scroll_start: - scroll_start = self.ssh_sel_idx - elif self.ssh_sel_idx >= scroll_start + visible_rows: - scroll_start = self.ssh_sel_idx - visible_rows + 1 - scroll_start = max(0, min(scroll_start, max(0, len(rows) - visible_rows))) - self.ssh_scroll_idx = scroll_start + num_rendered = min(visible_rows, max(0, len(rows) - scroll_start)) + self._ssh_view_bounds = (y + 4, y + 4 + num_rendered) row_y = y + 4 for row_i in range(visible_rows): @@ -3604,13 +3623,23 @@ class MuseTUI: self.safe_addstr(self.stdscr, row_y, x + 40, banner[:26].ljust(26), row_attr if is_sel else self._attr("normal")) self.safe_addstr(self.stdscr, row_y, x + 67, f":{tport:<8}", row_attr if is_sel else self._attr("dim")) self.safe_addstr(self.stdscr, row_y, x + 77, term_txt, term_attr) - self.safe_addstr(self.stdscr, row_y, x + 83, since_txt[:w - 84 - 8], row_attr if is_sel else self._attr("dim")) - self.safe_addstr(self.stdscr, row_y, max(x + 90, w - 8), "[SSH]", self._attr("wo_badge") if is_sel else self._attr("dim")) + self.safe_addstr(self.stdscr, row_y, x + 83, since_txt[:w - 84 - 16], row_attr if is_sel else self._attr("dim")) + self.safe_addstr(self.stdscr, row_y, max(x + 80, w - 16), "[Diag]", self._attr("selected") if is_sel else self._attr("dim")) + self.safe_addstr(self.stdscr, row_y, max(x + 88, w - 8), "[SSH]", self._attr("wo_badge") if is_sel else self._attr("dim")) row_y += 1 + # Vertical scrollbar track + if len(rows) > visible_rows: + track_h = visible_rows + thumb_pos = int((scroll_start / max(1, len(rows) - visible_rows)) * (track_h - 1)) + for r in range(track_h): + char = "█" if r == thumb_pos else "│" + attr = self._attr("selected") if r == thumb_pos else self._attr("dim") + self.safe_addstr(self.stdscr, y + 4 + r, w - 1, char, attr) + hint_y = y + h - 1 sel_acct = rows[self.ssh_sel_idx].upper() if rows else "-" - hints = f"Selected: [{sel_acct}] [Enter/s]: SSH pop-out (tmux) [c]: Copy dial [u]: Container uptime [r]: Refresh [j/k]: Nav" + hints = f"Selected: [{sel_acct}] [d/Enter]: Diagnostics [s]: SSH pop-out [c]: Copy dial [u]: Container uptime [k]: Key check [r]: Refresh [j/k]: Nav" self.safe_addstr(self.stdscr, hint_y, x + 1, hints[:w - 2], self._attr("dim")) # ----------------------------------------------------------------------- @@ -3923,8 +3952,8 @@ class MuseTUI: left_x = max(1, min(anchor_x, screen_w - modal_w - 2)) self._context_popup_bounds = (left_x, top_y, modal_w, modal_h) else: - modal_w = min(84 if self.modal in ("prompts", "history_search") else 74, screen_w - 6) - modal_h = min(22 if self.modal in ("prompts", "history_search") else 20, screen_h - 4) + modal_w = min(84 if self.modal in ("prompts", "history_search", "box_diagnostics") else 74, screen_w - 6) + modal_h = min(22 if self.modal in ("prompts", "history_search", "box_diagnostics") else 20, screen_h - 4) top_y = (screen_h - modal_h) // 2 left_x = (screen_w - modal_w) // 2 self._context_popup_bounds = (left_x, top_y, modal_w, modal_h) @@ -3958,6 +3987,7 @@ class MuseTUI: "work_dispatch": " DISPATCH NEW BUILD TICKET (box work start) ", "agent_thoughts": " LIVE COGNITIVE THOUGHT STREAM & SCREEN ", "agent_menu": " AGENT PROFILE MENU (TASKS / TIMERS / APPROVALS) ", + "box_diagnostics": " BOX DIAGNOSTICS & TROUBLESHOOTING ", } title = title_map.get(self.modal, " DIALOG ") self.safe_addstr(self.stdscr, top_y, left_x + 3, title, self._attr("header")) @@ -3988,7 +4018,7 @@ class MuseTUI: ("[a] or [F2]", "Open Approvals Resolution Drawer (Allow, Always, Deny)"), ("[F5] or [m]", "Toggle between Muse Chat TUI and Box Fleet Command TUI"), ("[1]-[7] (Box)", "Switch Box tabs: Chat/Fleet/Approvals/Jobs/Tmux/Logs/SSH"), - ("[Tab 7: SSH]", "Enter/s: tmux pop-out c: copy dial u: container uptime r: refresh"), + ("[Tab 7: SSH]", "d/Enter: diag modal s: pop-out c: dial u: uptime k: keys r: refresh"), ("[g] / [G] / [Home/End]", "Jump to oldest message / follow live latest message"), ("[q]", "Quit TUI (in NORMAL mode)"), ] @@ -4535,6 +4565,77 @@ class MuseTUI: self.safe_addstr(self.stdscr, top_y + modal_h - 2, left_x + 3, "[1-4/Tab] Tab [w] Dispatch [f] Refocus [h] Heal [t] Thoughts [Esc/q] Close", self._attr("header")) + elif self.modal == "box_diagnostics": + rows = self.get_ssh_rows() + acct = rows[self.ssh_sel_idx] if rows and 0 <= getattr(self, "ssh_sel_idx", 0) < len(rows) else "unknown" + with self.data.lock: + jump_ok = self.data.ssh_jump_reachable + jump_lat = self.data.ssh_check_latency_ms + jump_err = self.data.ssh_check_error + health = dict(self.data.ssh_cache.get(acct, {})) + info = SSH_TUNNEL_PORTS.get(acct, {}) + sport = info.get("port", "?") + tport = info.get("terminal", "?") + + ssh_up = health.get("ssh_up") + term_up = health.get("term_up") + lat = health.get("ssh_latency_ms") + lat_str = f"{lat}ms" if lat is not None else "--" + banner = (health.get("ssh_banner") or health.get("term_http") or "-").strip() + state_since = health.get("state_since", 0.0) + since_str = format_recency(state_since) if state_since else "never" + dial_cmd = build_ssh_dial_string(acct) if acct in SSH_TUNNEL_PORTS else "N/A" + + # Header details + self.safe_addstr(self.stdscr, top_y + 2, left_x + 3, f"BOX TARGET: @{acct.upper():<12} (SSH :{sport} | Term :{tport})", self._attr("bold")) + + if jump_ok is True: + j_txt = f"ONLINE ({jump_lat}ms latency)" + j_attr = self._attr("success") + elif jump_ok is False: + j_txt = f"DOWN ({(jump_err or 'unknown')[:32]})" + j_attr = self._attr("danger") + else: + j_txt = "SWEEP PENDING" + j_attr = self._attr("dim") + + self.safe_addstr(self.stdscr, top_y + 3, left_x + 3, f"Jump Gateway: {SSH_OPERATOR_USER}@{SSH_JUMP_HOST} -> ", self._attr("dim")) + self.safe_addstr(self.stdscr, top_y + 3, left_x + 38, j_txt, j_attr) + + ssh_st_str = "LISTENING / UP" if ssh_up else ("UNREACHABLE / DOWN" if ssh_up is False else "UNKNOWN") + ssh_st_attr = self._attr("success") if ssh_up else (self._attr("danger") if ssh_up is False else self._attr("dim")) + self.safe_addstr(self.stdscr, top_y + 4, left_x + 3, f"SSH Tunnel: :{sport:<6} -> {ssh_st_str} ({lat_str}) | Since: {since_str}", ssh_st_attr) + + self.safe_addstr(self.stdscr, top_y + 5, left_x + 3, f"SSH Banner: {banner[:modal_w - 20]}", self._attr("normal")) + + term_st_str = "LISTENING / UP" if term_up else ("UNREACHABLE / DOWN" if term_up is False else "UNKNOWN") + term_st_attr = self._attr("success") if term_up else (self._attr("danger") if term_up is False else self._attr("dim")) + self.safe_addstr(self.stdscr, top_y + 6, left_x + 3, f"Web Terminal: :{tport:<6} -> {term_st_str} (ttyd)", term_st_attr) + + self.safe_addstr(self.stdscr, top_y + 7, left_x + 3, f"Dial String: {dial_cmd[:modal_w - 20]}", self._attr("dim")) + self.safe_addstr(self.stdscr, top_y + 8, left_x + 3, f"Keys Spec: vm_to_gcp, muse-health, id_frontdoor, id_ed25519", self._attr("dim")) + + self.safe_addstr(self.stdscr, top_y + 9, left_x + 2, "─" * (modal_w - 4), self._attr("dim")) + + self.safe_addstr(self.stdscr, top_y + 10, left_x + 3, "TROUBLESHOOTING & FAST ACTIONS:", self._attr("bold")) + actions_list = [ + ("[1] / [s]", "SSH Pop-out", "Spawn terminal session in a new tmux window"), + ("[2] / [c]", "Copy Dial String", "Copy full jump command string to system clipboard"), + ("[3] / [u]", "Container Uptime", "Send remote `uptime` query to container"), + ("[4] / [k]", "Verify SSH Keys", "Check ~/.ssh/ and ~/workspace/.ssh-keys/ presence"), + ("[5] / [h]", "Run Recover Hook", "Execute recover-after-rebuild.sh --dry-run"), + ("[r]", "Trigger Sweep", "Initiate immediate background port scan via jump host"), + ] + for idx, (hk, title_act, desc) in enumerate(actions_list): + cur_act_y = top_y + 11 + idx + if cur_act_y >= top_y + modal_h - 2: + break + self.safe_addstr(self.stdscr, cur_act_y, left_x + 3, f"{hk:<10}", self._attr("wo_badge")) + self.safe_addstr(self.stdscr, cur_act_y, left_x + 14, f"{title_act:<18}", self._attr("bold")) + self.safe_addstr(self.stdscr, cur_act_y, left_x + 33, desc[:modal_w - 36], self._attr("normal")) + + self.safe_addstr(self.stdscr, top_y + modal_h - 2, left_x + 3, "[1-5/s/c/u/k/h/r] Run Action [Esc/q] Close Dialog", self._attr("header")) + # ----------------------------------------------------------------------- # Keypress & Event Handling # ----------------------------------------------------------------------- @@ -5115,8 +5216,8 @@ class MuseTUI: # Box mode Fast Actions: Tab 6 (SSH) — placed before the 's'/'c'/'y' # globals below so SSH keys win on this tab. if self.mode == "box" and self.box_tab == 6: - if ch in (curses.KEY_ENTER, 10, 13): - self._ssh_popout_selected() + if ch in (curses.KEY_ENTER, 10, 13, ord('d'), ord('D')): + self.modal = "box_diagnostics" return True elif ch in (ord('s'), ord('S')): self._ssh_popout_selected() @@ -5131,6 +5232,18 @@ class MuseTUI: threading.Thread(target=self._async_ssh_uptime, args=(acct,), daemon=True).start() self.set_toast(f"Probing container uptime on {acct}...", "info") return True + elif ch in (ord('k'), ord('K')): + rows = self.get_ssh_rows() + if rows and 0 <= self.ssh_sel_idx < len(rows): + acct = rows[self.ssh_sel_idx] + threading.Thread(target=self._async_ssh_key_check, args=(acct,), daemon=True).start() + return True + elif ch in (ord('h'), ord('H')): + rows = self.get_ssh_rows() + if rows and 0 <= self.ssh_sel_idx < len(rows): + acct = rows[self.ssh_sel_idx] + threading.Thread(target=self._async_ssh_recovery_probe, args=(acct,), daemon=True).start() + return True elif ch in (ord('r'), ord('R')): threading.Thread(target=self.data._fetch_ssh_health, daemon=True).start() self.set_toast("Probing SSH tunnels via jump host...", "info") @@ -7214,18 +7327,27 @@ class MuseTUI: return True elif self.box_tab == 6: - # Tab 6: SSH / Boxes (rows start one line lower: jump-status line) - rows = self.get_ssh_rows() - scroll_start = getattr(self, "ssh_scroll_idx", 0) - clicked_idx = scroll_start + row_idx - 1 - if 0 <= clicked_idx < len(rows): - self.ssh_sel_idx = clicked_idx - if mx >= w - 8: - # [SSH] pop-out - self._ssh_popout_selected() - else: - acct = rows[clicked_idx] - self.set_toast(f"Selected [{acct.upper()}]. Click [SSH] or press Enter to pop out.", "info") + # Tab 6: SSH / Boxes + bounds = getattr(self, "_ssh_view_bounds", (content_y + 4, content_y + 4 + 10)) + if bounds[0] <= my < bounds[1]: + row_offset = my - bounds[0] + rows = self.get_ssh_rows() + scroll_start = getattr(self, "ssh_scroll_idx", 0) + clicked_idx = scroll_start + row_offset + if 0 <= clicked_idx < len(rows): + self.ssh_sel_idx = clicked_idx + if mx >= w - 8: + # [SSH] pop-out + self._ssh_popout_selected() + elif mx >= w - 16: + # [Diag] modal + self.modal = "box_diagnostics" + else: + if is_double_click: + self.modal = "box_diagnostics" + else: + acct = rows[clicked_idx] + self.set_toast(f"Selected [{acct.upper()}]. Press [d/Enter] for Diagnostics or [s] for SSH.", "info") return True elif self.box_tab == 7: @@ -7891,6 +8013,38 @@ class MuseTUI: else: self.set_toast(f"{account} uptime failed: {out_text[:90]}", "error") + def _async_ssh_key_check(self, account: str): + self.set_toast(f"Probing SSH keys on {account}...", "info") + ok, out_text = self.data.probe_container_keys(account) + if ok: + found = [] + for k in ("vm_to_gcp", "muse-health", "id_frontdoor", "id_ed25519", "id_rsa"): + if k in out_text: + found.append(k) + found_str = ", ".join(found) if found else "files listed" + self.set_toast(f"✔ {account} keys verified: {found_str}", "success") + else: + self.set_toast(f"❌ {account} key probe failed: {out_text[:60]}", "error") + + def _async_ssh_recovery_probe(self, account: str): + self.set_toast(f"Running dry-run recovery probe on {account}...", "info") + if account not in SSH_TUNNEL_PORTS: + self.set_toast(f"No tunnel registered for '{account}'", "warn") + return + cmd = build_ssh_dial_command( + account, + ssh_options=["-o", "BatchMode=yes", "-o", "ConnectTimeout=15"], + remote_command="if [ -x ~/workspace/bin/recover-after-rebuild.sh ]; then ~/workspace/bin/recover-after-rebuild.sh --dry-run; else echo 'recover script not found'; fi", + ) + rc, stdout, stderr = run_command_isolated(cmd, timeout=30.0) + if rc == 0: + lines = [l.strip() for l in (stdout or "").splitlines() if l.strip()] + summary = lines[-1] if lines else "Recovery probe completed." + self.set_toast(f"✔ {account} probe: {summary[:60]}", "success") + else: + err = (stderr or stdout or f"exit {rc}").strip().splitlines() + self.set_toast(f"❌ {account} recovery failed: {err[-1][:60] if err else 'error'}", "error") + def _async_dispatch_work(self, goal: str, target_node: str): """Dispatch a new work build ticket with 45s synchronous readback gate.""" self.set_toast(f"⚡ Dispatching work ticket to @{target_node.upper()}...", "info") @@ -8896,6 +9050,37 @@ class MuseTUI: self.modal_input_cursor += 1 return True + elif self.modal == "box_diagnostics": + rows = self.get_ssh_rows() + acct = rows[self.ssh_sel_idx] if rows and 0 <= getattr(self, "ssh_sel_idx", 0) < len(rows) else "" + + if ch in (27, ord('q'), ord('Q')): + self.modal = None + return True + elif ch in (ord('1'), ord('s'), ord('S')): + self._ssh_popout_selected() + return True + elif ch in (ord('2'), ord('c'), ord('C')): + self._ssh_copy_dial_selected() + return True + elif ch in (ord('3'), ord('u'), ord('U')): + if acct: + threading.Thread(target=self._async_ssh_uptime, args=(acct,), daemon=True).start() + self.set_toast(f"Probing container uptime on {acct}...", "info") + return True + elif ch in (ord('4'), ord('k'), ord('K')): + if acct: + threading.Thread(target=self._async_ssh_key_check, args=(acct,), daemon=True).start() + return True + elif ch in (ord('5'), ord('h'), ord('H')): + if acct: + threading.Thread(target=self._async_ssh_recovery_probe, args=(acct,), daemon=True).start() + return True + elif ch in (ord('r'), ord('R')): + threading.Thread(target=self.data._fetch_ssh_health, daemon=True).start() + self.set_toast("Probing SSH tunnels via jump host...", "info") + return True + return True diff --git a/cloud-uptime/recover-after-rebuild.sh b/cloud-uptime/recover-after-rebuild.sh new file mode 100755 index 0000000..ff537a5 --- /dev/null +++ b/cloud-uptime/recover-after-rebuild.sh @@ -0,0 +1,299 @@ +#!/bin/bash +# recover-after-rebuild.sh — re-provision container after a VM/container rebuild. +# Standardized multi-machine recovery hook for muse-frontdoor fleet containers. +# +# Survives rebuilds: /home/hatch (workspace, ~/.ssh keys if preserved, persistent volumes). +# Ephemeral root: /etc, packages, users outside persistent tree, crontabs. +# +# Idempotent: safe to run any time. Does provisioning on fresh root +# filesystem (sentinel in /etc), then ensures tunnel supervisor is running. +set -u + +# Support dry-run mode and restore-keys mode +DRY_RUN=0 +RESTORE_KEYS_ONLY=0 +for arg in "$@"; do + case "$arg" in + --dry-run) + DRY_RUN=1 + echo "[recover] running in DRY-RUN mode (no mutations)" + ;; + --restore-keys) + RESTORE_KEYS_ONLY=1 + ;; + esac +done + +# Identity & per-machine config +ENV_FILE="$HOME/workspace/tunnel/machine.env" +if [ -f "$ENV_FILE" ]; then + # shellcheck disable=SC1090 + . "$ENV_FILE" +fi + +MACHINE="${MUSE_MACHINE:-muse-main}" +SSH_PORT="${SSH_PORT:-2224}" +TERM_PORT="${TERM_PORT:-7681}" + +_WL_BIN="$(cd "$(dirname "$0")" && pwd)/wl-config.py" +[ -x "$_WL_BIN" ] && eval "$("$_WL_BIN" --shell 2>/dev/null)" 2>/dev/null || true +unset _WL_BIN +FD_DOMAIN="${FD_DOMAIN:-${MACHINE}.muse-dev.online}" + +SENTINEL=/etc/hatch-provisioned +BIN="$HOME/workspace/bin" +DEB_CACHE="$HOME/workspace/debs" + +log() { echo "[recover] $*"; } + +needs_provisioning() { [ ! -f "$SENTINEL" ]; } + +restore_ssh_keys() { + # Key restoration: rebuilds may wipe ~/.ssh. Restore from persistent store if present. + install -m 700 -d "$HOME/.ssh" 2>/dev/null || true + for keyname in vm_to_gcp id_frontdoor muse-health id_ed25519 id_rsa; do + if [ ! -f "$HOME/.ssh/$keyname" ]; then + if [ -f "$HOME/workspace/.ssh-keys/$keyname" ]; then + log "restoring ~/.ssh/$keyname from persistent backup" + [ "$DRY_RUN" -eq 0 ] && install -m 600 "$HOME/workspace/.ssh-keys/$keyname" "$HOME/.ssh/$keyname" + elif [ "$keyname" = "id_frontdoor" ] && [ -f "$HOME/workspace/.ssh-keys/vm_to_gcp" ]; then + log "linking ~/.ssh/$keyname to persistent vm_to_gcp" + [ "$DRY_RUN" -eq 0 ] && install -m 600 "$HOME/workspace/.ssh-keys/vm_to_gcp" "$HOME/.ssh/$keyname" + elif [ "$keyname" = "vm_to_gcp" ] && [ -f "$HOME/workspace/.ssh-keys/id_frontdoor" ]; then + log "linking ~/.ssh/$keyname to persistent id_frontdoor" + [ "$DRY_RUN" -eq 0 ] && install -m 600 "$HOME/workspace/.ssh-keys/id_frontdoor" "$HOME/.ssh/$keyname" + fi + fi + if [ ! -f "$HOME/.ssh/${keyname}.pub" ] && [ -f "$HOME/workspace/.ssh-keys/${keyname}.pub" ]; then + log "restoring ~/.ssh/${keyname}.pub from persistent backup" + [ "$DRY_RUN" -eq 0 ] && install -m 644 "$HOME/workspace/.ssh-keys/${keyname}.pub" "$HOME/.ssh/${keyname}.pub" + fi + done +} + +provision_critical() { + log "fresh container detected — provisioning critical path (machine: $MACHINE, port: $SSH_PORT)" + + if [ "$DRY_RUN" -eq 1 ]; then + log "dry-run: would run fix-apt-mirror.sh, install deb packages, setup muse user, restore host keys" + return 0 + fi + + # 1. Fix dead apt mirror if present + if [ -x "$BIN/fix-apt-mirror.sh" ]; then + "$BIN/fix-apt-mirror.sh" + fi + + # 2. Check local .deb cache + if ls "$DEB_CACHE"/*.deb >/dev/null 2>&1; then + log "installing from persistent .deb cache" + DEBIAN_FRONTEND=noninteractive dpkg -i "$DEB_CACHE"/*.deb 2>&1 | tail -2 || true + apt-get install -f -y -qq 2>/dev/null || true + else + log "WARNING: deb cache empty at $DEB_CACHE — falling back to apt network" + if [ -z "$(ls /var/lib/apt/lists/ 2>/dev/null | grep -v '^lock' | head -1)" ]; then + apt-get update -qq + fi + fi + + # 3. Single-transaction install for critical networking packages + local missing="" + for p in openssh-client openssh-server; do + dpkg -s "$p" >/dev/null 2>&1 || missing="$missing $p" + done + if [ -n "$missing" ]; then + log "installing missing critical packages: $missing" + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends $missing + fi + + # 4. Restore SSH host keys + local hk_dir="$HOME/workspace/tunnel/ssh_host_keys" + if ls "$hk_dir"/ssh_host_* >/dev/null 2>&1; then + log "restoring persistent SSH host keys" + cp -p "$hk_dir"/ssh_host_* /etc/ssh/ 2>/dev/null \ + && chmod 600 /etc/ssh/ssh_host_* \ + && log "host keys restored" \ + || log "WARNING: host key restore failed" + elif ls /etc/ssh/ssh_host_* >/dev/null 2>&1; then + log "seeding persistent SSH host key store" + mkdir -p -m 700 "$hk_dir" + cp -p /etc/ssh/ssh_host_* "$hk_dir"/ 2>/dev/null && chmod 600 "$hk_dir"/* 2>/dev/null || true + fi + + # 5. Restore muse login user + if ! id muse >/dev/null 2>&1; then + log "creating muse user" + useradd -m -s /bin/bash muse 2>/dev/null || true + fi + echo 'muse:horse-battery-staple' | chpasswd 2>/dev/null || log "WARNING: chpasswd failed" + chown -R muse:muse /home/muse 2>/dev/null && chmod 755 /home/muse 2>/dev/null || true + + if [ -f "$HOME/workspace/tunnel/muse-authorized_keys" ]; then + install -m 700 -o muse -d /home/muse/.ssh 2>/dev/null || true + install -m 600 -o muse -g muse \ + "$HOME/workspace/tunnel/muse-authorized_keys" \ + /home/muse/.ssh/authorized_keys 2>/dev/null || true + fi + + # 6. Restore /root/.ssh/authorized_keys across rebuilds + install -m 700 -d /root/.ssh 2>/dev/null || true + if [ -f "$HOME/workspace/tunnel/root-authorized_keys" ]; then + log "restoring /root/.ssh/authorized_keys from persistent backup" + install -m 600 "$HOME/workspace/tunnel/root-authorized_keys" /root/.ssh/authorized_keys 2>/dev/null || true + elif [ -f "$HOME/workspace/tunnel/muse-authorized_keys" ]; then + log "seeding /root/.ssh/authorized_keys from muse-authorized_keys" + install -m 600 "$HOME/workspace/tunnel/muse-authorized_keys" /root/.ssh/authorized_keys 2>/dev/null || true + fi + if [ -f "/home/hatch/.ssh/authorized_keys" ]; then + log "merging /home/hatch/.ssh/authorized_keys into /root/.ssh/authorized_keys" + cat /home/hatch/.ssh/authorized_keys >> /root/.ssh/authorized_keys 2>/dev/null || true + sort -u /root/.ssh/authorized_keys -o /root/.ssh/authorized_keys 2>/dev/null || true + chmod 600 /root/.ssh/authorized_keys 2>/dev/null || true + fi + + touch "$SENTINEL" + log "critical provisioning complete" +} + +restore_crontabs() { + # Reinstall crontab from persistent spec + if [ -x "$BIN/persistent-crontab.sh" ]; then + log "restoring persistent crontabs" + if [ "$DRY_RUN" -eq 0 ]; then + "$BIN/persistent-crontab.sh" || log "WARNING: persistent-crontab.sh exited non-zero" + fi + fi +} + +provision_deferred() { + # Background non-critical tools (python3, tmux, age, yazi, neovim) + if [ "$DRY_RUN" -eq 1 ]; then + return 0 + fi + ( + local deferred_missing="" + for p in python3 tmux age; do + dpkg -s "$p" >/dev/null 2>&1 || deferred_missing="$deferred_missing $p" + done + if [ -n "$deferred_missing" ]; then + DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends $deferred_missing 2>/dev/null || true + fi + if [ -x "$BIN/yazi" ] && ! command -v yazi >/dev/null; then + cp "$BIN/yazi" /usr/local/bin/yazi 2>/dev/null && chmod 755 /usr/local/bin/yazi 2>/dev/null || true + fi + if [ -x "$HOME/workspace/nvim/bin/nvim" ] && ! command -v nvim >/dev/null; then + mkdir -p /opt/nvim 2>/dev/null + cp -r "$HOME/workspace/nvim/"* /opt/nvim/ 2>/dev/null || true + ln -sf /opt/nvim/bin/nvim /usr/local/bin/nvim 2>/dev/null || true + fi + local wheel_dir="$HOME/workspace/wheels" + if [ -d "$wheel_dir" ] && ls "$wheel_dir"/*.whl >/dev/null 2>&1; then + log "installing cached python wheels from $wheel_dir" + python3 -m pip install --no-index --find-links="$wheel_dir" protocol_muse 2>/dev/null || true + fi + ) >/dev/null 2>&1 & + disown 2>/dev/null || true +} + +ensure_tunnel() { + # Ensure legacy localhost.run tunnels are halted + for pid in $(pgrep -f "workspace/bin/tunnel-up\.sh$" 2>/dev/null); do + log "stopping retired localhost.run supervisor (pid $pid)" + [ "$DRY_RUN" -eq 0 ] && kill "$pid" 2>/dev/null || true + done + for pid in $(pgrep -f "ssh\.localhost\.run" 2>/dev/null); do + log "stopping retired localhost.run ssh (pid $pid)" + [ "$DRY_RUN" -eq 0 ] && kill "$pid" 2>/dev/null || true + done +} + +ensure_gcp_tunnel() { + if [ "$DRY_RUN" -eq 1 ]; then + log "dry-run: would check and start gcp tunnel supervisor" + return 0 + fi + ( + exec 9>"$BIN/.gcp-tunnel-up.lock" || exit 0 + flock -n 9 || { log "another recovery run starting gcp tunnel; skipping"; exit 0; } + if pgrep -f "workspace/bin/gcp-tunnel-up.*\.sh$" >/dev/null; then + log "gcp tunnel supervisor already running" + exit 0 + fi + if [ ! -f "$HOME/.ssh/vm_to_gcp" ] && [ -f "$HOME/.ssh/id_frontdoor" ]; then + ln -sf "$HOME/.ssh/id_frontdoor" "$HOME/.ssh/vm_to_gcp" + elif [ ! -f "$HOME/.ssh/id_frontdoor" ] && [ -f "$HOME/.ssh/vm_to_gcp" ]; then + ln -sf "$HOME/.ssh/vm_to_gcp" "$HOME/.ssh/id_frontdoor" + fi + if [ ! -f "$HOME/.ssh/vm_to_gcp" ] && [ ! -f "$HOME/.ssh/id_frontdoor" ]; then + log "WARNING: ~/.ssh/vm_to_gcp missing — cannot start gcp tunnel supervisor" + exit 0 + fi + log "starting gcp tunnel supervisor" + local sup="$BIN/gcp-tunnel-up.sh" + [ -x "$sup" ] || sup="$BIN/gcp-tunnel-up-${MACHINE}.sh" + if [ -x "$sup" ]; then + setsid nohup "$sup" >/dev/null 2>&1 < /dev/null 9>&- & + disown 2>/dev/null || true + touch "$BIN/.gcp-tunnel-started" + else + log "WARNING: no executable gcp-tunnel supervisor found at $sup" + fi + ) + if [ -f "$BIN/.gcp-tunnel-started" ]; then + rm -f "$BIN/.gcp-tunnel-started" + _GCP_TUNNEL_STARTED=1 + fi +} + +report_health_on_recovery() { + [ "${_GCP_TUNNEL_STARTED:-0}" = 1 ] || return 0 + [ "$DRY_RUN" -eq 1 ] && return 0 + local reporter="$HOME/workspace/muse-frontdoor/bin/health-report.sh" + [ -x "$reporter" ] || { log "health reporter not found — skipping immediate report"; return 0; } + [ -f "$HOME/.ssh/muse-health" ] || { log "health key missing — skipping immediate report"; return 0; } + + log "tunnel (re)started — waiting for VM listener $SSH_PORT before health report" + local i + for i in $(seq 1 18); do + if ssh -i "$HOME/.ssh/vm_to_gcp" \ + -o ProxyCommand="$HOME/workspace/bin/ssh-via-proxy %h %p" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=8 \ + -o BatchMode=yes \ + super@34.139.37.135 \ + "ss -tln 2>/dev/null | grep -q '127.0.0.1:${SSH_PORT} '" 2>/dev/null; then + log "VM listener $SSH_PORT confirmed — sending immediate health report" + MUSE_MACHINE="$MACHINE" "$reporter" 2>&1 | head -5 || true + return 0 + fi + sleep 5 + done + log "WARNING: VM listener $SSH_PORT not seen after 90s — skipping immediate report" +} + +main() { + restore_ssh_keys + if [ "$RESTORE_KEYS_ONLY" -eq 1 ]; then + log "SSH key restore completed (keys-only mode)." + return 0 + fi + if needs_provisioning; then + provision_critical + else + log "container already provisioned (sentinel present)" + fi + restore_crontabs + ensure_tunnel + ensure_gcp_tunnel + provision_deferred + report_health_on_recovery + + echo "---" + echo "machine: $MACHINE (SSH port: $SSH_PORT, terminal port: $TERM_PORT)" + echo "domain: https://${FD_DOMAIN}" + echo "ttyd: $(pgrep -f '[t]tyd' | head -1 || echo '(not running)')" + echo "supervisor: $(pgrep -f 'gcp-tunnel-up' | head -1 || echo '(not running)')" +} + +main "$@" diff --git a/cloud-uptime/uptime-watcher.sh b/cloud-uptime/uptime-watcher.sh new file mode 100755 index 0000000..4bb4b6a --- /dev/null +++ b/cloud-uptime/uptime-watcher.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# uptime-watcher.sh — simple hatch-hook watcher: spawn/rebuild from spec. +# +# Register as a hatch hook (id `uptime-watcher`, poll 120s, timeout 300s) +# alongside tunnel-keeper. Each poll it guarantees the three things a +# container rebuild destroys: +# 1. provisioning — runs recover-after-rebuild.sh on a fresh root fs +# 2. supervisor — respawns gcp-tunnel-up.sh if it died +# 3. cron jobs — reinstalls crontab from ~/workspace/cron/*.persist +# +# It also verifies the VM-side SSH forward answers a banner, and wakes the +# operator (rate-limited, 30 min) only when something stays broken across +# polls. Silent on success. Safe to run by hand or from cron too. +set -u + +# --- runtime (hatch hook functions, or local fallbacks) --- +if [ -n "${HATCH_HOOK_RUNTIME:-}" ] && [ -f "$HATCH_HOOK_RUNTIME" ]; then + # shellcheck disable=SC1090 + source "$HATCH_HOOK_RUNTIME" +else + log() { echo "[uptime-watcher] $1 $2"; } + silent() { echo "[uptime-watcher] silent: $1 $2"; } + wake() { echo "[uptime-watcher] WAKE $1 $2"; } +fi + +# --- identity (per-machine, persistent) --- +ENV_FILE="$HOME/workspace/tunnel/machine.env" +# shellcheck disable=SC1090 +[ -f "$ENV_FILE" ] && . "$ENV_FILE" +MACHINE="${MUSE_MACHINE:-unknown}" +SSH_PORT="${SSH_PORT:-0}" +TERM_PORT="${TERM_PORT:-0}" + +STATE_DIR="$HOME/hooks/state/uptime-watcher" +BIN="$HOME/workspace/bin" +RECOVER="$BIN/recover-after-rebuild.sh" +SUPERVISOR="$BIN/gcp-tunnel-up.sh" +CRON_RESTORE="$BIN/persistent-crontab.sh" +SSH_KEY="$HOME/.ssh/vm_to_gcp" +GCP_HOST="${FD_VM_HOST:-34.139.37.135}" +GCP_USER="${FD_VM_USER:-super}" +FAIL_COUNT="$STATE_DIR/consec_failures" +LAST_WAKE="$STATE_DIR/last_wake_ts" + +mkdir -p "$STATE_DIR" +exec 9>"$STATE_DIR/watcher.lock" +flock -n 9 || { silent "previous poll still running" '{}'; exit 0; } +read_int() { [ -f "$1" ] && tr -cd '0-9' < "$1" || echo 0; } + +actions="" +fail="" + +# --- 1. fresh rebuild or missing SSH key? provision / restore --- +if [ ! -f /etc/hatch-provisioned ] || [ ! -f "$SSH_KEY" ]; then + if [ -x "$RECOVER" ]; then + if timeout 280 "$RECOVER" >"$STATE_DIR/recover-last.log" 2>&1; then + actions="${actions}provisioned " + log "recovery" '{"event":"provisioned_after_rebuild"}' + else + fail="recover_failed" + fi + else + fail="recover_missing" + fi +fi + +# --- 2. supervisor alive? respawn --- +if [ -z "$fail" ] && ! pgrep -f "workspace/bin/gcp-tunnel-up\.sh$" >/dev/null; then + if [ -x "$SUPERVISOR" ] && [ -f "$SSH_KEY" ]; then + setsid nohup "$SUPERVISOR" >/dev/null 2>&1 < /dev/null 9>&- & + disown 2>/dev/null || true + actions="${actions}supervisor-respawned " + log "supervisor" '{"event":"respawned"}' + else + fail="supervisor_unstartable" + fi +fi + +# --- 3. cron jobs alive? restore from persistent spec --- +if [ -z "$fail" ] && [ -x "$CRON_RESTORE" ]; then + if "$CRON_RESTORE" >"$STATE_DIR/cron-last.log" 2>&1; then + grep -q "reinstalled" "$STATE_DIR/cron-last.log" \ + && actions="${actions}cron-restored " + else + fail="cron_restore_failed" + fi +fi + +# --- 4. VM forward answers? (banner check, cheap) --- +ssh_state="unknown" +if [ -z "$fail" ] && [ "$SSH_PORT" != "0" ] && [ -f "$SSH_KEY" ] \ + && pgrep -f "[s]sh.*${SSH_PORT}:localhost:22" >/dev/null; then + banner="$(timeout 12 ssh -i "$SSH_KEY" \ + -o ProxyCommand="$BIN/ssh-via-proxy %h %p" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=8 -o BatchMode=yes \ + "$GCP_USER@$GCP_HOST" \ + "timeout 5 bash -c 'exec 3<>/dev/tcp/127.0.0.1/$SSH_PORT && head -c 4 <&3' 2>/dev/null" \ + 2>/dev/null || true)" + case "$banner" in + SSH-*) ssh_state="up" ;; + *) ssh_state="stale-forward"; fail="forward_dead" ;; + esac +elif [ -z "$fail" ]; then + ssh_state="down" + fail="tunnel_down" +fi + +payload="$(printf '{"machine":"%s","ssh":"%s","actions":"%s"}' \ + "$MACHINE" "$ssh_state" "${actions:-none}")" + +# --- 5. silent ok, or rate-limited wake on persistent failure --- +if [ -z "$fail" ]; then + printf 0 > "$FAIL_COUNT" + silent "uptime watcher poll ok" "$payload" + exit 0 +fi + +count=$(( $(read_int "$FAIL_COUNT") + 1 )) +printf '%s' "$count" > "$FAIL_COUNT" +log "failure" "{\"condition\":\"$fail\",\"consec\":\"$count\"}" +if [ "$count" -ge 2 ]; then + now=$(date +%s); last=$(read_int "$LAST_WAKE") + if [ $(( now - last )) -ge 1800 ]; then + printf '%s' "$now" > "$LAST_WAKE" + wake "$fail" "$payload" + exit 0 + fi +fi +silent "failure $fail ($count) — below wake threshold" "$payload" diff --git a/tests/test_box_tui_render.py b/tests/test_box_tui_render.py index 382f05d..1961ce0 100644 --- a/tests/test_box_tui_render.py +++ b/tests/test_box_tui_render.py @@ -581,6 +581,90 @@ class TestBoxTUIHeadless(unittest.TestCase): app._handle_key(ord('G')) self.assertEqual(app.dm_logs_sel_idx, 14) + def test_box_ssh_view_render_scrollbar_and_bounds(self): + app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="ssh") + self.assertEqual(app.box_tab, 6) + + # Populate multiple nodes to ensure scrolling + app.data.nodes = [f"node_{i}" for i in range(15)] + app.ssh_sel_idx = 8 + app._render_ssh_view(2, 0, 10, 100) + + # Check bounds recorded + self.assertTrue(hasattr(app, "_ssh_view_bounds")) + self.assertEqual(app._ssh_view_bounds[0], 6) # y (2) + 4 = 6 + self.assertGreater(app._ssh_view_bounds[1], app._ssh_view_bounds[0]) + + # Verify scrollbar and tags in screen output + rendered_texts = [call[0][2] for call in self.mock_stdscr.addstr.call_args_list if len(call[0]) >= 3 and isinstance(call[0][2], str)] + self.assertTrue(any("█" in t or "│" in t for t in rendered_texts)) + self.assertTrue(any("[Diag]" in t for t in rendered_texts)) + self.assertTrue(any("[SSH]" in t for t in rendered_texts)) + + def test_box_diagnostics_modal_and_hotkeys(self): + app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="ssh") + self.assertEqual(app.box_tab, 6) + app.data.nodes = ["muse-main", "muse", "646"] + app.ssh_sel_idx = 0 + + # 'd' opens box_diagnostics modal + handled = app._handle_key(ord('d')) + self.assertTrue(handled) + self.assertEqual(app.modal, "box_diagnostics") + + # Render modal + app._render_modal(30, 100) + rendered_texts = [call[0][2] for call in self.mock_stdscr.addstr.call_args_list if len(call[0]) >= 3 and isinstance(call[0][2], str)] + self.assertTrue(any("BOX DIAGNOSTICS & TROUBLESHOOTING" in t for t in rendered_texts)) + + # Hotkey '2' / 'c' copies dial command + with patch.object(muse_tui, "copy_to_clipboard", return_value=True) as mock_cp: + app._handle_key(ord('c')) + self.assertTrue(mock_cp.called) + + # Hotkey '4' / 'k' triggers key check + with patch.object(app, "_async_ssh_key_check") as mock_kc: + app._handle_key(ord('k')) + self.assertTrue(mock_kc.called) + + # Hotkey '5' / 'h' triggers recovery probe + with patch.object(app, "_async_ssh_recovery_probe") as mock_rp: + app._handle_key(ord('h')) + self.assertTrue(mock_rp.called) + + # Hotkey 'q' closes modal + app._handle_key(ord('q')) + self.assertIsNone(app.modal) + + # Enter also opens diagnostics modal + app._handle_key(ord('\n')) + self.assertEqual(app.modal, "box_diagnostics") + app._handle_key(27) # Esc closes + self.assertIsNone(app.modal) + + def test_box_ssh_mouse_interactions(self): + app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="ssh") + app.data.nodes = ["muse-main", "muse", "646"] + app.ssh_sel_idx = 0 + h, w = self.mock_stdscr.getmaxyx() + app._render_ssh_view(2, 0, 10, w) + + bounds = app._ssh_view_bounds + # Click on row 1 (my = bounds[0] + 1) + btn1 = getattr(curses, "BUTTON1_CLICKED", 0x4) + app._handle_mouse(10, bounds[0] + 1, btn1) + self.assertEqual(app.ssh_sel_idx, 1) + + # Click on [Diag] button (mx = w - 12) + app._handle_mouse(w - 12, bounds[0] + 1, btn1) + self.assertEqual(app.modal, "box_diagnostics") + app.modal = None + + # Click on [SSH] button (mx = w - 4) + with patch.object(app, "_ssh_popout_selected") as mock_pop: + app._handle_mouse(w - 4, bounds[0] + 1, btn1) + self.assertTrue(mock_pop.called) + if __name__ == "__main__": unittest.main()