super/box
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(tui): upgrade Tab 7 SSH/Boxes with diagnostics modal, key verification, and recovery watcher hardening

Browse Source
This commit is contained in:
operator
2026-10-10 13:23:56 -04:00
parent e6e5616de6
commit f9f12b450e
4 changed files with 731 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/muse-tui.py
+218 -33
View File
@@ -1424,6 +1424,21 @@ class FleetDataManager:
err_lines = (stderr or stdout or f"exit {rc}").strip().splitlines()
return False, (err_lines[-1] if err_lines else f"exit {rc}")[:200]
def probe_container_keys(self, account: str) -> tuple[bool, str]:
"""On-demand probe: check presence of critical SSH keys inside container."""
if account not in SSH_TUNNEL_PORTS:
return False, f"No tunnel registered for '{account}'"
cmd = build_ssh_dial_command(
account,
ssh_options=["-o", "BatchMode=yes", "-o", "ConnectTimeout=12"],
remote_command="ls -1 ~/.ssh/ ~/workspace/.ssh-keys/ 2>/dev/null",
)
rc, stdout, stderr = run_command_isolated(cmd, timeout=25.0)
if rc == 0 and (stdout or "").strip():
return True, stdout.strip()
err_lines = (stderr or stdout or f"exit {rc}").strip().splitlines()
return False, (err_lines[-1] if err_lines else f"exit {rc}")[:200]
def _fetch_dm_logs(self):
log_path = REPO_ROOT / "dm-log.jsonl"
if not log_path.exists():
@@ -3521,7 +3536,21 @@ class MuseTUI:
return ssh_row_order(nodes, extras)
def _render_ssh_view(self, y: int, x: int, h: int, w: int):
self.safe_addstr(self.stdscr, y, x + 1, f"CONTAINER SSH TUNNEL HEALTH (jump: {SSH_OPERATOR_USER}@{SSH_JUMP_HOST})", self._attr("bold"))
rows = self.get_ssh_rows()
if self.ssh_sel_idx >= len(rows):
self.ssh_sel_idx = max(0, len(rows) - 1)
visible_rows = max(3, h - 6)
scroll_start = getattr(self, "ssh_scroll_idx", 0)
if self.ssh_sel_idx < scroll_start:
scroll_start = self.ssh_sel_idx
elif self.ssh_sel_idx >= scroll_start + visible_rows:
scroll_start = self.ssh_sel_idx - visible_rows + 1
scroll_start = max(0, min(scroll_start, max(0, len(rows) - visible_rows)))
self.ssh_scroll_idx = scroll_start
pos_tag = f" [{scroll_start + 1}-{min(len(rows), scroll_start + visible_rows)}/{len(rows)}]" if rows else ""
self.safe_addstr(self.stdscr, y, x + 1, f"CONTAINER SSH TUNNEL HEALTH (jump: {SSH_OPERATOR_USER}@{SSH_JUMP_HOST}){pos_tag}", self._attr("bold"))
with self.data.lock:
jump = self.data.ssh_jump_reachable
@@ -3542,18 +3571,8 @@ class MuseTUI:
self.safe_addstr(self.stdscr, y + 2, x + 1, " ACCOUNT SSH PORT SSH STATE LAT SSH BANNER / HOSTKEY TERM PORT TERM STATE SINCE", self._attr("dim"))
self.safe_addstr(self.stdscr, y + 3, x + 1, "─" * (w - 2), self._attr("dim"))
rows = self.get_ssh_rows()
if self.ssh_sel_idx >= len(rows):
self.ssh_sel_idx = max(0, len(rows) - 1)
visible_rows = max(3, h - 6)
scroll_start = getattr(self, "ssh_scroll_idx", 0)
if self.ssh_sel_idx < scroll_start:
scroll_start = self.ssh_sel_idx
elif self.ssh_sel_idx >= scroll_start + visible_rows:
scroll_start = self.ssh_sel_idx - visible_rows + 1
scroll_start = max(0, min(scroll_start, max(0, len(rows) - visible_rows)))
self.ssh_scroll_idx = scroll_start
num_rendered = min(visible_rows, max(0, len(rows) - scroll_start))
self._ssh_view_bounds = (y + 4, y + 4 + num_rendered)
row_y = y + 4
for row_i in range(visible_rows):
@@ -3604,13 +3623,23 @@ class MuseTUI:
self.safe_addstr(self.stdscr, row_y, x + 40, banner[:26].ljust(26), row_attr if is_sel else self._attr("normal"))
self.safe_addstr(self.stdscr, row_y, x + 67, f":{tport:<8}", row_attr if is_sel else self._attr("dim"))
self.safe_addstr(self.stdscr, row_y, x + 77, term_txt, term_attr)
self.safe_addstr(self.stdscr, row_y, x + 83, since_txt[:w - 84 - 8], row_attr if is_sel else self._attr("dim"))
self.safe_addstr(self.stdscr, row_y, max(x + 90, w - 8), "[SSH]", self._attr("wo_badge") if is_sel else self._attr("dim"))
self.safe_addstr(self.stdscr, row_y, x + 83, since_txt[:w - 84 - 16], row_attr if is_sel else self._attr("dim"))
self.safe_addstr(self.stdscr, row_y, max(x + 80, w - 16), "[Diag]", self._attr("selected") if is_sel else self._attr("dim"))
self.safe_addstr(self.stdscr, row_y, max(x + 88, w - 8), "[SSH]", self._attr("wo_badge") if is_sel else self._attr("dim"))
row_y += 1
# Vertical scrollbar track
if len(rows) > visible_rows:
track_h = visible_rows
thumb_pos = int((scroll_start / max(1, len(rows) - visible_rows)) * (track_h - 1))
for r in range(track_h):
char = "█" if r == thumb_pos else "│"
attr = self._attr("selected") if r == thumb_pos else self._attr("dim")
self.safe_addstr(self.stdscr, y + 4 + r, w - 1, char, attr)
hint_y = y + h - 1
sel_acct = rows[self.ssh_sel_idx].upper() if rows else "-"
hints = f"Selected: [{sel_acct}] [Enter/s]: SSH pop-out (tmux) [c]: Copy dial [u]: Container uptime [r]: Refresh [j/k]: Nav"
hints = f"Selected: [{sel_acct}] [d/Enter]: Diagnostics [s]: SSH pop-out [c]: Copy dial [u]: Container uptime [k]: Key check [r]: Refresh [j/k]: Nav"
self.safe_addstr(self.stdscr, hint_y, x + 1, hints[:w - 2], self._attr("dim"))
# -----------------------------------------------------------------------
@@ -3923,8 +3952,8 @@ class MuseTUI:
left_x = max(1, min(anchor_x, screen_w - modal_w - 2))
self._context_popup_bounds = (left_x, top_y, modal_w, modal_h)
else:
modal_w = min(84 if self.modal in ("prompts", "history_search") else 74, screen_w - 6)
modal_h = min(22 if self.modal in ("prompts", "history_search") else 20, screen_h - 4)
modal_w = min(84 if self.modal in ("prompts", "history_search", "box_diagnostics") else 74, screen_w - 6)
modal_h = min(22 if self.modal in ("prompts", "history_search", "box_diagnostics") else 20, screen_h - 4)
top_y = (screen_h - modal_h) // 2
left_x = (screen_w - modal_w) // 2
self._context_popup_bounds = (left_x, top_y, modal_w, modal_h)
@@ -3958,6 +3987,7 @@ class MuseTUI:
"work_dispatch": " DISPATCH NEW BUILD TICKET (box work start) ",
"agent_thoughts": " LIVE COGNITIVE THOUGHT STREAM & SCREEN ",
"agent_menu": " AGENT PROFILE MENU (TASKS / TIMERS / APPROVALS) ",
"box_diagnostics": " BOX DIAGNOSTICS & TROUBLESHOOTING ",
}
title = title_map.get(self.modal, " DIALOG ")
self.safe_addstr(self.stdscr, top_y, left_x + 3, title, self._attr("header"))
@@ -3988,7 +4018,7 @@ class MuseTUI:
("[a] or [F2]", "Open Approvals Resolution Drawer (Allow, Always, Deny)"),
("[F5] or [m]", "Toggle between Muse Chat TUI and Box Fleet Command TUI"),
("[1]-[7] (Box)", "Switch Box tabs: Chat/Fleet/Approvals/Jobs/Tmux/Logs/SSH"),
("[Tab 7: SSH]", "Enter/s: tmux pop-out c: copy dial u: container uptime r: refresh"),
("[Tab 7: SSH]", "d/Enter: diag modal s: pop-out c: dial u: uptime k: keys r: refresh"),
("[g] / [G] / [Home/End]", "Jump to oldest message / follow live latest message"),
("[q]", "Quit TUI (in NORMAL mode)"),
]
@@ -4535,6 +4565,77 @@ class MuseTUI:
self.safe_addstr(self.stdscr, top_y + modal_h - 2, left_x + 3, "[1-4/Tab] Tab [w] Dispatch [f] Refocus [h] Heal [t] Thoughts [Esc/q] Close", self._attr("header"))
elif self.modal == "box_diagnostics":
rows = self.get_ssh_rows()
acct = rows[self.ssh_sel_idx] if rows and 0 <= getattr(self, "ssh_sel_idx", 0) < len(rows) else "unknown"
with self.data.lock:
jump_ok = self.data.ssh_jump_reachable
jump_lat = self.data.ssh_check_latency_ms
jump_err = self.data.ssh_check_error
health = dict(self.data.ssh_cache.get(acct, {}))
info = SSH_TUNNEL_PORTS.get(acct, {})
sport = info.get("port", "?")
tport = info.get("terminal", "?")
ssh_up = health.get("ssh_up")
term_up = health.get("term_up")
lat = health.get("ssh_latency_ms")
lat_str = f"{lat}ms" if lat is not None else "--"
banner = (health.get("ssh_banner") or health.get("term_http") or "-").strip()
state_since = health.get("state_since", 0.0)
since_str = format_recency(state_since) if state_since else "never"
dial_cmd = build_ssh_dial_string(acct) if acct in SSH_TUNNEL_PORTS else "N/A"
# Header details
self.safe_addstr(self.stdscr, top_y + 2, left_x + 3, f"BOX TARGET: @{acct.upper():<12} (SSH :{sport} | Term :{tport})", self._attr("bold"))
if jump_ok is True:
j_txt = f"ONLINE ({jump_lat}ms latency)"
j_attr = self._attr("success")
elif jump_ok is False:
j_txt = f"DOWN ({(jump_err or 'unknown')[:32]})"
j_attr = self._attr("danger")
else:
j_txt = "SWEEP PENDING"
j_attr = self._attr("dim")
self.safe_addstr(self.stdscr, top_y + 3, left_x + 3, f"Jump Gateway: {SSH_OPERATOR_USER}@{SSH_JUMP_HOST} -> ", self._attr("dim"))
self.safe_addstr(self.stdscr, top_y + 3, left_x + 38, j_txt, j_attr)
ssh_st_str = "LISTENING / UP" if ssh_up else ("UNREACHABLE / DOWN" if ssh_up is False else "UNKNOWN")
ssh_st_attr = self._attr("success") if ssh_up else (self._attr("danger") if ssh_up is False else self._attr("dim"))
self.safe_addstr(self.stdscr, top_y + 4, left_x + 3, f"SSH Tunnel: :{sport:<6} -> {ssh_st_str} ({lat_str}) | Since: {since_str}", ssh_st_attr)
self.safe_addstr(self.stdscr, top_y + 5, left_x + 3, f"SSH Banner: {banner[:modal_w - 20]}", self._attr("normal"))
term_st_str = "LISTENING / UP" if term_up else ("UNREACHABLE / DOWN" if term_up is False else "UNKNOWN")
term_st_attr = self._attr("success") if term_up else (self._attr("danger") if term_up is False else self._attr("dim"))
self.safe_addstr(self.stdscr, top_y + 6, left_x + 3, f"Web Terminal: :{tport:<6} -> {term_st_str} (ttyd)", term_st_attr)
self.safe_addstr(self.stdscr, top_y + 7, left_x + 3, f"Dial String: {dial_cmd[:modal_w - 20]}", self._attr("dim"))
self.safe_addstr(self.stdscr, top_y + 8, left_x + 3, f"Keys Spec: vm_to_gcp, muse-health, id_frontdoor, id_ed25519", self._attr("dim"))
self.safe_addstr(self.stdscr, top_y + 9, left_x + 2, "─" * (modal_w - 4), self._attr("dim"))
self.safe_addstr(self.stdscr, top_y + 10, left_x + 3, "TROUBLESHOOTING & FAST ACTIONS:", self._attr("bold"))
actions_list = [
("[1] / [s]", "SSH Pop-out", "Spawn terminal session in a new tmux window"),
("[2] / [c]", "Copy Dial String", "Copy full jump command string to system clipboard"),
("[3] / [u]", "Container Uptime", "Send remote `uptime` query to container"),
("[4] / [k]", "Verify SSH Keys", "Check ~/.ssh/ and ~/workspace/.ssh-keys/ presence"),
("[5] / [h]", "Run Recover Hook", "Execute recover-after-rebuild.sh --dry-run"),
("[r]", "Trigger Sweep", "Initiate immediate background port scan via jump host"),
]
for idx, (hk, title_act, desc) in enumerate(actions_list):
cur_act_y = top_y + 11 + idx
if cur_act_y >= top_y + modal_h - 2:
break
self.safe_addstr(self.stdscr, cur_act_y, left_x + 3, f"{hk:<10}", self._attr("wo_badge"))
self.safe_addstr(self.stdscr, cur_act_y, left_x + 14, f"{title_act:<18}", self._attr("bold"))
self.safe_addstr(self.stdscr, cur_act_y, left_x + 33, desc[:modal_w - 36], self._attr("normal"))
self.safe_addstr(self.stdscr, top_y + modal_h - 2, left_x + 3, "[1-5/s/c/u/k/h/r] Run Action [Esc/q] Close Dialog", self._attr("header"))
# -----------------------------------------------------------------------
# Keypress & Event Handling
# -----------------------------------------------------------------------
@@ -5115,8 +5216,8 @@ class MuseTUI:
# Box mode Fast Actions: Tab 6 (SSH) — placed before the 's'/'c'/'y'
# globals below so SSH keys win on this tab.
if self.mode == "box" and self.box_tab == 6:
if ch in (curses.KEY_ENTER, 10, 13):
self._ssh_popout_selected()
if ch in (curses.KEY_ENTER, 10, 13, ord('d'), ord('D')):
self.modal = "box_diagnostics"
return True
elif ch in (ord('s'), ord('S')):
self._ssh_popout_selected()
@@ -5131,6 +5232,18 @@ class MuseTUI:
threading.Thread(target=self._async_ssh_uptime, args=(acct,), daemon=True).start()
self.set_toast(f"Probing container uptime on {acct}...", "info")
return True
elif ch in (ord('k'), ord('K')):
rows = self.get_ssh_rows()
if rows and 0 <= self.ssh_sel_idx < len(rows):
acct = rows[self.ssh_sel_idx]
threading.Thread(target=self._async_ssh_key_check, args=(acct,), daemon=True).start()
return True
elif ch in (ord('h'), ord('H')):
rows = self.get_ssh_rows()
if rows and 0 <= self.ssh_sel_idx < len(rows):
acct = rows[self.ssh_sel_idx]
threading.Thread(target=self._async_ssh_recovery_probe, args=(acct,), daemon=True).start()
return True
elif ch in (ord('r'), ord('R')):
threading.Thread(target=self.data._fetch_ssh_health, daemon=True).start()
self.set_toast("Probing SSH tunnels via jump host...", "info")
@@ -7214,18 +7327,27 @@ class MuseTUI:
return True
elif self.box_tab == 6:
# Tab 6: SSH / Boxes (rows start one line lower: jump-status line)
rows = self.get_ssh_rows()
scroll_start = getattr(self, "ssh_scroll_idx", 0)
clicked_idx = scroll_start + row_idx - 1
if 0 <= clicked_idx < len(rows):
self.ssh_sel_idx = clicked_idx
if mx >= w - 8:
# [SSH] pop-out
self._ssh_popout_selected()
else:
acct = rows[clicked_idx]
self.set_toast(f"Selected [{acct.upper()}]. Click [SSH] or press Enter to pop out.", "info")
# Tab 6: SSH / Boxes
bounds = getattr(self, "_ssh_view_bounds", (content_y + 4, content_y + 4 + 10))
if bounds[0] <= my < bounds[1]:
row_offset = my - bounds[0]
rows = self.get_ssh_rows()
scroll_start = getattr(self, "ssh_scroll_idx", 0)
clicked_idx = scroll_start + row_offset
if 0 <= clicked_idx < len(rows):
self.ssh_sel_idx = clicked_idx
if mx >= w - 8:
# [SSH] pop-out
self._ssh_popout_selected()
elif mx >= w - 16:
# [Diag] modal
self.modal = "box_diagnostics"
else:
if is_double_click:
self.modal = "box_diagnostics"
else:
acct = rows[clicked_idx]
self.set_toast(f"Selected [{acct.upper()}]. Press [d/Enter] for Diagnostics or [s] for SSH.", "info")
return True
elif self.box_tab == 7:
@@ -7891,6 +8013,38 @@ class MuseTUI:
else:
self.set_toast(f"{account} uptime failed: {out_text[:90]}", "error")
def _async_ssh_key_check(self, account: str):
self.set_toast(f"Probing SSH keys on {account}...", "info")
ok, out_text = self.data.probe_container_keys(account)
if ok:
found = []
for k in ("vm_to_gcp", "muse-health", "id_frontdoor", "id_ed25519", "id_rsa"):
if k in out_text:
found.append(k)
found_str = ", ".join(found) if found else "files listed"
self.set_toast(f"✔ {account} keys verified: {found_str}", "success")
else:
self.set_toast(f"❌ {account} key probe failed: {out_text[:60]}", "error")
def _async_ssh_recovery_probe(self, account: str):
self.set_toast(f"Running dry-run recovery probe on {account}...", "info")
if account not in SSH_TUNNEL_PORTS:
self.set_toast(f"No tunnel registered for '{account}'", "warn")
return
cmd = build_ssh_dial_command(
account,
ssh_options=["-o", "BatchMode=yes", "-o", "ConnectTimeout=15"],
remote_command="if [ -x ~/workspace/bin/recover-after-rebuild.sh ]; then ~/workspace/bin/recover-after-rebuild.sh --dry-run; else echo 'recover script not found'; fi",
)
rc, stdout, stderr = run_command_isolated(cmd, timeout=30.0)
if rc == 0:
lines = [l.strip() for l in (stdout or "").splitlines() if l.strip()]
summary = lines[-1] if lines else "Recovery probe completed."
self.set_toast(f"✔ {account} probe: {summary[:60]}", "success")
else:
err = (stderr or stdout or f"exit {rc}").strip().splitlines()
self.set_toast(f"❌ {account} recovery failed: {err[-1][:60] if err else 'error'}", "error")
def _async_dispatch_work(self, goal: str, target_node: str):
"""Dispatch a new work build ticket with 45s synchronous readback gate."""
self.set_toast(f"⚡ Dispatching work ticket to @{target_node.upper()}...", "info")
@@ -8896,6 +9050,37 @@ class MuseTUI:
self.modal_input_cursor += 1
return True
elif self.modal == "box_diagnostics":
rows = self.get_ssh_rows()
acct = rows[self.ssh_sel_idx] if rows and 0 <= getattr(self, "ssh_sel_idx", 0) < len(rows) else ""
if ch in (27, ord('q'), ord('Q')):
self.modal = None
return True
elif ch in (ord('1'), ord('s'), ord('S')):
self._ssh_popout_selected()
return True
elif ch in (ord('2'), ord('c'), ord('C')):
self._ssh_copy_dial_selected()
return True
elif ch in (ord('3'), ord('u'), ord('U')):
if acct:
threading.Thread(target=self._async_ssh_uptime, args=(acct,), daemon=True).start()
self.set_toast(f"Probing container uptime on {acct}...", "info")
return True
elif ch in (ord('4'), ord('k'), ord('K')):
if acct:
threading.Thread(target=self._async_ssh_key_check, args=(acct,), daemon=True).start()
return True
elif ch in (ord('5'), ord('h'), ord('H')):
if acct:
threading.Thread(target=self._async_ssh_recovery_probe, args=(acct,), daemon=True).start()
return True
elif ch in (ord('r'), ord('R')):
threading.Thread(target=self.data._fetch_ssh_health, daemon=True).start()
self.set_toast("Probing SSH tunnels via jump host...", "info")
return True
return True
cloud-uptime/recover-after-rebuild.sh
Executable
+299
View File
@@ -0,0 +1,299 @@
#!/bin/bash
# recover-after-rebuild.sh — re-provision container after a VM/container rebuild.
# Standardized multi-machine recovery hook for muse-frontdoor fleet containers.
#
# Survives rebuilds: /home/hatch (workspace, ~/.ssh keys if preserved, persistent volumes).
# Ephemeral root: /etc, packages, users outside persistent tree, crontabs.
#
# Idempotent: safe to run any time. Does provisioning on fresh root
# filesystem (sentinel in /etc), then ensures tunnel supervisor is running.
set -u
# Support dry-run mode and restore-keys mode
DRY_RUN=0
RESTORE_KEYS_ONLY=0
for arg in "$@"; do
case "$arg" in
--dry-run)
DRY_RUN=1
echo "[recover] running in DRY-RUN mode (no mutations)"
;;
--restore-keys)
RESTORE_KEYS_ONLY=1
;;
esac
done
# Identity & per-machine config
ENV_FILE="$HOME/workspace/tunnel/machine.env"
if [ -f "$ENV_FILE" ]; then
# shellcheck disable=SC1090
. "$ENV_FILE"
fi
MACHINE="${MUSE_MACHINE:-muse-main}"
SSH_PORT="${SSH_PORT:-2224}"
TERM_PORT="${TERM_PORT:-7681}"
_WL_BIN="$(cd "$(dirname "$0")" && pwd)/wl-config.py"
[ -x "$_WL_BIN" ] && eval "$("$_WL_BIN" --shell 2>/dev/null)" 2>/dev/null || true
unset _WL_BIN
FD_DOMAIN="${FD_DOMAIN:-${MACHINE}.muse-dev.online}"
SENTINEL=/etc/hatch-provisioned
BIN="$HOME/workspace/bin"
DEB_CACHE="$HOME/workspace/debs"
log() { echo "[recover] $*"; }
needs_provisioning() { [ ! -f "$SENTINEL" ]; }
restore_ssh_keys() {
# Key restoration: rebuilds may wipe ~/.ssh. Restore from persistent store if present.
install -m 700 -d "$HOME/.ssh" 2>/dev/null || true
for keyname in vm_to_gcp id_frontdoor muse-health id_ed25519 id_rsa; do
if [ ! -f "$HOME/.ssh/$keyname" ]; then
if [ -f "$HOME/workspace/.ssh-keys/$keyname" ]; then
log "restoring ~/.ssh/$keyname from persistent backup"
[ "$DRY_RUN" -eq 0 ] && install -m 600 "$HOME/workspace/.ssh-keys/$keyname" "$HOME/.ssh/$keyname"
elif [ "$keyname" = "id_frontdoor" ] && [ -f "$HOME/workspace/.ssh-keys/vm_to_gcp" ]; then
log "linking ~/.ssh/$keyname to persistent vm_to_gcp"
[ "$DRY_RUN" -eq 0 ] && install -m 600 "$HOME/workspace/.ssh-keys/vm_to_gcp" "$HOME/.ssh/$keyname"
elif [ "$keyname" = "vm_to_gcp" ] && [ -f "$HOME/workspace/.ssh-keys/id_frontdoor" ]; then
log "linking ~/.ssh/$keyname to persistent id_frontdoor"
[ "$DRY_RUN" -eq 0 ] && install -m 600 "$HOME/workspace/.ssh-keys/id_frontdoor" "$HOME/.ssh/$keyname"
fi
fi
if [ ! -f "$HOME/.ssh/${keyname}.pub" ] && [ -f "$HOME/workspace/.ssh-keys/${keyname}.pub" ]; then
log "restoring ~/.ssh/${keyname}.pub from persistent backup"
[ "$DRY_RUN" -eq 0 ] && install -m 644 "$HOME/workspace/.ssh-keys/${keyname}.pub" "$HOME/.ssh/${keyname}.pub"
fi
done
}
provision_critical() {
log "fresh container detected — provisioning critical path (machine: $MACHINE, port: $SSH_PORT)"
if [ "$DRY_RUN" -eq 1 ]; then
log "dry-run: would run fix-apt-mirror.sh, install deb packages, setup muse user, restore host keys"
return 0
fi
# 1. Fix dead apt mirror if present
if [ -x "$BIN/fix-apt-mirror.sh" ]; then
"$BIN/fix-apt-mirror.sh"
fi
# 2. Check local .deb cache
if ls "$DEB_CACHE"/*.deb >/dev/null 2>&1; then
log "installing from persistent .deb cache"
DEBIAN_FRONTEND=noninteractive dpkg -i "$DEB_CACHE"/*.deb 2>&1 | tail -2 || true
apt-get install -f -y -qq 2>/dev/null || true
else
log "WARNING: deb cache empty at $DEB_CACHE — falling back to apt network"
if [ -z "$(ls /var/lib/apt/lists/ 2>/dev/null | grep -v '^lock' | head -1)" ]; then
apt-get update -qq
fi
fi
# 3. Single-transaction install for critical networking packages
local missing=""
for p in openssh-client openssh-server; do
dpkg -s "$p" >/dev/null 2>&1 || missing="$missing $p"
done
if [ -n "$missing" ]; then
log "installing missing critical packages: $missing"
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends $missing
fi
# 4. Restore SSH host keys
local hk_dir="$HOME/workspace/tunnel/ssh_host_keys"
if ls "$hk_dir"/ssh_host_* >/dev/null 2>&1; then
log "restoring persistent SSH host keys"
cp -p "$hk_dir"/ssh_host_* /etc/ssh/ 2>/dev/null \
&& chmod 600 /etc/ssh/ssh_host_* \
&& log "host keys restored" \
|| log "WARNING: host key restore failed"
elif ls /etc/ssh/ssh_host_* >/dev/null 2>&1; then
log "seeding persistent SSH host key store"
mkdir -p -m 700 "$hk_dir"
cp -p /etc/ssh/ssh_host_* "$hk_dir"/ 2>/dev/null && chmod 600 "$hk_dir"/* 2>/dev/null || true
fi
# 5. Restore muse login user
if ! id muse >/dev/null 2>&1; then
log "creating muse user"
useradd -m -s /bin/bash muse 2>/dev/null || true
fi
echo 'muse:horse-battery-staple' | chpasswd 2>/dev/null || log "WARNING: chpasswd failed"
chown -R muse:muse /home/muse 2>/dev/null && chmod 755 /home/muse 2>/dev/null || true
if [ -f "$HOME/workspace/tunnel/muse-authorized_keys" ]; then
install -m 700 -o muse -d /home/muse/.ssh 2>/dev/null || true
install -m 600 -o muse -g muse \
"$HOME/workspace/tunnel/muse-authorized_keys" \
/home/muse/.ssh/authorized_keys 2>/dev/null || true
fi
# 6. Restore /root/.ssh/authorized_keys across rebuilds
install -m 700 -d /root/.ssh 2>/dev/null || true
if [ -f "$HOME/workspace/tunnel/root-authorized_keys" ]; then
log "restoring /root/.ssh/authorized_keys from persistent backup"
install -m 600 "$HOME/workspace/tunnel/root-authorized_keys" /root/.ssh/authorized_keys 2>/dev/null || true
elif [ -f "$HOME/workspace/tunnel/muse-authorized_keys" ]; then
log "seeding /root/.ssh/authorized_keys from muse-authorized_keys"
install -m 600 "$HOME/workspace/tunnel/muse-authorized_keys" /root/.ssh/authorized_keys 2>/dev/null || true
fi
if [ -f "/home/hatch/.ssh/authorized_keys" ]; then
log "merging /home/hatch/.ssh/authorized_keys into /root/.ssh/authorized_keys"
cat /home/hatch/.ssh/authorized_keys >> /root/.ssh/authorized_keys 2>/dev/null || true
sort -u /root/.ssh/authorized_keys -o /root/.ssh/authorized_keys 2>/dev/null || true
chmod 600 /root/.ssh/authorized_keys 2>/dev/null || true
fi
touch "$SENTINEL"
log "critical provisioning complete"
}
restore_crontabs() {
# Reinstall crontab from persistent spec
if [ -x "$BIN/persistent-crontab.sh" ]; then
log "restoring persistent crontabs"
if [ "$DRY_RUN" -eq 0 ]; then
"$BIN/persistent-crontab.sh" || log "WARNING: persistent-crontab.sh exited non-zero"
fi
fi
}
provision_deferred() {
# Background non-critical tools (python3, tmux, age, yazi, neovim)
if [ "$DRY_RUN" -eq 1 ]; then
return 0
fi
(
local deferred_missing=""
for p in python3 tmux age; do
dpkg -s "$p" >/dev/null 2>&1 || deferred_missing="$deferred_missing $p"
done
if [ -n "$deferred_missing" ]; then
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends $deferred_missing 2>/dev/null || true
fi
if [ -x "$BIN/yazi" ] && ! command -v yazi >/dev/null; then
cp "$BIN/yazi" /usr/local/bin/yazi 2>/dev/null && chmod 755 /usr/local/bin/yazi 2>/dev/null || true
fi
if [ -x "$HOME/workspace/nvim/bin/nvim" ] && ! command -v nvim >/dev/null; then
mkdir -p /opt/nvim 2>/dev/null
cp -r "$HOME/workspace/nvim/"* /opt/nvim/ 2>/dev/null || true
ln -sf /opt/nvim/bin/nvim /usr/local/bin/nvim 2>/dev/null || true
fi
local wheel_dir="$HOME/workspace/wheels"
if [ -d "$wheel_dir" ] && ls "$wheel_dir"/*.whl >/dev/null 2>&1; then
log "installing cached python wheels from $wheel_dir"
python3 -m pip install --no-index --find-links="$wheel_dir" protocol_muse 2>/dev/null || true
fi
) >/dev/null 2>&1 &
disown 2>/dev/null || true
}
ensure_tunnel() {
# Ensure legacy localhost.run tunnels are halted
for pid in $(pgrep -f "workspace/bin/tunnel-up\.sh$" 2>/dev/null); do
log "stopping retired localhost.run supervisor (pid $pid)"
[ "$DRY_RUN" -eq 0 ] && kill "$pid" 2>/dev/null || true
done
for pid in $(pgrep -f "ssh\.localhost\.run" 2>/dev/null); do
log "stopping retired localhost.run ssh (pid $pid)"
[ "$DRY_RUN" -eq 0 ] && kill "$pid" 2>/dev/null || true
done
}
ensure_gcp_tunnel() {
if [ "$DRY_RUN" -eq 1 ]; then
log "dry-run: would check and start gcp tunnel supervisor"
return 0
fi
(
exec 9>"$BIN/.gcp-tunnel-up.lock" || exit 0
flock -n 9 || { log "another recovery run starting gcp tunnel; skipping"; exit 0; }
if pgrep -f "workspace/bin/gcp-tunnel-up.*\.sh$" >/dev/null; then
log "gcp tunnel supervisor already running"
exit 0
fi
if [ ! -f "$HOME/.ssh/vm_to_gcp" ] && [ -f "$HOME/.ssh/id_frontdoor" ]; then
ln -sf "$HOME/.ssh/id_frontdoor" "$HOME/.ssh/vm_to_gcp"
elif [ ! -f "$HOME/.ssh/id_frontdoor" ] && [ -f "$HOME/.ssh/vm_to_gcp" ]; then
ln -sf "$HOME/.ssh/vm_to_gcp" "$HOME/.ssh/id_frontdoor"
fi
if [ ! -f "$HOME/.ssh/vm_to_gcp" ] && [ ! -f "$HOME/.ssh/id_frontdoor" ]; then
log "WARNING: ~/.ssh/vm_to_gcp missing — cannot start gcp tunnel supervisor"
exit 0
fi
log "starting gcp tunnel supervisor"
local sup="$BIN/gcp-tunnel-up.sh"
[ -x "$sup" ] || sup="$BIN/gcp-tunnel-up-${MACHINE}.sh"
if [ -x "$sup" ]; then
setsid nohup "$sup" >/dev/null 2>&1 < /dev/null 9>&- &
disown 2>/dev/null || true
touch "$BIN/.gcp-tunnel-started"
else
log "WARNING: no executable gcp-tunnel supervisor found at $sup"
fi
)
if [ -f "$BIN/.gcp-tunnel-started" ]; then
rm -f "$BIN/.gcp-tunnel-started"
_GCP_TUNNEL_STARTED=1
fi
}
report_health_on_recovery() {
[ "${_GCP_TUNNEL_STARTED:-0}" = 1 ] || return 0
[ "$DRY_RUN" -eq 1 ] && return 0
local reporter="$HOME/workspace/muse-frontdoor/bin/health-report.sh"
[ -x "$reporter" ] || { log "health reporter not found — skipping immediate report"; return 0; }
[ -f "$HOME/.ssh/muse-health" ] || { log "health key missing — skipping immediate report"; return 0; }
log "tunnel (re)started — waiting for VM listener $SSH_PORT before health report"
local i
for i in $(seq 1 18); do
if ssh -i "$HOME/.ssh/vm_to_gcp" \
-o ProxyCommand="$HOME/workspace/bin/ssh-via-proxy %h %p" \
-o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null \
-o ConnectTimeout=8 \
-o BatchMode=yes \
super@34.139.37.135 \
"ss -tln 2>/dev/null | grep -q '127.0.0.1:${SSH_PORT} '" 2>/dev/null; then
log "VM listener $SSH_PORT confirmed — sending immediate health report"
MUSE_MACHINE="$MACHINE" "$reporter" 2>&1 | head -5 || true
return 0
fi
sleep 5
done
log "WARNING: VM listener $SSH_PORT not seen after 90s — skipping immediate report"
}
main() {
restore_ssh_keys
if [ "$RESTORE_KEYS_ONLY" -eq 1 ]; then
log "SSH key restore completed (keys-only mode)."
return 0
fi
if needs_provisioning; then
provision_critical
else
log "container already provisioned (sentinel present)"
fi
restore_crontabs
ensure_tunnel
ensure_gcp_tunnel
provision_deferred
report_health_on_recovery
echo "---"
echo "machine: $MACHINE (SSH port: $SSH_PORT, terminal port: $TERM_PORT)"
echo "domain: https://${FD_DOMAIN}"
echo "ttyd: $(pgrep -f '[t]tyd' | head -1 || echo '(not running)')"
echo "supervisor: $(pgrep -f 'gcp-tunnel-up' | head -1 || echo '(not running)')"
}
main "$@"
cloud-uptime/uptime-watcher.sh
Executable
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env bash
# uptime-watcher.sh — simple hatch-hook watcher: spawn/rebuild from spec.
#
# Register as a hatch hook (id `uptime-watcher`, poll 120s, timeout 300s)
# alongside tunnel-keeper. Each poll it guarantees the three things a
# container rebuild destroys:
# 1. provisioning — runs recover-after-rebuild.sh on a fresh root fs
# 2. supervisor — respawns gcp-tunnel-up.sh if it died
# 3. cron jobs — reinstalls crontab from ~/workspace/cron/*.persist
#
# It also verifies the VM-side SSH forward answers a banner, and wakes the
# operator (rate-limited, 30 min) only when something stays broken across
# polls. Silent on success. Safe to run by hand or from cron too.
set -u
# --- runtime (hatch hook functions, or local fallbacks) ---
if [ -n "${HATCH_HOOK_RUNTIME:-}" ] && [ -f "$HATCH_HOOK_RUNTIME" ]; then
# shellcheck disable=SC1090
source "$HATCH_HOOK_RUNTIME"
else
log() { echo "[uptime-watcher] $1 $2"; }
silent() { echo "[uptime-watcher] silent: $1 $2"; }
wake() { echo "[uptime-watcher] WAKE $1 $2"; }
fi
# --- identity (per-machine, persistent) ---
ENV_FILE="$HOME/workspace/tunnel/machine.env"
# shellcheck disable=SC1090
[ -f "$ENV_FILE" ] && . "$ENV_FILE"
MACHINE="${MUSE_MACHINE:-unknown}"
SSH_PORT="${SSH_PORT:-0}"
TERM_PORT="${TERM_PORT:-0}"
STATE_DIR="$HOME/hooks/state/uptime-watcher"
BIN="$HOME/workspace/bin"
RECOVER="$BIN/recover-after-rebuild.sh"
SUPERVISOR="$BIN/gcp-tunnel-up.sh"
CRON_RESTORE="$BIN/persistent-crontab.sh"
SSH_KEY="$HOME/.ssh/vm_to_gcp"
GCP_HOST="${FD_VM_HOST:-34.139.37.135}"
GCP_USER="${FD_VM_USER:-super}"
FAIL_COUNT="$STATE_DIR/consec_failures"
LAST_WAKE="$STATE_DIR/last_wake_ts"
mkdir -p "$STATE_DIR"
exec 9>"$STATE_DIR/watcher.lock"
flock -n 9 || { silent "previous poll still running" '{}'; exit 0; }
read_int() { [ -f "$1" ] && tr -cd '0-9' < "$1" || echo 0; }
actions=""
fail=""
# --- 1. fresh rebuild or missing SSH key? provision / restore ---
if [ ! -f /etc/hatch-provisioned ] || [ ! -f "$SSH_KEY" ]; then
if [ -x "$RECOVER" ]; then
if timeout 280 "$RECOVER" >"$STATE_DIR/recover-last.log" 2>&1; then
actions="${actions}provisioned "
log "recovery" '{"event":"provisioned_after_rebuild"}'
else
fail="recover_failed"
fi
else
fail="recover_missing"
fi
fi
# --- 2. supervisor alive? respawn ---
if [ -z "$fail" ] && ! pgrep -f "workspace/bin/gcp-tunnel-up\.sh$" >/dev/null; then
if [ -x "$SUPERVISOR" ] && [ -f "$SSH_KEY" ]; then
setsid nohup "$SUPERVISOR" >/dev/null 2>&1 < /dev/null 9>&- &
disown 2>/dev/null || true
actions="${actions}supervisor-respawned "
log "supervisor" '{"event":"respawned"}'
else
fail="supervisor_unstartable"
fi
fi
# --- 3. cron jobs alive? restore from persistent spec ---
if [ -z "$fail" ] && [ -x "$CRON_RESTORE" ]; then
if "$CRON_RESTORE" >"$STATE_DIR/cron-last.log" 2>&1; then
grep -q "reinstalled" "$STATE_DIR/cron-last.log" \
&& actions="${actions}cron-restored "
else
fail="cron_restore_failed"
fi
fi
# --- 4. VM forward answers? (banner check, cheap) ---
ssh_state="unknown"
if [ -z "$fail" ] && [ "$SSH_PORT" != "0" ] && [ -f "$SSH_KEY" ] \
&& pgrep -f "[s]sh.*${SSH_PORT}:localhost:22" >/dev/null; then
banner="$(timeout 12 ssh -i "$SSH_KEY" \
-o ProxyCommand="$BIN/ssh-via-proxy %h %p" \
-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-o ConnectTimeout=8 -o BatchMode=yes \
"$GCP_USER@$GCP_HOST" \
"timeout 5 bash -c 'exec 3<>/dev/tcp/127.0.0.1/$SSH_PORT && head -c 4 <&3' 2>/dev/null" \
2>/dev/null || true)"
case "$banner" in
SSH-*) ssh_state="up" ;;
*) ssh_state="stale-forward"; fail="forward_dead" ;;
esac
elif [ -z "$fail" ]; then
ssh_state="down"
fail="tunnel_down"
fi
payload="$(printf '{"machine":"%s","ssh":"%s","actions":"%s"}' \
"$MACHINE" "$ssh_state" "${actions:-none}")"
# --- 5. silent ok, or rate-limited wake on persistent failure ---
if [ -z "$fail" ]; then
printf 0 > "$FAIL_COUNT"
silent "uptime watcher poll ok" "$payload"
exit 0
fi
count=$(( $(read_int "$FAIL_COUNT") + 1 ))
printf '%s' "$count" > "$FAIL_COUNT"
log "failure" "{\"condition\":\"$fail\",\"consec\":\"$count\"}"
if [ "$count" -ge 2 ]; then
now=$(date +%s); last=$(read_int "$LAST_WAKE")
if [ $(( now - last )) -ge 1800 ]; then
printf '%s' "$now" > "$LAST_WAKE"
wake "$fail" "$payload"
exit 0
fi
fi
silent "failure $fail ($count) — below wake threshold" "$payload"
tests/test_box_tui_render.py
+84
View File
@@ -581,6 +581,90 @@ class TestBoxTUIHeadless(unittest.TestCase):
app._handle_key(ord('G'))
self.assertEqual(app.dm_logs_sel_idx, 14)
def test_box_ssh_view_render_scrollbar_and_bounds(self):
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="ssh")
self.assertEqual(app.box_tab, 6)
# Populate multiple nodes to ensure scrolling
app.data.nodes = [f"node_{i}" for i in range(15)]
app.ssh_sel_idx = 8
app._render_ssh_view(2, 0, 10, 100)
# Check bounds recorded
self.assertTrue(hasattr(app, "_ssh_view_bounds"))
self.assertEqual(app._ssh_view_bounds[0], 6) # y (2) + 4 = 6
self.assertGreater(app._ssh_view_bounds[1], app._ssh_view_bounds[0])
# Verify scrollbar and tags in screen output
rendered_texts = [call[0][2] for call in self.mock_stdscr.addstr.call_args_list if len(call[0]) >= 3 and isinstance(call[0][2], str)]
self.assertTrue(any("█" in t or "│" in t for t in rendered_texts))
self.assertTrue(any("[Diag]" in t for t in rendered_texts))
self.assertTrue(any("[SSH]" in t for t in rendered_texts))
def test_box_diagnostics_modal_and_hotkeys(self):
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="ssh")
self.assertEqual(app.box_tab, 6)
app.data.nodes = ["muse-main", "muse", "646"]
app.ssh_sel_idx = 0
# 'd' opens box_diagnostics modal
handled = app._handle_key(ord('d'))
self.assertTrue(handled)
self.assertEqual(app.modal, "box_diagnostics")
# Render modal
app._render_modal(30, 100)
rendered_texts = [call[0][2] for call in self.mock_stdscr.addstr.call_args_list if len(call[0]) >= 3 and isinstance(call[0][2], str)]
self.assertTrue(any("BOX DIAGNOSTICS & TROUBLESHOOTING" in t for t in rendered_texts))
# Hotkey '2' / 'c' copies dial command
with patch.object(muse_tui, "copy_to_clipboard", return_value=True) as mock_cp:
app._handle_key(ord('c'))
self.assertTrue(mock_cp.called)
# Hotkey '4' / 'k' triggers key check
with patch.object(app, "_async_ssh_key_check") as mock_kc:
app._handle_key(ord('k'))
self.assertTrue(mock_kc.called)
# Hotkey '5' / 'h' triggers recovery probe
with patch.object(app, "_async_ssh_recovery_probe") as mock_rp:
app._handle_key(ord('h'))
self.assertTrue(mock_rp.called)
# Hotkey 'q' closes modal
app._handle_key(ord('q'))
self.assertIsNone(app.modal)
# Enter also opens diagnostics modal
app._handle_key(ord('\n'))
self.assertEqual(app.modal, "box_diagnostics")
app._handle_key(27) # Esc closes
self.assertIsNone(app.modal)
def test_box_ssh_mouse_interactions(self):
app = muse_tui.MuseTUI(self.mock_stdscr, initial_mode="box", initial_tab="ssh")
app.data.nodes = ["muse-main", "muse", "646"]
app.ssh_sel_idx = 0
h, w = self.mock_stdscr.getmaxyx()
app._render_ssh_view(2, 0, 10, w)
bounds = app._ssh_view_bounds
# Click on row 1 (my = bounds[0] + 1)
btn1 = getattr(curses, "BUTTON1_CLICKED", 0x4)
app._handle_mouse(10, bounds[0] + 1, btn1)
self.assertEqual(app.ssh_sel_idx, 1)
# Click on [Diag] button (mx = w - 12)
app._handle_mouse(w - 12, bounds[0] + 1, btn1)
self.assertEqual(app.modal, "box_diagnostics")
app.modal = None
# Click on [SSH] button (mx = w - 4)
with patch.object(app, "_ssh_popout_selected") as mock_pop:
app._handle_mouse(w - 4, bounds[0] + 1, btn1)
self.assertTrue(mock_pop.called)
if __name__ == "__main__":
unittest.main()