super/box
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(tui): upgrade Tab 7 SSH/Boxes with diagnostics modal, key verification, and recovery watcher hardening

Browse Source
This commit is contained in:
operator
2026-10-10 13:23:56 -04:00
parent e6e5616de6
commit f9f12b450e
4 changed files with 731 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/muse-tui.py
+218 -33
View File
@@ -1424,6 +1424,21 @@ class FleetDataManager:
err_lines = (stderr or stdout or f"exit {rc}").strip().splitlines()
return False, (err_lines[-1] if err_lines else f"exit {rc}")[:200]
def probe_container_keys(self, account: str) -> tuple[bool, str]:
"""On-demand probe: check presence of critical SSH keys inside container."""
if account not in SSH_TUNNEL_PORTS:
return False, f"No tunnel registered for '{account}'"
cmd = build_ssh_dial_command(
account,
ssh_options=["-o", "BatchMode=yes", "-o", "ConnectTimeout=12"],
remote_command="ls -1 ~/.ssh/ ~/workspace/.ssh-keys/ 2>/dev/null",
)
rc, stdout, stderr = run_command_isolated(cmd, timeout=25.0)
if rc == 0 and (stdout or "").strip():
return True, stdout.strip()
err_lines = (stderr or stdout or f"exit {rc}").strip().splitlines()
return False, (err_lines[-1] if err_lines else f"exit {rc}")[:200]
def _fetch_dm_logs(self):
log_path = REPO_ROOT / "dm-log.jsonl"
if not log_path.exists():
@@ -3521,7 +3536,21 @@ class MuseTUI:
return ssh_row_order(nodes, extras)
def _render_ssh_view(self, y: int, x: int, h: int, w: int):
self.safe_addstr(self.stdscr, y, x + 1, f"CONTAINER SSH TUNNEL HEALTH (jump: {SSH_OPERATOR_USER}@{SSH_JUMP_HOST})", self._attr("bold"))
rows = self.get_ssh_rows()
if self.ssh_sel_idx >= len(rows):
self.ssh_sel_idx = max(0, len(rows) - 1)
visible_rows = max(3, h - 6)
scroll_start = getattr(self, "ssh_scroll_idx", 0)
if self.ssh_sel_idx < scroll_start:
scroll_start = self.ssh_sel_idx
elif self.ssh_sel_idx >= scroll_start + visible_rows:
scroll_start = self.ssh_sel_idx - visible_rows + 1
scroll_start = max(0, min(scroll_start, max(0, len(rows) - visible_rows)))
self.ssh_scroll_idx = scroll_start
pos_tag = f" [{scroll_start + 1}-{min(len(rows), scroll_start + visible_rows)}/{len(rows)}]" if rows else ""
self.safe_addstr(self.stdscr, y, x + 1, f"CONTAINER SSH TUNNEL HEALTH (jump: {SSH_OPERATOR_USER}@{SSH_JUMP_HOST}){pos_tag}", self._attr("bold"))
with self.data.lock:
jump = self.data.ssh_jump_reachable
@@ -3542,18 +3571,8 @@ class MuseTUI:
self.safe_addstr(self.stdscr, y + 2, x + 1, " ACCOUNT SSH PORT SSH STATE LAT SSH BANNER / HOSTKEY TERM PORT TERM STATE SINCE", self._attr("dim"))
self.safe_addstr(self.stdscr, y + 3, x + 1, "─" * (w - 2), self._attr("dim"))
rows = self.get_ssh_rows()
if self.ssh_sel_idx >= len(rows):
self.ssh_sel_idx = max(0, len(rows) - 1)
visible_rows = max(3, h - 6)
scroll_start = getattr(self, "ssh_scroll_idx", 0)
if self.ssh_sel_idx < scroll_start:
scroll_start = self.ssh_sel_idx
elif self.ssh_sel_idx >= scroll_start + visible_rows:
scroll_start = self.ssh_sel_idx - visible_rows + 1
scroll_start = max(0, min(scroll_start, max(0, len(rows) - visible_rows)))
self.ssh_scroll_idx = scroll_start
num_rendered = min(visible_rows, max(0, len(rows) - scroll_start))
self._ssh_view_bounds = (y + 4, y + 4 + num_rendered)
row_y = y + 4
for row_i in range(visible_rows):
@@ -3604,13 +3623,23 @@ class MuseTUI:
self.safe_addstr(self.stdscr, row_y, x + 40, banner[:26].ljust(26), row_attr if is_sel else self._attr("normal"))
self.safe_addstr(self.stdscr, row_y, x + 67, f":{tport:<8}", row_attr if is_sel else self._attr("dim"))
self.safe_addstr(self.stdscr, row_y, x + 77, term_txt, term_attr)
self.safe_addstr(self.stdscr, row_y, x + 83, since_txt[:w - 84 - 8], row_attr if is_sel else self._attr("dim"))
self.safe_addstr(self.stdscr, row_y, max(x + 90, w - 8), "[SSH]", self._attr("wo_badge") if is_sel else self._attr("dim"))
self.safe_addstr(self.stdscr, row_y, x + 83, since_txt[:w - 84 - 16], row_attr if is_sel else self._attr("dim"))
self.safe_addstr(self.stdscr, row_y, max(x + 80, w - 16), "[Diag]", self._attr("selected") if is_sel else self._attr("dim"))
self.safe_addstr(self.stdscr, row_y, max(x + 88, w - 8), "[SSH]", self._attr("wo_badge") if is_sel else self._attr("dim"))
row_y += 1
# Vertical scrollbar track
if len(rows) > visible_rows:
track_h = visible_rows
thumb_pos = int((scroll_start / max(1, len(rows) - visible_rows)) * (track_h - 1))
for r in range(track_h):
char = "█" if r == thumb_pos else "│"
attr = self._attr("selected") if r == thumb_pos else self._attr("dim")
self.safe_addstr(self.stdscr, y + 4 + r, w - 1, char, attr)
hint_y = y + h - 1
sel_acct = rows[self.ssh_sel_idx].upper() if rows else "-"
hints = f"Selected: [{sel_acct}] [Enter/s]: SSH pop-out (tmux) [c]: Copy dial [u]: Container uptime [r]: Refresh [j/k]: Nav"
hints = f"Selected: [{sel_acct}] [d/Enter]: Diagnostics [s]: SSH pop-out [c]: Copy dial [u]: Container uptime [k]: Key check [r]: Refresh [j/k]: Nav"
self.safe_addstr(self.stdscr, hint_y, x + 1, hints[:w - 2], self._attr("dim"))
# -----------------------------------------------------------------------
@@ -3923,8 +3952,8 @@ class MuseTUI:
left_x = max(1, min(anchor_x, screen_w - modal_w - 2))
self._context_popup_bounds = (left_x, top_y, modal_w, modal_h)
else:
modal_w = min(84 if self.modal in ("prompts", "history_search") else 74, screen_w - 6)
modal_h = min(22 if self.modal in ("prompts", "history_search") else 20, screen_h - 4)
modal_w = min(84 if self.modal in ("prompts", "history_search", "box_diagnostics") else 74, screen_w - 6)
modal_h = min(22 if self.modal in ("prompts", "history_search", "box_diagnostics") else 20, screen_h - 4)
top_y = (screen_h - modal_h) // 2
left_x = (screen_w - modal_w) // 2
self._context_popup_bounds = (left_x, top_y, modal_w, modal_h)
@@ -3958,6 +3987,7 @@ class MuseTUI:
"work_dispatch": " DISPATCH NEW BUILD TICKET (box work start) ",
"agent_thoughts": " LIVE COGNITIVE THOUGHT STREAM & SCREEN ",
"agent_menu": " AGENT PROFILE MENU (TASKS / TIMERS / APPROVALS) ",
"box_diagnostics": " BOX DIAGNOSTICS & TROUBLESHOOTING ",
}
title = title_map.get(self.modal, " DIALOG ")
self.safe_addstr(self.stdscr, top_y, left_x + 3, title, self._attr("header"))
@@ -3988,7 +4018,7 @@ class MuseTUI:
("[a] or [F2]", "Open Approvals Resolution Drawer (Allow, Always, Deny)"),
("[F5] or [m]", "Toggle between Muse Chat TUI and Box Fleet Command TUI"),
("[1]-[7] (Box)", "Switch Box tabs: Chat/Fleet/Approvals/Jobs/Tmux/Logs/SSH"),
("[Tab 7: SSH]", "Enter/s: tmux pop-out c: copy dial u: container uptime r: refresh"),
("[Tab 7: SSH]", "d/Enter: diag modal s: pop-out c: dial u: uptime k: keys r: refresh"),
("[g] / [G] / [Home/End]", "Jump to oldest message / follow live latest message"),
("[q]", "Quit TUI (in NORMAL mode)"),
]
@@ -4535,6 +4565,77 @@ class MuseTUI:
self.safe_addstr(self.stdscr, top_y + modal_h - 2, left_x + 3, "[1-4/Tab] Tab [w] Dispatch [f] Refocus [h] Heal [t] Thoughts [Esc/q] Close", self._attr("header"))
elif self.modal == "box_diagnostics":
rows = self.get_ssh_rows()
acct = rows[self.ssh_sel_idx] if rows and 0 <= getattr(self, "ssh_sel_idx", 0) < len(rows) else "unknown"
with self.data.lock:
jump_ok = self.data.ssh_jump_reachable
jump_lat = self.data.ssh_check_latency_ms
jump_err = self.data.ssh_check_error
health = dict(self.data.ssh_cache.get(acct, {}))
info = SSH_TUNNEL_PORTS.get(acct, {})
sport = info.get("port", "?")
tport = info.get("terminal", "?")
ssh_up = health.get("ssh_up")
term_up = health.get("term_up")
lat = health.get("ssh_latency_ms")
lat_str = f"{lat}ms" if lat is not None else "--"
banner = (health.get("ssh_banner") or health.get("term_http") or "-").strip()
state_since = health.get("state_since", 0.0)
since_str = format_recency(state_since) if state_since else "never"
dial_cmd = build_ssh_dial_string(acct) if acct in SSH_TUNNEL_PORTS else "N/A"
# Header details
self.safe_addstr(self.stdscr, top_y + 2, left_x + 3, f"BOX TARGET: @{acct.upper():<12} (SSH :{sport} | Term :{tport})", self._attr("bold"))
if jump_ok is True:
j_txt = f"ONLINE ({jump_lat}ms latency)"
j_attr = self._attr("success")
elif jump_ok is False:
j_txt = f"DOWN ({(jump_err or 'unknown')[:32]})"
j_attr = self._attr("danger")
else:
j_txt = "SWEEP PENDING"
j_attr = self._attr("dim")
self.safe_addstr(self.stdscr, top_y + 3, left_x + 3, f"Jump Gateway: {SSH_OPERATOR_USER}@{SSH_JUMP_HOST} -> ", self._attr("dim"))
self.safe_addstr(self.stdscr, top_y + 3, left_x + 38, j_txt, j_attr)
ssh_st_str = "LISTENING / UP" if ssh_up else ("UNREACHABLE / DOWN" if ssh_up is False else "UNKNOWN")
ssh_st_attr = self._attr("success") if ssh_up else (self._attr("danger") if ssh_up is False else self._attr("dim"))
self.safe_addstr(self.stdscr, top_y + 4, left_x + 3, f"SSH Tunnel: :{sport:<6} -> {ssh_st_str} ({lat_str}) | Since: {since_str}", ssh_st_attr)
self.safe_addstr(self.stdscr, top_y + 5, left_x + 3, f"SSH Banner: {banner[:modal_w - 20]}", self._attr("normal"))
term_st_str = "LISTENING / UP" if term_up else ("UNREACHABLE / DOWN" if term_up is False else "UNKNOWN")
term_st_attr = self._attr("success") if term_up else (self._attr("danger") if term_up is False else self._attr("dim"))
self.safe_addstr(self.stdscr, top_y + 6, left_x + 3, f"Web Terminal: :{tport:<6} -> {term_st_str} (ttyd)", term_st_attr)
self.safe_addstr(self.stdscr, top_y + 7, left_x + 3, f"Dial String: {dial_cmd[:modal_w - 20]}", self._attr("dim"))
self.safe_addstr(self.stdscr, top_y + 8, left_x + 3, f"Keys Spec: vm_to_gcp, muse-health, id_frontdoor, id_ed25519", self._attr("dim"))
self.safe_addstr(self.stdscr, top_y + 9, left_x + 2, "─" * (modal_w - 4), self._attr("dim"))
self.safe_addstr(self.stdscr, top_y + 10, left_x + 3, "TROUBLESHOOTING & FAST ACTIONS:", self._attr("bold"))
actions_list = [
("[1] / [s]", "SSH Pop-out", "Spawn terminal session in a new tmux window"),
("[2] / [c]", "Copy Dial String", "Copy full jump command string to system clipboard"),
("[3] / [u]", "Container Uptime", "Send remote `uptime` query to container"),
("[4] / [k]", "Verify SSH Keys", "Check ~/.ssh/ and ~/workspace/.ssh-keys/ presence"),
("[5] / [h]", "Run Recover Hook", "Execute recover-after-rebuild.sh --dry-run"),
("[r]", "Trigger Sweep", "Initiate immediate background port scan via jump host"),
]
for idx, (hk, title_act, desc) in enumerate(actions_list):
cur_act_y = top_y + 11 + idx
if cur_act_y >= top_y + modal_h - 2:
break
self.safe_addstr(self.stdscr, cur_act_y, left_x + 3, f"{hk:<10}", self._attr("wo_badge"))
self.safe_addstr(self.stdscr, cur_act_y, left_x + 14, f"{title_act:<18}", self._attr("bold"))
self.safe_addstr(self.stdscr, cur_act_y, left_x + 33, desc[:modal_w - 36], self._attr("normal"))
self.safe_addstr(self.stdscr, top_y + modal_h - 2, left_x + 3, "[1-5/s/c/u/k/h/r] Run Action [Esc/q] Close Dialog", self._attr("header"))
# -----------------------------------------------------------------------
# Keypress & Event Handling
# -----------------------------------------------------------------------
@@ -5115,8 +5216,8 @@ class MuseTUI:
# Box mode Fast Actions: Tab 6 (SSH) — placed before the 's'/'c'/'y'
# globals below so SSH keys win on this tab.
if self.mode == "box" and self.box_tab == 6:
if ch in (curses.KEY_ENTER, 10, 13):
self._ssh_popout_selected()
if ch in (curses.KEY_ENTER, 10, 13, ord('d'), ord('D')):
self.modal = "box_diagnostics"
return True
elif ch in (ord('s'), ord('S')):
self._ssh_popout_selected()
@@ -5131,6 +5232,18 @@ class MuseTUI:
threading.Thread(target=self._async_ssh_uptime, args=(acct,), daemon=True).start()
self.set_toast(f"Probing container uptime on {acct}...", "info")
return True
elif ch in (ord('k'), ord('K')):
rows = self.get_ssh_rows()
if rows and 0 <= self.ssh_sel_idx < len(rows):
acct = rows[self.ssh_sel_idx]
threading.Thread(target=self._async_ssh_key_check, args=(acct,), daemon=True).start()
return True
elif ch in (ord('h'), ord('H')):
rows = self.get_ssh_rows()
if rows and 0 <= self.ssh_sel_idx < len(rows):
acct = rows[self.ssh_sel_idx]
threading.Thread(target=self._async_ssh_recovery_probe, args=(acct,), daemon=True).start()
return True
elif ch in (ord('r'), ord('R')):
threading.Thread(target=self.data._fetch_ssh_health, daemon=True).start()
self.set_toast("Probing SSH tunnels via jump host...", "info")
@@ -7214,18 +7327,27 @@ class MuseTUI:
return True
elif self.box_tab == 6:
# Tab 6: SSH / Boxes (rows start one line lower: jump-status line)
rows = self.get_ssh_rows()
scroll_start = getattr(self, "ssh_scroll_idx", 0)
clicked_idx = scroll_start + row_idx - 1
if 0 <= clicked_idx < len(rows):
self.ssh_sel_idx = clicked_idx
if mx >= w - 8:
# [SSH] pop-out
self._ssh_popout_selected()
else:
acct = rows[clicked_idx]
self.set_toast(f"Selected [{acct.upper()}]. Click [SSH] or press Enter to pop out.", "info")
# Tab 6: SSH / Boxes
bounds = getattr(self, "_ssh_view_bounds", (content_y + 4, content_y + 4 + 10))
if bounds[0] <= my < bounds[1]:
row_offset = my - bounds[0]
rows = self.get_ssh_rows()
scroll_start = getattr(self, "ssh_scroll_idx", 0)
clicked_idx = scroll_start + row_offset
if 0 <= clicked_idx < len(rows):
self.ssh_sel_idx = clicked_idx
if mx >= w - 8:
# [SSH] pop-out
self._ssh_popout_selected()
elif mx >= w - 16:
# [Diag] modal
self.modal = "box_diagnostics"
else:
if is_double_click:
self.modal = "box_diagnostics"
else:
acct = rows[clicked_idx]
self.set_toast(f"Selected [{acct.upper()}]. Press [d/Enter] for Diagnostics or [s] for SSH.", "info")
return True
elif self.box_tab == 7:
@@ -7891,6 +8013,38 @@ class MuseTUI:
else:
self.set_toast(f"{account} uptime failed: {out_text[:90]}", "error")
def _async_ssh_key_check(self, account: str):
self.set_toast(f"Probing SSH keys on {account}...", "info")
ok, out_text = self.data.probe_container_keys(account)
if ok:
found = []
for k in ("vm_to_gcp", "muse-health", "id_frontdoor", "id_ed25519", "id_rsa"):
if k in out_text:
found.append(k)
found_str = ", ".join(found) if found else "files listed"
self.set_toast(f"✔ {account} keys verified: {found_str}", "success")
else:
self.set_toast(f"❌ {account} key probe failed: {out_text[:60]}", "error")
def _async_ssh_recovery_probe(self, account: str):
self.set_toast(f"Running dry-run recovery probe on {account}...", "info")
if account not in SSH_TUNNEL_PORTS:
self.set_toast(f"No tunnel registered for '{account}'", "warn")
return
cmd = build_ssh_dial_command(
account,
ssh_options=["-o", "BatchMode=yes", "-o", "ConnectTimeout=15"],
remote_command="if [ -x ~/workspace/bin/recover-after-rebuild.sh ]; then ~/workspace/bin/recover-after-rebuild.sh --dry-run; else echo 'recover script not found'; fi",
)
rc, stdout, stderr = run_command_isolated(cmd, timeout=30.0)
if rc == 0:
lines = [l.strip() for l in (stdout or "").splitlines() if l.strip()]
summary = lines[-1] if lines else "Recovery probe completed."
self.set_toast(f"✔ {account} probe: {summary[:60]}", "success")
else:
err = (stderr or stdout or f"exit {rc}").strip().splitlines()
self.set_toast(f"❌ {account} recovery failed: {err[-1][:60] if err else 'error'}", "error")
def _async_dispatch_work(self, goal: str, target_node: str):
"""Dispatch a new work build ticket with 45s synchronous readback gate."""
self.set_toast(f"⚡ Dispatching work ticket to @{target_node.upper()}...", "info")
@@ -8896,6 +9050,37 @@ class MuseTUI:
self.modal_input_cursor += 1
return True
elif self.modal == "box_diagnostics":
rows = self.get_ssh_rows()
acct = rows[self.ssh_sel_idx] if rows and 0 <= getattr(self, "ssh_sel_idx", 0) < len(rows) else ""
if ch in (27, ord('q'), ord('Q')):
self.modal = None
return True
elif ch in (ord('1'), ord('s'), ord('S')):
self._ssh_popout_selected()
return True
elif ch in (ord('2'), ord('c'), ord('C')):
self._ssh_copy_dial_selected()
return True
elif ch in (ord('3'), ord('u'), ord('U')):
if acct:
threading.Thread(target=self._async_ssh_uptime, args=(acct,), daemon=True).start()
self.set_toast(f"Probing container uptime on {acct}...", "info")
return True
elif ch in (ord('4'), ord('k'), ord('K')):
if acct:
threading.Thread(target=self._async_ssh_key_check, args=(acct,), daemon=True).start()
return True
elif ch in (ord('5'), ord('h'), ord('H')):
if acct:
threading.Thread(target=self._async_ssh_recovery_probe, args=(acct,), daemon=True).start()
return True
elif ch in (ord('r'), ord('R')):
threading.Thread(target=self.data._fetch_ssh_health, daemon=True).start()
self.set_toast("Probing SSH tunnels via jump host...", "info")
return True
return True