From c0113c1ebf3879bae18a491d6dfaef955e587884 Mon Sep 17 00:00:00 2001
From: opm <opm@muse-dev.online>
Date: Fri, 9 Oct 2026 21:56:08 +0000
Subject: [PATCH] feat(ssh): add node SSH dial-in verification script

Adds bin/verify-node-ssh.sh: checks reverse-tunnel listeners and SSH
auth for each fleet node port from the VM. Distinguishes dark nodes
(no listener) from auth failures (authorized_keys perms/keys).

Verification 2026-10-09:
- muse/2225, 646/2226, pip/2227, muse-main/2224, opm/2228: LISTEN
- def/2229, dev/2230: DARK (no reverse tunnel)
- All listening nodes reject VM super key (expected: nodes authorize
  per-operator/id_frontdoor keys, not the VM super key)

Fixes #211
---
 bin/verify-node-ssh.sh | 60 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100755 bin/verify-node-ssh.sh

diff --git a/bin/verify-node-ssh.sh b/bin/verify-node-ssh.sh
new file mode 100755
index 0000000..8ccb74d
--- /dev/null
+++ b/bin/verify-node-ssh.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+# verify-node-ssh.sh — verify container SSH dial-in readiness across fleet nodes.
+# Checks from the VM: reverse-tunnel listeners + SSH auth for each node port.
+#
+# Port map (docs/OPERATOR-DRIVE-RUNBOOK.md):
+#   muse-main 2224 | muse 2225 | 646 2226 | pip 2227 | opm 2228 | def 2229 | dev 2230
+#
+# What it checks per node:
+#   1. Reverse-tunnel listener on 127.0.0.1:<port> (dark node = no listener)
+#   2. SSH dial-in with BatchMode (auth failure = authorized_keys perms/key issue)
+#
+# Common root causes (see #211):
+#   - sshd requires non-group-writable authorized_keys (must be 600)
+#   - stale /run/nologin blocks logins
+#   - missing id_frontdoor keys on dark nodes
+#
+# Usage: run on the VM (super@34.139.37.135), or via:
+#   ssh-vm.sh "bash -s" < verify-node-ssh.sh
+set -u
+
+# node:port pairs to check
+NODES="muse:2225 646:2226 pip:2227 def:2229 dev:2230 muse-main:2224 opm:2228"
+
+fail=0
+for pair in $NODES; do
+    node="${pair%%:*}"
+    port="${pair##*:}"
+
+    # 1. listener check
+    if ss -tln 2>/dev/null | grep -q "127.0.0.1:${port} "; then
+        listener="LISTEN"
+    else
+        listener="DARK (no listener)"
+    fi
+
+    # 2. auth check (only if listening)
+    if [ "$listener" = "LISTEN" ]; then
+        out=$(timeout 15 ssh -o StrictHostKeyChecking=no -o BatchMode=yes \
+            -o ConnectTimeout=10 -p "$port" hatch@127.0.0.1 'echo OK' 2>&1)
+        case "$out" in
+            OK)            auth="OK" ;;
+            *"Permission denied"*) auth="AUTH-FAIL (check authorized_keys perms/keys)" ;;
+            *"Connection refused"*) auth="REFUSED (tunnel died after listen check)" ;;
+            *)             auth="OTHER: $(echo "$out" | head -1 | cut -c1-60)" ;;
+        esac
+    else
+        auth="SKIP"
+    fi
+
+    printf '%-10s port %-5s listener: %-22s auth: %s\n' "$node" "$port" "$listener" "$auth"
+    [ "$listener" = "DARK (no listener)" ] && fail=1
+    case "$auth" in AUTH-FAIL*) fail=1 ;; esac
+done
+
+if [ "$fail" -eq 0 ]; then
+    echo "ALL NODES REACHABLE"
+else
+    echo "ISSUES FOUND (see above)"
+fi
+exit "$fail"
-- 
2.54.0