61 lines
2.0 KiB
Bash
61 lines
2.0 KiB
Bash
|
#!/bin/bash
|
||
|
|
# verify-node-ssh.sh — verify container SSH dial-in readiness across fleet nodes.
|
||
|
|
# Checks from the VM: reverse-tunnel listeners + SSH auth for each node port.
|
||
|
|
#
|
||
|
|
# Port map (docs/OPERATOR-DRIVE-RUNBOOK.md):
|
||
|
|
# muse-main 2224 | muse 2225 | 646 2226 | pip 2227 | opm 2228 | def 2229 | dev 2230
|
||
|
|
#
|
||
|
|
# What it checks per node:
|
||
|
|
# 1. Reverse-tunnel listener on 127.0.0.1:<port> (dark node = no listener)
|
||
|
|
# 2. SSH dial-in with BatchMode (auth failure = authorized_keys perms/key issue)
|
||
|
|
#
|
||
|
|
# Common root causes (see #211):
|
||
|
|
# - sshd requires non-group-writable authorized_keys (must be 600)
|
||
|
|
# - stale /run/nologin blocks logins
|
||
|
|
# - missing id_frontdoor keys on dark nodes
|
||
|
|
#
|
||
|
|
# Usage: run on the VM (super@34.139.37.135), or via:
|
||
|
|
# ssh-vm.sh "bash -s" < verify-node-ssh.sh
|
||
|
|
set -u
|
||
|
|
|
||
|
|
# node:port pairs to check
|
||
|
|
NODES="muse:2225 646:2226 pip:2227 def:2229 dev:2230 muse-main:2224 opm:2228"
|
||
|
|
|
||
|
|
fail=0
|
||
|
|
for pair in $NODES; do
|
||
|
|
node="${pair%%:*}"
|
||
|
|
port="${pair##*:}"
|
||
|
|
|
||
|
|
# 1. listener check
|
||
|
|
if ss -tln 2>/dev/null | grep -q "127.0.0.1:${port} "; then
|
||
|
|
listener="LISTEN"
|
||
|
|
else
|
||
|
|
listener="DARK (no listener)"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# 2. auth check (only if listening)
|
||
|
|
if [ "$listener" = "LISTEN" ]; then
|
||
|
|
out=$(timeout 15 ssh -o StrictHostKeyChecking=no -o BatchMode=yes \
|
||
|
|
-o ConnectTimeout=10 -p "$port" hatch@127.0.0.1 'echo OK' 2>&1)
|
||
|
|
case "$out" in
|
||
|
|
OK) auth="OK" ;;
|
||
|
|
*"Permission denied"*) auth="AUTH-FAIL (check authorized_keys perms/keys)" ;;
|
||
|
|
*"Connection refused"*) auth="REFUSED (tunnel died after listen check)" ;;
|
||
|
|
*) auth="OTHER: $(echo "$out" | head -1 | cut -c1-60)" ;;
|
||
|
|
esac
|
||
|
|
else
|
||
|
|
auth="SKIP"
|
||
|
|
fi
|
||
|
|
|
||
|
|
printf '%-10s port %-5s listener: %-22s auth: %s\n' "$node" "$port" "$listener" "$auth"
|
||
|
|
[ "$listener" = "DARK (no listener)" ] && fail=1
|
||
|
|
case "$auth" in AUTH-FAIL*) fail=1 ;; esac
|
||
|
|
done
|
||
|
|
|
||
|
|
if [ "$fail" -eq 0 ]; then
|
||
|
|
echo "ALL NODES REACHABLE"
|
||
|
|
else
|
||
|
|
echo "ISSUES FOUND (see above)"
|
||
|
|
fi
|
||
|
|
exit "$fail"
|