NetVM: veth+NAT plumbing for isolated netns (endpoint bypass, no routing loop)

bin/netvm-node-down.sh

@@ -1,8 +1,14 @@
 #!/usr/bin/env bash
-# Tear down a node's Warp egress. Run as root.
+# netvm-node-down.sh <node> — tear down a node's Warp egress. Run as root.
 set -euo pipefail
 NODE="${1:?usage: netvm-node-down.sh <node>}"
 NETNS="warp-${NODE}"
-ip netns exec "$NETNS" ip link set "wg-${NODE}" down 2>/dev/null || true
+VETH="veth-${NODE}"
+IDX=$(( $(echo -n "$NODE" | cksum | cut -d' ' -f1) % 60 + 10 ))
+SUB="10.201.${IDX}.0/30"
+nsexec() { ip netns exec "$NETNS" "$@"; }
+nsexec ip link set "wg-${NODE}" down 2>/dev/null || true
+ip link del "$VETH" 2>/dev/null || true   # also drops the netns-side peer
+iptables -t nat -D POSTROUTING -s "$SUB" -j MASQUERADE 2>/dev/null || true
 ip netns del "$NETNS" 2>/dev/null || true
 echo "node=$NODE down"

bin/netvm-node-up.sh

@@ -1,23 +1,79 @@
 #!/usr/bin/env bash
-# Bring up a node's Warp egress. Run as root.
+# netvm-node-up.sh <node> — bring up a node's Warp egress in its own netns.
-# The WireGuard config at /etc/netvm/<node>.conf is human-generated
+# Run as root (operator: sudo -n via the allowlist).
-# (a credential). This script manages lifecycle only — it never creates
+#
-# or copies identities.
+# Per-node layout:
+#   netns warp-<node>; veth pair veth-<node> <-> vpeer-<node> (10.201.X.0/30)
+#   with host NAT; WireGuard handshake packets route via the veth gateway
+#   (bypassing the tunnel — else they'd loop into it); everything else
+#   defaults through the tunnel.
 set -euo pipefail
 NODE="${1:?usage: netvm-node-up.sh <node>}"
 NETNS="warp-${NODE}"
+WG="wg-${NODE}"
+VETH="veth-${NODE}"
+VPEER="vpeer-${NODE}"
 CONF="/etc/netvm/${NODE}.conf"
-[ -f "$CONF" ] || { echo "missing $CONF — human generates it once (wgcf), root-owned 0600"; exit 1; }
+[ -f "$CONF" ] || { echo "missing $CONF (human: netvm-new-identity.sh $NODE)"; exit 1; }
 chmod 600 "$CONF"
+
+# deterministic /30 per node for the veth link
+IDX=$(( $(echo -n "$NODE" | cksum | cut -d' ' -f1) % 60 + 10 ))
+GW="10.201.${IDX}.1"; PEER_IP="10.201.${IDX}.2"; SUB="10.201.${IDX}.0/30"
+
+nsexec() { ip netns exec "$NETNS" "$@"; }
+
 ip netns add "$NETNS" 2>/dev/null || true
-if ! ip netns exec "$NETNS" ip link show "wg-${NODE}" >/dev/null 2>&1; then
+
-  ip link add "wg-${NODE}" type wireguard 2>/dev/null || true
+# veth pair host <-> netns
-  ip link set "wg-${NODE}" netns "$NETNS"
+if ! nsexec ip link show "$VPEER" >/dev/null 2>&1; then
+  ip link del "$VETH" 2>/dev/null || true
+  ip link add "$VETH" type veth peer name "$VPEER"
+  ip link set "$VPEER" netns "$NETNS"
+  ip addr add "${GW}/30" dev "$VETH" 2>/dev/null || true
+  ip link set "$VETH" up
+  nsexec ip addr add "${PEER_IP}/30" dev "$VPEER" 2>/dev/null || true
+  nsexec ip link set "$VPEER" up
 fi
-ip netns exec "$NETNS" wg setconf "wg-${NODE}" "$CONF"
+nsexec ip link set lo up
-ip netns exec "$NETNS" ip link set lo up
+
-ip netns exec "$NETNS" ip link set "wg-${NODE}" up
+# host NAT + forwarding for the veth subnet
-# NOTE: addresses/routes come from the generated config (wgcf carries them).
+sysctl -qw net.ipv4.ip_forward=1
-EGRESS=$(ip netns exec "$NETNS" curl -s --max-time 15 ifconfig.me || true)
+iptables -t nat -C POSTROUTING -s "$SUB" -j MASQUERADE 2>/dev/null || \
-echo "node=$NODE netns=$NETNS egress=${EGRESS:-UNREACHABLE}"
+  iptables -t nat -A POSTROUTING -s "$SUB" -j MASQUERADE
-[ -n "$EGRESS" ] || { echo "egress check failed"; exit 1; }
+
+# endpoint bypasses the tunnel (else the handshake routes into itself)
+ENDPOINT=$(grep -oP '^\s*Endpoint\s*=\s*\K[^:;#]+' "$CONF" | head -1)
+for eip in $(getent ahostsv4 "$ENDPOINT" | awk '{print $1}' | sort -u); do
+  nsexec ip route replace "$eip" via "$GW"
+done
+
+# wireguard interface (wg setconf rejects wg-quick extensions: strip them)
+if ! nsexec ip link show "$WG" >/dev/null 2>&1; then
+  ip link add "$WG" type wireguard
+  ip link set "$WG" netns "$NETNS"
+fi
+STRIPPED=$(mktemp)
+grep -vE '^\s*(Address|DNS)\s*=' "$CONF" > "$STRIPPED"
+nsexec wg setconf "$WG" "$STRIPPED"
+rm -f "$STRIPPED"
+MTU=$(grep -oP '^\s*MTU\s*=\s*\K\d+' "$CONF" | head -1); MTU=${MTU:-1280}
+nsexec ip link set "$WG" mtu "$MTU"
+for a in $(grep -oP '^\s*Address\s*=\s*\K\S+' "$CONF" | tr ',' ' '); do
+  if [[ "$a" == *:* ]]; then nsexec ip -6 addr add "$a" dev "$WG" 2>/dev/null || true
+  else nsexec ip addr add "$a" dev "$WG" 2>/dev/null || true; fi
+done
+nsexec ip link set "$WG" up
+nsexec ip route replace default dev "$WG"
+nsexec ip -6 route replace default dev "$WG" 2>/dev/null || true
+
+# verify: handshake = tunnel up (definitive); egress IP best-effort (no DNS in netns)
+sleep 3
+HS=$(nsexec wg show "$WG" latest-handshakes | awk '{print $2}')
+if [ -z "$HS" ] || [ "$HS" = "0" ]; then
+  echo "no handshake yet (endpoint=$ENDPOINT) — may still be negotiating"
+else
+  echo "handshake ok"
+fi
+EGRESS=$(nsexec curl -sk --max-time 15 'https://[2606:4700:4700::1111]/cdn-cgi/trace' 2>/dev/null | grep -oP '^ip=\K.*' || true)
+echo "node=$NODE netns=$NETNS veth=$SUB egress=${EGRESS:-unknown}"