super/box
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(watchdog): eliminate SIGPIPE+pipefail phantom failures in health checks

Browse Source 
echo "$list" | grep -q under set -o pipefail exits 141 whenever grep
matches before echo finishes writing, so healthy browsers were reported
'CDP up but no page target' and killed every 2 min fleet-wide (load 19+).
Use [[ == *glob* ]] (no pipe, no race) for the page/muse.ai stages, and
grep -c (reads to EOF, never early-exits) for the netns check.
This commit is contained in:
operator-main
2026-10-06 07:28:20 +00:00
parent 7444b52763
commit 345eb09559
2 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/chromebox-watchdog.sh
+2 -2
View File
@@ -64,9 +64,9 @@ healthy() {
local list local list
list="$(cdp_list)" \ list="$(cdp_list)" \
|| { HEALTH_FAIL_REASON="CDP unreachable on :$CDP_PORT"; return 1; } || { HEALTH_FAIL_REASON="CDP unreachable on :$CDP_PORT"; return 1; }
echo "$list" | grep -q '"type": "page"' \ [[ "$list" == *'"type": "page"'* ]] \
|| { HEALTH_FAIL_REASON="CDP up but no page target in list"; return 1; } || { HEALTH_FAIL_REASON="CDP up but no page target in list"; return 1; }
echo "$list" | grep -E -q '"url": "https://muse\.ai' \ [[ "$list" == *'"url": "https://muse.ai'* ]] \
|| { HEALTH_FAIL_REASON="CDP up but not on muse.ai"; return 1; } || { HEALTH_FAIL_REASON="CDP up but not on muse.ai"; return 1; }
warp_egress_healthy || return 1 warp_egress_healthy || return 1
return 0 return 0
bin/netvm-exec.sh
+1 -1
View File
@@ -9,5 +9,5 @@ NODE="${1:?usage: netvm-exec.sh <node> -- <cmd> [args...]}"; shift
[ "${1:-}" = "--" ] && shift [ "${1:-}" = "--" ] && shift
[ $# -gt 0 ] || { echo "usage: netvm-exec.sh <node> -- <cmd> [args...]"; exit 1; } [ $# -gt 0 ] || { echo "usage: netvm-exec.sh <node> -- <cmd> [args...]"; exit 1; }
[ -f "/etc/netvm/${NODE}.conf" ] || { echo "no warp identity for '$NODE' (human: netvm-new-identity.sh $NODE)"; exit 1; } [ -f "/etc/netvm/${NODE}.conf" ] || { echo "no warp identity for '$NODE' (human: netvm-new-identity.sh $NODE)"; exit 1; }
ip netns list 2>/dev/null | grep -q "^warp-${NODE}" || { echo "node '$NODE' is not up (netvm-node-up.sh $NODE)"; exit 1; } [ "$(ip netns list 2>/dev/null | grep -c "^warp-${NODE}")" -ge 1 ] || { echo "node '$NODE' is not up (netvm-node-up.sh $NODE)"; exit 1; }
exec sudo -n "$NETVM_BIN/netvm-enter.sh" "$NODE" "$(id -u)" "$(id -g)" "$HOME" -- "$@" exec sudo -n "$NETVM_BIN/netvm-enter.sh" "$NODE" "$(id -u)" "$(id -g)" "$HOME" -- "$@"