From e3806cf46f925a6e9f699ac5103e0f2e8029e864 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 1 Aug 2026 05:40:31 +0800 Subject: [PATCH] fix(test): kill timeout-resistant unit shards (#3631) Co-Authored-By: Vyacheslav Zakharov --- scripts/run-unit-parallel.sh | 15 ++++++++++----- test/scripts/run-unit-parallel.test.ts | 21 +++++++++++++++++---- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/scripts/run-unit-parallel.sh b/scripts/run-unit-parallel.sh index fb6deeade..b65210938 100755 --- a/scripts/run-unit-parallel.sh +++ b/scripts/run-unit-parallel.sh @@ -13,7 +13,8 @@ # # Env overrides: # SHARDS=N same as --shards -# GBRAIN_TEST_SHARD_TIMEOUT per-shard wallclock cap, seconds (default 600) +# GBRAIN_TEST_SHARD_TIMEOUT per-shard wallclock cap, seconds (default 1500) +# GBRAIN_TEST_SHARD_KILL_AFTER grace after TERM before KILL (default 30) # GBRAIN_TEST_MAX_CONCURRENCY passed through to bun test (default 4) # # Output files (workspace-local; falls back to /tmp if .context/ unwritable): @@ -79,6 +80,10 @@ INTRA_CONC="${MAX_CONCURRENCY_OVERRIDE:-${GBRAIN_TEST_MAX_CONCURRENCY:-4}}" # 4-shard wallclock; real hangs still hit it. Override via # GBRAIN_TEST_SHARD_TIMEOUT=N. SHARD_TIMEOUT="${GBRAIN_TEST_SHARD_TIMEOUT:-1500}" +SHARD_KILL_AFTER="${GBRAIN_TEST_SHARD_KILL_AFTER:-30}" +if ! printf '%s' "$SHARD_KILL_AFTER" | grep -qE '^[0-9]+$' || [ "$SHARD_KILL_AFTER" -lt 1 ]; then + echo "ERROR: invalid shard kill-after: $SHARD_KILL_AFTER" >&2; exit 2 +fi # ────────────────────────────────────────────────────────────────────────── # Output directories. Prefer workspace-local .context/, fall back to /tmp. @@ -109,7 +114,7 @@ elif command -v timeout >/dev/null 2>&1; then TIMEOUT_BIN="timeout" fi START_TS=$(date +%s) -echo "[unit-parallel] N=$N shards | --max-concurrency=$INTRA_CONC | timeout=${SHARD_TIMEOUT}s | logs=$LOG_DIR" >&2 +echo "[unit-parallel] N=$N shards | --max-concurrency=$INTRA_CONC | timeout=${SHARD_TIMEOUT}s | kill-after=${SHARD_KILL_AFTER}s | logs=$LOG_DIR" >&2 if [ "$DRY_RUN" = "1" ]; then echo "[unit-parallel] dry-run: would spawn $N shards with the above settings." @@ -129,7 +134,7 @@ for i in $(seq 1 "$N"); do ( SHARD_LOG="$LOG_DIR/shard-$i.log" if [ -n "$TIMEOUT_BIN" ]; then - "$TIMEOUT_BIN" "${SHARD_TIMEOUT}s" \ + "$TIMEOUT_BIN" --signal=TERM --kill-after="${SHARD_KILL_AFTER}s" "${SHARD_TIMEOUT}s" \ env SHARD="$i/$N" \ bash scripts/run-unit-shard.sh --max-concurrency="$INTRA_CONC" \ > "$SHARD_LOG" 2>&1 @@ -140,7 +145,7 @@ for i in $(seq 1 "$N"); do > "$SHARD_LOG" 2>&1 & pid=$! ( sleep "$SHARD_TIMEOUT" && kill -TERM "$pid" 2>/dev/null && \ - sleep 5 && kill -KILL "$pid" 2>/dev/null ) & + sleep "$SHARD_KILL_AFTER" && kill -KILL "$pid" 2>/dev/null ) & cap_pid=$! wait "$pid" 2>/dev/null # Capture the shard's exit code from ITS `wait`, before any watchdog @@ -158,7 +163,7 @@ for i in $(seq 1 "$N"); do wait "$cap_pid" 2>/dev/null fi echo "$rc" > "$LOG_DIR/shard-$i.exit" - [ "$rc" = "124" ] && echo "WEDGED" > "$LOG_DIR/shard-$i.wedged" + { [ "$rc" = "124" ] || [ "$rc" = "137" ]; } && echo "WEDGED" > "$LOG_DIR/shard-$i.wedged" ) & SHARD_PIDS+=($!) done diff --git a/test/scripts/run-unit-parallel.test.ts b/test/scripts/run-unit-parallel.test.ts index 4227ba655..3656764a4 100644 --- a/test/scripts/run-unit-parallel.test.ts +++ b/test/scripts/run-unit-parallel.test.ts @@ -14,10 +14,9 @@ * containing one passing and one failing test, override the discovery * roots via env-vars, and run with --shards=2. * - * NOT covered here: the heartbeat (timing-sensitive, not load-bearing - * for correctness) and timeout / WEDGED markers (require synthesizing a - * hung test which is fragile across machines). Those rely on the live - * smoke tests captured in CHANGELOG measurements. + * NOT covered behaviorally here: the heartbeat and a real hung Bun process + * (both timing-sensitive). The timeout escalation wiring is covered as a + * source contract below and exercised separately by a process-leak smoke. */ import { describe, it, expect, beforeAll, afterAll } from 'bun:test'; @@ -155,6 +154,20 @@ describe('failing-on-purpose', () => { }); }); +describe('run-unit-parallel.sh timeout escalation contract', () => { + it('gives a timed-out shard 30 seconds after TERM, then forces KILL', () => { + const source = readFileSync(PARALLEL_SH_SRC, 'utf-8'); + expect(source).toContain('SHARD_KILL_AFTER="${GBRAIN_TEST_SHARD_KILL_AFTER:-30}"'); + expect(source).toContain('--signal=TERM --kill-after="${SHARD_KILL_AFTER}s"'); + expect(source).toContain('sleep "$SHARD_KILL_AFTER" && kill -KILL "$pid"'); + }); + + it('marks both ordinary timeout and forced-KILL timeout exits as wedged', () => { + const source = readFileSync(PARALLEL_SH_SRC, 'utf-8'); + expect(source).toContain('[ "$rc" = "124" ] || [ "$rc" = "137" ]'); + }); +}); + describe('run-unit-parallel.sh no-timeout-binary fallback (rc from shard wait, not watchdog teardown)', () => { // Forces the no-gtimeout/no-timeout branch by running the wrapper under a // curated PATH that has every tool the scripts call EXCEPT timeout