fix(test): kill timeout-resistant unit shards (#3631)

Co-Authored-By: Vyacheslav Zakharov <vyacheslav.zakharov@avers.kz>

scripts/run-unit-parallel.sh

@@ -13,7 +13,8 @@
 #
 # Env overrides:
 #   SHARDS=N                     same as --shards
-#   GBRAIN_TEST_SHARD_TIMEOUT    per-shard wallclock cap, seconds (default 600)
+#   GBRAIN_TEST_SHARD_TIMEOUT    per-shard wallclock cap, seconds (default 1500)
+#   GBRAIN_TEST_SHARD_KILL_AFTER grace after TERM before KILL (default 30)
 #   GBRAIN_TEST_MAX_CONCURRENCY  passed through to bun test (default 4)
 #
 # Output files (workspace-local; falls back to /tmp if .context/ unwritable):
@@ -79,6 +80,10 @@ INTRA_CONC="${MAX_CONCURRENCY_OVERRIDE:-${GBRAIN_TEST_MAX_CONCURRENCY:-4}}"
 # 4-shard wallclock; real hangs still hit it. Override via
 # GBRAIN_TEST_SHARD_TIMEOUT=N.
 SHARD_TIMEOUT="${GBRAIN_TEST_SHARD_TIMEOUT:-1500}"
+SHARD_KILL_AFTER="${GBRAIN_TEST_SHARD_KILL_AFTER:-30}"
+if ! printf '%s' "$SHARD_KILL_AFTER" | grep -qE '^[0-9]+$' || [ "$SHARD_KILL_AFTER" -lt 1 ]; then
+  echo "ERROR: invalid shard kill-after: $SHARD_KILL_AFTER" >&2; exit 2
+fi
 
 # ──────────────────────────────────────────────────────────────────────────
 # Output directories. Prefer workspace-local .context/, fall back to /tmp.
@@ -109,7 +114,7 @@ elif command -v timeout >/dev/null 2>&1; then TIMEOUT_BIN="timeout"
 fi
 
 START_TS=$(date +%s)
-echo "[unit-parallel] N=$N shards | --max-concurrency=$INTRA_CONC | timeout=${SHARD_TIMEOUT}s | logs=$LOG_DIR" >&2
+echo "[unit-parallel] N=$N shards | --max-concurrency=$INTRA_CONC | timeout=${SHARD_TIMEOUT}s | kill-after=${SHARD_KILL_AFTER}s | logs=$LOG_DIR" >&2
 
 if [ "$DRY_RUN" = "1" ]; then
   echo "[unit-parallel] dry-run: would spawn $N shards with the above settings."
@@ -129,7 +134,7 @@ for i in $(seq 1 "$N"); do
   (
     SHARD_LOG="$LOG_DIR/shard-$i.log"
     if [ -n "$TIMEOUT_BIN" ]; then
-      "$TIMEOUT_BIN" "${SHARD_TIMEOUT}s" \
+      "$TIMEOUT_BIN" --signal=TERM --kill-after="${SHARD_KILL_AFTER}s" "${SHARD_TIMEOUT}s" \
         env SHARD="$i/$N" \
         bash scripts/run-unit-shard.sh --max-concurrency="$INTRA_CONC" \
         > "$SHARD_LOG" 2>&1
@@ -140,7 +145,7 @@ for i in $(seq 1 "$N"); do
         > "$SHARD_LOG" 2>&1 &
       pid=$!
       ( sleep "$SHARD_TIMEOUT" && kill -TERM "$pid" 2>/dev/null && \
-        sleep 5 && kill -KILL "$pid" 2>/dev/null ) &
+        sleep "$SHARD_KILL_AFTER" && kill -KILL "$pid" 2>/dev/null ) &
       cap_pid=$!
       wait "$pid" 2>/dev/null
       # Capture the shard's exit code from ITS `wait`, before any watchdog
@@ -158,7 +163,7 @@ for i in $(seq 1 "$N"); do
       wait "$cap_pid" 2>/dev/null
     fi
     echo "$rc" > "$LOG_DIR/shard-$i.exit"
-    [ "$rc" = "124" ] && echo "WEDGED" > "$LOG_DIR/shard-$i.wedged"
+    { [ "$rc" = "124" ] || [ "$rc" = "137" ]; } && echo "WEDGED" > "$LOG_DIR/shard-$i.wedged"
   ) &
   SHARD_PIDS+=($!)
 done

test/scripts/run-unit-parallel.test.ts

@@ -14,10 +14,9 @@
  * containing one passing and one failing test, override the discovery
  * roots via env-vars, and run with --shards=2.
  *
- * NOT covered here: the heartbeat (timing-sensitive, not load-bearing
- * for correctness) and timeout / WEDGED markers (require synthesizing a
- * hung test which is fragile across machines). Those rely on the live
- * smoke tests captured in CHANGELOG measurements.
+ * NOT covered behaviorally here: the heartbeat and a real hung Bun process
+ * (both timing-sensitive). The timeout escalation wiring is covered as a
+ * source contract below and exercised separately by a process-leak smoke.
  */
 
 import { describe, it, expect, beforeAll, afterAll } from 'bun:test';
@@ -155,6 +154,20 @@ describe('failing-on-purpose', () => {
   });
 });
 
+describe('run-unit-parallel.sh timeout escalation contract', () => {
+  it('gives a timed-out shard 30 seconds after TERM, then forces KILL', () => {
+    const source = readFileSync(PARALLEL_SH_SRC, 'utf-8');
+    expect(source).toContain('SHARD_KILL_AFTER="${GBRAIN_TEST_SHARD_KILL_AFTER:-30}"');
+    expect(source).toContain('--signal=TERM --kill-after="${SHARD_KILL_AFTER}s"');
+    expect(source).toContain('sleep "$SHARD_KILL_AFTER" && kill -KILL "$pid"');
+  });
+
+  it('marks both ordinary timeout and forced-KILL timeout exits as wedged', () => {
+    const source = readFileSync(PARALLEL_SH_SRC, 'utf-8');
+    expect(source).toContain('[ "$rc" = "124" ] || [ "$rc" = "137" ]');
+  });
+});
+
 describe('run-unit-parallel.sh no-timeout-binary fallback (rc from shard wait, not watchdog teardown)', () => {
   // Forces the no-gtimeout/no-timeout branch by running the wrapper under a
   // curated PATH that has every tool the scripts call EXCEPT timeout